superlocalmemory 4.0.5 → 4.0.6

package/src/superlocalmemory/code_graph/parser.py

@@ -31,6 +31,7 @@ from superlocalmemory.code_graph.models import (
     NodeKind,
     ParseResult,
 )
+from superlocalmemory.code_graph.resolver import ImportResolver
 from superlocalmemory.storage.models import _new_id
 
 logger = logging.getLogger(__name__)
@@ -65,6 +66,140 @@ def _sha256(data: bytes) -> str:
     return hashlib.sha256(data).hexdigest()
 
 
+def _assign_edge_sources(
+    nodes: list[GraphNode],
+    edges: list[GraphEdge],
+) -> list[GraphEdge]:
+    """Replace ``__unresolved__`` source_node_ids on CALLS edges.
+
+    Locates the innermost function/method whose line-range contains the call
+    site.  Falls back to the file-level node when no function wraps the call
+    (e.g. top-level script code).  IMPORTS and other non-CALLS edges whose
+    source is still ``__unresolved__`` are dropped — they cannot be resolved
+    without storing the module path in the edge, which is a separate defect.
+
+    Returns a new list of edges.
+    """
+    # ── build per-file line-range index ────────────────────────────────
+    # {file_path: [(line_start, line_end, node_id), ...]}
+    # Sorted by range size ascending so the NARROWEST (innermost) match wins.
+    file_ranges: dict[str, list[tuple[int, int, str]]] = {}
+    file_node_ids: dict[str, str] = {}      # file_path → FILE node_id
+
+    for node in nodes:
+        if node.kind == NodeKind.FILE:
+            file_node_ids[node.file_path] = node.node_id
+        elif node.kind in (NodeKind.FUNCTION, NodeKind.METHOD):
+            fp = node.file_path
+            file_ranges.setdefault(fp, []).append(
+                (node.line_start, node.line_end, node.node_id)
+            )
+
+    for fp in file_ranges:
+        file_ranges[fp].sort(key=lambda t: t[1] - t[0])   # narrowest first
+
+    resolved: list[GraphEdge] = []
+    dropped = 0
+
+    for edge in edges:
+        if edge.source_node_id != "__unresolved__":
+            resolved.append(edge)
+            continue
+
+        if edge.kind != EdgeKind.CALLS:
+            # IMPORTS and other edges: source unknown — drop.
+            # Root cause: extract_imports() does not store the module path in
+            # extra_json, so targets are also unresolvable.  Tracked as a
+            # separate pre-existing defect; fixing it here would require an
+            # extractor change beyond this PR scope.
+            dropped += 1
+            continue
+
+        # ── find innermost enclosing function ──────────────────────────
+        enclosing_id: str | None = None
+        for ls, le, nid in file_ranges.get(edge.file_path, []):
+            if ls <= edge.line <= le:
+                enclosing_id = nid
+                break                      # narrowest first → stop at first hit
+
+        # ── fallback: top-level call, use FILE node ─────────────────────
+        if enclosing_id is None:
+            enclosing_id = file_node_ids.get(edge.file_path)
+
+        if enclosing_id is None:
+            logger.debug(
+                "_assign_edge_sources: no source node for call at %s:%d — dropping",
+                edge.file_path, edge.line,
+            )
+            dropped += 1
+            continue
+
+        resolved.append(GraphEdge(
+            edge_id=edge.edge_id,
+            kind=edge.kind,
+            source_node_id=enclosing_id,
+            target_node_id=edge.target_node_id,
+            file_path=edge.file_path,
+            line=edge.line,
+            confidence=edge.confidence,
+            extra_json=edge.extra_json,
+            created_at=edge.created_at,
+            updated_at=edge.updated_at,
+        ))
+
+    if dropped:
+        logger.debug(
+            "_assign_edge_sources: dropped %d edge(s) with unresolvable source",
+            dropped,
+        )
+
+    return resolved
+
+
+def _clean_and_resolve_edges(
+    nodes: list[GraphNode],
+    edges: list[GraphEdge],
+    import_maps_by_file: dict[str, dict[str, tuple[str, str]]],
+    repo_root: Path,
+    config: CodeGraphConfig,
+) -> list[GraphEdge]:
+    """Full resolution pipeline applied after parse_all collects all nodes.
+
+    1. Assign real source_node_ids to CALLS edges (line-range lookup).
+    2. Drop IMPORTS edges whose endpoints are still ``__unresolved__``.
+    3. Run ImportResolver.resolve_call_targets to replace ``__call__<name>``
+       targets with real node_ids or drop the edge (external calls).
+    4. Final defensive pass: drop any edge whose endpoints are still absent
+       from the full node set (belt-and-braces, should be a no-op after 1-3).
+
+    Returns a clean edge list safe to hand to GraphStore.
+    """
+    # Step 1: fix sources
+    edges = _assign_edge_sources(nodes, edges)
+
+    # Step 2: resolve targets
+    resolver = ImportResolver(repo_root, config)
+    edges = resolver.resolve_call_targets(nodes, edges, import_maps_by_file)
+
+    # Step 3: final drop of any remaining stragglers
+    valid_ids = {n.node_id for n in nodes}
+    clean: list[GraphEdge] = []
+    stray = 0
+    for e in edges:
+        if e.source_node_id in valid_ids and e.target_node_id in valid_ids:
+            clean.append(e)
+        else:
+            stray += 1
+
+    if stray:
+        logger.debug(
+            "_clean_and_resolve_edges: final pass dropped %d stray edge(s)",
+            stray,
+        )
+
+    return clean
+
+
 def _make_qualified_name(
     file_path: str, name: str, parent_name: str | None
 ) -> str:
@@ -111,11 +246,12 @@ def _parse_file_standalone(
         else:
             return {"nodes": [], "edges": [], "errors": [f"Unsupported language: {language}"]}
 
-        nodes, edges = extractor.extract()
+        nodes, edges, import_map = extractor.extract_with_import_map()
 
         return {
             "nodes": nodes,
             "edges": edges,
+            "import_map": import_map,
             "errors": [],
         }
     except Exception as exc:
@@ -191,7 +327,7 @@ class CodeParser:
         file_path: Path,
         source_bytes: bytes,
         language: str,
-    ) -> tuple[list[GraphNode], list[GraphEdge]]:
+    ) -> tuple[list[GraphNode], list[GraphEdge], dict[str, tuple[str, str]]]:
         """Parse a single file and return extracted nodes and edges.
 
         Raises UnsupportedLanguageError if language is not supported.
@@ -235,7 +371,7 @@ class CodeParser:
             from superlocalmemory.code_graph.extractors.typescript import TypeScriptExtractor
             extractor = TypeScriptExtractor(root, source_bytes, file_path_str, self._config)
 
-        extracted_nodes, extracted_edges = extractor.extract()
+        extracted_nodes, extracted_edges, file_import_map = extractor.extract_with_import_map()
 
         # Check if test file
         is_test = _is_test_file(file_path_str, self._config)
@@ -277,7 +413,7 @@ class CodeParser:
 
         all_edges = extracted_edges + contains_edges + tested_by_edges
 
-        return all_nodes, all_edges
+        return all_nodes, all_edges, file_import_map
 
     def parse_all(
         self, repo_root: Path
@@ -293,6 +429,11 @@ class CodeParser:
         all_nodes: list[GraphNode] = []
         all_edges: list[GraphEdge] = []
         all_file_records: list[FileRecord] = []
+        # Keyed by relative file_path string; populated by the parallel path
+        # (the sequential path goes through parse_file which does not expose
+        # the per-file import map, so it stays empty — resolver Strategy 2/3
+        # still works without it).
+        import_maps_by_file: dict[str, dict[str, tuple[str, str]]] = {}
 
         # Read files and prepare tasks
         tasks: list[tuple[Path, bytes, str]] = []
@@ -313,10 +454,16 @@ class CodeParser:
 
         # Parse with ProcessPoolExecutor for parallel CPU-bound work
         # For small numbers of files, run sequentially to avoid overhead
-        if len(tasks) <= 2:
+        # Use the sequential (in-process) path for small repos or when the
+        # caller explicitly requests single-worker execution.  parallel_workers=1
+        # is the only way to guarantee in-process parsing (no subprocess spawn).
+        if len(tasks) <= 2 or self._config.parallel_workers == 1:
             for rel_path, source_bytes, language in tasks:
                 try:
-                    nodes, edges = self.parse_file(rel_path, source_bytes, language)
+                    nodes, edges, file_import_map = self.parse_file(
+                        rel_path, source_bytes, language
+                    )
+                    import_maps_by_file[str(rel_path)] = file_import_map
                     all_nodes.extend(nodes)
                     all_edges.extend(edges)
                     all_file_records.append(FileRecord(
@@ -330,105 +477,138 @@ class CodeParser:
                     ))
                 except Exception as exc:
                     logger.warning("Failed to parse %s: %s", rel_path, exc)
-            return all_nodes, all_edges, all_file_records
+            # import_maps_by_file is now populated on the sequential path:
+            # parse_file returns the per-file import map (Strategy 1 works
+            # for small repos / parallel_workers=1 runs too).
+        else:
+            # Parallel execution
+            parse_failures = 0   # count of futures that raised (worker crashes)
+            config_dict = {
+                field_name: getattr(self._config, field_name)
+                for field_name in CodeGraphConfig.__dataclass_fields__
+                if not isinstance(getattr(self._config, field_name), Path)
+            }
+            # Convert Path fields to strings
+            config_dict["repo_root"] = str(self._config.repo_root)
+
+            workers = min(self._config.parallel_workers, len(tasks))
+            with ProcessPoolExecutor(max_workers=workers) as executor:
+                future_map = {}
+                for rel_path, source_bytes, language in tasks:
+                    future = executor.submit(
+                        _parse_file_standalone,
+                        str(rel_path),
+                        source_bytes,
+                        language,
+                        config_dict,
+                    )
+                    future_map[future] = (rel_path, source_bytes, language)
+
+                for future in as_completed(future_map):
+                    rel_path, source_bytes, language = future_map[future]
+                    try:
+                        result = future.result(timeout=self._config.parse_timeout_seconds)
+                    except Exception as exc:
+                        logger.warning("Parse failed for %s: %s", rel_path, exc)
+                        parse_failures += 1
+                        continue
 
-        # Parallel execution
-        config_dict = {
-            field_name: getattr(self._config, field_name)
-            for field_name in CodeGraphConfig.__dataclass_fields__
-            if not isinstance(getattr(self._config, field_name), Path)
-        }
-        # Convert Path fields to strings
-        config_dict["repo_root"] = str(self._config.repo_root)
+                    if result["errors"]:
+                        for err in result["errors"]:
+                            logger.warning("Parse error in %s: %s", rel_path, err)
+                        if not result["nodes"]:
+                            continue
+
+                    file_nodes = result["nodes"]
+                    file_edges = result["edges"]
+                    # Collect per-file import map for Strategy 1 resolution.
+                    import_maps_by_file[str(rel_path)] = result.get("import_map", {})
+
+                    # Build the full parse result with file node and CONTAINS edges
+                    file_path_str = str(rel_path)
+                    content_hash = _sha256(source_bytes)
+
+                    file_node = GraphNode(
+                        node_id=_new_id(),
+                        kind=NodeKind.FILE,
+                        name=rel_path.name,
+                        qualified_name=file_path_str,
+                        file_path=file_path_str,
+                        line_start=0,
+                        line_end=0,
+                        language=language,
+                        content_hash=content_hash,
+                    )
 
-        workers = min(self._config.parallel_workers, len(tasks))
-        with ProcessPoolExecutor(max_workers=workers) as executor:
-            future_map = {}
-            for rel_path, source_bytes, language in tasks:
-                future = executor.submit(
-                    _parse_file_standalone,
-                    str(rel_path),
-                    source_bytes,
-                    language,
-                    config_dict,
-                )
-                future_map[future] = (rel_path, source_bytes, language)
-
-            for future in as_completed(future_map):
-                rel_path, source_bytes, language = future_map[future]
-                try:
-                    result = future.result(timeout=self._config.parse_timeout_seconds)
-                except Exception as exc:
-                    logger.warning("Parse failed for %s: %s", rel_path, exc)
-                    continue
+                    is_test = _is_test_file(file_path_str, self._config)
+                    if is_test:
+                        marked: list[GraphNode] = []
+                        for n in file_nodes:
+                            if n.kind in (NodeKind.FUNCTION, NodeKind.METHOD):
+                                marked.append(GraphNode(
+                                    node_id=n.node_id, kind=n.kind, name=n.name,
+                                    qualified_name=n.qualified_name,
+                                    file_path=n.file_path,
+                                    line_start=n.line_start, line_end=n.line_end,
+                                    language=n.language, parent_name=n.parent_name,
+                                    signature=n.signature, docstring=n.docstring,
+                                    is_test=True, content_hash=n.content_hash,
+                                    extra_json=n.extra_json,
+                                ))
+                            else:
+                                marked.append(n)
+                        file_nodes = marked
+
+                    contains = self._generate_contains_edges(file_node, file_nodes)
+                    tested_by = self._generate_tested_by_edges(file_nodes, file_edges)
+
+                    final_nodes = [file_node] + file_nodes
+                    final_edges = file_edges + contains + tested_by
+
+                    all_nodes.extend(final_nodes)
+                    all_edges.extend(final_edges)
+
+                    try:
+                        mtime = (repo_root / rel_path).stat().st_mtime
+                    except OSError:
+                        mtime = 0.0
 
-                if result["errors"]:
-                    for err in result["errors"]:
-                        logger.warning("Parse error in %s: %s", rel_path, err)
-                    if not result["nodes"]:
-                        continue
+                    all_file_records.append(FileRecord(
+                        file_path=file_path_str,
+                        content_hash=content_hash,
+                        mtime=mtime,
+                        language=language,
+                        node_count=len(final_nodes),
+                        edge_count=len(final_edges),
+                        last_indexed=time.time(),
+                    ))
 
-                file_nodes = result["nodes"]
-                file_edges = result["edges"]
-
-                # Build the full parse result with file node and CONTAINS edges
-                file_path_str = str(rel_path)
-                content_hash = _sha256(source_bytes)
-
-                file_node = GraphNode(
-                    node_id=_new_id(),
-                    kind=NodeKind.FILE,
-                    name=rel_path.name,
-                    qualified_name=file_path_str,
-                    file_path=file_path_str,
-                    line_start=0,
-                    line_end=0,
-                    language=language,
-                    content_hash=content_hash,
-                )
-
-                is_test = _is_test_file(file_path_str, self._config)
-                if is_test:
-                    marked: list[GraphNode] = []
-                    for n in file_nodes:
-                        if n.kind in (NodeKind.FUNCTION, NodeKind.METHOD):
-                            marked.append(GraphNode(
-                                node_id=n.node_id, kind=n.kind, name=n.name,
-                                qualified_name=n.qualified_name,
-                                file_path=n.file_path,
-                                line_start=n.line_start, line_end=n.line_end,
-                                language=n.language, parent_name=n.parent_name,
-                                signature=n.signature, docstring=n.docstring,
-                                is_test=True, content_hash=n.content_hash,
-                                extra_json=n.extra_json,
-                            ))
-                        else:
-                            marked.append(n)
-                    file_nodes = marked
-
-                contains = self._generate_contains_edges(file_node, file_nodes)
-                tested_by = self._generate_tested_by_edges(file_nodes, file_edges)
-
-                final_nodes = [file_node] + file_nodes
-                final_edges = file_edges + contains + tested_by
-
-                all_nodes.extend(final_nodes)
-                all_edges.extend(final_edges)
+            # ── Fail-fast: majority pool failure ──────────────────────────────
+            # >50% failure rate means worker processes were killed (OOM, missing
+            # deps in subprocess env) — NOT a legitimate empty repo.  Returning
+            # silently would let build_code_graph report success with zero nodes,
+            # which is worse than the original FK crash.
+            # Threshold rationale: ≥50% is a clear systemic failure; it allows
+            # for a tail of legitimately-malformed files without triggering on
+            # ordinary parse errors.
+            if parse_failures > 0 and len(tasks) > 0:
+                failure_rate = parse_failures / len(tasks)
+                if failure_rate > 0.50:
+                    raise RuntimeError(
+                        f"Parsing aborted: {parse_failures}/{len(tasks)} files "
+                        f"failed ({failure_rate:.0%}). Likely cause: process pool "
+                        f"worker crash or missing tree-sitter grammars in subprocess. "
+                        f"Set CodeGraphConfig(parallel_workers=1) for in-process parsing."
+                    )
 
-                try:
-                    mtime = (repo_root / rel_path).stat().st_mtime
-                except OSError:
-                    mtime = 0.0
-
-                all_file_records.append(FileRecord(
-                    file_path=file_path_str,
-                    content_hash=content_hash,
-                    mtime=mtime,
-                    language=language,
-                    node_count=len(final_nodes),
-                    edge_count=len(final_edges),
-                    last_indexed=time.time(),
-                ))
+        # ── Resolution pipeline ────────────────────────────────────────────────
+        # Resolve placeholder edge endpoints before returning.  Ensures that
+        # every edge exiting parse_all has both endpoints in the returned node
+        # set, so GraphStore never encounters a FOREIGN KEY constraint failure.
+        # This is Fix A: wiring resolver.py into the parse pipeline.
+        all_edges = _clean_and_resolve_edges(
+            all_nodes, all_edges, import_maps_by_file, repo_root, self._config
+        )
 
         return all_nodes, all_edges, all_file_records