codeanalyzer-python 0.3.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

codeanalyzer/semantic_analysis/pycg/pycg_analysis.py

@@ -39,9 +39,12 @@ project-relative dotted paths so they align with the symbol table.
 # re-enters PyCG's hook before its import graph is ready.  Pre-importing
 # these modules at import time ensures they're already in sys.modules when
 # PyCG's hook is active, preventing the re-entrant ImportManagerError.
+import fcntl
+import hashlib
 import importlib.metadata  # noqa: F401
 import importlib.util  # noqa: F401
 import contextlib
+import os
 import json  # noqa: F401
 import shutil
 import signal
@@ -85,6 +88,15 @@ from codeanalyzer.semantic_analysis.pycg.shard_planner import plan_shards
 from codeanalyzer.utils import ProgressBar, logger
 
 
+def _shard_root_path(files: List[str], project_dir: Path) -> Path:
+    """Content-derived mini-project root for a shard: same project + same file
+    set → same path on every run (determinism, issue #99)."""
+    digest = hashlib.sha1(
+        "\0".join([str(project_dir), *sorted(files)]).encode("utf-8")
+    ).hexdigest()[:16]
+    return Path(tempfile.gettempdir()) / f"canpy_pycg_shard_{digest}"
+
+
 def _materialize_shard_root(
     files: List[str],
     project_dir: Path,
@@ -103,10 +115,21 @@ def _materialize_shard_root(
 
     The caller owns the returned *root* and must ``shutil.rmtree`` it.
     """
-    root = Path(tempfile.mkdtemp(prefix="canpy_pycg_shard_"))
+    # Deterministic root: PyCG's capped fixpoint (--pycg-max-iter) is
+    # order-sensitive, and its internal state keys on absolute module paths —
+    # a random mkdtemp suffix changes those strings every run and shifts the
+    # iteration frontier, making the emitted edge set vary run-to-run
+    # (issue #99). Deriving the directory name from the shard's content keeps
+    # the path (and thus the analysis input) identical across runs. Callers
+    # that may run concurrently on the same shard serialize on the sidecar
+    # lock (see _shard_symlink_root).
+    root = _shard_root_path(files, project_dir)
+    if root.exists():
+        shutil.rmtree(root, ignore_errors=True)
+    root.mkdir(parents=True, exist_ok=True)
     entry_points: List[str] = []
     linked_inits: Set[Path] = set()
-    for f in files:
+    for f in sorted(files):
         src = Path(f).resolve()
         try:
             rel = src.relative_to(project_dir)
@@ -142,12 +165,27 @@ def _shard_symlink_root(
     """Context-manager wrapper around :func:`_materialize_shard_root`.
 
     Yields ``(root, entry_points)`` and removes the temp tree on exit.
+
+    The root path is content-derived (determinism, issue #99), so two
+    concurrent analyses of the same shard — e.g. a test suite and a manual
+    run on one project — would collide on it (one rmtree's the tree the
+    other is mid-analysis on). An exclusive flock on a sidecar lockfile
+    serializes them; distinct projects/shards hash to distinct roots and
+    never contend.
     """
-    root, entry_points = _materialize_shard_root(files, project_dir)
+    digest_root = _shard_root_path(files, project_dir)
+    lock_path = digest_root.with_name(digest_root.name + ".lock")
+    lock_fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR)
     try:
-        yield root, entry_points
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+        root, entry_points = _materialize_shard_root(files, project_dir)
+        try:
+            yield root, entry_points
+        finally:
+            shutil.rmtree(root, ignore_errors=True)
     finally:
-        shutil.rmtree(root, ignore_errors=True)
+        fcntl.flock(lock_fd, fcntl.LOCK_UN)
+        os.close(lock_fd)
 
 
 def _pycg_shard_worker(
@@ -431,14 +469,14 @@ class PyCG:
         """Sum weights of duplicate ``(source, target)`` pairs across shards."""
         merged: Dict[tuple, PyCallEdge] = {}
         for edge in edges:
-            key = (edge.source, edge.target)
+            key = (edge.src, edge.dst)
             if key in merged:
                 existing = merged[key]
                 merged[key] = PyCallEdge(
                     source=existing.source,
                     target=existing.target,
                     weight=existing.weight + edge.weight,
-                    provenance=existing.provenance,
+                    prov=existing.prov,
                 )
             else:
                 merged[key] = edge
@@ -469,7 +507,9 @@ class PyCG:
             ):
                 continue
             paths.append(str(p))
-        return paths
+        # Sorted for run-to-run stability: rglob yields filesystem order, and
+        # PyCG's capped fixpoint is sensitive to entry-point order (issue #99).
+        return sorted(paths)
 
     # ------------------------------------------------------------------
     # Package-root helpers for sharding
@@ -564,7 +604,7 @@ class PyCG:
             edge_counts[(resolver.resolve(src), resolver.resolve(dst))] += 1
 
         return [
-            PyCallEdge(source=src, target=dst, weight=count, provenance=["pycg"])
+            PyCallEdge(src=src, dst=dst, weight=count, prov=["pycg"])
             for (src, dst), count in edge_counts.items()
         ]
 
@@ -712,11 +752,14 @@ class PyCG:
         """
         import os
         import ray
+        from codeanalyzer.core import _ensure_ray
+        _ensure_ray()
 
         os.environ.setdefault("RAY_IGNORE_UNHANDLED_ERRORS", "1")
         remote_fn = ray.remote(_pycg_shard_worker)
 
         roots: List[Path] = []
+        lock_fds: List[int] = []
         futures: List[Any] = []
         meta: Dict[Any, List[str]] = {}  # ObjectRef -> shard file list
         edges_all: List[PyCallEdge] = []
@@ -724,6 +767,16 @@ class PyCG:
         try:
             with ProgressBar(len(shards), "Building call graph shards (parallel)", item_label="shards") as progress:
                 for files in shards:
+                    # Deterministic roots can collide across concurrent
+                    # analyses of the same project — the driver holds each
+                    # shard's sidecar lock for the whole Ray fan-out (released
+                    # in the finally below with the root cleanup).
+                    lock_fd = os.open(
+                        str(_shard_root_path(files, self.project_dir).with_suffix(".lock")),
+                        os.O_CREAT | os.O_RDWR,
+                    )
+                    fcntl.flock(lock_fd, fcntl.LOCK_EX)
+                    lock_fds.append(lock_fd)
                     root, eps = _materialize_shard_root(files, self.project_dir)
                     roots.append(root)
                     fut = remote_fn.remote(eps, str(root), "", self.max_iter)
@@ -751,7 +804,7 @@ class PyCG:
                     try:
                         triples = ray.get(fut)
                         edges_all.extend(
-                            PyCallEdge(source=s, target=t, weight=w, provenance=["pycg"])
+                            PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
                             for s, t, w in triples
                         )
                     except Exception:
@@ -765,6 +818,12 @@ class PyCG:
         finally:
             for root in roots:
                 shutil.rmtree(root, ignore_errors=True)
+            for fd in lock_fds:
+                try:
+                    fcntl.flock(fd, fcntl.LOCK_UN)
+                    os.close(fd)
+                except OSError:
+                    pass
         return edges_all, runaways
 
     def _build_sharded(
@@ -841,14 +900,14 @@ class PyCG:
         # Merge duplicate (source, target) pairs that appear in multiple shards.
         merged: Dict[tuple, PyCallEdge] = {}
         for edge in all_edges:
-            key = (edge.source, edge.target)
+            key = (edge.src, edge.dst)
             if key in merged:
                 existing = merged[key]
                 merged[key] = PyCallEdge(
                     source=existing.source,
                     target=existing.target,
                     weight=existing.weight + edge.weight,
-                    provenance=existing.provenance,
+                    prov=existing.prov,
                 )
             else:
                 merged[key] = edge
@@ -870,6 +929,8 @@ class PyCG:
         """
         import os
         import ray
+        from codeanalyzer.core import _ensure_ray
+        _ensure_ray()
 
         # force-cancel kills worker processes; suppress Ray's "worker died
         # unexpectedly" noise since the death is intentional here.
@@ -923,7 +984,7 @@ class PyCG:
                 try:
                     triples = ray.get(fut)
                     edges = [
-                        PyCallEdge(source=s, target=t, weight=w, provenance=["pycg"])
+                        PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
                         for s, t, w in triples
                     ]
                     all_edges.extend(edges)
@@ -956,14 +1017,14 @@ class PyCG:
 
         merged: Dict[tuple, PyCallEdge] = {}
         for edge in all_edges:
-            key = (edge.source, edge.target)
+            key = (edge.src, edge.dst)
             if key in merged:
                 existing = merged[key]
                 merged[key] = PyCallEdge(
                     source=existing.source,
                     target=existing.target,
                     weight=existing.weight + edge.weight,
-                    provenance=existing.provenance,
+                    prov=existing.prov,
                 )
             else:
                 merged[key] = edge
@@ -984,7 +1045,7 @@ class PyCG:
         symbol_table: Dict[str, PyModule],
         jedi_edges: Optional[List[PyCallEdge]] = None,
     ) -> List[PyCallEdge]:
-        """Run PyCG and return ``PyCallEdge`` entries with ``provenance=["pycg"]``.
+        """Run PyCG and return ``PyCallEdge`` entries with ``prov=["pycg"]``.
 
         Edges are coalesced on ``(source, target)`` — ``weight`` equals the
         number of times PyCG reports the same (caller, callee) pair (always 1

codeanalyzer/semantic_analysis/pycg/shard_planner.py

@@ -66,17 +66,17 @@ logger = logging.getLogger(__name__)
 
 def _walk_callable_sigs(c: PyCallable) -> Iterator[str]:
     yield c.signature
-    for inner in c.inner_callables.values():
+    for inner in c.callables.values():
         yield from _walk_callable_sigs(inner)
-    for inner_cls in c.inner_classes.values():
+    for inner_cls in c.types.values():
         yield from _walk_class_sigs(inner_cls)
 
 
 def _walk_class_sigs(cls: PyClass) -> Iterator[str]:
     yield cls.signature
-    for method in cls.methods.values():
+    for method in cls.callables.values():
         yield from _walk_callable_sigs(method)
-    for inner in cls.inner_classes.values():
+    for inner in cls.types.values():
         yield from _walk_class_sigs(inner)
 
 
@@ -95,7 +95,7 @@ def _signature_to_file(symbol_table: Dict[str, PyModule]) -> Dict[str, str]:
         for fn in module.functions.values():
             for sig in _walk_callable_sigs(fn):
                 sig_to_file[sig] = module.file_path
-        for cls in module.classes.values():
+        for cls in module.types.values():
             for sig in _walk_class_sigs(cls):
                 sig_to_file[sig] = module.file_path
     return sig_to_file
@@ -152,8 +152,8 @@ def build_module_graph(
         g.add_node(module.file_path, module_name=module.module_name)
 
     for edge in jedi_edges:
-        src = sig_to_file.get(edge.source)
-        dst = sig_to_file.get(edge.target)
+        src = sig_to_file.get(edge.src)
+        dst = sig_to_file.get(edge.dst)
         if src is None or dst is None or src == dst:
             continue
         if g.has_edge(src, dst):

codeanalyzer/syntactic_analysis/symbol_table_builder.py

@@ -23,6 +23,8 @@ from codeanalyzer.schema.py_schema import (
     PyModule,
     PySymbol,
     PyVariableDeclaration,
+    Span,
+    byte_offsets,
 )
 
 
@@ -55,13 +57,26 @@ class SymbolTableBuilder:
         relative = Path(script_path).relative_to(self.project_dir)
         return ".".join(relative.with_suffix("").parts) + f".{name}"
 
+    @staticmethod
+    def _first_definition(definitions):
+        """Deterministic pick from Jedi's inference candidates.
+
+        On a union-typed receiver Jedi may return several candidates whose
+        ORDER varies run to run (issue #99 — e.g. ``o.seek`` on
+        ``_IOBase | BufferedRandom | TextIOWrapper``); taking whichever came
+        first made the emitted call graph nondeterministic. Sort on the
+        stable identity (full_name, then name) and take the smallest."""
+        if not definitions:
+            return None
+        return min(definitions, key=lambda d: (d.full_name or "", d.name or ""))
+
     @staticmethod
     def _infer_type(script: Script, line: int, column: int) -> str:
         """Tries to infer the type at a given position using Jedi."""
         try:
-            inference = script.infer(line=line, column=column)
-            if inference:
-                return inference[0].name  # or .full_name
+            d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column))
+            if d is not None:
+                return d.name  # or .full_name
         except Exception:
             pass
         return None
@@ -80,9 +95,9 @@ class SymbolTableBuilder:
             Optional[str]: The fully qualified name if available, else None.
         """
         try:
-            definitions = script.infer(line=line, column=column)
-            if definitions:
-                return definitions[0].full_name
+            d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column))
+            if d is not None:
+                return d.full_name
         except Exception:
             pass
         return None
@@ -101,10 +116,9 @@ class SymbolTableBuilder:
         the call graph.
         """
         try:
-            definitions = script.infer(line=line, column=column)
-            if not definitions:
+            d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column))
+            if d is None:
                 return None, False
-            d = definitions[0]
             is_class = (d.type == "class")
             full = d.full_name
             if is_class and full:
@@ -142,11 +156,17 @@ class SymbolTableBuilder:
         as the callee's own name.
         """
         try:
-            definitions = script.infer(line=line, column=column)
-            if definitions:
-                results = definitions[0].execute()
-                if results:
-                    return results[0].name
+            d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column))
+            if d is not None:
+                # Drop NoneType results before picking: Jedi flaps between
+                # yielding {NoneType} and {} for the None arm of an Optional
+                # return (issue #99), and when a real type is also in the
+                # union it is the informative choice — a bare NoneType says
+                # nothing a missing return_type doesn't.
+                results = [r for r in d.execute() if r.name != "NoneType"]
+                r = SymbolTableBuilder._first_definition(results)
+                if r is not None:
+                    return r.name
         except Exception:
             pass
         return None
@@ -177,11 +197,12 @@ class SymbolTableBuilder:
             PyModule.builder()
             .file_path(str(py_file))
             .module_name(py_file.stem)
+            .source(source)
             .comments(self._pycomments(module, source))
             .imports(self._imports(module))
             .variables(self._module_variables(module, script))
-            .classes(self._add_class(module, script))
-            .functions(self._callables(module, script))
+            .types(self._add_class(module, script, source))
+            .functions(self._callables(module, script, source))
             .content_hash(content_hash)
             .last_modified(last_modified)
             .file_size(file_size)
@@ -237,7 +258,7 @@ class SymbolTableBuilder:
 
         return imports
 
-    def _add_class(self, node: AST, script: Script, prefix: str = "") -> Dict[str, PyClass]:
+    def _add_class(self, node: AST, script: Script, source: str, prefix: str = "") -> Dict[str, PyClass]:
         classes: Dict[str, PyClass] = {}
 
         for child in ast.iter_child_nodes(node):
@@ -248,6 +269,14 @@ class SymbolTableBuilder:
             start_line = child.lineno
             end_line = getattr(child, "end_lineno", start_line + len(child.body))
             code = ast.unparse(child).strip()
+            span = Span(
+                start=(child.lineno, child.col_offset),
+                end=(getattr(child, "end_lineno", child.lineno),
+                     getattr(child, "end_col_offset", child.col_offset)),
+                bytes=byte_offsets(source, child.lineno, child.col_offset,
+                                   getattr(child, "end_lineno", child.lineno),
+                                   getattr(child, "end_col_offset", child.col_offset)),
+            )
 
             # Try resolving full signature with Jedi
             if prefix:
@@ -265,18 +294,18 @@ class SymbolTableBuilder:
                 PyClass.builder()
                 .name(class_name)
                 .signature(signature)
+                .span(span)
                 .start_line(start_line)
                 .end_line(end_line)
-                .code(code)
                 .comments(self._pycomments(child, code))
                 .base_classes([
                     ast.unparse(base)
                     for base in child.bases
                     if isinstance(base, ast.expr)
                 ])
-                .methods(self._callables(child, script, prefix=signature))  # Pass class signature as prefix
+                .callables(self._callables(child, script, source, prefix=signature))  # Pass class signature as prefix
                 .attributes(self._class_attributes(child, script))
-                .inner_classes(self._add_class(child, script, prefix=signature))  # Pass class signature as prefix
+                .types(self._add_class(child, script, source, prefix=signature))  # Pass class signature as prefix
                 .build()
             )
 
@@ -285,7 +314,7 @@ class SymbolTableBuilder:
         return classes
 
 
-    def _callables(self, node: AST, script: Script, prefix: str = "") -> Dict[str, PyCallable]:
+    def _callables(self, node: AST, script: Script, source: str, prefix: str = "") -> Dict[str, PyCallable]:
         callables: Dict[str, PyCallable] = {}
 
         for child in ast.iter_child_nodes(node):
@@ -294,6 +323,14 @@ class SymbolTableBuilder:
                 start_line = child.lineno
                 end_line = getattr(child, "end_lineno", start_line + len(child.body))
                 code = ast.unparse(child).strip()
+                span = Span(
+                    start=(child.lineno, child.col_offset),
+                    end=(getattr(child, "end_lineno", child.lineno),
+                         getattr(child, "end_col_offset", child.col_offset)),
+                    bytes=byte_offsets(source, child.lineno, child.col_offset,
+                                       getattr(child, "end_lineno", child.lineno),
+                                       getattr(child, "end_col_offset", child.col_offset)),
+                )
                 decorators = [ast.unparse(d) for d in child.decorator_list]
                 
                 if prefix:
@@ -318,8 +355,8 @@ class SymbolTableBuilder:
                     .name(method_name)  # Use the actual method name, not the full signature
                     .path(str(script.path))
                     .signature(signature)  # Use the full signature here
+                    .span(span)
                     .decorators(decorators)
-                    .code(code)
                     .start_line(start_line)
                     .end_line(end_line)
                     .code_start_line(child.body[0].lineno if child.body else start_line)
@@ -333,8 +370,8 @@ class SymbolTableBuilder:
                         if child.returns else self._infer_type(script, child.lineno, child.col_offset)
                     )
                     .comments(self._pycomments(child, code))
-                    .inner_callables(self._callables(child, script, signature))  # Pass current signature as prefix
-                    .inner_classes(self._add_class(child, script, signature))    # Pass current signature as prefix
+                    .callables(self._callables(child, script, source, signature))  # Pass current signature as prefix
+                    .types(self._add_class(child, script, source, signature))    # Pass current signature as prefix
                     .build()
                 )
 
@@ -955,9 +992,10 @@ class SymbolTableBuilder:
 
         if script:
             try:
-                definitions = script.infer(line=lineno, column=col_offset)
-                if definitions:
-                    d = definitions[0]
+                d = SymbolTableBuilder._first_definition(
+                    script.infer(line=lineno, column=col_offset)
+                )
+                if d is not None:
                     inferred_type = d.name
                     qname = d.full_name
                     if d.type == "function":