codeanalyzer-python 0.1.13__py3-none-any.whl → 0.1.14__py3-none-any.whl

codeanalyzer/__main__.py

@@ -27,10 +27,6 @@ def main(
             case_sensitive=False,
         ),
     ] = OutputFormat.JSON,
-    analysis_level: Annotated[
-        int,
-        typer.Option("-a", "--analysis-level", help="1: symbol table, 2: call graph."),
-    ] = 1,
     using_codeql: Annotated[
         bool, typer.Option("--codeql/--no-codeql", help="Enable CodeQL-based analysis.")
     ] = False,
@@ -82,7 +78,6 @@ def main(
         input=input,
         output=output,
         format=format,
-        analysis_level=analysis_level,
         using_codeql=using_codeql,
         using_ray=using_ray,
         rebuild_analysis=rebuild_analysis,

codeanalyzer/core.py

@@ -9,7 +9,14 @@ from typing import Any, Dict, Optional, Union, List
 import ray
 from codeanalyzer.utils import logger
 from codeanalyzer.schema import PyApplication, PyModule, model_dump_json, model_validate_json
+from codeanalyzer.schema.py_schema import PyCallEdge
+from codeanalyzer.semantic_analysis.call_graph import (
+    jedi_call_graph_edges,
+    merge_edges,
+    resolve_unresolved_constructors,
+)
 from codeanalyzer.semantic_analysis.codeql import CodeQLLoader
+from codeanalyzer.semantic_analysis.codeql.codeql_analysis import CodeQL
 from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
 from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
 from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
@@ -49,7 +56,6 @@ class Codeanalyzer:
 
     def __init__(self, options: AnalysisOptions) -> None:
         self.options = options
-        self.analysis_depth = options.analysis_level
         self.project_dir = Path(options.input).resolve()
         self.skip_tests = options.skip_tests
         self.using_codeql = options.using_codeql
@@ -60,6 +66,7 @@ class Codeanalyzer:
         self.clear_cache = options.clear_cache
         self.db_path: Optional[Path] = None
         self.codeql_bin: Optional[Path] = None
+        self.codeql_packs_dir: Optional[Path] = None
         self.virtualenv: Optional[Path] = None
         self.using_ray: bool = options.using_ray
         self.file_name: Optional[Path] = options.file_name
@@ -292,6 +299,15 @@ class Codeanalyzer:
 
         if self.using_codeql:
             logger.info(f"(Re-)initializing CodeQL analysis for {self.project_dir}")
+
+            # Resolve the CLI binary before anything else uses it: DB build
+            # below needs it, and so does every subsequent query run.
+            self.codeql_bin = self._ensure_codeql_bin()
+            # Download the standard query library pack (idempotent). The
+            # CLI install ships only the language extractors; the
+            # ``codeql/python-all`` library pack must be fetched separately.
+            self.codeql_packs_dir = self._ensure_codeql_packs(self.codeql_bin)
+
             cache_root = self.cache_dir / "codeql"
             cache_root.mkdir(parents=True, exist_ok=True)
             self.db_path = cache_root / f"{self.project_dir.name}-db"
@@ -310,19 +326,6 @@ class Codeanalyzer:
             if self.rebuild_analysis or not is_cache_valid():
                 logger.info("Creating new CodeQL database...")
 
-                codeql_in_path = shutil.which("codeql")
-                if codeql_in_path:
-                    self.codeql_bin = Path(codeql_in_path)
-                else:
-                    self.codeql_bin = CodeQLLoader.download_and_extract_codeql(
-                        self.cache_dir / "codeql" / "bin"
-                    )
-
-                if not shutil.which(str(self.codeql_bin)):
-                    raise FileNotFoundError(
-                        f"CodeQL binary not executable: {self.codeql_bin}"
-                    )
-
                 cmd = [
                     str(self.codeql_bin),
                     "database",
@@ -375,8 +378,27 @@ class Codeanalyzer:
         # Build symbol table from cached application if available (if no available, the build a new one)
         symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
 
+        # Build the call graph in four steps:
+        #   1. Run CodeQL (when enabled). Produces resolved edges with
+        #      ``provenance=["codeql"]`` and augments ``PyCallsite``s
+        #      in-place — filling ``callee_signature`` for sites Jedi
+        #      couldn't resolve.
+        #   2. Heuristic fallback for constructor calls neither Jedi nor
+        #      CodeQL could resolve (commonly classes nested inside
+        #      functions). Walks the symbol table by class short-name +
+        #      scope and writes ``<class>.__init__`` into the site.
+        #   3. Derive Jedi edges from the now-fully-augmented symbol
+        #      table — these reflect every resolution the symbol table
+        #      contains, regardless of which pass put it there.
+        #   4. Merge with CodeQL edges; provenance unions for edges both
+        #      backends saw.
+        codeql_edges = self._get_call_graph(symbol_table, augment_sites=True)
+        resolve_unresolved_constructors(symbol_table)
+        jedi_edges = jedi_call_graph_edges(symbol_table)
+        call_graph = merge_edges(jedi_edges, codeql_edges)
+
         # Recreate pyapplication
-        app = PyApplication.builder().symbol_table(symbol_table).build()
+        app = PyApplication.builder().symbol_table(symbol_table).call_graph(call_graph).build()
         
         # Save to cache
         self._save_analysis_cache(app, cache_file)
@@ -579,7 +601,120 @@ class Codeanalyzer:
         logger.info("✅ Symbol table generation complete.")
         return symbol_table
 
-    def _get_call_graph(self) -> Dict[str, Any]:
-        """Retrieve call graph from CodeQL database."""
-        logger.warning("Call graph extraction not yet implemented.")
-        return {}
+    def _ensure_codeql_packs(self, codeql_bin: Path) -> Path:
+        """Materialize a qlpack that depends on ``codeql/python-all``.
+
+        The CodeQL CLI install ships only the language extractors — query
+        library packs (and their transitive dependencies like
+        ``codeql/concepts``) must be resolved separately. The canonical
+        way is to declare the dependency in a ``qlpack.yml`` and run
+        ``codeql pack install`` in that directory; CodeQL writes a
+        ``codeql-pack.lock.yml`` and downloads everything needed.
+
+        We do this once per project under ``<cache_dir>/codeql/qlpack/``
+        and return that directory. The query runner then writes its
+        temporary ``.ql`` file inside this pack — colocation makes
+        ``import python`` resolve without any ``--additional-packs`` or
+        ``--search-path`` gymnastics.
+        """
+        pack_dir = self.cache_dir / "codeql" / "qlpack"
+        pack_dir.mkdir(parents=True, exist_ok=True)
+        qlpack_yml = pack_dir / "qlpack.yml"
+        lock_file = pack_dir / "codeql-pack.lock.yml"
+
+        if not qlpack_yml.exists():
+            qlpack_yml.write_text(
+                "name: codeanalyzer-deps\n"
+                "version: 1.0.0\n"
+                "dependencies:\n"
+                '  codeql/python-all: "*"\n'
+            )
+
+        if lock_file.exists():
+            logger.debug(f"CodeQL pack dependencies already installed in {pack_dir}")
+            return pack_dir
+
+        logger.info(f"Installing CodeQL pack dependencies in {pack_dir}.")
+        proc = subprocess.Popen(
+            [str(codeql_bin), "pack", "install", str(pack_dir)],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        _, err = proc.communicate()
+        if proc.returncode != 0:
+            raise CodeQLExceptions.CodeQLDatabaseBuildException(
+                f"Failed to install CodeQL pack dependencies:\n"
+                f"{(err or b'').decode(errors='replace')}"
+            )
+        return pack_dir
+
+    def _ensure_codeql_bin(self) -> Path:
+        """Locate (or download) the CodeQL CLI binary into the project cache.
+
+        Resolution order:
+          1. An existing binary inside ``<cache_dir>/codeql/bin/`` —
+             reused across runs on the same project.
+          2. ``codeql`` already on the user's PATH — picked up verbatim.
+          3. Otherwise, download into ``<cache_dir>/codeql/bin/``.
+
+        The project-local cache is preferred over PATH so the version we
+        installed earlier wins over whatever the OS ships — keeps behavior
+        deterministic when the user has both.
+        """
+        bin_root = self.cache_dir / "codeql" / "bin"
+        bin_root.mkdir(parents=True, exist_ok=True)
+
+        existing = next(
+            (p for p in bin_root.rglob("codeql") if p.is_file()),
+            None,
+        )
+        if existing and os.access(existing, os.X_OK):
+            logger.debug(f"Reusing cached CodeQL CLI at {existing}")
+            return existing.resolve()
+
+        on_path = shutil.which("codeql")
+        if on_path:
+            logger.debug(f"Using CodeQL CLI from PATH at {on_path}")
+            return Path(on_path)
+
+        logger.info(f"CodeQL CLI not found; downloading into {bin_root}.")
+        downloaded = CodeQLLoader.download_and_extract_codeql(bin_root)
+        if not downloaded.exists() or not os.access(downloaded, os.X_OK):
+            raise FileNotFoundError(
+                f"CodeQL binary not executable after download: {downloaded}"
+            )
+        return downloaded
+
+    def _get_call_graph(
+        self,
+        symbol_table: Dict[str, PyModule],
+        augment_sites: bool = False,
+    ) -> List[PyCallEdge]:
+        """Build CodeQL-resolved call edges and optionally augment sites.
+
+        Returns an empty list when CodeQL isn't enabled or the database
+        isn't available. Edges carry ``provenance=["codeql"]`` — merge
+        with Jedi-derived edges via ``call_graph.merge_edges``.
+
+        When ``augment_sites`` is True, also mutates
+        ``PyCallable.call_sites`` in the symbol table to backfill
+        ``callee_signature`` for sites Jedi couldn't resolve. The single
+        CodeQL query is shared (cached on the ``CodeQL`` instance) so
+        this costs no extra DB work.
+        """
+        if not self.using_codeql or self.db_path is None:
+            return []
+        try:
+            cq = CodeQL(
+                self.project_dir,
+                self.db_path,
+                codeql_bin=self.codeql_bin,
+                codeql_packs_dir=self.codeql_packs_dir,
+            )
+            edges = cq.build_call_graph_edges(symbol_table)
+            if augment_sites:
+                cq.augment_call_sites(symbol_table)
+            return edges
+        except Exception as exc:
+            logger.warning(f"CodeQL call-graph extraction failed: {exc}")
+            return []

codeanalyzer/options/options.py

@@ -14,7 +14,6 @@ class AnalysisOptions:
     input: Path
     output: Optional[Path] = None
     format: OutputFormat = OutputFormat.JSON
-    analysis_level: int = 1
     using_codeql: bool = False
     using_ray: bool = False
     rebuild_analysis: bool = False

codeanalyzer/schema/py_schema.py

@@ -339,9 +339,29 @@ class PyModule(BaseModel):
     file_size: Optional[int] = None
 
 
+@builder
+@msgpk
+class PyCallEdge(BaseModel):
+    """Identity-only call-graph edge with weight.
+
+    Mirrors Java's ``CallDependency``. ``source`` and ``target`` are
+    ``PyCallable.signature`` strings — nodes of the graph are the existing
+    ``PyCallable`` entries in the symbol table, not a separate vertex type.
+    Rich per-call metadata (receiver, arguments, location, ...) lives on
+    ``PyCallsite`` inside the source ``PyCallable.call_sites``.
+    """
+
+    source: str  # caller's PyCallable.signature
+    target: str  # callee's PyCallable.signature
+    type: Literal["CALL_DEP"] = "CALL_DEP"
+    weight: int = 1
+    provenance: List[Literal["jedi", "codeql", "joern"]] = []
+
+
 @builder
 @msgpk
 class PyApplication(BaseModel):
     """Represents a Python application."""
 
     symbol_table: Dict[str, PyModule]
+    call_graph: List[PyCallEdge] = []

codeanalyzer/semantic_analysis/call_graph.py

@@ -0,0 +1,266 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""Adapters between the persisted call-graph schema and ``networkx``.
+
+The schema persists the call graph as ``List[PyCallEdge]`` with signatures
+referencing ``PyCallable`` entries already in the symbol table. These
+helpers rehydrate it into a ``networkx.DiGraph`` for in-process queries
+(paths, callers, callees) and reduce a built ``DiGraph`` back to the
+serializable edge list.
+"""
+
+from collections import Counter
+from typing import Dict, Iterator, List, Tuple
+
+import networkx as nx
+
+from codeanalyzer.schema.py_schema import (
+    PyApplication,
+    PyCallable,
+    PyCallEdge,
+    PyClass,
+    PyModule,
+)
+
+
+def _walk_class_callables(cls: PyClass) -> Iterator[PyCallable]:
+    for method in cls.methods.values():
+        yield from _walk_callable(method)
+    for inner in cls.inner_classes.values():
+        yield from _walk_class_callables(inner)
+
+
+def _walk_callable(c: PyCallable) -> Iterator[PyCallable]:
+    yield c
+    for inner in c.inner_callables.values():
+        yield from _walk_callable(inner)
+    for inner_cls in c.inner_classes.values():
+        yield from _walk_class_callables(inner_cls)
+
+
+def _walk_module_callables(module: PyModule) -> Iterator[PyCallable]:
+    for fn in module.functions.values():
+        yield from _walk_callable(fn)
+    for cls in module.classes.values():
+        yield from _walk_class_callables(cls)
+
+
+def iter_callables_in_symbol_table(
+    symbol_table: Dict[str, PyModule],
+) -> Iterator[PyCallable]:
+    """Yield every ``PyCallable`` in a symbol table, recursively."""
+    for module in symbol_table.values():
+        yield from _walk_module_callables(module)
+
+
+def _walk_classes_in_class(cls: PyClass) -> Iterator[PyClass]:
+    yield cls
+    for inner in cls.inner_classes.values():
+        yield from _walk_classes_in_class(inner)
+    # Classes can live inside methods (e.g. a factory method that defines
+    # a helper class). Recurse through every method's callable subtree.
+    for method in cls.methods.values():
+        yield from _walk_classes_in_callable(method)
+
+
+def _walk_classes_in_callable(c: PyCallable) -> Iterator[PyClass]:
+    for inner_cls in c.inner_classes.values():
+        yield from _walk_classes_in_class(inner_cls)
+    for inner in c.inner_callables.values():
+        yield from _walk_classes_in_callable(inner)
+
+
+def iter_classes_in_symbol_table(
+    symbol_table: Dict[str, PyModule],
+) -> Iterator[PyClass]:
+    """Yield every ``PyClass`` in a symbol table, recursively — including
+    inner classes, classes nested in functions, and classes nested in
+    class methods."""
+    for module in symbol_table.values():
+        for cls in module.classes.values():
+            yield from _walk_classes_in_class(cls)
+        for fn in module.functions.values():
+            yield from _walk_classes_in_callable(fn)
+
+
+def iter_callables(app: PyApplication) -> Iterator[PyCallable]:
+    """Yield every ``PyCallable`` in the application, recursively."""
+    yield from iter_callables_in_symbol_table(app.symbol_table)
+
+
+def callables_by_signature(app: PyApplication) -> Dict[str, PyCallable]:
+    """Flat ``signature -> PyCallable`` index for O(1) node lookup."""
+    return {c.signature: c for c in iter_callables(app)}
+
+
+def to_digraph(app: PyApplication) -> nx.DiGraph:
+    """Build a ``networkx.DiGraph`` from a ``PyApplication``.
+
+    Nodes are keyed by ``PyCallable.signature``. Nodes for in-source
+    callables carry a ``callable`` attribute holding the full
+    ``PyCallable`` and ``ghost=False``. Endpoints referenced by edges
+    but absent from the symbol table — RPC targets, third-party
+    libraries, framework callbacks, dynamically resolved callees — are
+    added as **ghost** nodes (``callable=None``, ``ghost=True``) so the
+    edges are preserved.
+
+    Edges carry ``type``, ``weight``, and ``provenance`` attributes.
+    """
+    g = nx.DiGraph()
+    by_sig = callables_by_signature(app)
+    for sig, c in by_sig.items():
+        g.add_node(sig, callable=c, ghost=False)
+    for e in app.call_graph:
+        for sig in (e.source, e.target):
+            if sig not in g.nodes:
+                g.add_node(sig, callable=None, ghost=True)
+        g.add_edge(
+            e.source,
+            e.target,
+            type=e.type,
+            weight=e.weight,
+            provenance=list(e.provenance),
+        )
+    return g
+
+
+def from_digraph(g: nx.DiGraph) -> list:
+    """Reduce a ``DiGraph`` to the persisted ``List[PyCallEdge]`` form.
+
+    Only edges are extracted; nodes are not serialized here — they are
+    expected to already exist as ``PyCallable`` entries in the symbol
+    table. Edge attributes default to ``CALL_DEP`` / weight 1 / empty
+    provenance when missing.
+    """
+    edges = []
+    for src, dst, data in g.edges(data=True):
+        edges.append(
+            PyCallEdge(
+                source=src,
+                target=dst,
+                type=data.get("type", "CALL_DEP"),
+                weight=int(data.get("weight", 1)),
+                provenance=list(data.get("provenance", [])),
+            )
+        )
+    return edges
+
+
+def jedi_call_graph_edges(
+    symbol_table: Dict[str, PyModule],
+) -> List[PyCallEdge]:
+    """Derive ``PyCallEdge`` entries from Jedi's per-callable ``call_sites``.
+
+    For every ``PyCallable`` in the symbol table, each ``PyCallsite`` whose
+    ``callee_signature`` is resolved (non-empty) contributes an edge
+    ``caller.signature -> site.callee_signature``. Sites where Jedi failed
+    to resolve the callee (``callee_signature`` is ``None`` or empty) are
+    skipped — they have no anchor to put on the graph.
+
+    Edges are coalesced on ``(source, target)``: ``weight`` is the count of
+    matching sites. Provenance is always ``["jedi"]``; combine with
+    CodeQL-derived edges via ``merge_edges``.
+    """
+    counts: Counter = Counter()
+    for caller in iter_callables_in_symbol_table(symbol_table):
+        for site in caller.call_sites:
+            if not site.callee_signature:
+                continue
+            counts[(caller.signature, site.callee_signature)] += 1
+
+    return [
+        PyCallEdge(source=src, target=dst, weight=n, provenance=["jedi"])
+        for (src, dst), n in counts.items()
+    ]
+
+
+def resolve_unresolved_constructors(symbol_table: Dict[str, PyModule]) -> int:
+    """Fill in ``PyCallsite.callee_signature`` for unresolved constructor sites.
+
+    When both Jedi and CodeQL fail to resolve a constructor call (commonly
+    for classes nested inside functions or methods, where static-analysis
+    points-to is weakest), Jedi still flags the site as
+    ``is_constructor_call=True`` with ``method_name`` set to the class's
+    short name. This pass does the resolution heuristically:
+
+    1. Build a ``short_name -> [PyClass]`` index from all classes in the
+       symbol table.
+    2. For each unresolved constructor site under a caller ``C``, look up
+       candidates by ``site.method_name`` and prefer the class whose
+       ``signature`` is the longest prefix-ancestor of ``C.signature`` —
+       this approximates Python's LEGB scoping for nested classes.
+    3. Set ``callee_signature = f"{class.signature}.__init__"``.
+
+    Returns the number of sites resolved. Best-effort; sites with no
+    matching class or ambiguous candidates with no scope tiebreaker are
+    left as-is.
+    """
+    by_name: Dict[str, List[PyClass]] = {}
+    for cls in iter_classes_in_symbol_table(symbol_table):
+        by_name.setdefault(cls.name, []).append(cls)
+
+    resolved = 0
+    for caller in iter_callables_in_symbol_table(symbol_table):
+        for site in caller.call_sites:
+            if not site.is_constructor_call or site.callee_signature:
+                continue
+            candidates = by_name.get(site.method_name)
+            if not candidates:
+                continue
+
+            # Prefer the class whose signature is the longest prefix of
+            # the caller's signature (closest enclosing scope).
+            def scope_score(c: PyClass, _caller_sig: str = caller.signature) -> int:
+                cls_sig = c.signature
+                parent_sig = cls_sig.rsplit(".", 1)[0] if "." in cls_sig else ""
+                # Score = length of parent_sig if it's a prefix of caller's
+                # signature, else -1 (not in scope, lowest priority).
+                if parent_sig and _caller_sig.startswith(parent_sig):
+                    return len(parent_sig)
+                # Module-level class (parent_sig is the module path) — give
+                # it a base score so it still wins over no match.
+                return 0 if not parent_sig else -1
+
+            best = max(candidates, key=scope_score)
+            if scope_score(best) < 0:
+                # No candidate is reachable from caller's scope.
+                continue
+
+            site.callee_signature = f"{best.signature}.__init__"
+            resolved += 1
+
+    return resolved
+
+
+def merge_edges(*edge_lists: list) -> list:
+    """Merge multiple ``List[PyCallEdge]`` into one.
+
+    Edges with the same ``(source, target)`` are coalesced: weights sum,
+    provenance is the sorted union. Useful for combining edges produced
+    by different backends (e.g. Jedi + CodeQL).
+    """
+    by_key: Dict[Tuple[str, str], PyCallEdge] = {}
+    for edges in edge_lists:
+        for e in edges:
+            k = (e.source, e.target)
+            if k in by_key:
+                cur = by_key[k]
+                cur.weight += e.weight
+                cur.provenance = sorted(set(cur.provenance) | set(e.provenance))
+            else:
+                by_key[k] = e.model_copy()
+    return list(by_key.values())