codeanalyzer-python 0.1.14__py3-none-any.whl → 0.2.0__py3-none-any.whl

codeanalyzer/neo4j/rows.py

@@ -0,0 +1,176 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""The output-agnostic intermediate between :func:`project` and the two writers
+(cypher snapshot / bolt incremental). Pure data — no I/O, no driver. A
+:class:`GraphRows` is a deterministic, deduped bag of nodes and edges that both
+writers consume identically.
+
+Property values are restricted to Neo4j-legal shapes: primitives and homogeneous
+arrays of primitives. ``None`` values are pruned (in Neo4j a null property is
+simply absence).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
+
+# A property value: a primitive, or a homogeneous list of primitives.
+Scalar = Union[str, int, float, bool]
+Prop = Union[Scalar, List[str], List[int], List[float], List[bool]]
+Props = Dict[str, Prop]
+
+
+@dataclass(frozen=True)
+class NodeRef:
+    """How an edge addresses one of its endpoints: the label + key property to
+    MATCH on, and the value."""
+
+    label: str  # the label carrying the uniqueness constraint (e.g. "PySymbol", "PyModule")
+    key_prop: str  # "signature" | "file_key" | "name" | "id"
+    value: str
+
+
+@dataclass
+class NodeRow:
+    labels: List[str]  # labels[0] is the constrained MERGE label; the rest are SET as extra labels
+    key_prop: str
+    value: str
+    props: Props
+
+
+@dataclass
+class EdgeRow:
+    type: str
+    from_ref: NodeRef
+    to_ref: NodeRef
+    props: Props
+
+
+@dataclass
+class GraphRows:
+    nodes: List[NodeRow] = field(default_factory=list)
+    edges: List[EdgeRow] = field(default_factory=list)
+
+
+def prune(p: Dict[str, Optional[Prop]]) -> Props:
+    """Drop ``None`` entries — in Neo4j a null property means "absent", so we
+    never store one. Empty lists are kept (a present-but-empty array is legal)."""
+    return {k: v for k, v in p.items() if v is not None}
+
+
+class RowBuilder:
+    """Accumulates nodes/edges with ``MERGE`` semantics in memory, so the same
+    node touched many times (a hot external symbol, a canonical decorator)
+    collapses to one row, and cross-reference edges to a target that never
+    materialized are dropped (the "edge-only-when-resolved" rule).
+    """
+
+    def __init__(self) -> None:
+        self._nodes: Dict[str, NodeRow] = {}  # key: f"{labels[0]} {value}"
+        self._edges: List[EdgeRow] = []
+        self._deferred: List[EdgeRow] = []  # edges gated against node existence at finish()
+        self._keys: set = set()  # every node value seen, for resolved-gating
+
+    def node(self, labels: List[str], key_prop: str, value: str, props: Props) -> NodeRef:
+        """Upsert a node. Re-seeing the same ``(labels[0], value)`` merges props
+        (last write wins) and unions labels — the in-memory analog of
+        ``MERGE (n:Label {key}) SET n += props``."""
+        node_id = f"{labels[0]} {value}"
+        existing = self._nodes.get(node_id)
+        if existing is not None:
+            existing.props.update(props)
+            for label in labels:
+                if label not in existing.labels:
+                    existing.labels.append(label)
+        else:
+            self._nodes[node_id] = NodeRow(list(labels), key_prop, value, dict(props))
+        self._keys.add(value)
+        return NodeRef(labels[0], key_prop, value)
+
+    def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None) -> None:
+        """An edge whose endpoints are known to exist (both ends emitted this run)."""
+        self._edges.append(EdgeRow(type_, from_ref, to_ref, dict(props or {})))
+
+    def edge_to_symbol(
+        self, type_: str, from_ref: NodeRef, target_signature: str, props: Optional[Props] = None
+    ) -> None:
+        """An edge to a ``:PySymbol`` target that may be external/library code not
+        present in the graph. Deferred and kept only if the target signature was
+        actually emitted as a node — so PY_EXTENDS / PY_RESOLVES_TO never dangle (the
+        string fallback lives on the source node's props)."""
+        self._deferred.append(
+            EdgeRow(
+                type_,
+                from_ref,
+                NodeRef("PySymbol", "signature", target_signature),
+                dict(props or {}),
+            )
+        )
+
+    def has_key(self, value: str) -> bool:
+        return value in self._keys
+
+    def finish(self) -> GraphRows:
+        for e in self._deferred:
+            if e.to_ref.value in self._keys:
+                self._edges.append(e)
+        nodes = sorted(self._nodes.values(), key=lambda n: f"{n.labels[0]} {n.value}")
+        edges = sorted(self._edges, key=lambda e: f"{e.type} {e.from_ref.value} {e.to_ref.value}")
+        return GraphRows(nodes, edges)
+
+
+# ----------------------------------------------------------------------------------------------
+# Cypher literal rendering (used by the snapshot writer; the bolt writer passes params instead).
+# ----------------------------------------------------------------------------------------------
+
+
+def cypher_value(v: Prop) -> str:
+    """Render a property value as a Cypher literal."""
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if isinstance(v, str):
+        return _cypher_string(v)
+    if isinstance(v, (int, float)):
+        # bools are handled above; int/float fall through here.
+        if isinstance(v, float) and (v != v or v in (float("inf"), float("-inf"))):
+            return "null"
+        return repr(v) if isinstance(v, float) else str(v)
+    if isinstance(v, list):
+        return "[" + ", ".join(cypher_value(x) for x in v) + "]"
+    return "null"
+
+
+def cypher_map(props: Props) -> str:
+    """Render a props map as a Cypher map literal: ``{key: value, ...}``.
+    Keys are valid identifiers."""
+    return "{" + ", ".join(f"{k}: {cypher_value(v)}" for k, v in props.items()) + "}"
+
+
+def _cypher_string(s: str) -> str:
+    escaped = (
+        s.replace("\\", "\\\\")
+        .replace("'", "\\'")
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+        .replace("\t", "\\t")
+    )
+    return f"'{escaped}'"
+
+
+def chunk(items: list, size: int) -> list:
+    """Split a list into chunks of at most ``size`` (UNWIND batch sizing)."""
+    return [items[i : i + size] for i in range(0, len(items), size)]

codeanalyzer/neo4j/schema.py

@@ -0,0 +1,39 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""The Cypher DDL — uniqueness constraints and indexes — shared by both writers.
+Run BEFORE any load so MERGE uses an index seek (not a label scan) and the
+identity invariant is enforced by the database. Every statement is idempotent
+(``IF NOT EXISTS``).
+"""
+from typing import List
+
+CONSTRAINTS: List[str] = [
+    "CREATE CONSTRAINT py_symbol_sig IF NOT EXISTS FOR (s:PySymbol) REQUIRE s.signature IS UNIQUE",
+    "CREATE CONSTRAINT py_app_name IF NOT EXISTS FOR (a:PyApplication) REQUIRE a.name IS UNIQUE",
+    "CREATE CONSTRAINT py_module_key IF NOT EXISTS FOR (m:PyModule) REQUIRE m.file_key IS UNIQUE",
+    "CREATE CONSTRAINT py_package_name IF NOT EXISTS FOR (p:PyPackage) REQUIRE p.name IS UNIQUE",
+    "CREATE CONSTRAINT py_decorator_name IF NOT EXISTS FOR (d:PyDecorator) REQUIRE d.name IS UNIQUE",
+    "CREATE CONSTRAINT py_callsite_id IF NOT EXISTS FOR (c:PyCallSite) REQUIRE c.id IS UNIQUE",
+    "CREATE CONSTRAINT py_attribute_id IF NOT EXISTS FOR (a:PyAttribute) REQUIRE a.id IS UNIQUE",
+    "CREATE CONSTRAINT py_variable_id IF NOT EXISTS FOR (v:PyVariable) REQUIRE v.id IS UNIQUE",
+]
+
+INDEXES: List[str] = [
+    "CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
+    "CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
+    "CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
+]

codeanalyzer/options/__init__.py

@@ -1,3 +1,3 @@
-from .options import AnalysisOptions
+from .options import AnalysisOptions, EmitTarget, OutputFormat
 
-__all__ = ["AnalysisOptions"]
+__all__ = ["AnalysisOptions", "EmitTarget", "OutputFormat"]

codeanalyzer/options/options.py

@@ -9,11 +9,31 @@ class OutputFormat(str, Enum):
     MSGPACK = "msgpack"
 
 
+class EmitTarget(str, Enum):
+    """Output target selected by ``--emit``.
+
+    - ``json``   : the canonical ``analysis.json`` (symbol table + call graph).
+    - ``neo4j``  : project the analysis into a labeled property graph — a
+                   ``graph.cypher`` snapshot, or a live Bolt push with ``--neo4j-uri``.
+    - ``schema`` : the machine-readable, version-stamped Neo4j schema contract.
+    """
+
+    JSON = "json"
+    NEO4J = "neo4j"
+    SCHEMA = "schema"
+
+
 @dataclass
 class AnalysisOptions:
     input: Path
     output: Optional[Path] = None
     format: OutputFormat = OutputFormat.JSON
+    emit: EmitTarget = EmitTarget.JSON
+    app_name: Optional[str] = None
+    neo4j_uri: Optional[str] = None
+    neo4j_user: str = "neo4j"
+    neo4j_password: str = "neo4j"
+    neo4j_database: Optional[str] = None
     using_codeql: bool = False
     using_ray: bool = False
     rebuild_analysis: bool = False

codeanalyzer/semantic_analysis/codeql/codeql_analysis.py

@@ -32,6 +32,67 @@ from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQuer
 from codeanalyzer.utils import logger
 
 
+class _CallableResolver:
+    """Maps a CodeQL endpoint ``(file, start_line, name, arity)`` to a Jedi
+    ``PyCallable``.
+
+    Resolution ladder:
+      1. exact ``(abs_path, start_line)`` — the precise join;
+      2. on miss, candidates sharing ``(abs_path, short_name)``: a single
+         candidate is taken directly; otherwise prefer those whose
+         parameter count equals the CodeQL positional arity, then the
+         nearest ``start_line``;
+      3. no name match -> ``None`` (caller row skipped / callee becomes
+         a ghost node).
+
+    Step 2 recovers edges the ``(file, line)`` join silently drops when
+    CodeQL and Jedi disagree on a definition's start line (e.g. decorator
+    handling). Jedi's ``parameters`` counts every declared slot (incl.
+    ``*args``/``**kwargs``/keyword-only) whereas CodeQL's arity is
+    positional only, so the arity filter is exact for plain signatures
+    and otherwise yields to the nearest-line tiebreak.
+    """
+
+    def __init__(self) -> None:
+        self._by_loc: Dict[Tuple[str, int], Any] = {}
+        self._by_name: Dict[Tuple[str, str], List[Any]] = {}
+
+    @staticmethod
+    def _abs(path: str) -> str:
+        try:
+            return str(Path(path).resolve())
+        except (OSError, RuntimeError):
+            return path
+
+    @classmethod
+    def from_symbol_table(
+        cls, symbol_table: Dict[str, PyModule]
+    ) -> "_CallableResolver":
+        resolver = cls()
+        for c in iter_callables_in_symbol_table(symbol_table):
+            abs_path = cls._abs(c.path)
+            resolver._by_loc[(abs_path, c.start_line)] = c
+            resolver._by_name.setdefault((abs_path, c.name), []).append(c)
+        return resolver
+
+    def resolve(
+        self, file: str, start_line: int, name: str, arity: int
+    ) -> Any:
+        exact = self._by_loc.get((file, start_line))
+        if exact is not None:
+            return exact
+        if not name:
+            return None
+        candidates = self._by_name.get((file, name))
+        if not candidates:
+            return None
+        if len(candidates) == 1:
+            return candidates[0]
+        arity_matched = [c for c in candidates if len(c.parameters) == arity]
+        pool = arity_matched or candidates
+        return min(pool, key=lambda c: abs(c.start_line - start_line))
+
+
 class CodeQL:
     """A class for building the application view of a Python application using CodeQL.
 
@@ -99,9 +160,14 @@ class CodeQL:
             # codeql/python-all 7.x — it returns the ``CallNode`` (CFG)
             # whose target was resolved to that ``Value``. Cleaner than
             # poking at ``pointsTo`` directly.
-            "from CallNode call, Function caller, FunctionValue calleeVal",
+            # ``callee`` is bound to the FunctionValue's scope so the
+            # endpoint emits the same Function-level facts (name, arity,
+            # location) the post-processor needs for the name+arity
+            # fallback when the (file, start_line) join misses.
+            "from CallNode call, Function caller, FunctionValue calleeVal, Function callee",
             "where",
             "  call.getScope() = caller and",
+            "  callee = calleeVal.getScope() and",
             "  (",
             # Direct function / bound-method call:  foo()  or  obj.foo()
             "    call = calleeVal.getACall()",
@@ -115,15 +181,20 @@ class CodeQL:
             "    )",
             "  )",
             "select",
-            # --- Caller endpoint --- (joins to PyCallable via file + start_line)
+            # --- Caller endpoint --- (joins to PyCallable: exact by
+            #     (file, start_line), else by (file, name) + arity)
             "  caller.getLocation().getFile().getAbsolutePath(),",
             "  caller.getLocation().getStartLine(),",
             "  caller.getQualifiedName(),",
+            "  caller.getName(),",
+            "  count(caller.getArg(_)),",
             # --- Callee endpoint --- (file/line may live in a library stub;
             #     post-processor classifies as in-source or ghost)
-            "  calleeVal.getScope().getLocation().getFile().getAbsolutePath(),",
-            "  calleeVal.getScope().getLocation().getStartLine(),",
+            "  callee.getLocation().getFile().getAbsolutePath(),",
+            "  callee.getLocation().getStartLine(),",
             "  calleeVal.getQualifiedName(),",
+            "  callee.getName(),",
+            "  count(callee.getArg(_)),",
             # --- Call-site location --- (for PyCallsite augmentation)
             "  call.getLocation().getStartLine(),",
             "  call.getLocation().getStartColumn(),",
@@ -149,9 +220,13 @@ class CodeQL:
                     "caller_file",
                     "caller_start_line",
                     "caller_qname",
+                    "caller_name",
+                    "caller_arity",
                     "callee_file",
                     "callee_start_line",
                     "callee_qname",
+                    "callee_name",
+                    "callee_arity",
                     "call_start_line",
                     "call_start_column",
                     "call_end_line",
@@ -162,24 +237,15 @@ class CodeQL:
         return df
 
     @staticmethod
-    def _build_callable_location_index(
+    def _build_callable_resolver(
         symbol_table: Dict[str, PyModule],
-    ) -> Dict[Tuple[str, int], "PyCallable"]:
-        """Build ``(absolute_file_path, start_line) -> PyCallable`` from Jedi.
+    ) -> _CallableResolver:
+        """Build the endpoint -> ``PyCallable`` resolver from Jedi.
 
         Paths are resolved so they match CodeQL's ``getAbsolutePath()``
         regardless of symlinks or the current working directory.
         """
-        from codeanalyzer.schema.py_schema import PyCallable  # local to avoid cycle
-
-        index: Dict[Tuple[str, int], PyCallable] = {}
-        for c in iter_callables_in_symbol_table(symbol_table):
-            try:
-                abs_path = str(Path(c.path).resolve())
-            except (OSError, RuntimeError):
-                abs_path = c.path
-            index[(abs_path, c.start_line)] = c
-        return index
+        return _CallableResolver.from_symbol_table(symbol_table)
 
     def _iter_resolved_rows(
         self, symbol_table: Dict[str, PyModule]
@@ -194,19 +260,27 @@ class CodeQL:
         df = self._query_call_edges()
         if df.empty:
             return
-        location_index = self._build_callable_location_index(symbol_table)
+        resolver = self._build_callable_resolver(symbol_table)
 
         skipped_unknown_caller = 0
         ghost_callees = 0
         for row in df.itertuples(index=False):
-            caller_key = (row.caller_file, int(row.caller_start_line))
-            caller = location_index.get(caller_key)
+            caller = resolver.resolve(
+                row.caller_file,
+                int(row.caller_start_line),
+                row.caller_name,
+                int(row.caller_arity),
+            )
             if caller is None:
                 skipped_unknown_caller += 1
                 continue
 
-            callee_key = (row.callee_file, int(row.callee_start_line))
-            callee = location_index.get(callee_key)
+            callee = resolver.resolve(
+                row.callee_file,
+                int(row.callee_start_line),
+                row.callee_name,
+                int(row.callee_arity),
+            )
             if callee is not None:
                 target_sig = callee.signature
             else:
@@ -267,20 +341,28 @@ class CodeQL:
         Returns:
             Number of ``PyCallsite`` entries augmented.
         """
-        location_index = self._build_callable_location_index(symbol_table)
+        resolver = self._build_callable_resolver(symbol_table)
         df = self._query_call_edges()
         if df.empty:
             return 0
 
         augmented = 0
         for row in df.itertuples(index=False):
-            caller_key = (row.caller_file, int(row.caller_start_line))
-            caller = location_index.get(caller_key)
+            caller = resolver.resolve(
+                row.caller_file,
+                int(row.caller_start_line),
+                row.caller_name,
+                int(row.caller_arity),
+            )
             if caller is None:
                 continue
 
-            callee_key = (row.callee_file, int(row.callee_start_line))
-            callee = location_index.get(callee_key)
+            callee = resolver.resolve(
+                row.callee_file,
+                int(row.callee_start_line),
+                row.callee_name,
+                int(row.callee_arity),
+            )
             resolved_sig = callee.signature if callee is not None else row.callee_qname
 
             call_start = int(row.call_start_line)