codeanalyzer-python 0.1.13__py3-none-any.whl → 0.2.0__py3-none-any.whl

codeanalyzer/neo4j/project.py

@@ -0,0 +1,322 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""``project()`` — the pure projection from the canonical :class:`PyApplication`
+IR to graph rows. It walks the same recursive symbol table the call-graph builder
+walks, but instead of collecting callables it emits nodes + edges. No I/O: the
+writers (cypher snapshot / bolt incremental) consume the returned
+:class:`GraphRows`.
+
+Modelling decisions (mirror of the TypeScript backend):
+  - signature-keyed declarations (PyClass, PyCallable) carry a shared ``:PySymbol``
+    label (the global-identity / MERGE key).
+  - call sites, decorators, class attributes and variables are first-class nodes.
+  - call-graph endpoints absent from the symbol table become ``:PyExternal`` ghost
+    nodes, so RPC / third-party / framework edges are preserved (matching the
+    analyzer's own ghost-node behaviour).
+  - every project-owned node carries an internal ``_module`` provenance prop, so
+    the incremental writer can delete exactly what a re-analyzed module emitted.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, List, Optional
+
+from codeanalyzer.neo4j.catalog import SCHEMA_VERSION
+from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
+from codeanalyzer.schema import (
+    PyApplication,
+    PyCallable,
+    PyClass,
+    PyClassAttribute,
+    PyComment,
+    PyModule,
+    PyVariableDeclaration,
+)
+from codeanalyzer.schema.py_schema import PyCallsite
+
+
+def project(app: PyApplication, app_name: str) -> GraphRows:
+    b = RowBuilder()
+
+    app_ref = b.node(["PyApplication"], "name", app_name, {"schema_version": SCHEMA_VERSION})
+
+    for file_key, mod in app.symbol_table.items():
+        mod_ref = b.node(["PyModule"], "file_key", file_key, _module_props(mod, file_key))
+        b.edge("PY_HAS_MODULE", app_ref, mod_ref)
+        _project_module_body(b, file_key, mod_ref, mod)
+
+    # The aggregated :PY_CALLS twin. Endpoints not present in the symbol table become
+    # :PyExternal ghost nodes (the analyzer already preserves them as ghost nodes).
+    for e in app.call_graph:
+        src = _call_endpoint(b, e.source)
+        tgt = _call_endpoint(b, e.target)
+        b.edge("PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.provenance or [])))
+
+    return b.finish()
+
+
+def _sym(signature: str) -> NodeRef:
+    return NodeRef("PySymbol", "signature", signature)
+
+
+def _call_endpoint(b: RowBuilder, signature: str) -> NodeRef:
+    """A call-graph endpoint: a known callable already emitted, or a phantom
+    :PyExternal symbol materialized on demand for a ghost target."""
+    if b.has_key(signature):
+        return _sym(signature)
+    name = signature.rsplit(".", 1)[-1] if "." in signature else signature
+    return b.node(["PySymbol", "PyExternal"], "signature", signature, {"name": name})
+
+
+# ----------------------------------------------------------------------------------------------
+# Module body
+# ----------------------------------------------------------------------------------------------
+
+
+def _project_module_body(b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule) -> None:
+    for fn in (mod.functions or {}).values():
+        _project_callable(b, file_key, mod_ref, "PY_DECLARES", fn)
+    for cl in (mod.classes or {}).values():
+        _project_class(b, file_key, mod_ref, "PY_DECLARES", cl)
+    for v in mod.variables or []:
+        _project_variable(b, file_key, mod_ref, file_key, v)
+    _project_imports(b, mod_ref, mod)
+
+
+def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
+    # Per-target-module aggregation: collapse all bindings for a given imported
+    # module into one PY_IMPORTS edge to a shared :PyPackage node.
+    agg: dict = {}
+    for im in mod.imports or []:
+        if not im.module:
+            continue  # relative `from . import x` — no resolvable package
+        a = agg.setdefault(im.module, {"names": set(), "aliases": set()})
+        if im.name:
+            a["names"].add(im.name)
+        if im.alias:
+            a["aliases"].add(im.alias)
+    for module_name, a in agg.items():
+        pkg = b.node(["PyPackage"], "name", module_name, {})
+        b.edge(
+            "PY_IMPORTS",
+            mod_ref,
+            pkg,
+            prune(
+                {
+                    "imported_names": sorted(a["names"]) or None,
+                    "aliases": sorted(a["aliases"]) or None,
+                }
+            ),
+        )
+
+
+# ----------------------------------------------------------------------------------------------
+# Declarations
+# ----------------------------------------------------------------------------------------------
+
+
+def _project_class(
+    b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass
+) -> None:
+    ref = b.node(["PySymbol", "PyClass"], "signature", cl.signature, _class_props(cl, file_key))
+    b.edge(parent_rel, parent, ref)
+
+    for base in cl.base_classes or []:
+        b.edge_to_symbol("PY_EXTENDS", ref, base)
+
+    for m in (cl.methods or {}).values():
+        _project_callable(b, file_key, ref, "PY_HAS_METHOD", m)
+    for a in (cl.attributes or {}).values():
+        _project_attribute(b, file_key, ref, cl.signature, a)
+    for ic in (cl.inner_classes or {}).values():
+        _project_class(b, file_key, ref, "PY_DECLARES", ic)
+
+
+def _project_callable(
+    b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable
+) -> None:
+    ref = b.node(["PySymbol", "PyCallable"], "signature", c.signature, _callable_props(c, file_key))
+    b.edge(owner_rel, owner, ref)
+
+    for d in c.decorators or []:
+        _project_decorator(b, ref, d)
+
+    for s in c.call_sites or []:
+        # Key off the relative file (a call site lives in its callable's file) so ids stay portable.
+        cs_id = f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
+        cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
+        b.edge("PY_HAS_CALLSITE", ref, cs)
+        if s.callee_signature:
+            b.edge_to_symbol("PY_RESOLVES_TO", cs, s.callee_signature)
+
+    for v in c.local_variables or []:
+        _project_variable(b, file_key, ref, c.signature, v)
+    for ic in (c.inner_callables or {}).values():
+        _project_callable(b, file_key, ref, "PY_DECLARES", ic)
+    for cl in (c.inner_classes or {}).values():
+        _project_class(b, file_key, ref, "PY_DECLARES", cl)
+
+
+def _project_attribute(
+    b: RowBuilder, file_key: str, owner: NodeRef, owner_sig: str, a: PyClassAttribute
+) -> None:
+    attr_id = f"{owner_sig}.{a.name}"
+    ref = b.node(["PyAttribute"], "id", attr_id, _attribute_props(a, attr_id, file_key))
+    b.edge("PY_HAS_ATTRIBUTE", owner, ref)
+
+
+def _project_variable(
+    b: RowBuilder, file_key: str, owner: NodeRef, owner_id: str, v: PyVariableDeclaration
+) -> None:
+    var_id = f"{owner_id}#{v.name}@{v.start_line}"
+    ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
+    b.edge("PY_DECLARES_VAR", owner, ref)
+
+
+def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
+    dec = b.node(["PyDecorator"], "name", decorator, {"name": decorator})
+    b.edge("PY_DECORATED_BY", on, dec)
+
+
+# ----------------------------------------------------------------------------------------------
+# Property flattening
+# ----------------------------------------------------------------------------------------------
+
+
+def _module_props(mod: PyModule, file_key: str) -> Props:
+    return prune(
+        {
+            "module_name": mod.module_name,
+            "content_hash": mod.content_hash,
+            "last_modified": mod.last_modified,
+            "file_size": mod.file_size,
+            "_module": file_key,
+        }
+    )
+
+
+def _class_props(cl: PyClass, file_key: str) -> Props:
+    return prune(
+        {
+            "name": cl.name,
+            "code": cl.code,
+            "base_classes": list(cl.base_classes or []),
+            "docstring": _docstring_of(cl.comments),
+            "start_line": cl.start_line,
+            "end_line": cl.end_line,
+            "_module": file_key,
+        }
+    )
+
+
+def _callable_props(c: PyCallable, file_key: str) -> Props:
+    return prune(
+        {
+            "name": c.name,
+            "path": c.path,
+            "return_type": c.return_type,
+            "cyclomatic_complexity": c.cyclomatic_complexity,
+            "code": c.code,
+            "code_start_line": c.code_start_line,
+            "start_line": c.start_line,
+            "end_line": c.end_line,
+            "docstring": _docstring_of(c.comments),
+            "decorators": list(c.decorators or []),
+            "parameters_json": _stringify_if(c.parameters),
+            "accessed_symbols_json": _stringify_if(c.accessed_symbols),
+            "_module": file_key,
+        }
+    )
+
+
+def _attribute_props(a: PyClassAttribute, attr_id: str, file_key: str) -> Props:
+    return prune(
+        {
+            "id": attr_id,
+            "name": a.name,
+            "type": a.type,
+            "docstring": _docstring_of(a.comments),
+            "start_line": a.start_line,
+            "end_line": a.end_line,
+            "_module": file_key,
+        }
+    )
+
+
+def _variable_props(v: PyVariableDeclaration, var_id: str, file_key: str) -> Props:
+    return prune(
+        {
+            "id": var_id,
+            "name": v.name,
+            "type": v.type,
+            "initializer": v.initializer,
+            "scope": v.scope,
+            "start_line": v.start_line,
+            "end_line": v.end_line,
+            "_module": file_key,
+        }
+    )
+
+
+def _call_site_props(s: PyCallsite, file_key: str) -> Props:
+    cs_id = f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
+    return prune(
+        {
+            "id": cs_id,
+            "method_name": s.method_name,
+            "receiver_expr": s.receiver_expr,
+            "receiver_type": s.receiver_type,
+            "argument_types": list(s.argument_types or []),
+            "return_type": s.return_type,
+            "callee_signature": s.callee_signature,
+            "is_constructor_call": s.is_constructor_call,
+            "start_line": s.start_line,
+            "start_column": s.start_column,
+            "end_line": s.end_line,
+            "end_column": s.end_column,
+            "_module": file_key,
+        }
+    )
+
+
+def _call_edge_props(weight: int, provenance: List[str]) -> Props:
+    return prune({"weight": weight, "provenance": list(provenance)})
+
+
+def _docstring_of(comments: Optional[List[PyComment]]) -> Optional[str]:
+    docs = [c.content for c in (comments or []) if c.is_docstring]
+    return "\n".join(docs) if docs else None
+
+
+def _stringify_if(value: Any) -> Optional[str]:
+    """JSON-encode a list/dict of pydantic models, or None when empty."""
+    if value is None:
+        return None
+    if isinstance(value, (list, dict)) and len(value) == 0:
+        return None
+    return json.dumps(value, default=_jsonable, sort_keys=True)
+
+
+def _jsonable(o: Any) -> Any:
+    if hasattr(o, "model_dump"):
+        return o.model_dump()
+    if hasattr(o, "dict"):
+        return o.dict()
+    if isinstance(o, Path):
+        return str(o)
+    return str(o)

codeanalyzer/neo4j/rows.py

@@ -0,0 +1,176 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""The output-agnostic intermediate between :func:`project` and the two writers
+(cypher snapshot / bolt incremental). Pure data — no I/O, no driver. A
+:class:`GraphRows` is a deterministic, deduped bag of nodes and edges that both
+writers consume identically.
+
+Property values are restricted to Neo4j-legal shapes: primitives and homogeneous
+arrays of primitives. ``None`` values are pruned (in Neo4j a null property is
+simply absence).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
+
+# A property value: a primitive, or a homogeneous list of primitives.
+Scalar = Union[str, int, float, bool]
+Prop = Union[Scalar, List[str], List[int], List[float], List[bool]]
+Props = Dict[str, Prop]
+
+
+@dataclass(frozen=True)
+class NodeRef:
+    """How an edge addresses one of its endpoints: the label + key property to
+    MATCH on, and the value."""
+
+    label: str  # the label carrying the uniqueness constraint (e.g. "PySymbol", "PyModule")
+    key_prop: str  # "signature" | "file_key" | "name" | "id"
+    value: str
+
+
+@dataclass
+class NodeRow:
+    labels: List[str]  # labels[0] is the constrained MERGE label; the rest are SET as extra labels
+    key_prop: str
+    value: str
+    props: Props
+
+
+@dataclass
+class EdgeRow:
+    type: str
+    from_ref: NodeRef
+    to_ref: NodeRef
+    props: Props
+
+
+@dataclass
+class GraphRows:
+    nodes: List[NodeRow] = field(default_factory=list)
+    edges: List[EdgeRow] = field(default_factory=list)
+
+
+def prune(p: Dict[str, Optional[Prop]]) -> Props:
+    """Drop ``None`` entries — in Neo4j a null property means "absent", so we
+    never store one. Empty lists are kept (a present-but-empty array is legal)."""
+    return {k: v for k, v in p.items() if v is not None}
+
+
+class RowBuilder:
+    """Accumulates nodes/edges with ``MERGE`` semantics in memory, so the same
+    node touched many times (a hot external symbol, a canonical decorator)
+    collapses to one row, and cross-reference edges to a target that never
+    materialized are dropped (the "edge-only-when-resolved" rule).
+    """
+
+    def __init__(self) -> None:
+        self._nodes: Dict[str, NodeRow] = {}  # key: f"{labels[0]} {value}"
+        self._edges: List[EdgeRow] = []
+        self._deferred: List[EdgeRow] = []  # edges gated against node existence at finish()
+        self._keys: set = set()  # every node value seen, for resolved-gating
+
+    def node(self, labels: List[str], key_prop: str, value: str, props: Props) -> NodeRef:
+        """Upsert a node. Re-seeing the same ``(labels[0], value)`` merges props
+        (last write wins) and unions labels — the in-memory analog of
+        ``MERGE (n:Label {key}) SET n += props``."""
+        node_id = f"{labels[0]} {value}"
+        existing = self._nodes.get(node_id)
+        if existing is not None:
+            existing.props.update(props)
+            for label in labels:
+                if label not in existing.labels:
+                    existing.labels.append(label)
+        else:
+            self._nodes[node_id] = NodeRow(list(labels), key_prop, value, dict(props))
+        self._keys.add(value)
+        return NodeRef(labels[0], key_prop, value)
+
+    def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None) -> None:
+        """An edge whose endpoints are known to exist (both ends emitted this run)."""
+        self._edges.append(EdgeRow(type_, from_ref, to_ref, dict(props or {})))
+
+    def edge_to_symbol(
+        self, type_: str, from_ref: NodeRef, target_signature: str, props: Optional[Props] = None
+    ) -> None:
+        """An edge to a ``:PySymbol`` target that may be external/library code not
+        present in the graph. Deferred and kept only if the target signature was
+        actually emitted as a node — so PY_EXTENDS / PY_RESOLVES_TO never dangle (the
+        string fallback lives on the source node's props)."""
+        self._deferred.append(
+            EdgeRow(
+                type_,
+                from_ref,
+                NodeRef("PySymbol", "signature", target_signature),
+                dict(props or {}),
+            )
+        )
+
+    def has_key(self, value: str) -> bool:
+        return value in self._keys
+
+    def finish(self) -> GraphRows:
+        for e in self._deferred:
+            if e.to_ref.value in self._keys:
+                self._edges.append(e)
+        nodes = sorted(self._nodes.values(), key=lambda n: f"{n.labels[0]} {n.value}")
+        edges = sorted(self._edges, key=lambda e: f"{e.type} {e.from_ref.value} {e.to_ref.value}")
+        return GraphRows(nodes, edges)
+
+
+# ----------------------------------------------------------------------------------------------
+# Cypher literal rendering (used by the snapshot writer; the bolt writer passes params instead).
+# ----------------------------------------------------------------------------------------------
+
+
+def cypher_value(v: Prop) -> str:
+    """Render a property value as a Cypher literal."""
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if isinstance(v, str):
+        return _cypher_string(v)
+    if isinstance(v, (int, float)):
+        # bools are handled above; int/float fall through here.
+        if isinstance(v, float) and (v != v or v in (float("inf"), float("-inf"))):
+            return "null"
+        return repr(v) if isinstance(v, float) else str(v)
+    if isinstance(v, list):
+        return "[" + ", ".join(cypher_value(x) for x in v) + "]"
+    return "null"
+
+
+def cypher_map(props: Props) -> str:
+    """Render a props map as a Cypher map literal: ``{key: value, ...}``.
+    Keys are valid identifiers."""
+    return "{" + ", ".join(f"{k}: {cypher_value(v)}" for k, v in props.items()) + "}"
+
+
+def _cypher_string(s: str) -> str:
+    escaped = (
+        s.replace("\\", "\\\\")
+        .replace("'", "\\'")
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+        .replace("\t", "\\t")
+    )
+    return f"'{escaped}'"
+
+
+def chunk(items: list, size: int) -> list:
+    """Split a list into chunks of at most ``size`` (UNWIND batch sizing)."""
+    return [items[i : i + size] for i in range(0, len(items), size)]

codeanalyzer/neo4j/schema.py

@@ -0,0 +1,39 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""The Cypher DDL — uniqueness constraints and indexes — shared by both writers.
+Run BEFORE any load so MERGE uses an index seek (not a label scan) and the
+identity invariant is enforced by the database. Every statement is idempotent
+(``IF NOT EXISTS``).
+"""
+from typing import List
+
+CONSTRAINTS: List[str] = [
+    "CREATE CONSTRAINT py_symbol_sig IF NOT EXISTS FOR (s:PySymbol) REQUIRE s.signature IS UNIQUE",
+    "CREATE CONSTRAINT py_app_name IF NOT EXISTS FOR (a:PyApplication) REQUIRE a.name IS UNIQUE",
+    "CREATE CONSTRAINT py_module_key IF NOT EXISTS FOR (m:PyModule) REQUIRE m.file_key IS UNIQUE",
+    "CREATE CONSTRAINT py_package_name IF NOT EXISTS FOR (p:PyPackage) REQUIRE p.name IS UNIQUE",
+    "CREATE CONSTRAINT py_decorator_name IF NOT EXISTS FOR (d:PyDecorator) REQUIRE d.name IS UNIQUE",
+    "CREATE CONSTRAINT py_callsite_id IF NOT EXISTS FOR (c:PyCallSite) REQUIRE c.id IS UNIQUE",
+    "CREATE CONSTRAINT py_attribute_id IF NOT EXISTS FOR (a:PyAttribute) REQUIRE a.id IS UNIQUE",
+    "CREATE CONSTRAINT py_variable_id IF NOT EXISTS FOR (v:PyVariable) REQUIRE v.id IS UNIQUE",
+]
+
+INDEXES: List[str] = [
+    "CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
+    "CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
+    "CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
+]

codeanalyzer/options/__init__.py

@@ -1,3 +1,3 @@
-from .options import AnalysisOptions
+from .options import AnalysisOptions, EmitTarget, OutputFormat
 
-__all__ = ["AnalysisOptions"]
+__all__ = ["AnalysisOptions", "EmitTarget", "OutputFormat"]

codeanalyzer/options/options.py

@@ -9,12 +9,31 @@ class OutputFormat(str, Enum):
     MSGPACK = "msgpack"
 
 
+class EmitTarget(str, Enum):
+    """Output target selected by ``--emit``.
+
+    - ``json``   : the canonical ``analysis.json`` (symbol table + call graph).
+    - ``neo4j``  : project the analysis into a labeled property graph — a
+                   ``graph.cypher`` snapshot, or a live Bolt push with ``--neo4j-uri``.
+    - ``schema`` : the machine-readable, version-stamped Neo4j schema contract.
+    """
+
+    JSON = "json"
+    NEO4J = "neo4j"
+    SCHEMA = "schema"
+
+
 @dataclass
 class AnalysisOptions:
     input: Path
     output: Optional[Path] = None
     format: OutputFormat = OutputFormat.JSON
-    analysis_level: int = 1
+    emit: EmitTarget = EmitTarget.JSON
+    app_name: Optional[str] = None
+    neo4j_uri: Optional[str] = None
+    neo4j_user: str = "neo4j"
+    neo4j_password: str = "neo4j"
+    neo4j_database: Optional[str] = None
     using_codeql: bool = False
     using_ray: bool = False
     rebuild_analysis: bool = False

codeanalyzer/schema/py_schema.py

@@ -339,9 +339,29 @@ class PyModule(BaseModel):
     file_size: Optional[int] = None
 
 
+@builder
+@msgpk
+class PyCallEdge(BaseModel):
+    """Identity-only call-graph edge with weight.
+
+    Mirrors Java's ``CallDependency``. ``source`` and ``target`` are
+    ``PyCallable.signature`` strings — nodes of the graph are the existing
+    ``PyCallable`` entries in the symbol table, not a separate vertex type.
+    Rich per-call metadata (receiver, arguments, location, ...) lives on
+    ``PyCallsite`` inside the source ``PyCallable.call_sites``.
+    """
+
+    source: str  # caller's PyCallable.signature
+    target: str  # callee's PyCallable.signature
+    type: Literal["CALL_DEP"] = "CALL_DEP"
+    weight: int = 1
+    provenance: List[Literal["jedi", "codeql", "joern"]] = []
+
+
 @builder
 @msgpk
 class PyApplication(BaseModel):
     """Represents a Python application."""
 
     symbol_table: Dict[str, PyModule]
+    call_graph: List[PyCallEdge] = []