codeanalyzer-python 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

codeanalyzer/neo4j/schema.py

@@ -14,26 +14,308 @@
 # limitations under the License.
 ################################################################################
 
-"""The Cypher DDL — uniqueness constraints and indexes — shared by both writers.
-Run BEFORE any load so MERGE uses an index seek (not a label scan) and the
-identity invariant is enforced by the database. Every statement is idempotent
-(``IF NOT EXISTS``).
 """
-from typing import List
-
-CONSTRAINTS: List[str] = [
-    "CREATE CONSTRAINT py_symbol_sig IF NOT EXISTS FOR (s:PySymbol) REQUIRE s.signature IS UNIQUE",
-    "CREATE CONSTRAINT py_app_name IF NOT EXISTS FOR (a:PyApplication) REQUIRE a.name IS UNIQUE",
-    "CREATE CONSTRAINT py_module_key IF NOT EXISTS FOR (m:PyModule) REQUIRE m.file_key IS UNIQUE",
-    "CREATE CONSTRAINT py_package_name IF NOT EXISTS FOR (p:PyPackage) REQUIRE p.name IS UNIQUE",
-    "CREATE CONSTRAINT py_decorator_name IF NOT EXISTS FOR (d:PyDecorator) REQUIRE d.name IS UNIQUE",
-    "CREATE CONSTRAINT py_callsite_id IF NOT EXISTS FOR (c:PyCallSite) REQUIRE c.id IS UNIQUE",
-    "CREATE CONSTRAINT py_attribute_id IF NOT EXISTS FOR (a:PyAttribute) REQUIRE a.id IS UNIQUE",
-    "CREATE CONSTRAINT py_variable_id IF NOT EXISTS FOR (v:PyVariable) REQUIRE v.id IS UNIQUE",
+The declarative Neo4j schema — the single in-repo source of truth for the graph contract: node
+labels with their keys and typed properties, relationship types and their endpoints, and the
+Cypher DDL (uniqueness constraints + indexes). The constraints are DERIVED from the node labels
+(one per distinct mergeLabel/key) so a new label brings its own constraint — there is no second
+list to keep in sync. `--emit schema` serializes all of this to a machine-readable schema.json,
+and the conformance test (``test/test_neo4j_schema.py``) asserts the real emitter never produces a
+label / relationship / property that isn't declared here — so this file cannot silently drift
+from :mod:`codeanalyzer.neo4j.project`.
+
+SCHEMA_VERSION is the contract version: bump MAJOR on a breaking change (renamed/removed label,
+relationship or key), MINOR on an additive change (new label/rel/property). It is stamped onto
+the :PyApplication node of every emitted graph so any consumer can detect a producer/consumer
+mismatch at runtime.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+SCHEMA_VERSION = "2.0.0"
+
+# PropType ∈ {"string", "integer", "float", "boolean", "string[]", "integer[]"}.
+
+
+@dataclass
+class NodeLabel:
+    label: str  # the specific label (also the catalog key)
+    merge_label: str  # the label the uniqueness constraint / MERGE is on
+    key: str
+    properties: Dict[str, str]
+
+
+@dataclass
+class RelType:
+    type: str
+    from_labels: List[str]
+    to_labels: List[str]
+    properties: Dict[str, str] = field(default_factory=dict)
+
+
+# Labels layered onto a node in addition to its primary/specific label.
+MARKER_LABELS: List[str] = []
+
+_SPAN = {"start_line": "integer", "end_line": "integer"}
+
+
+NODE_LABELS: List[NodeLabel] = [
+    NodeLabel(
+        "PyApplication",
+        "PyApplication",
+        "name",
+        {
+            "name": "string",
+            "schema_version": "string",
+            "analyzer_name": "string",
+            "analyzer_version": "string",
+            "repo_uri": "string",
+            "source_revision": "string",
+            "repo_dirty": "boolean",
+        },
+    ),
+    NodeLabel(
+        "PyModule",
+        "PyModule",
+        "id",
+        {
+            "id": "string",
+            "file_key": "string",
+            "module_name": "string",
+            "content_hash": "string",
+            "last_modified": "float",
+            "file_size": "integer",
+            "_module": "string",
+        },
+    ),
+    NodeLabel(
+        "PyClass",
+        "PySymbol",
+        "id",
+        {
+            "id": "string",
+            "signature": "string",
+            "name": "string",
+            "code": "string",
+            "base_classes": "string[]",
+            "docstring": "string",
+            **_SPAN,
+            "_module": "string",
+        },
+    ),
+    NodeLabel(
+        "PyCallable",
+        "PySymbol",
+        "id",
+        {
+            "id": "string",
+            "signature": "string",
+            "name": "string",
+            "path": "string",
+            "return_type": "string",
+            "cyclomatic_complexity": "integer",
+            "code": "string",
+            "code_start_line": "integer",
+            **_SPAN,
+            "docstring": "string",
+            "decorators": "string[]",
+            "parameters_json": "string",
+            "accessed_symbols_json": "string",
+            "_module": "string",
+        },
+    ),
+    NodeLabel(
+        "PyExternal",
+        "PySymbol",
+        "id",
+        {"id": "string", "name": "string", "module": "string"},
+    ),
+    NodeLabel("PyPackage", "PyPackage", "name", {"name": "string"}),
+    NodeLabel(
+        "PyDecorator",
+        "PyDecorator",
+        "name",
+        {"name": "string"},
+    ),
+    NodeLabel(
+        "PyCallSite",
+        "PyCallSite",
+        "id",
+        {
+            "id": "string",
+            "method_name": "string",
+            "receiver_expr": "string",
+            "receiver_type": "string",
+            "argument_types": "string[]",
+            "arguments_json": "string",
+            "return_type": "string",
+            "callee_signature": "string",
+            "is_constructor_call": "boolean",
+            "start_line": "integer",
+            "start_column": "integer",
+            "end_line": "integer",
+            "end_column": "integer",
+            "_module": "string",
+        },
+    ),
+    NodeLabel(
+        "PyAttribute",
+        "PyAttribute",
+        "id",
+        {
+            "id": "string",
+            "name": "string",
+            "type": "string",
+            "initializer": "string",
+            "docstring": "string",
+            **_SPAN,
+            "_module": "string",
+        },
+    ),
+    NodeLabel(
+        "PyVariable",
+        "PyVariable",
+        "id",
+        {
+            "id": "string",
+            "name": "string",
+            "type": "string",
+            "initializer": "string",
+            "scope": "string",
+            **_SPAN,
+            "_module": "string",
+        },
+    ),
+    # Level-3 CPG overlay (present only at -a 3). The dataflow vocabulary is
+    # shared cross-language in *shape* (same suffixes, props, semantics) but
+    # namespaced per language like every other row family — a multi-language
+    # Neo4j database must never mingle one analyzer's dependence edges with
+    # another's. `id` = "<signature>#<node_id>"; parameter-passing nodes
+    # (formal/actual in/out) ride the same label with `var`/`call_node`.
+    NodeLabel(
+        "PyCFGNode",
+        "PyCFGNode",
+        "id",
+        {
+            "id": "string",
+            "kind": "string",
+            "var": "string",
+            "call_node": "string",
+            **_SPAN,
+            "_module": "string",
+        },
+    ),
 ]
 
+_DECL_TARGETS = ["PyClass", "PyCallable"]
+
+
+REL_TYPES: List[RelType] = [
+    RelType("PY_HAS_MODULE", ["PyApplication"], ["PyModule"]),
+    RelType("PY_DECLARES", ["PyModule", "PyClass", "PyCallable"], _DECL_TARGETS),
+    RelType("PY_HAS_METHOD", ["PyClass"], ["PyCallable"]),
+    RelType("PY_HAS_ATTRIBUTE", ["PyClass"], ["PyAttribute"]),
+    RelType("PY_DECLARES_VAR", ["PyModule", "PyCallable"], ["PyVariable"]),
+    RelType("PY_HAS_CALLSITE", ["PyCallable"], ["PyCallSite"]),
+    RelType("PY_RESOLVES_TO", ["PyCallSite"], ["PyCallable", "PyExternal"]),
+    RelType(
+        "PY_CALLS",
+        ["PyCallable", "PyExternal"],
+        ["PyCallable", "PyExternal"],
+        {"weight": "integer", "prov": "string[]"},
+    ),
+    RelType("PY_EXTENDS", ["PyClass"], ["PyClass"]),
+    RelType(
+        "PY_IMPORTS",
+        ["PyModule"],
+        ["PyModule", "PyPackage"],
+        {"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
+    ),
+    RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
+    # Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
+    # PY_-namespaced so per-language SDK backends can scope their queries.
+    RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]),
+    # ``_k`` is the relationship-identity discriminant (internal, underscore-
+    # prefixed like ``_module``): PY_CFG_NEXT merges per ``kind`` (a conditional's
+    # true/false pair), PY_DDG per ``(var, prov)`` (one dependence per variable,
+    # and the ssa/points-to split) — a plain endpoint-pair MERGE would collapse
+    # legitimately-distinct edges.
+    RelType("PY_CFG_NEXT", ["PyCFGNode"], ["PyCFGNode"], {"kind": "string", "_k": "string"}),
+    RelType("PY_CDG", ["PyCFGNode"], ["PyCFGNode"]),
+    RelType("PY_DDG", ["PyCFGNode"], ["PyCFGNode"], {"var": "string", "prov": "string[]", "_k": "string"}),
+    RelType("PY_PARAM_IN", ["PyCFGNode"], ["PyCFGNode"], {"var": "string"}),
+    RelType("PY_PARAM_OUT", ["PyCFGNode"], ["PyCFGNode"], {"var": "string"}),
+    RelType("PY_SUMMARY", ["PyCFGNode"], ["PyCFGNode"]),
+]
+
+
+def uniqueness_constraints() -> list[str]:
+    """One uniqueness constraint per distinct (merge_label, key)."""
+    seen: set[tuple[str, str]] = set()
+    out: list[str] = []
+
+    for node in NODE_LABELS:
+        identifier = (node.merge_label, node.key)
+        if identifier in seen:
+            continue
+
+        seen.add(identifier)
+        out.append(
+            f"CREATE CONSTRAINT {node.merge_label.lower()}_{node.key} "
+            f"IF NOT EXISTS FOR (x:{node.merge_label}) "
+            f"REQUIRE x.{node.key} IS UNIQUE"
+        )
+
+    return out
+
+
+CONSTRAINTS: List[str] = uniqueness_constraints()
+
 INDEXES: List[str] = [
     "CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
     "CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
     "CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
 ]
+
+
+@dataclass
+class SchemaDocument:
+    schema_version: str
+    generator: str
+    marker_labels: List[str]
+    node_labels: List[NodeLabel]
+    relationship_types: List[RelType]
+    constraints: List[str]
+    indexes: List[str]
+
+
+def build_schema_document() -> dict:
+    """Build the full machine-readable schema document emitted by ``--emit schema``."""
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "generator": "codeanalyzer-python",
+        "marker_labels": list(MARKER_LABELS),
+        "node_labels": [
+            {
+                "label": n.label,
+                "merge_label": n.merge_label,
+                "key": n.key,
+                "properties": n.properties,
+            }
+            for n in NODE_LABELS
+        ],
+        "relationship_types": [
+            {
+                "type": r.type,
+                "from": r.from_labels,
+                "to": r.to_labels,
+                "properties": r.properties,
+            }
+            for r in REL_TYPES
+        ],
+        "constraints": list(CONSTRAINTS),
+        "indexes": list(INDEXES),
+    }

codeanalyzer/options/options.py

@@ -49,6 +49,10 @@ class AnalysisOptions:
     neo4j_password: str = "neo4j"
     neo4j_database: Optional[str] = None
     analysis_level: int = 1
+    # Level-3 dataflow knobs: which program graphs to emit (csv of
+    # cfg|dfg|pdg|sdg) and the access-path k-limit.
+    graphs: str = "cfg,dfg,pdg,sdg"
+    graph_field_depth: int = 3
     using_ray: bool = False
     rebuild_analysis: bool = False
     skip_tests: bool = True

codeanalyzer/provenance.py

@@ -0,0 +1,61 @@
+"""Provenance capture: repository (git) and analyzer identity for a snapshot.
+
+Runs git only as a subprocess query against the analyzed project directory —
+never mutates anything. Absence of git (no repo, no binary) degrades to None.
+"""
+from __future__ import annotations
+
+import subprocess
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+from pathlib import Path
+from typing import Optional, Union
+from urllib.parse import urlsplit, urlunsplit
+
+from codeanalyzer.schema.py_schema import PyAnalyzerInfo, PyRepositoryInfo
+
+
+def _strip_userinfo(uri: str) -> str:
+    """Drop credentials from URL-style remotes; scp-style git@host:path is a
+    username, not a secret, and parses with no netloc — left intact."""
+    parts = urlsplit(uri)
+    if parts.netloc and "@" in parts.netloc:
+        return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2]))
+    return uri
+
+
+def _git(project_dir: Union[Path, str], *args: str) -> Optional[str]:
+    """One git query; None on any failure (no repo, no git, timeout)."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(project_dir), *args],
+            capture_output=True, text=True, timeout=10, check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if result.returncode != 0:
+        return None
+    return result.stdout.strip()
+
+
+def repository_info(project_dir: Union[Path, str]) -> Optional[PyRepositoryInfo]:
+    """Git provenance of ``project_dir``, or None when it isn't a git checkout."""
+    revision = _git(project_dir, "rev-parse", "HEAD")
+    if not revision:
+        return None
+    raw_uri = _git(project_dir, "remote", "get-url", "origin")
+    uri = _strip_userinfo(raw_uri) if raw_uri else None
+    status = _git(project_dir, "status", "--porcelain", "--untracked-files=no")
+    return PyRepositoryInfo(uri=uri, revision=revision, dirty=bool(status))
+
+
+def analyzer_info(analysis_level: int) -> PyAnalyzerInfo:
+    """Identity + configuration of this analyzer run."""
+    try:
+        version = _pkg_version("codeanalyzer-python")
+    except PackageNotFoundError:
+        version = "unknown"
+    return PyAnalyzerInfo(
+        name="codeanalyzer-python",
+        version=version,
+        config={"analysis_level": analysis_level},
+    )

codeanalyzer/schema/__init__.py

@@ -2,6 +2,12 @@ from importlib.metadata import version, PackageNotFoundError
 from packaging.version import parse as parse_version
 
 from .py_schema import (
+    Analysis,
+    BodyNode,
+    CdgEdge,
+    CfgEdge,
+    DdgEdge,
+    ParamEdge,
     PyApplication,
     PyCallable,
     PyCallableParameter,
@@ -12,9 +18,12 @@ from .py_schema import (
     PyImport,
     PyModule,
     PyVariableDeclaration,
+    Span,
+    SummaryEdge,
 )
 
 __all__ = [
+    "Analysis",
     "PyApplication",
     "PyExternalSymbol",
     "PyImport",
@@ -25,6 +34,13 @@ __all__ = [
     "PyCallable",
     "PyClassAttribute",
     "PyCallableParameter",
+    "Span",
+    "BodyNode",
+    "CfgEdge",
+    "CdgEdge",
+    "DdgEdge",
+    "SummaryEdge",
+    "ParamEdge",
 ]
 
 try:
@@ -44,6 +60,7 @@ if not PYDANTIC_V2:
         PyClass=PyClass,
         PyModule=PyModule
     )
+    Analysis.update_forward_refs(PyApplication=PyApplication)
     
 # Compatibility helpers for Pydantic v1/v2
 def model_dump_json(model, **kwargs):
@@ -55,6 +72,8 @@ def model_dump_json(model, **kwargs):
         v1_kwargs = {}
         if 'indent' in kwargs:
             v1_kwargs['indent'] = kwargs['indent']
+        if 'exclude_none' in kwargs:
+            v1_kwargs['exclude_none'] = kwargs['exclude_none']
         if 'separators' in kwargs:
             # In v1, separators is passed to dumps_kwargs
             v1_kwargs['separators'] = kwargs['separators']

codeanalyzer/schema/assign_ids.py

@@ -0,0 +1,37 @@
+"""Walk the symbol-table tree and stamp every node with its can:// id."""
+from __future__ import annotations
+from typing import Dict
+from codeanalyzer.schema import ids
+from codeanalyzer.schema.py_schema import PyApplication, PyModule, PyClass, PyCallable
+
+
+def assign_ids(app: PyApplication, app_name: str) -> Dict[str, str]:
+    """Sets `.id` on the app + every module/class/callable. Returns a
+    `signature -> can://id` map for later stages (identity layer input)."""
+    app.id = ids.application_id(app_name); app.kind = "application"
+    sig_to_id: Dict[str, str] = {}
+
+    def do_callable(parent_id: str, c: PyCallable) -> None:
+        seg = ids.callable_sig_segment(c.name, [p.name for p in c.parameters])
+        c.id = ids.child_id(parent_id, seg)
+        sig_to_id[c.signature] = c.id
+        for ic in (c.callables or {}).values():
+            do_callable(c.id, ic)
+        for icl in (c.types or {}).values():
+            do_class(c.id, icl)
+
+    def do_class(parent_id: str, cl: PyClass) -> None:
+        cl.id = ids.child_id(parent_id, cl.name); cl.kind = "class"
+        sig_to_id[cl.signature] = cl.id
+        for m in (cl.callables or {}).values():
+            do_callable(cl.id, m)
+        for ic in (cl.types or {}).values():
+            do_class(cl.id, ic)
+
+    for file_key, mod in app.symbol_table.items():
+        mod.id = ids.module_id(app_name, file_key); mod.kind = "module"
+        for fn in (mod.functions or {}).values():
+            do_callable(mod.id, fn)
+        for cl in (mod.types or {}).values():
+            do_class(mod.id, cl)
+    return sig_to_id

codeanalyzer/schema/call_graph_ids.py

@@ -0,0 +1,12 @@
+"""Re-identify call-graph edge endpoints onto canonical can:// ids so the JSON
+call_graph agrees with the Neo4j PY_CALLS projection. Declared endpoints map
+through sig_to_id; external/library endpoints keep their dotted signature
+(they have no can:// id)."""
+from __future__ import annotations
+from codeanalyzer.schema.py_schema import PyApplication
+
+
+def reidentify_call_graph(app: PyApplication, sig_to_id: dict) -> None:
+    for edge in app.call_graph or []:
+        edge.src = sig_to_id.get(edge.src, edge.src)
+        edge.dst = sig_to_id.get(edge.dst, edge.dst)

codeanalyzer/schema/ids.py

@@ -0,0 +1,23 @@
+"""Canonical `can://` id construction for schema v2 (durable ids, ≥ callable).
+Ordinal ids (< callable) are `ordinal_id(callable_id, tag)`. Pure functions;
+ids are opaque handles (the <file> segment itself contains '/')."""
+from __future__ import annotations
+from typing import List
+
+_SCHEME = "can://python"
+
+def application_id(app_name: str) -> str:
+    return f"{_SCHEME}/{app_name}"
+
+def module_id(app_name: str, file_key: str) -> str:
+    rel = file_key.replace("\\", "/").lstrip("./")
+    return f"{application_id(app_name)}/{rel}"
+
+def child_id(parent_id: str, segment: str) -> str:
+    return f"{parent_id}/{segment}"
+
+def callable_sig_segment(name: str, param_names: List[str]) -> str:
+    return f"{name}({','.join(param_names)})"
+
+def ordinal_id(callable_id: str, tag: str) -> str:
+    return f"{callable_id}@{tag}"

codeanalyzer/schema/l1_body.py

@@ -0,0 +1,29 @@
+"""L1 body population: materialize `call` nodes from existing call sites.
+`callee` is left None here — the sanctioned null→id refinement happens at L2."""
+from __future__ import annotations
+from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable, BodyNode, Span, byte_offsets
+
+def _do_callable(source: str, c: PyCallable) -> None:
+    for cs in c.call_sites or []:
+        key = f"{cs.start_line}:{cs.start_column}"
+        span = Span(start=(cs.start_line, cs.start_column),
+                    end=(cs.end_line, cs.end_column),
+                    bytes=byte_offsets(source, cs.start_line, cs.start_column, cs.end_line, cs.end_column)) if source else None
+        c.body[key] = BodyNode(kind="call", span=span, callee=None)
+    for ic in (c.callables or {}).values():
+        _do_callable(source, ic)
+    for icl in (c.types or {}).values():
+        _do_class(source, icl)
+
+def _do_class(source: str, cl: PyClass) -> None:
+    for m in (cl.callables or {}).values():
+        _do_callable(source, m)
+    for ic in (cl.types or {}).values():
+        _do_class(source, ic)
+
+def populate_l1_body(app: PyApplication) -> None:
+    for mod in app.symbol_table.values():
+        for fn in (mod.functions or {}).values():
+            _do_callable(mod.source, fn)
+        for cl in (mod.types or {}).values():
+            _do_class(mod.source, cl)

codeanalyzer/schema/l2_callees.py

@@ -0,0 +1,36 @@
+"""L2 refinement: fill each L1 `call` body node's `callee` (null→id) from the
+call site's resolved signature — the one sanctioned value change. A declared
+target becomes its can:// id; an external/library target keeps its dotted
+signature; an unresolved call site leaves `callee` absent."""
+from __future__ import annotations
+from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable
+
+
+def _do_callable(c: PyCallable, sig_to_id: dict) -> None:
+    for cs in c.call_sites or []:
+        if cs.callee_signature is None:
+            continue
+        key = f"{cs.start_line}:{cs.start_column}"
+        node = c.body.get(key)
+        if node is None or node.kind != "call":
+            continue
+        node.callee = sig_to_id.get(cs.callee_signature, cs.callee_signature)
+    for ic in (c.callables or {}).values():
+        _do_callable(ic, sig_to_id)
+    for icl in (c.types or {}).values():
+        _do_class(icl, sig_to_id)
+
+
+def _do_class(cl: PyClass, sig_to_id: dict) -> None:
+    for m in (cl.callables or {}).values():
+        _do_callable(m, sig_to_id)
+    for ic in (cl.types or {}).values():
+        _do_class(ic, sig_to_id)
+
+
+def backfill_callees(app: PyApplication, sig_to_id: dict) -> None:
+    for mod in app.symbol_table.values():
+        for fn in (mod.functions or {}).values():
+            _do_callable(fn, sig_to_id)
+        for cl in (mod.types or {}).values():
+            _do_class(cl, sig_to_id)