codeanalyzer-python 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

codeanalyzer/schema/py_schema.py

@@ -22,85 +22,8 @@ for static analysis purposes.
 from __future__ import annotations
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
-import gzip
-
 from pydantic import BaseModel
 from typing_extensions import Literal
-import msgpack
-
-
-def msgpk(cls):
-    """
-    Decorator that adds MessagePack serialization methods to Pydantic models.
-
-    Adds methods:
-        - to_msgpack_bytes() -> bytes: Serialize to compact binary format
-        - from_msgpack_bytes(data: bytes) -> cls: Deserialize from binary format
-        - to_msgpack_dict() -> dict: Convert to msgpack-compatible dict
-        - from_msgpack_dict(data: dict) -> cls: Create instance from msgpack dict
-    """
-
-    def _prepare_for_serialization(obj: Any) -> Any:
-        """Convert objects to serialization-friendly format."""
-        if isinstance(obj, Path):
-            return str(obj)
-        elif isinstance(obj, dict):
-            return {
-                _prepare_for_serialization(k): _prepare_for_serialization(v)
-                for k, v in obj.items()
-            }
-        elif isinstance(obj, list):
-            return [_prepare_for_serialization(item) for item in obj]
-        elif isinstance(obj, tuple):
-            return tuple(_prepare_for_serialization(item) for item in obj)
-        elif isinstance(obj, set):
-            return [_prepare_for_serialization(item) for item in obj]
-        elif hasattr(obj, "model_dump"):  # Pydantic model
-            return _prepare_for_serialization(obj.model_dump())
-        else:
-            return obj
-
-    def to_msgpack_bytes(self) -> bytes:
-        """Serialize the model to compact binary format using MessagePack + gzip."""
-        data = _prepare_for_serialization(self.model_dump())
-        msgpack_data = msgpack.packb(data, use_bin_type=True)
-        return gzip.compress(msgpack_data)
-
-    @classmethod
-    def from_msgpack_bytes(cls_obj, data: bytes):
-        """Deserialize from MessagePack + gzip binary format."""
-        decompressed_data = gzip.decompress(data)
-        obj_dict = msgpack.unpackb(decompressed_data, raw=False)
-        return cls_obj.model_validate(obj_dict)
-
-    def to_msgpack_dict(self) -> dict:
-        """Convert to msgpack-compatible dictionary format."""
-        return _prepare_for_serialization(self.model_dump())
-
-    @classmethod
-    def from_msgpack_dict(cls_obj, data: dict):
-        """Create instance from msgpack-compatible dictionary."""
-        return cls_obj.model_validate(data)
-
-    def get_msgpack_size(self) -> int:
-        """Get the size of the msgpack serialization in bytes."""
-        return len(self.to_msgpack_bytes())
-
-    def get_compression_ratio(self) -> float:
-        """Get compression ratio compared to JSON."""
-        json_size = len(self.model_dump_json().encode("utf-8"))
-        msgpack_gzip_size = self.get_msgpack_size()
-        return msgpack_gzip_size / json_size if json_size > 0 else 1.0
-
-    # Add methods to the class
-    cls.to_msgpack_bytes = to_msgpack_bytes
-    cls.from_msgpack_bytes = from_msgpack_bytes
-    cls.to_msgpack_dict = to_msgpack_dict
-    cls.from_msgpack_dict = from_msgpack_dict
-    cls.get_msgpack_size = get_msgpack_size
-    cls.get_compression_ratio = get_compression_ratio
-
-    return cls
 
 
 def builder(cls):
@@ -192,7 +115,6 @@ def byte_offsets(source: str, start_line: int, start_col: int,
 
 
 @builder
-@msgpk
 class Span(BaseModel):
     """Where a node lives in source. `start`/`end` are [line, col] (1-based line,
     0-based col, ast semantics); `bytes` are utf-8 offsets into module.source."""
@@ -202,7 +124,6 @@ class Span(BaseModel):
 
 
 @builder
-@msgpk
 class BodyNode(BaseModel):
     """A node in a callable's `body`: an AST region (statement/call/branch/…) or
     a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
@@ -211,40 +132,46 @@ class BodyNode(BaseModel):
     callee: Optional[str] = None   # only on `call` nodes; the sanctioned null→id slot
     of: Optional[str] = None       # param vertices: the variable/return they carry
     parent: Optional[str] = None   # actuals: owning callsite ordinal id
+    # Call-site detail (#120). Previously reachable only through the parallel
+    # `PyCallable.call_sites` list, which emitted the same fact a second time under
+    # an unrelated id scheme. `method_name` and `is_constructor_call` are carried
+    # rather than derived from `callee`: measured across `requests` and `flask`,
+    # 20-28% of call sites never resolve a callee, so deriving them would lose them
+    # on one call in four.
+    method_name: Optional[str] = None
+    receiver_expr: Optional[str] = None
+    receiver_type: Optional[str] = None
+    return_type: Optional[str] = None
+    is_constructor_call: Optional[bool] = None
+    arguments: List["PyCallArgument"] = []
 
 
 @builder
-@msgpk
 class CfgEdge(BaseModel):
     src: str; dst: str; kind: str = "fallthrough"
 
 
 @builder
-@msgpk
 class CdgEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class DdgEdge(BaseModel):
     src: str; dst: str; var: Optional[str] = None; prov: List[str] = []
 
 
 @builder
-@msgpk
 class SummaryEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class ParamEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class PyImport(BaseModel):
     """Represents a Python import statement."""
 
@@ -259,7 +186,6 @@ class PyImport(BaseModel):
 
 
 @builder
-@msgpk
 class PyComment(BaseModel):
     """Represents a Python comment."""
 
@@ -272,7 +198,6 @@ class PyComment(BaseModel):
 
 
 @builder
-@msgpk
 class PySymbol(BaseModel):
     """Represents a symbol used or declared in Python code."""
 
@@ -287,7 +212,6 @@ class PySymbol(BaseModel):
 
 
 @builder
-@msgpk
 class PyVariableDeclaration(BaseModel):
     """Represents a Python variable declaration."""
 
@@ -306,13 +230,66 @@ class PyVariableDeclaration(BaseModel):
 
 
 @builder
-@msgpk
+class PyDecorator(BaseModel):
+    """One decorator application, structured rather than a source string (#128).
+
+    ``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``);
+    ``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is
+    absent when it cannot be resolved. ``expression`` keeps the full unparsed source
+    so nothing is lost for decorators too complex to decompose.
+    """
+
+    name: str
+    qualified_name: Optional[str] = None
+    positional_arguments: List[str] = []
+    keyword_arguments: Dict[str, str] = {}
+    expression: str = ""
+    span: Optional[Span] = None
+
+
+@builder
+class PyEntrypoint(BaseModel):
+    """One way a callable or class is invoked from outside the application (#27).
+
+    A node may hold several: two ``@app.route`` decorators, or a function that
+    is both a Celery task and a CLI command. ``confidence`` lets a consumer
+    threshold on evidence quality rather than inheriting this analyzer's
+    judgement.
+    """
+
+    framework: str
+    confidence: str = "certain"   # "declared" | "certain" | "heuristic"
+    rule: str = ""                # rules.yml `id:`, or an engine name
+    ruleset: str = "shipped"      # "shipped" | "user:<path>"
+    evidence: Optional[str] = None
+    route: Optional[str] = None
+    http_methods: List[str] = []
+    via: Optional[str] = None     # can:// id of the routed node dispatching here
+
+
+@builder
+class PyEntrypointReport(BaseModel):
+    """Coverage and failure record for the entrypoint pass (#27).
+
+    The pass under-approximates by design, so silence is its failure mode.
+    This is what makes a gap visible instead of indistinguishable from
+    "this project has no entrypoints".
+    """
+
+    frameworks_detected: List[str] = []
+    rulesets: List[str] = []
+    unresolved: Dict[str, int] = {}
+    errors: List[str] = []
+
+
+@builder
 class PyCallableParameter(BaseModel):
     """Represents a parameter of a Python callable (function/method)."""
 
     name: str
     type: Optional[str] = None
     default_value: Optional[str] = None
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
     start_column: int = -1
@@ -320,7 +297,6 @@ class PyCallableParameter(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallArgument(BaseModel):
     """One call-site argument: AST category + inferred type, kept separate.
 
@@ -332,8 +308,14 @@ class PyCallArgument(BaseModel):
     inferred_type: Optional[str] = None
 
 
+# BodyNode.arguments forward-references PyCallArgument (defined later);
+# pydantic v1 resolves string annotations only when told to, while v2
+# rebuilds automatically (and its update_forward_refs shim rejects localns).
+if not hasattr(BodyNode, "model_rebuild"):  # pydantic v1
+    BodyNode.update_forward_refs(PyCallArgument=PyCallArgument)
+
+
 @builder
-@msgpk
 class PyCallsite(BaseModel):
     """Represents a Python call site (function or method invocation) with contextual metadata."""
 
@@ -352,7 +334,6 @@ class PyCallsite(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallable(BaseModel):
     """Represents a Python callable (function/method)."""
 
@@ -363,13 +344,27 @@ class PyCallable(BaseModel):
     kind: str = "function"
     span: Optional[Span] = None
     comments: List[PyComment] = []
-    decorators: List[str] = []
+    decorators: List[PyDecorator] = []
+    # Language-level modifiers on the declaration itself (#130). `async` is the
+    # only one Python has today. It lives here rather than in `kind` because it
+    # is orthogonal to every kind -- an async method is both -- so encoding it
+    # in the discriminant would need async_function, async_method,
+    # async_generator and so on, combinatorially.
+    modifiers: List[str] = []
+    entrypoints: List[PyEntrypoint] = []
+    is_entrypoint: bool = False
     parameters: List[PyCallableParameter] = []
     return_type: Optional[str] = None
     start_line: int = -1
     end_line: int = -1
     code_start_line: int = -1
     accessed_symbols: List[PySymbol] = []
+    # Internal (#120): the Jedi-produced record that `l1_body` derives `body{}`
+    # call nodes from, and that `call_graph.py`, `l2_callees.py` and the dataflow
+    # builder all read. It is stripped at EMIT time (see `wire_json`), not with a
+    # field-level `exclude`: the analysis cache round-trips through the same
+    # serializer, so excluding it would drop it from the cache too and a warm-cache
+    # run would rebuild with no call sites at all.
     call_sites: List[PyCallsite] = []
     callables: Dict[str, "PyCallable"] = {}  # nested callables (closures)
     types: Dict[str, "PyClass"] = {}  # nested (local) classes
@@ -389,7 +384,6 @@ class PyCallable(BaseModel):
 
 
 @builder
-@msgpk
 class PyClassAttribute(BaseModel):
     """Represents a Python class attribute."""
 
@@ -397,12 +391,12 @@ class PyClassAttribute(BaseModel):
     type: Optional[str] = None
     initializer: Optional[str] = None
     comments: List[PyComment] = []
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
 
 
 @builder
-@msgpk
 class PyClass(BaseModel):
     """Represents a Python class."""
 
@@ -413,6 +407,9 @@ class PyClass(BaseModel):
     span: Optional[Span] = None
     comments: List[PyComment] = []
     base_classes: List[str] = []
+    decorators: List[PyDecorator] = []
+    entrypoints: List[PyEntrypoint] = []
+    is_entrypoint: bool = False
     callables: Dict[str, PyCallable] = {}  # methods, keystone containment name
     attributes: Dict[str, PyClassAttribute] = {}
     types: Dict[str, "PyClass"] = {}  # inner classes, keystone containment name
@@ -425,7 +422,6 @@ class PyClass(BaseModel):
 
 
 @builder
-@msgpk
 class PyModule(BaseModel):
     """Represents a Python module."""
 
@@ -446,7 +442,6 @@ class PyModule(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallEdge(BaseModel):
     """Identity-only call-graph edge with weight (keystone shape: the list name
     IS the edge type, so there is no ``type`` field).
@@ -460,11 +455,10 @@ class PyCallEdge(BaseModel):
     src: str  # caller callable id
     dst: str  # callee callable (or external) id
     weight: int = 1
-    prov: List[Literal["jedi", "pycg", "joern"]] = []
+    prov: List[Literal["jedi", "defuse"]] = []
 
 
 @builder
-@msgpk
 class PyExternalSymbol(BaseModel):
     """A call-graph target outside the analyzed project -- an imported library or
     builtin member. An edge-endpoint id home, not a tree node: keyed in
@@ -477,7 +471,6 @@ class PyExternalSymbol(BaseModel):
 
 
 @builder
-@msgpk
 class PyRepositoryInfo(BaseModel):
     """Where the analyzed source came from: git provenance captured at analysis time."""
 
@@ -487,7 +480,6 @@ class PyRepositoryInfo(BaseModel):
 
 
 @builder
-@msgpk
 class PyAnalyzerInfo(BaseModel):
     """Which analyzer produced this snapshot, and how it was configured.
     Lives on the ``Analysis`` envelope (keystone ``analyzer{name,version}``;
@@ -499,7 +491,6 @@ class PyAnalyzerInfo(BaseModel):
 
 
 @builder
-@msgpk
 class PyApplication(BaseModel):
     """Represents a Python application."""
 
@@ -511,6 +502,8 @@ class PyApplication(BaseModel):
     # builtin members), keyed by signature. Populated by the analyzer so every
     # backend (JSON and Neo4j) shares one authoritative external-symbol set.
     external_symbols: Dict[str, PyExternalSymbol] = {}
+    # Coverage/failure record for the entrypoint pass; see PyEntrypointReport (#27).
+    entrypoint_report: PyEntrypointReport = PyEntrypointReport()
     # Git provenance of the analyzed checkout, captured at analysis time.
     repository: Optional[PyRepositoryInfo] = None
     # Interprocedural parameter-passing edges (formal↔actual); populated at L4.
@@ -519,7 +512,6 @@ class PyApplication(BaseModel):
 
 
 @builder
-@msgpk
 class Analysis(BaseModel):
     """v2 payload root: envelope + the application tree node. ``k_limit`` is an
     L3+ envelope key (None below the dataflow levels; exclude_none drops it)."""

codeanalyzer/semantic_analysis/call_graph.py

@@ -28,6 +28,8 @@ from typing import Dict, Iterator, List, Tuple
 
 import networkx as nx
 
+from codeanalyzer.semantic_analysis.defuse_linker import _module_qual
+from codeanalyzer.schema import model_copy
 from codeanalyzer.schema.py_schema import (
     PyApplication,
     PyCallable,
@@ -170,7 +172,7 @@ def jedi_call_graph_edges(
 
     Edges are coalesced on ``(source, target)``: ``weight`` is the count of
     matching sites. Provenance is always ``["jedi"]``; combine with
-    PyCG-derived edges via ``merge_edges``.
+    defuse-linker edges via ``merge_edges``.
     """
     counts: Counter = Counter()
     for caller in iter_callables_in_symbol_table(symbol_table):
@@ -253,10 +255,24 @@ def filter_external_edges(
     retained; only lib→lib edges are dropped.  The app symbol set is built by
     walking every callable in the symbol table recursively (including nested
     functions and closures via ``callables``) plus every class, so
-    PyCG-discovered closure nodes are correctly recognised as app symbols.
+    resolver-discovered closure nodes are correctly recognised as app symbols.
+
+    Module names count as app symbols too (#131). A resolver attributes a call in
+    module scope to the MODULE -- ``app -> functools.reduce`` for a module-level
+    ``functools.reduce(...)``, or for a decorator applied to a top-level
+    definition, since a decorator executes in its enclosing scope. Without the
+    module names here, both endpoints looked third-party and every module-scope
+    call to a library target was discarded as lib→lib.
     """
     app_symbols: set = {c.signature for c in iter_callables_in_symbol_table(symbol_table)}
     app_symbols.update(cls.signature for cls in iter_classes_in_symbol_table(symbol_table))
+    app_symbols.update(
+        mod.module_name for mod in symbol_table.values() if mod.module_name
+    )
+    # Dotted module quals too: the defuse linker attributes module- and
+    # class-scope calls to "pkg.module" (collision-free across packages),
+    # which the bare `module_name` stems above never match.
+    app_symbols.update(_module_qual(key) for key in symbol_table)
 
     return [
         e for e in edges
@@ -269,7 +285,7 @@ def merge_edges(*edge_lists: list) -> list:
 
     Edges with the same ``(source, target)`` are coalesced: weights sum,
     provenance is the sorted union. Useful for combining edges produced
-    by different backends (e.g. Jedi + PyCG).
+    by different backends (e.g. Jedi + the defuse linker).
     """
     by_key: Dict[Tuple[str, str], PyCallEdge] = {}
     for edges in edge_lists:
@@ -280,5 +296,5 @@ def merge_edges(*edge_lists: list) -> list:
                 cur.weight += e.weight
                 cur.prov = sorted(set(cur.prov) | set(e.prov))
             else:
-                by_key[k] = e.model_copy()
+                by_key[k] = model_copy(e)
     return list(by_key.values())