codeanalyzer-python 1.1.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

codeanalyzer/schema/py_schema.py

@@ -22,85 +22,8 @@ for static analysis purposes.
 from __future__ import annotations
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
-import gzip
-
 from pydantic import BaseModel
 from typing_extensions import Literal
-import msgpack
-
-
-def msgpk(cls):
-    """
-    Decorator that adds MessagePack serialization methods to Pydantic models.
-
-    Adds methods:
-        - to_msgpack_bytes() -> bytes: Serialize to compact binary format
-        - from_msgpack_bytes(data: bytes) -> cls: Deserialize from binary format
-        - to_msgpack_dict() -> dict: Convert to msgpack-compatible dict
-        - from_msgpack_dict(data: dict) -> cls: Create instance from msgpack dict
-    """
-
-    def _prepare_for_serialization(obj: Any) -> Any:
-        """Convert objects to serialization-friendly format."""
-        if isinstance(obj, Path):
-            return str(obj)
-        elif isinstance(obj, dict):
-            return {
-                _prepare_for_serialization(k): _prepare_for_serialization(v)
-                for k, v in obj.items()
-            }
-        elif isinstance(obj, list):
-            return [_prepare_for_serialization(item) for item in obj]
-        elif isinstance(obj, tuple):
-            return tuple(_prepare_for_serialization(item) for item in obj)
-        elif isinstance(obj, set):
-            return [_prepare_for_serialization(item) for item in obj]
-        elif hasattr(obj, "model_dump"):  # Pydantic model
-            return _prepare_for_serialization(obj.model_dump())
-        else:
-            return obj
-
-    def to_msgpack_bytes(self) -> bytes:
-        """Serialize the model to compact binary format using MessagePack + gzip."""
-        data = _prepare_for_serialization(self.model_dump())
-        msgpack_data = msgpack.packb(data, use_bin_type=True)
-        return gzip.compress(msgpack_data)
-
-    @classmethod
-    def from_msgpack_bytes(cls_obj, data: bytes):
-        """Deserialize from MessagePack + gzip binary format."""
-        decompressed_data = gzip.decompress(data)
-        obj_dict = msgpack.unpackb(decompressed_data, raw=False)
-        return cls_obj.model_validate(obj_dict)
-
-    def to_msgpack_dict(self) -> dict:
-        """Convert to msgpack-compatible dictionary format."""
-        return _prepare_for_serialization(self.model_dump())
-
-    @classmethod
-    def from_msgpack_dict(cls_obj, data: dict):
-        """Create instance from msgpack-compatible dictionary."""
-        return cls_obj.model_validate(data)
-
-    def get_msgpack_size(self) -> int:
-        """Get the size of the msgpack serialization in bytes."""
-        return len(self.to_msgpack_bytes())
-
-    def get_compression_ratio(self) -> float:
-        """Get compression ratio compared to JSON."""
-        json_size = len(self.model_dump_json().encode("utf-8"))
-        msgpack_gzip_size = self.get_msgpack_size()
-        return msgpack_gzip_size / json_size if json_size > 0 else 1.0
-
-    # Add methods to the class
-    cls.to_msgpack_bytes = to_msgpack_bytes
-    cls.from_msgpack_bytes = from_msgpack_bytes
-    cls.to_msgpack_dict = to_msgpack_dict
-    cls.from_msgpack_dict = from_msgpack_dict
-    cls.get_msgpack_size = get_msgpack_size
-    cls.get_compression_ratio = get_compression_ratio
-
-    return cls
 
 
 def builder(cls):
@@ -192,7 +115,6 @@ def byte_offsets(source: str, start_line: int, start_col: int,
 
 
 @builder
-@msgpk
 class Span(BaseModel):
     """Where a node lives in source. `start`/`end` are [line, col] (1-based line,
     0-based col, ast semantics); `bytes` are utf-8 offsets into module.source."""
@@ -202,7 +124,6 @@ class Span(BaseModel):
 
 
 @builder
-@msgpk
 class BodyNode(BaseModel):
     """A node in a callable's `body`: an AST region (statement/call/branch/…) or
     a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
@@ -211,40 +132,46 @@ class BodyNode(BaseModel):
     callee: Optional[str] = None   # only on `call` nodes; the sanctioned null→id slot
     of: Optional[str] = None       # param vertices: the variable/return they carry
     parent: Optional[str] = None   # actuals: owning callsite ordinal id
+    # Call-site detail (#120). Previously reachable only through the parallel
+    # `PyCallable.call_sites` list, which emitted the same fact a second time under
+    # an unrelated id scheme. `method_name` and `is_constructor_call` are carried
+    # rather than derived from `callee`: measured across `requests` and `flask`,
+    # 20-28% of call sites never resolve a callee, so deriving them would lose them
+    # on one call in four.
+    method_name: Optional[str] = None
+    receiver_expr: Optional[str] = None
+    receiver_type: Optional[str] = None
+    return_type: Optional[str] = None
+    is_constructor_call: Optional[bool] = None
+    arguments: List["PyCallArgument"] = []
 
 
 @builder
-@msgpk
 class CfgEdge(BaseModel):
     src: str; dst: str; kind: str = "fallthrough"
 
 
 @builder
-@msgpk
 class CdgEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class DdgEdge(BaseModel):
     src: str; dst: str; var: Optional[str] = None; prov: List[str] = []
 
 
 @builder
-@msgpk
 class SummaryEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class ParamEdge(BaseModel):
     src: str; dst: str
 
 
 @builder
-@msgpk
 class PyImport(BaseModel):
     """Represents a Python import statement."""
 
@@ -259,7 +186,6 @@ class PyImport(BaseModel):
 
 
 @builder
-@msgpk
 class PyComment(BaseModel):
     """Represents a Python comment."""
 
@@ -272,7 +198,6 @@ class PyComment(BaseModel):
 
 
 @builder
-@msgpk
 class PySymbol(BaseModel):
     """Represents a symbol used or declared in Python code."""
 
@@ -287,7 +212,6 @@ class PySymbol(BaseModel):
 
 
 @builder
-@msgpk
 class PyVariableDeclaration(BaseModel):
     """Represents a Python variable declaration."""
 
@@ -306,13 +230,66 @@ class PyVariableDeclaration(BaseModel):
 
 
 @builder
-@msgpk
+class PyDecorator(BaseModel):
+    """One decorator application, structured rather than a source string (#128).
+
+    ``name`` is the spelling as written (``lru_cache``, ``builtins.staticmethod``);
+    ``qualified_name`` is Jedi's resolution of it (``functools.lru_cache``) and is
+    absent when it cannot be resolved. ``expression`` keeps the full unparsed source
+    so nothing is lost for decorators too complex to decompose.
+    """
+
+    name: str
+    qualified_name: Optional[str] = None
+    positional_arguments: List[str] = []
+    keyword_arguments: Dict[str, str] = {}
+    expression: str = ""
+    span: Optional[Span] = None
+
+
+@builder
+class PyEntrypoint(BaseModel):
+    """One way a callable or class is invoked from outside the application (#27).
+
+    A node may hold several: two ``@app.route`` decorators, or a function that
+    is both a Celery task and a CLI command. ``confidence`` lets a consumer
+    threshold on evidence quality rather than inheriting this analyzer's
+    judgement.
+    """
+
+    framework: str
+    confidence: str = "certain"   # "declared" | "certain" | "heuristic"
+    rule: str = ""                # rules.yml `id:`, or an engine name
+    ruleset: str = "shipped"      # "shipped" | "user:<path>"
+    evidence: Optional[str] = None
+    route: Optional[str] = None
+    http_methods: List[str] = []
+    via: Optional[str] = None     # can:// id of the routed node dispatching here
+
+
+@builder
+class PyEntrypointReport(BaseModel):
+    """Coverage and failure record for the entrypoint pass (#27).
+
+    The pass under-approximates by design, so silence is its failure mode.
+    This is what makes a gap visible instead of indistinguishable from
+    "this project has no entrypoints".
+    """
+
+    frameworks_detected: List[str] = []
+    rulesets: List[str] = []
+    unresolved: Dict[str, int] = {}
+    errors: List[str] = []
+
+
+@builder
 class PyCallableParameter(BaseModel):
     """Represents a parameter of a Python callable (function/method)."""
 
     name: str
     type: Optional[str] = None
     default_value: Optional[str] = None
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
     start_column: int = -1
@@ -320,7 +297,6 @@ class PyCallableParameter(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallArgument(BaseModel):
     """One call-site argument: AST category + inferred type, kept separate.
 
@@ -330,10 +306,27 @@ class PyCallArgument(BaseModel):
 
     ast_kind: str
     inferred_type: Optional[str] = None
+    # Literal capture (#162): populated only for an `ast.Constant` argument
+    # whose value is str/int/float/bool/None, JSON-encoded (`json.dumps`) --
+    # decode with `json.loads` to recover the Python constant. `None` for
+    # every non-constant argument (and for a Constant of another type, e.g.
+    # bytes/complex/Ellipsis).
+    value: Optional[str] = None
+    # Bare-identifier capture (#162): the `id` of an `ast.Name` argument
+    # (e.g. `f(KEY)` -> `"KEY"`) -- ships alongside `value` as the dataflow
+    # tier's join point back to the variable's own definitions. `None` for
+    # every other argument shape.
+    name: Optional[str] = None
+
+
+# BodyNode.arguments forward-references PyCallArgument (defined later);
+# pydantic v1 resolves string annotations only when told to, while v2
+# rebuilds automatically (and its update_forward_refs shim rejects localns).
+if not hasattr(BodyNode, "model_rebuild"):  # pydantic v1
+    BodyNode.update_forward_refs(PyCallArgument=PyCallArgument)
 
 
 @builder
-@msgpk
 class PyCallsite(BaseModel):
     """Represents a Python call site (function or method invocation) with contextual metadata."""
 
@@ -352,7 +345,6 @@ class PyCallsite(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallable(BaseModel):
     """Represents a Python callable (function/method)."""
 
@@ -363,13 +355,27 @@ class PyCallable(BaseModel):
     kind: str = "function"
     span: Optional[Span] = None
     comments: List[PyComment] = []
-    decorators: List[str] = []
+    decorators: List[PyDecorator] = []
+    # Language-level modifiers on the declaration itself (#130). `async` is the
+    # only one Python has today. It lives here rather than in `kind` because it
+    # is orthogonal to every kind -- an async method is both -- so encoding it
+    # in the discriminant would need async_function, async_method,
+    # async_generator and so on, combinatorially.
+    modifiers: List[str] = []
+    entrypoints: List[PyEntrypoint] = []
+    is_entrypoint: bool = False
     parameters: List[PyCallableParameter] = []
     return_type: Optional[str] = None
     start_line: int = -1
     end_line: int = -1
     code_start_line: int = -1
     accessed_symbols: List[PySymbol] = []
+    # Internal (#120): the Jedi-produced record that `l1_body` derives `body{}`
+    # call nodes from, and that `call_graph.py`, `l2_callees.py` and the dataflow
+    # builder all read. It is stripped at EMIT time (see `wire_json`), not with a
+    # field-level `exclude`: the analysis cache round-trips through the same
+    # serializer, so excluding it would drop it from the cache too and a warm-cache
+    # run would rebuild with no call sites at all.
     call_sites: List[PyCallsite] = []
     callables: Dict[str, "PyCallable"] = {}  # nested callables (closures)
     types: Dict[str, "PyClass"] = {}  # nested (local) classes
@@ -389,7 +395,6 @@ class PyCallable(BaseModel):
 
 
 @builder
-@msgpk
 class PyClassAttribute(BaseModel):
     """Represents a Python class attribute."""
 
@@ -397,12 +402,12 @@ class PyClassAttribute(BaseModel):
     type: Optional[str] = None
     initializer: Optional[str] = None
     comments: List[PyComment] = []
+    decorators: List[PyDecorator] = []
     start_line: int = -1
     end_line: int = -1
 
 
 @builder
-@msgpk
 class PyClass(BaseModel):
     """Represents a Python class."""
 
@@ -413,6 +418,9 @@ class PyClass(BaseModel):
     span: Optional[Span] = None
     comments: List[PyComment] = []
     base_classes: List[str] = []
+    decorators: List[PyDecorator] = []
+    entrypoints: List[PyEntrypoint] = []
+    is_entrypoint: bool = False
     callables: Dict[str, PyCallable] = {}  # methods, keystone containment name
     attributes: Dict[str, PyClassAttribute] = {}
     types: Dict[str, "PyClass"] = {}  # inner classes, keystone containment name
@@ -425,7 +433,6 @@ class PyClass(BaseModel):
 
 
 @builder
-@msgpk
 class PyModule(BaseModel):
     """Represents a Python module."""
 
@@ -446,7 +453,6 @@ class PyModule(BaseModel):
 
 
 @builder
-@msgpk
 class PyCallEdge(BaseModel):
     """Identity-only call-graph edge with weight (keystone shape: the list name
     IS the edge type, so there is no ``type`` field).
@@ -460,11 +466,10 @@ class PyCallEdge(BaseModel):
     src: str  # caller callable id
     dst: str  # callee callable (or external) id
     weight: int = 1
-    prov: List[Literal["jedi", "pycg", "joern"]] = []
+    prov: List[Literal["jedi", "defuse"]] = []
 
 
 @builder
-@msgpk
 class PyExternalSymbol(BaseModel):
     """A call-graph target outside the analyzed project -- an imported library or
     builtin member. An edge-endpoint id home, not a tree node: keyed in
@@ -477,7 +482,104 @@ class PyExternalSymbol(BaseModel):
 
 
 @builder
-@msgpk
+class PyConfigKey(BaseModel):
+    """A configuration key flattened out of a config-bearing ``PyArtifact``
+    (#152). Graph vocabulary stays neutral (label ``ConfigKey``, edge
+    ``DEFINES_CONFIG``) -- the ``Py`` prefix here is only the ``PyArtifact``
+    naming precedent, not a Python-specific claim. L1 data, identical at
+    every analysis level; nested under the owning artifact, containment
+    mirrors ``DEFINES_CONFIG``."""
+
+    id: str = ""  # <artifact-id>@key/<dotted.key>
+    key: str  # dotted path; numeric segments for arrays, e.g. "services.web.ports.0"
+    namespace: str  # env|yaml|json|toml|ini|properties|dockerfile
+    value: Optional[str] = None  # populated only when options.artifact_text is on
+    span: Optional[Span] = None  # into the artifact's source; best-effort for yaml/json/toml
+    references: List[str] = []  # raw recognized tokens, order of appearance, deduplicated
+
+
+@builder
+class PyArtifact(BaseModel):
+    """Any non-`.py` project file (config, manifest, CI, container spec, or
+    plain data/binary) -- never dropped from the walk. Captured broadly (node
+    + verbatim ``source``); *meaning* is extracted narrowly -- only
+    ``dependency-manifest`` roles feed ``dependencies`` today. ``id`` is
+    language-neutral (``can://artifact/<app>/<path>``)."""
+
+    id: str = ""
+    kind: str = "artifact"
+    path: str  # repo-relative POSIX path (also the map key)
+    format: str  # toml|yaml|json|ini|properties|requirements|dockerfile|text|binary
+    roles: List[str] = []
+    size_bytes: int = 0
+    sha256: str = ""  # always the full file's hash, even when source is truncated/empty
+    source: str = ""  # verbatim by default; "" for binary or when capture is disabled
+    text_truncated: bool = False  # True when `source` is a prefix, not the full file
+    extraction: str = "none"  # none|partial|full
+    config_keys: List[PyConfigKey] = []  # flattened config keys (#152); [] when not namespace-eligible
+
+
+@builder
+class PyConfigUseEdge(BaseModel):
+    """One resolved config read (#162): a detector-matched call's key
+    argument closed on exactly one string literal that matches a declared
+    ``PyConfigKey``. ``src`` is the call's GLOBAL ordinal id
+    (``<callable-id>@<local-id>``); ``dst`` is the matched ``PyConfigKey.id``
+    -- application scope, mirroring ``param_in`` (endpoints span callables/
+    artifacts). Superset-monotonic across levels, same additive contract as
+    the DDG's ``prov`` widening: literal (``-a 2``+) subset of +dataflow
+    (``-a 3``/``-a 4``)."""
+
+    src: str
+    dst: str
+    prov: List[Literal["literal", "dataflow"]] = []
+
+
+@builder
+class PyConfigRead(BaseModel):
+    """A detector-matched call whose key did not close on exactly one string
+    literal -- first-class so a config read nobody can trace is as visible
+    as one that resolves (#162). ``key`` is the decoded literal text only
+    when it IS a literal but matches no declared ``PyConfigKey``
+    (``reason="undefined-key"``); ``None`` for a key that never closed on a
+    literal at all (``reason="non-literal"``). ``prov`` lists every tier
+    that was attempted before giving up."""
+
+    site: str  # GLOBAL ordinal id
+    callee: str  # external id (can://.../@external/<module>/<name>)
+    key: Optional[str] = None
+    reason: Literal["non-literal", "undefined-key"]
+    prov: List[Literal["literal", "dataflow"]] = []
+
+
+@builder
+class PyDependency(BaseModel):
+    """One declared third-party dependency, evidence-tagged via ``prov``."""
+
+    name: str  # PEP 503 normalized
+    ecosystem: str = "pypi"  # SDK symmetry with purl (#152 rider); the only ecosystem this analyzer emits
+    spec: str = ""
+    kind: str = "runtime"  # runtime|dev|optional|build
+    extras: List[str] = []
+    declared_in: str = ""  # PyArtifact id
+    # False for lockfile-only (transitive) dependencies -- pinned in a lock
+    # with no manifest declaration (#152 reconciliation).
+    direct: bool = True
+    locked_version: Optional[str] = None
+    provides_imports: List[str] = []
+    prov: List[str] = []  # declared|lockfile|installed-metadata|heuristic
+
+
+@builder
+class PyImportBinding(BaseModel):
+    """A top-level import no declared dependency accounts for."""
+
+    module: str
+    bound_to: Optional[str] = None  # best-effort distribution name
+    prov: List[str] = []
+
+
+@builder
 class PyRepositoryInfo(BaseModel):
     """Where the analyzed source came from: git provenance captured at analysis time."""
 
@@ -487,7 +589,6 @@ class PyRepositoryInfo(BaseModel):
 
 
 @builder
-@msgpk
 class PyAnalyzerInfo(BaseModel):
     """Which analyzer produced this snapshot, and how it was configured.
     Lives on the ``Analysis`` envelope (keystone ``analyzer{name,version}``;
@@ -499,7 +600,6 @@ class PyAnalyzerInfo(BaseModel):
 
 
 @builder
-@msgpk
 class PyApplication(BaseModel):
     """Represents a Python application."""
 
@@ -511,15 +611,25 @@ class PyApplication(BaseModel):
     # builtin members), keyed by signature. Populated by the analyzer so every
     # backend (JSON and Neo4j) shares one authoritative external-symbol set.
     external_symbols: Dict[str, PyExternalSymbol] = {}
+    # Non-code artifacts, declared dependencies, and undeclared imports
+    # (spec 2026-08-27). L1 data: identical at every analysis level.
+    artifacts: Dict[str, PyArtifact] = {}
+    dependencies: List[PyDependency] = []
+    unresolved_imports: List[PyImportBinding] = []
+    # Coverage/failure record for the entrypoint pass; see PyEntrypointReport (#27).
+    entrypoint_report: PyEntrypointReport = PyEntrypointReport()
     # Git provenance of the analyzed checkout, captured at analysis time.
     repository: Optional[PyRepositoryInfo] = None
     # Interprocedural parameter-passing edges (formal↔actual); populated at L4.
     param_in: List[ParamEdge] = []
     param_out: List[ParamEdge] = []
+    # config_use (#162): PY_USES_CONFIG edges + first-class unresolved reads.
+    # Literal tier from L2; dataflow tiers widen the set at L3/L4 (additive).
+    config_uses: List[PyConfigUseEdge] = []
+    config_reads_unresolved: List[PyConfigRead] = []
 
 
 @builder
-@msgpk
 class Analysis(BaseModel):
     """v2 payload root: envelope + the application tree node. ``k_limit`` is an
     L3+ envelope key (None below the dataflow levels; exclude_none drops it)."""

codeanalyzer/semantic_analysis/call_graph.py

@@ -28,6 +28,8 @@ from typing import Dict, Iterator, List, Tuple
 
 import networkx as nx
 
+from codeanalyzer.semantic_analysis.defuse_linker import _module_qual
+from codeanalyzer.schema import model_copy
 from codeanalyzer.schema.py_schema import (
     PyApplication,
     PyCallable,
@@ -170,7 +172,7 @@ def jedi_call_graph_edges(
 
     Edges are coalesced on ``(source, target)``: ``weight`` is the count of
     matching sites. Provenance is always ``["jedi"]``; combine with
-    PyCG-derived edges via ``merge_edges``.
+    defuse-linker edges via ``merge_edges``.
     """
     counts: Counter = Counter()
     for caller in iter_callables_in_symbol_table(symbol_table):
@@ -253,10 +255,24 @@ def filter_external_edges(
     retained; only lib→lib edges are dropped.  The app symbol set is built by
     walking every callable in the symbol table recursively (including nested
     functions and closures via ``callables``) plus every class, so
-    PyCG-discovered closure nodes are correctly recognised as app symbols.
+    resolver-discovered closure nodes are correctly recognised as app symbols.
+
+    Module names count as app symbols too (#131). A resolver attributes a call in
+    module scope to the MODULE -- ``app -> functools.reduce`` for a module-level
+    ``functools.reduce(...)``, or for a decorator applied to a top-level
+    definition, since a decorator executes in its enclosing scope. Without the
+    module names here, both endpoints looked third-party and every module-scope
+    call to a library target was discarded as lib→lib.
     """
     app_symbols: set = {c.signature for c in iter_callables_in_symbol_table(symbol_table)}
     app_symbols.update(cls.signature for cls in iter_classes_in_symbol_table(symbol_table))
+    app_symbols.update(
+        mod.module_name for mod in symbol_table.values() if mod.module_name
+    )
+    # Dotted module quals too: the defuse linker attributes module- and
+    # class-scope calls to "pkg.module" (collision-free across packages),
+    # which the bare `module_name` stems above never match.
+    app_symbols.update(_module_qual(key) for key in symbol_table)
 
     return [
         e for e in edges
@@ -269,7 +285,7 @@ def merge_edges(*edge_lists: list) -> list:
 
     Edges with the same ``(source, target)`` are coalesced: weights sum,
     provenance is the sorted union. Useful for combining edges produced
-    by different backends (e.g. Jedi + PyCG).
+    by different backends (e.g. Jedi + the defuse linker).
     """
     by_key: Dict[Tuple[str, str], PyCallEdge] = {}
     for edges in edge_lists:
@@ -280,5 +296,5 @@ def merge_edges(*edge_lists: list) -> list:
                 cur.weight += e.weight
                 cur.prov = sorted(set(cur.prov) | set(e.prov))
             else:
-                by_key[k] = e.model_copy()
+                by_key[k] = model_copy(e)
     return list(by_key.values())