codeanalyzer-python 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +77 -4
- codeanalyzer/core.py +174 -72
- codeanalyzer/dataflow/__init__.py +35 -0
- codeanalyzer/dataflow/access_paths.py +563 -0
- codeanalyzer/dataflow/alias.py +93 -0
- codeanalyzer/dataflow/builder.py +688 -0
- codeanalyzer/dataflow/cfg.py +605 -0
- codeanalyzer/dataflow/defuse.py +113 -0
- codeanalyzer/dataflow/dominance.py +140 -0
- codeanalyzer/dataflow/identity.py +91 -0
- codeanalyzer/dataflow/pdg.py +100 -0
- codeanalyzer/dataflow/scalpel_oracle.py +269 -0
- codeanalyzer/dataflow/scc.py +91 -0
- codeanalyzer/dataflow/sdg.py +424 -0
- codeanalyzer/dataflow/slicing.py +93 -0
- codeanalyzer/dataflow/summaries.py +217 -0
- codeanalyzer/dataflow/syntactic.py +26 -0
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/bolt.py +19 -4
- codeanalyzer/neo4j/cypher.py +9 -3
- codeanalyzer/neo4j/emit.py +10 -5
- codeanalyzer/neo4j/project.py +307 -60
- codeanalyzer/neo4j/rows.py +18 -15
- codeanalyzer/neo4j/schema.py +297 -15
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/__init__.py +19 -0
- codeanalyzer/schema/assign_ids.py +37 -0
- codeanalyzer/schema/call_graph_ids.py +12 -0
- codeanalyzer/schema/ids.py +23 -0
- codeanalyzer/schema/l1_body.py +29 -0
- codeanalyzer/schema/l2_callees.py +36 -0
- codeanalyzer/schema/py_schema.py +175 -26
- codeanalyzer/semantic_analysis/call_graph.py +24 -27
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +10 -10
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +7 -7
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +103 -20
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/METADATA +233 -43
- codeanalyzer_python-1.0.0.dist-info/RECORD +59 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer_python-0.3.0.dist-info/RECORD +0 -38
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -36,7 +36,7 @@ import json
|
|
|
36
36
|
from pathlib import Path
|
|
37
37
|
from typing import Any, List, Optional
|
|
38
38
|
|
|
39
|
-
from codeanalyzer.neo4j.
|
|
39
|
+
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
40
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
41
41
|
from codeanalyzer.schema import (
|
|
42
42
|
PyApplication,
|
|
@@ -50,45 +50,234 @@ from codeanalyzer.schema import (
|
|
|
50
50
|
from codeanalyzer.schema.py_schema import PyCallsite
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
def project(app: PyApplication, app_name: str
|
|
53
|
+
def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
54
|
+
analyzer: Optional[Any] = None) -> GraphRows:
|
|
55
|
+
"""``analyzer`` is the envelope-level ``PyAnalyzerInfo`` (the keystone home
|
|
56
|
+
for analyzer identity); the caller that holds the ``Analysis`` envelope
|
|
57
|
+
passes it through so the :PyApplication node carries it as props."""
|
|
54
58
|
b = RowBuilder()
|
|
55
59
|
|
|
56
|
-
app_ref = b.node(
|
|
60
|
+
app_ref = b.node(
|
|
61
|
+
["PyApplication"],
|
|
62
|
+
"name",
|
|
63
|
+
app_name,
|
|
64
|
+
prune(
|
|
65
|
+
{
|
|
66
|
+
"schema_version": SCHEMA_VERSION,
|
|
67
|
+
"analyzer_name": analyzer.name if analyzer else None,
|
|
68
|
+
"analyzer_version": analyzer.version if analyzer else None,
|
|
69
|
+
"repo_uri": app.repository.uri if app.repository else None,
|
|
70
|
+
"source_revision": app.repository.revision if app.repository else None,
|
|
71
|
+
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
72
|
+
}
|
|
73
|
+
),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Endpoints listed in app.external_symbols become :PyExternal ghost nodes; the
|
|
77
|
+
# rest are declared :PySymbol nodes emitted here (keyed by their can:// id,
|
|
78
|
+
# resolved through ``sig_to_id``). Both the module-body projection (for
|
|
79
|
+
# PY_EXTENDS / PY_RESOLVES_TO) and the PY_CALLS twin below share this split.
|
|
80
|
+
externals = app.external_symbols or {}
|
|
81
|
+
|
|
82
|
+
# file key → module can:// id, so resolved PY_IMPORTS edges land on the v2
|
|
83
|
+
# module merge key (id) rather than the legacy file_key property.
|
|
84
|
+
module_id_by_key = {k: m.id for k, m in app.symbol_table.items()}
|
|
57
85
|
|
|
58
86
|
for file_key, mod in app.symbol_table.items():
|
|
59
|
-
mod_ref = b.node(["PyModule"], "
|
|
87
|
+
mod_ref = b.node(["PyModule"], "id", mod.id, _module_props(mod, file_key))
|
|
60
88
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
61
|
-
_project_module_body(b, file_key, mod_ref, mod)
|
|
89
|
+
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key)
|
|
62
90
|
|
|
63
|
-
# The aggregated :PY_CALLS twin.
|
|
64
|
-
# :PyExternal ghost nodes; the rest are declared :PySymbol nodes already emitted.
|
|
65
|
-
externals = app.external_symbols or {}
|
|
91
|
+
# The aggregated :PY_CALLS twin.
|
|
66
92
|
for e in app.call_graph:
|
|
67
|
-
src = _call_endpoint(b, e.
|
|
68
|
-
tgt = _call_endpoint(b, e.
|
|
69
|
-
b.edge(
|
|
93
|
+
src = _call_endpoint(b, e.src, externals, sig_to_id)
|
|
94
|
+
tgt = _call_endpoint(b, e.dst, externals, sig_to_id)
|
|
95
|
+
b.edge(
|
|
96
|
+
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.prov or []))
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
100
|
+
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
101
|
+
_project_program_graphs(b, app)
|
|
70
102
|
|
|
71
103
|
return b.finish()
|
|
72
104
|
|
|
73
105
|
|
|
74
|
-
|
|
75
|
-
|
|
106
|
+
# ----------------------------------------------------------------------------------------------
|
|
107
|
+
# Level-3 CPG overlay
|
|
108
|
+
# ----------------------------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
112
|
+
"""The globally-unique PyCFGNode merge key for a callable's body node: the
|
|
113
|
+
callable's ``can://`` id joined to its LOCAL body key with a single ``@``.
|
|
114
|
+
The synthetic bookends already carry the leading ``@`` (``"@entry"``/
|
|
115
|
+
``"@exit"``); real statements are bare ``"line:col"`` and gain the ``@``.
|
|
116
|
+
|
|
117
|
+
This MUST agree with :meth:`IdentityMap.global_id` for the same node, so the
|
|
118
|
+
JSON ``body``/``cfg`` projection and this Neo4j projection land on one node
|
|
119
|
+
identity (two-projection agreement)."""
|
|
120
|
+
return (
|
|
121
|
+
f"{callable_id}{local_key}"
|
|
122
|
+
if local_key.startswith("@")
|
|
123
|
+
else f"{callable_id}@{local_key}"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _cfg_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
128
|
+
return NodeRef("PyCFGNode", "id", _global_ordinal(callable_id, local_key))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
132
|
+
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
133
|
+
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
134
|
+
|
|
135
|
+
Node label ``PyCFGNode`` (merge key ``id`` = the GLOBAL ordinal
|
|
136
|
+
``<callable can:// id>@<local body key>`` — identical to the JSON body key
|
|
137
|
+
prefixed with the callable id, so the two projections agree). Edges:
|
|
138
|
+
``PY_HAS_CFG_NODE`` from the owning callable, ``PY_CFG_NEXT`` (prop ``kind``)
|
|
139
|
+
over the CFG, ``PY_CDG`` over control dependence, and ``PY_DDG`` (props
|
|
140
|
+
``var``/``prov``) over data dependence. The vocabulary is cross-language in
|
|
141
|
+
shape but PY_-namespaced like every other row family, so a multi-language
|
|
142
|
+
database never mingles analyzers' dependence edges.
|
|
76
143
|
|
|
144
|
+
L4 (``-a 4``) layers the interprocedural delta onto the same node label:
|
|
145
|
+
parameter-passing vertices (``formal_in``/``formal_out``/``actual_in``/
|
|
146
|
+
``actual_out``) carry ``var`` (the variable/return they model, from
|
|
147
|
+
``BodyNode.of``) and ``call_node`` (the owning callsite local id, from
|
|
148
|
+
``BodyNode.parent``) instead of span-derived lines; ``PY_SUMMARY`` runs over
|
|
149
|
+
each callable's transitive pass-throughs (LOCAL ids → global refs), and the
|
|
150
|
+
app-level ``PY_PARAM_IN``/``PY_PARAM_OUT`` edges connect actual↔formal
|
|
151
|
+
vertices across callables (endpoints are already GLOBAL ordinals matching the
|
|
152
|
+
emitted ``PyCFGNode`` keys). All idempotent under MERGE — no-ops below L4."""
|
|
153
|
+
from codeanalyzer.semantic_analysis.call_graph import _walk_module_callables
|
|
77
154
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
155
|
+
for file_key, mod in app.symbol_table.items():
|
|
156
|
+
for c in _walk_module_callables(mod):
|
|
157
|
+
if not c.id:
|
|
158
|
+
continue # unstamped callable — assign_ids must run first
|
|
159
|
+
owner = _sym(c.id) # the :PyCallable node, keyed by its can:// id
|
|
160
|
+
for local_key, node in (c.body or {}).items():
|
|
161
|
+
span = node.span
|
|
162
|
+
# L4 param vertices carry the variable they model (``of``) and
|
|
163
|
+
# their owning callsite (``parent``) instead of span lines; both
|
|
164
|
+
# are None on ordinary statement nodes and pruned away there.
|
|
165
|
+
ref = b.node(
|
|
166
|
+
["PyCFGNode"],
|
|
167
|
+
"id",
|
|
168
|
+
_global_ordinal(c.id, local_key),
|
|
169
|
+
prune(
|
|
170
|
+
{
|
|
171
|
+
"kind": node.kind,
|
|
172
|
+
"start_line": span.start[0] if span else None,
|
|
173
|
+
"end_line": span.end[0] if span else None,
|
|
174
|
+
"var": node.of,
|
|
175
|
+
"call_node": node.parent,
|
|
176
|
+
"_module": file_key,
|
|
177
|
+
}
|
|
178
|
+
),
|
|
179
|
+
)
|
|
180
|
+
b.edge("PY_HAS_CFG_NODE", owner, ref)
|
|
181
|
+
for e in c.cfg or []:
|
|
182
|
+
# kind-discriminated: a conditional's true/false pair between one
|
|
183
|
+
# endpoint pair must stay two relationships, not one MERGE.
|
|
184
|
+
b.edge(
|
|
185
|
+
"PY_CFG_NEXT",
|
|
186
|
+
_cfg_ref(c.id, e.src),
|
|
187
|
+
_cfg_ref(c.id, e.dst),
|
|
188
|
+
{"kind": e.kind},
|
|
189
|
+
key=e.kind,
|
|
190
|
+
)
|
|
191
|
+
for e in c.cdg or []:
|
|
192
|
+
b.edge("PY_CDG", _cfg_ref(c.id, e.src), _cfg_ref(c.id, e.dst))
|
|
193
|
+
for e in c.ddg or []:
|
|
194
|
+
# (var, prov)-discriminated: the DDG legitimately carries several
|
|
195
|
+
# edges between one statement pair (one per variable, and the
|
|
196
|
+
# ssa/points-to split) — a plain endpoint-pair MERGE collapses
|
|
197
|
+
# them and silently drops dependences.
|
|
198
|
+
b.edge(
|
|
199
|
+
"PY_DDG",
|
|
200
|
+
_cfg_ref(c.id, e.src),
|
|
201
|
+
_cfg_ref(c.id, e.dst),
|
|
202
|
+
prune({"var": e.var, "prov": list(e.prov) if e.prov else None}),
|
|
203
|
+
key=f"{e.var or ''}|{','.join(e.prov or [])}",
|
|
204
|
+
)
|
|
205
|
+
# L4 intraprocedural summaries (transitive actual_in → actual_out
|
|
206
|
+
# pass-throughs); LOCAL ids resolved to global PyCFGNode refs.
|
|
207
|
+
for e in c.summary or []:
|
|
208
|
+
b.edge("PY_SUMMARY", _cfg_ref(c.id, e.src), _cfg_ref(c.id, e.dst))
|
|
209
|
+
|
|
210
|
+
# L4 interprocedural parameter passing, emitted once at the app scope. The
|
|
211
|
+
# endpoints are ALREADY global ordinals (emit_l4 resolved them through the
|
|
212
|
+
# endpoint functions' identity maps), so they land on the very PyCFGNode ids
|
|
213
|
+
# projected above — a formal_in global id equals _global_ordinal(callee.id,
|
|
214
|
+
# "@formal_in:0"). No dangling references.
|
|
215
|
+
for e in app.param_in or []:
|
|
216
|
+
b.edge(
|
|
217
|
+
"PY_PARAM_IN",
|
|
218
|
+
NodeRef("PyCFGNode", "id", e.src),
|
|
219
|
+
NodeRef("PyCFGNode", "id", e.dst),
|
|
220
|
+
)
|
|
221
|
+
for e in app.param_out or []:
|
|
222
|
+
b.edge(
|
|
223
|
+
"PY_PARAM_OUT",
|
|
224
|
+
NodeRef("PyCFGNode", "id", e.src),
|
|
225
|
+
NodeRef("PyCFGNode", "id", e.dst),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _sym(can_id: str) -> NodeRef:
|
|
230
|
+
return NodeRef("PySymbol", "id", can_id)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _symbol_ref(signature: str, externals: dict, sig_to_id: dict) -> NodeRef:
|
|
234
|
+
"""Resolve a call/inheritance target to the NodeRef under which it was (or
|
|
235
|
+
will be) emitted: a declared symbol by its can:// id, otherwise a
|
|
236
|
+
signature-keyed :PySymbol (external ghost)."""
|
|
237
|
+
can_id = sig_to_id.get(signature)
|
|
238
|
+
if can_id is not None:
|
|
239
|
+
return NodeRef("PySymbol", "id", can_id)
|
|
240
|
+
return NodeRef("PySymbol", "signature", signature)
|
|
81
241
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
242
|
+
|
|
243
|
+
def _call_endpoint(
|
|
244
|
+
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict
|
|
245
|
+
) -> NodeRef:
|
|
246
|
+
"""A call-graph endpoint: a declared callable already emitted (keyed by its
|
|
247
|
+
canonical ``can://`` id, resolved through ``sig_to_id``), or an external symbol
|
|
248
|
+
(imported library / builtin member) materialized as a :PyExternal ghost.
|
|
249
|
+
|
|
250
|
+
Classification is authoritative -- it comes from ``app.external_symbols``
|
|
251
|
+
(keyed by ``can://…/@external/…`` id), not a "present in the graph" heuristic --
|
|
252
|
+
so an imported module name (which exists only as a :PyPackage) can never shadow
|
|
253
|
+
the call target. A declared endpoint resolves to its ``can://`` id (either
|
|
254
|
+
already re-identified on the edge, or resolved through ``sig_to_id``); anything
|
|
255
|
+
neither declared nor listed falls back to an id-keyed :PyExternal ghost rather
|
|
256
|
+
than raising."""
|
|
86
257
|
ext = externals.get(signature)
|
|
87
|
-
if ext is None
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
258
|
+
if ext is None:
|
|
259
|
+
can_id = sig_to_id.get(signature)
|
|
260
|
+
if can_id is not None:
|
|
261
|
+
ext = externals.get(can_id)
|
|
262
|
+
if ext is None:
|
|
263
|
+
return _sym(can_id)
|
|
264
|
+
elif signature.startswith("can://") and "/@external/" not in signature:
|
|
265
|
+
# An already re-identified declared endpoint (post reidentify_call_graph).
|
|
266
|
+
return _sym(signature)
|
|
267
|
+
if ext is not None:
|
|
268
|
+
return b.node(
|
|
269
|
+
["PySymbol", "PyExternal"],
|
|
270
|
+
"id",
|
|
271
|
+
ext.id or signature,
|
|
272
|
+
prune({"name": ext.name, "module": ext.module}),
|
|
273
|
+
)
|
|
274
|
+
name = signature.rsplit(".", 1)[-1] if "." in signature else signature
|
|
275
|
+
return b.node(
|
|
276
|
+
["PySymbol", "PyExternal"],
|
|
277
|
+
"id",
|
|
278
|
+
signature,
|
|
279
|
+
prune({"name": name}),
|
|
280
|
+
)
|
|
92
281
|
|
|
93
282
|
|
|
94
283
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -96,36 +285,67 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
96
285
|
# ----------------------------------------------------------------------------------------------
|
|
97
286
|
|
|
98
287
|
|
|
99
|
-
def _project_module_body(
|
|
288
|
+
def _project_module_body(
|
|
289
|
+
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule,
|
|
290
|
+
externals: dict, sig_to_id: dict, module_id_by_key: dict,
|
|
291
|
+
) -> None:
|
|
100
292
|
for fn in (mod.functions or {}).values():
|
|
101
|
-
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn)
|
|
102
|
-
for cl in (mod.
|
|
103
|
-
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl)
|
|
293
|
+
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id)
|
|
294
|
+
for cl in (mod.types or {}).values():
|
|
295
|
+
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id)
|
|
104
296
|
for v in mod.variables or []:
|
|
105
297
|
_project_variable(b, file_key, mod_ref, file_key, v)
|
|
106
|
-
_project_imports(b, mod_ref, mod)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule
|
|
110
|
-
|
|
111
|
-
#
|
|
298
|
+
_project_imports(b, mod_ref, mod, module_id_by_key)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
|
|
302
|
+
module_id_by_key: dict) -> None:
|
|
303
|
+
# At most one PY_IMPORTS edge per (module, target) pair -- mirrors PY_CALLS,
|
|
304
|
+
# which pre-aggregates for the same reason: both writers MERGE edges on
|
|
305
|
+
# (type, from, to) and SET their props, so a second row for the same pair
|
|
306
|
+
# would silently overwrite the first in Neo4j instead of adding an edge.
|
|
307
|
+
# Buckets key on the edge's target identity (the resolved module, or the
|
|
308
|
+
# spelling itself for unresolved/external imports), so the SAME target
|
|
309
|
+
# imported under different spellings (``from pkg import util``,
|
|
310
|
+
# ``from . import util as u``, ``from .util import helper``) collapses
|
|
311
|
+
# onto one edge; the raw spellings ride along as a ``spellings`` array.
|
|
312
|
+
# Resolved internal imports point at the real :PyModule; externals keep
|
|
313
|
+
# the shared :PyPackage. Unresolved *relative* spellings (".", ".foo")
|
|
314
|
+
# name no package -- they are dropped from the graph (the spelling
|
|
315
|
+
# survives in analysis.json), instead of minting bogus
|
|
316
|
+
# :PyPackage{name: "."} nodes.
|
|
112
317
|
agg: dict = {}
|
|
113
318
|
for im in mod.imports or []:
|
|
114
319
|
if not im.module:
|
|
115
|
-
continue
|
|
116
|
-
|
|
320
|
+
continue
|
|
321
|
+
if im.resolved_module is None and im.module.startswith("."):
|
|
322
|
+
continue
|
|
323
|
+
key = im.resolved_module or im.module
|
|
324
|
+
a = agg.setdefault(
|
|
325
|
+
key, {"spellings": set(), "names": set(), "aliases": set(), "resolved": im.resolved_module}
|
|
326
|
+
)
|
|
327
|
+
a["spellings"].add(im.module)
|
|
117
328
|
if im.name:
|
|
118
329
|
a["names"].add(im.name)
|
|
119
330
|
if im.alias:
|
|
120
331
|
a["aliases"].add(im.alias)
|
|
121
|
-
for
|
|
122
|
-
|
|
332
|
+
for key, a in agg.items():
|
|
333
|
+
resolved_id = module_id_by_key.get(a["resolved"]) if a["resolved"] is not None else None
|
|
334
|
+
if resolved_id is not None:
|
|
335
|
+
target = NodeRef("PyModule", "id", resolved_id)
|
|
336
|
+
elif a["resolved"] is None:
|
|
337
|
+
target = b.node(["PyPackage"], "name", key, {})
|
|
338
|
+
else:
|
|
339
|
+
# resolved to a file key that is not in this symbol table (partial
|
|
340
|
+
# run) — keep the module target via its legacy file_key property.
|
|
341
|
+
target = NodeRef("PyModule", "file_key", a["resolved"])
|
|
123
342
|
b.edge(
|
|
124
343
|
"PY_IMPORTS",
|
|
125
344
|
mod_ref,
|
|
126
|
-
|
|
345
|
+
target,
|
|
127
346
|
prune(
|
|
128
347
|
{
|
|
348
|
+
"spellings": sorted(a["spellings"]),
|
|
129
349
|
"imported_names": sorted(a["names"]) or None,
|
|
130
350
|
"aliases": sorted(a["aliases"]) or None,
|
|
131
351
|
}
|
|
@@ -139,26 +359,36 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
|
139
359
|
|
|
140
360
|
|
|
141
361
|
def _project_class(
|
|
142
|
-
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass
|
|
362
|
+
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
|
|
363
|
+
externals: dict, sig_to_id: dict,
|
|
143
364
|
) -> None:
|
|
144
|
-
ref = b.node(
|
|
365
|
+
ref = b.node(
|
|
366
|
+
["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key)
|
|
367
|
+
)
|
|
145
368
|
b.edge(parent_rel, parent, ref)
|
|
146
369
|
|
|
147
370
|
for base in cl.base_classes or []:
|
|
148
|
-
|
|
371
|
+
if base:
|
|
372
|
+
b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
|
|
149
373
|
|
|
150
|
-
for m in (cl.
|
|
151
|
-
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m)
|
|
374
|
+
for m in (cl.callables or {}).values():
|
|
375
|
+
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id)
|
|
152
376
|
for a in (cl.attributes or {}).values():
|
|
153
377
|
_project_attribute(b, file_key, ref, cl.signature, a)
|
|
154
|
-
for ic in (cl.
|
|
155
|
-
_project_class(b, file_key, ref, "PY_DECLARES", ic)
|
|
378
|
+
for ic in (cl.types or {}).values():
|
|
379
|
+
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
|
|
156
380
|
|
|
157
381
|
|
|
158
382
|
def _project_callable(
|
|
159
|
-
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable
|
|
383
|
+
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
|
|
384
|
+
externals: dict, sig_to_id: dict,
|
|
160
385
|
) -> None:
|
|
161
|
-
ref = b.node(
|
|
386
|
+
ref = b.node(
|
|
387
|
+
["PySymbol", "PyCallable"],
|
|
388
|
+
"id",
|
|
389
|
+
c.id,
|
|
390
|
+
_callable_props(c, file_key),
|
|
391
|
+
)
|
|
162
392
|
b.edge(owner_rel, owner, ref)
|
|
163
393
|
|
|
164
394
|
for d in c.decorators or []:
|
|
@@ -166,18 +396,23 @@ def _project_callable(
|
|
|
166
396
|
|
|
167
397
|
for s in c.call_sites or []:
|
|
168
398
|
# Key off the relative file (a call site lives in its callable's file) so ids stay portable.
|
|
169
|
-
cs_id =
|
|
399
|
+
cs_id = (
|
|
400
|
+
f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
|
|
401
|
+
)
|
|
170
402
|
cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
|
|
171
403
|
b.edge("PY_HAS_CALLSITE", ref, cs)
|
|
172
404
|
if s.callee_signature:
|
|
173
|
-
b.edge_to_symbol(
|
|
405
|
+
b.edge_to_symbol(
|
|
406
|
+
"PY_RESOLVES_TO", cs,
|
|
407
|
+
_symbol_ref(s.callee_signature, externals, sig_to_id),
|
|
408
|
+
)
|
|
174
409
|
|
|
175
410
|
for v in c.local_variables or []:
|
|
176
411
|
_project_variable(b, file_key, ref, c.signature, v)
|
|
177
|
-
for ic in (c.
|
|
178
|
-
_project_callable(b, file_key, ref, "PY_DECLARES", ic)
|
|
179
|
-
for cl in (c.
|
|
180
|
-
_project_class(b, file_key, ref, "PY_DECLARES", cl)
|
|
412
|
+
for ic in (c.callables or {}).values():
|
|
413
|
+
_project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
|
|
414
|
+
for cl in (c.types or {}).values():
|
|
415
|
+
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id)
|
|
181
416
|
|
|
182
417
|
|
|
183
418
|
def _project_attribute(
|
|
@@ -189,7 +424,11 @@ def _project_attribute(
|
|
|
189
424
|
|
|
190
425
|
|
|
191
426
|
def _project_variable(
|
|
192
|
-
b: RowBuilder,
|
|
427
|
+
b: RowBuilder,
|
|
428
|
+
file_key: str,
|
|
429
|
+
owner: NodeRef,
|
|
430
|
+
owner_id: str,
|
|
431
|
+
v: PyVariableDeclaration,
|
|
193
432
|
) -> None:
|
|
194
433
|
var_id = f"{owner_id}#{v.name}@{v.start_line}"
|
|
195
434
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
@@ -209,6 +448,8 @@ def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
|
|
|
209
448
|
def _module_props(mod: PyModule, file_key: str) -> Props:
|
|
210
449
|
return prune(
|
|
211
450
|
{
|
|
451
|
+
"id": mod.id,
|
|
452
|
+
"file_key": file_key,
|
|
212
453
|
"module_name": mod.module_name,
|
|
213
454
|
"content_hash": mod.content_hash,
|
|
214
455
|
"last_modified": mod.last_modified,
|
|
@@ -221,8 +462,10 @@ def _module_props(mod: PyModule, file_key: str) -> Props:
|
|
|
221
462
|
def _class_props(cl: PyClass, file_key: str) -> Props:
|
|
222
463
|
return prune(
|
|
223
464
|
{
|
|
465
|
+
"id": cl.id,
|
|
466
|
+
"signature": cl.signature,
|
|
224
467
|
"name": cl.name,
|
|
225
|
-
"code": cl
|
|
468
|
+
"code": getattr(cl, "code", None),
|
|
226
469
|
"base_classes": list(cl.base_classes or []),
|
|
227
470
|
"docstring": _docstring_of(cl.comments),
|
|
228
471
|
"start_line": cl.start_line,
|
|
@@ -235,11 +478,13 @@ def _class_props(cl: PyClass, file_key: str) -> Props:
|
|
|
235
478
|
def _callable_props(c: PyCallable, file_key: str) -> Props:
|
|
236
479
|
return prune(
|
|
237
480
|
{
|
|
481
|
+
"id": c.id,
|
|
482
|
+
"signature": c.signature,
|
|
238
483
|
"name": c.name,
|
|
239
484
|
"path": c.path,
|
|
240
485
|
"return_type": c.return_type,
|
|
241
486
|
"cyclomatic_complexity": c.cyclomatic_complexity,
|
|
242
|
-
"code": c
|
|
487
|
+
"code": getattr(c, "code", None),
|
|
243
488
|
"code_start_line": c.code_start_line,
|
|
244
489
|
"start_line": c.start_line,
|
|
245
490
|
"end_line": c.end_line,
|
|
@@ -258,6 +503,7 @@ def _attribute_props(a: PyClassAttribute, attr_id: str, file_key: str) -> Props:
|
|
|
258
503
|
"id": attr_id,
|
|
259
504
|
"name": a.name,
|
|
260
505
|
"type": a.type,
|
|
506
|
+
"initializer": a.initializer,
|
|
261
507
|
"docstring": _docstring_of(a.comments),
|
|
262
508
|
"start_line": a.start_line,
|
|
263
509
|
"end_line": a.end_line,
|
|
@@ -290,6 +536,7 @@ def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
|
290
536
|
"receiver_expr": s.receiver_expr,
|
|
291
537
|
"receiver_type": s.receiver_type,
|
|
292
538
|
"argument_types": list(s.argument_types or []),
|
|
539
|
+
"arguments_json": _stringify_if(s.arguments),
|
|
293
540
|
"return_type": s.return_type,
|
|
294
541
|
"callee_signature": s.callee_signature,
|
|
295
542
|
"is_constructor_call": s.is_constructor_call,
|
|
@@ -302,8 +549,8 @@ def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
|
302
549
|
)
|
|
303
550
|
|
|
304
551
|
|
|
305
|
-
def _call_edge_props(weight: int,
|
|
306
|
-
return prune({"weight": weight, "
|
|
552
|
+
def _call_edge_props(weight: int, prov: List[str]) -> Props:
|
|
553
|
+
return prune({"weight": weight, "prov": list(prov)})
|
|
307
554
|
|
|
308
555
|
|
|
309
556
|
def _docstring_of(comments: Optional[List[PyComment]]) -> Optional[str]:
|
codeanalyzer/neo4j/rows.py
CHANGED
|
@@ -58,6 +58,12 @@ class EdgeRow:
|
|
|
58
58
|
from_ref: NodeRef
|
|
59
59
|
to_ref: NodeRef
|
|
60
60
|
props: Props
|
|
61
|
+
# Optional relationship discriminant: when set, the MERGE is on
|
|
62
|
+
# ``{_k: key}`` so several legitimately-distinct edges of one type may
|
|
63
|
+
# coexist between the same endpoint pair (e.g. per-variable PY_DDG edges,
|
|
64
|
+
# or the true/false PY_CFG_NEXT pair of a conditional). ``None`` keeps the
|
|
65
|
+
# plain endpoint-pair MERGE.
|
|
66
|
+
key: Optional[str] = None
|
|
61
67
|
|
|
62
68
|
|
|
63
69
|
@dataclass
|
|
@@ -105,25 +111,22 @@ class RowBuilder:
|
|
|
105
111
|
self._keys.add((labels[0], value))
|
|
106
112
|
return NodeRef(labels[0], key_prop, value)
|
|
107
113
|
|
|
108
|
-
def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None
|
|
109
|
-
|
|
110
|
-
|
|
114
|
+
def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None,
|
|
115
|
+
key: Optional[str] = None) -> None:
|
|
116
|
+
"""An edge whose endpoints are known to exist (both ends emitted this run).
|
|
117
|
+
``key`` sets the relationship discriminant (see :class:`EdgeRow`)."""
|
|
118
|
+
self._edges.append(EdgeRow(type_, from_ref, to_ref, dict(props or {}), key))
|
|
111
119
|
|
|
112
120
|
def edge_to_symbol(
|
|
113
|
-
self, type_: str, from_ref: NodeRef,
|
|
121
|
+
self, type_: str, from_ref: NodeRef, target_ref: NodeRef, props: Optional[Props] = None
|
|
114
122
|
) -> None:
|
|
115
123
|
"""An edge to a ``:PySymbol`` target that may be external/library code not
|
|
116
|
-
present in the graph.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
from_ref,
|
|
123
|
-
NodeRef("PySymbol", "signature", target_signature),
|
|
124
|
-
dict(props or {}),
|
|
125
|
-
)
|
|
126
|
-
)
|
|
124
|
+
present in the graph. The target is an already-resolved :class:`NodeRef`
|
|
125
|
+
(a declared symbol by its can:// id, or a signature-keyed external ghost).
|
|
126
|
+
Deferred and kept only if that ``(label, value)`` was actually emitted as a
|
|
127
|
+
node — so PY_EXTENDS / PY_RESOLVES_TO never dangle (the string fallback lives
|
|
128
|
+
on the source node's props)."""
|
|
129
|
+
self._deferred.append(EdgeRow(type_, from_ref, target_ref, dict(props or {})))
|
|
127
130
|
|
|
128
131
|
def has_key(self, label: str, value: str) -> bool:
|
|
129
132
|
"""Whether a node with this ``(merge_label, value)`` identity was emitted."""
|