codeanalyzer-python 0.3.1__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +51 -4
- codeanalyzer/core.py +153 -82
- codeanalyzer/dataflow/__init__.py +35 -0
- codeanalyzer/dataflow/access_paths.py +563 -0
- codeanalyzer/dataflow/alias.py +93 -0
- codeanalyzer/dataflow/builder.py +688 -0
- codeanalyzer/dataflow/cfg.py +605 -0
- codeanalyzer/dataflow/defuse.py +113 -0
- codeanalyzer/dataflow/dominance.py +140 -0
- codeanalyzer/dataflow/identity.py +91 -0
- codeanalyzer/dataflow/pdg.py +100 -0
- codeanalyzer/dataflow/scalpel_oracle.py +269 -0
- codeanalyzer/dataflow/scc.py +91 -0
- codeanalyzer/dataflow/sdg.py +424 -0
- codeanalyzer/dataflow/slicing.py +93 -0
- codeanalyzer/dataflow/summaries.py +217 -0
- codeanalyzer/dataflow/syntactic.py +26 -0
- codeanalyzer/neo4j/bolt.py +19 -4
- codeanalyzer/neo4j/cypher.py +9 -3
- codeanalyzer/neo4j/emit.py +8 -3
- codeanalyzer/neo4j/project.py +241 -60
- codeanalyzer/neo4j/rows.py +18 -15
- codeanalyzer/neo4j/schema.py +43 -7
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/schema/__init__.py +19 -0
- codeanalyzer/schema/assign_ids.py +37 -0
- codeanalyzer/schema/call_graph_ids.py +12 -0
- codeanalyzer/schema/ids.py +23 -0
- codeanalyzer/schema/l1_body.py +29 -0
- codeanalyzer/schema/l2_callees.py +36 -0
- codeanalyzer/schema/py_schema.py +141 -30
- codeanalyzer/semantic_analysis/call_graph.py +24 -27
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +10 -10
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +7 -7
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +29 -10
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/METADATA +230 -43
- codeanalyzer_python-1.0.0.dist-info/RECORD +59 -0
- codeanalyzer_python-0.3.1.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -50,7 +50,11 @@ from codeanalyzer.schema import (
|
|
|
50
50
|
from codeanalyzer.schema.py_schema import PyCallsite
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
def project(app: PyApplication, app_name: str
|
|
53
|
+
def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
54
|
+
analyzer: Optional[Any] = None) -> GraphRows:
|
|
55
|
+
"""``analyzer`` is the envelope-level ``PyAnalyzerInfo`` (the keystone home
|
|
56
|
+
for analyzer identity); the caller that holds the ``Analysis`` envelope
|
|
57
|
+
passes it through so the :PyApplication node carries it as props."""
|
|
54
58
|
b = RowBuilder()
|
|
55
59
|
|
|
56
60
|
app_ref = b.node(
|
|
@@ -60,8 +64,8 @@ def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
|
60
64
|
prune(
|
|
61
65
|
{
|
|
62
66
|
"schema_version": SCHEMA_VERSION,
|
|
63
|
-
"analyzer_name":
|
|
64
|
-
"analyzer_version":
|
|
67
|
+
"analyzer_name": analyzer.name if analyzer else None,
|
|
68
|
+
"analyzer_version": analyzer.version if analyzer else None,
|
|
65
69
|
"repo_uri": app.repository.uri if app.repository else None,
|
|
66
70
|
"source_revision": app.repository.revision if app.repository else None,
|
|
67
71
|
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
@@ -69,52 +73,210 @@ def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
|
69
73
|
),
|
|
70
74
|
)
|
|
71
75
|
|
|
76
|
+
# Endpoints listed in app.external_symbols become :PyExternal ghost nodes; the
|
|
77
|
+
# rest are declared :PySymbol nodes emitted here (keyed by their can:// id,
|
|
78
|
+
# resolved through ``sig_to_id``). Both the module-body projection (for
|
|
79
|
+
# PY_EXTENDS / PY_RESOLVES_TO) and the PY_CALLS twin below share this split.
|
|
80
|
+
externals = app.external_symbols or {}
|
|
81
|
+
|
|
82
|
+
# file key → module can:// id, so resolved PY_IMPORTS edges land on the v2
|
|
83
|
+
# module merge key (id) rather than the legacy file_key property.
|
|
84
|
+
module_id_by_key = {k: m.id for k, m in app.symbol_table.items()}
|
|
85
|
+
|
|
72
86
|
for file_key, mod in app.symbol_table.items():
|
|
73
|
-
mod_ref = b.node(
|
|
74
|
-
["PyModule"], "file_key", file_key, _module_props(mod, file_key)
|
|
75
|
-
)
|
|
87
|
+
mod_ref = b.node(["PyModule"], "id", mod.id, _module_props(mod, file_key))
|
|
76
88
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
77
|
-
_project_module_body(b, file_key, mod_ref, mod)
|
|
89
|
+
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key)
|
|
78
90
|
|
|
79
|
-
# The aggregated :PY_CALLS twin.
|
|
80
|
-
# :PyExternal ghost nodes; the rest are declared :PySymbol nodes already emitted.
|
|
81
|
-
externals = app.external_symbols or {}
|
|
91
|
+
# The aggregated :PY_CALLS twin.
|
|
82
92
|
for e in app.call_graph:
|
|
83
|
-
src = _call_endpoint(b, e.
|
|
84
|
-
tgt = _call_endpoint(b, e.
|
|
93
|
+
src = _call_endpoint(b, e.src, externals, sig_to_id)
|
|
94
|
+
tgt = _call_endpoint(b, e.dst, externals, sig_to_id)
|
|
85
95
|
b.edge(
|
|
86
|
-
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.
|
|
96
|
+
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.prov or []))
|
|
87
97
|
)
|
|
88
98
|
|
|
99
|
+
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
100
|
+
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
101
|
+
_project_program_graphs(b, app)
|
|
102
|
+
|
|
89
103
|
return b.finish()
|
|
90
104
|
|
|
91
105
|
|
|
92
|
-
|
|
93
|
-
|
|
106
|
+
# ----------------------------------------------------------------------------------------------
|
|
107
|
+
# Level-3 CPG overlay
|
|
108
|
+
# ----------------------------------------------------------------------------------------------
|
|
94
109
|
|
|
95
110
|
|
|
96
|
-
def
|
|
97
|
-
"""
|
|
98
|
-
|
|
111
|
+
def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
112
|
+
"""The globally-unique PyCFGNode merge key for a callable's body node: the
|
|
113
|
+
callable's ``can://`` id joined to its LOCAL body key with a single ``@``.
|
|
114
|
+
The synthetic bookends already carry the leading ``@`` (``"@entry"``/
|
|
115
|
+
``"@exit"``); real statements are bare ``"line:col"`` and gain the ``@``.
|
|
99
116
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
name = (
|
|
108
|
-
ext.name
|
|
109
|
-
if ext is not None
|
|
110
|
-
else (signature.rsplit(".", 1)[-1] if "." in signature else signature)
|
|
117
|
+
This MUST agree with :meth:`IdentityMap.global_id` for the same node, so the
|
|
118
|
+
JSON ``body``/``cfg`` projection and this Neo4j projection land on one node
|
|
119
|
+
identity (two-projection agreement)."""
|
|
120
|
+
return (
|
|
121
|
+
f"{callable_id}{local_key}"
|
|
122
|
+
if local_key.startswith("@")
|
|
123
|
+
else f"{callable_id}@{local_key}"
|
|
111
124
|
)
|
|
112
|
-
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _cfg_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
128
|
+
return NodeRef("PyCFGNode", "id", _global_ordinal(callable_id, local_key))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
132
|
+
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
133
|
+
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
134
|
+
|
|
135
|
+
Node label ``PyCFGNode`` (merge key ``id`` = the GLOBAL ordinal
|
|
136
|
+
``<callable can:// id>@<local body key>`` — identical to the JSON body key
|
|
137
|
+
prefixed with the callable id, so the two projections agree). Edges:
|
|
138
|
+
``PY_HAS_CFG_NODE`` from the owning callable, ``PY_CFG_NEXT`` (prop ``kind``)
|
|
139
|
+
over the CFG, ``PY_CDG`` over control dependence, and ``PY_DDG`` (props
|
|
140
|
+
``var``/``prov``) over data dependence. The vocabulary is cross-language in
|
|
141
|
+
shape but PY_-namespaced like every other row family, so a multi-language
|
|
142
|
+
database never mingles analyzers' dependence edges.
|
|
143
|
+
|
|
144
|
+
L4 (``-a 4``) layers the interprocedural delta onto the same node label:
|
|
145
|
+
parameter-passing vertices (``formal_in``/``formal_out``/``actual_in``/
|
|
146
|
+
``actual_out``) carry ``var`` (the variable/return they model, from
|
|
147
|
+
``BodyNode.of``) and ``call_node`` (the owning callsite local id, from
|
|
148
|
+
``BodyNode.parent``) instead of span-derived lines; ``PY_SUMMARY`` runs over
|
|
149
|
+
each callable's transitive pass-throughs (LOCAL ids → global refs), and the
|
|
150
|
+
app-level ``PY_PARAM_IN``/``PY_PARAM_OUT`` edges connect actual↔formal
|
|
151
|
+
vertices across callables (endpoints are already GLOBAL ordinals matching the
|
|
152
|
+
emitted ``PyCFGNode`` keys). All idempotent under MERGE — no-ops below L4."""
|
|
153
|
+
from codeanalyzer.semantic_analysis.call_graph import _walk_module_callables
|
|
154
|
+
|
|
155
|
+
for file_key, mod in app.symbol_table.items():
|
|
156
|
+
for c in _walk_module_callables(mod):
|
|
157
|
+
if not c.id:
|
|
158
|
+
continue # unstamped callable — assign_ids must run first
|
|
159
|
+
owner = _sym(c.id) # the :PyCallable node, keyed by its can:// id
|
|
160
|
+
for local_key, node in (c.body or {}).items():
|
|
161
|
+
span = node.span
|
|
162
|
+
# L4 param vertices carry the variable they model (``of``) and
|
|
163
|
+
# their owning callsite (``parent``) instead of span lines; both
|
|
164
|
+
# are None on ordinary statement nodes and pruned away there.
|
|
165
|
+
ref = b.node(
|
|
166
|
+
["PyCFGNode"],
|
|
167
|
+
"id",
|
|
168
|
+
_global_ordinal(c.id, local_key),
|
|
169
|
+
prune(
|
|
170
|
+
{
|
|
171
|
+
"kind": node.kind,
|
|
172
|
+
"start_line": span.start[0] if span else None,
|
|
173
|
+
"end_line": span.end[0] if span else None,
|
|
174
|
+
"var": node.of,
|
|
175
|
+
"call_node": node.parent,
|
|
176
|
+
"_module": file_key,
|
|
177
|
+
}
|
|
178
|
+
),
|
|
179
|
+
)
|
|
180
|
+
b.edge("PY_HAS_CFG_NODE", owner, ref)
|
|
181
|
+
for e in c.cfg or []:
|
|
182
|
+
# kind-discriminated: a conditional's true/false pair between one
|
|
183
|
+
# endpoint pair must stay two relationships, not one MERGE.
|
|
184
|
+
b.edge(
|
|
185
|
+
"PY_CFG_NEXT",
|
|
186
|
+
_cfg_ref(c.id, e.src),
|
|
187
|
+
_cfg_ref(c.id, e.dst),
|
|
188
|
+
{"kind": e.kind},
|
|
189
|
+
key=e.kind,
|
|
190
|
+
)
|
|
191
|
+
for e in c.cdg or []:
|
|
192
|
+
b.edge("PY_CDG", _cfg_ref(c.id, e.src), _cfg_ref(c.id, e.dst))
|
|
193
|
+
for e in c.ddg or []:
|
|
194
|
+
# (var, prov)-discriminated: the DDG legitimately carries several
|
|
195
|
+
# edges between one statement pair (one per variable, and the
|
|
196
|
+
# ssa/points-to split) — a plain endpoint-pair MERGE collapses
|
|
197
|
+
# them and silently drops dependences.
|
|
198
|
+
b.edge(
|
|
199
|
+
"PY_DDG",
|
|
200
|
+
_cfg_ref(c.id, e.src),
|
|
201
|
+
_cfg_ref(c.id, e.dst),
|
|
202
|
+
prune({"var": e.var, "prov": list(e.prov) if e.prov else None}),
|
|
203
|
+
key=f"{e.var or ''}|{','.join(e.prov or [])}",
|
|
204
|
+
)
|
|
205
|
+
# L4 intraprocedural summaries (transitive actual_in → actual_out
|
|
206
|
+
# pass-throughs); LOCAL ids resolved to global PyCFGNode refs.
|
|
207
|
+
for e in c.summary or []:
|
|
208
|
+
b.edge("PY_SUMMARY", _cfg_ref(c.id, e.src), _cfg_ref(c.id, e.dst))
|
|
209
|
+
|
|
210
|
+
# L4 interprocedural parameter passing, emitted once at the app scope. The
|
|
211
|
+
# endpoints are ALREADY global ordinals (emit_l4 resolved them through the
|
|
212
|
+
# endpoint functions' identity maps), so they land on the very PyCFGNode ids
|
|
213
|
+
# projected above — a formal_in global id equals _global_ordinal(callee.id,
|
|
214
|
+
# "@formal_in:0"). No dangling references.
|
|
215
|
+
for e in app.param_in or []:
|
|
216
|
+
b.edge(
|
|
217
|
+
"PY_PARAM_IN",
|
|
218
|
+
NodeRef("PyCFGNode", "id", e.src),
|
|
219
|
+
NodeRef("PyCFGNode", "id", e.dst),
|
|
220
|
+
)
|
|
221
|
+
for e in app.param_out or []:
|
|
222
|
+
b.edge(
|
|
223
|
+
"PY_PARAM_OUT",
|
|
224
|
+
NodeRef("PyCFGNode", "id", e.src),
|
|
225
|
+
NodeRef("PyCFGNode", "id", e.dst),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _sym(can_id: str) -> NodeRef:
|
|
230
|
+
return NodeRef("PySymbol", "id", can_id)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _symbol_ref(signature: str, externals: dict, sig_to_id: dict) -> NodeRef:
|
|
234
|
+
"""Resolve a call/inheritance target to the NodeRef under which it was (or
|
|
235
|
+
will be) emitted: a declared symbol by its can:// id, otherwise a
|
|
236
|
+
signature-keyed :PySymbol (external ghost)."""
|
|
237
|
+
can_id = sig_to_id.get(signature)
|
|
238
|
+
if can_id is not None:
|
|
239
|
+
return NodeRef("PySymbol", "id", can_id)
|
|
240
|
+
return NodeRef("PySymbol", "signature", signature)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _call_endpoint(
|
|
244
|
+
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict
|
|
245
|
+
) -> NodeRef:
|
|
246
|
+
"""A call-graph endpoint: a declared callable already emitted (keyed by its
|
|
247
|
+
canonical ``can://`` id, resolved through ``sig_to_id``), or an external symbol
|
|
248
|
+
(imported library / builtin member) materialized as a :PyExternal ghost.
|
|
249
|
+
|
|
250
|
+
Classification is authoritative -- it comes from ``app.external_symbols``
|
|
251
|
+
(keyed by ``can://…/@external/…`` id), not a "present in the graph" heuristic --
|
|
252
|
+
so an imported module name (which exists only as a :PyPackage) can never shadow
|
|
253
|
+
the call target. A declared endpoint resolves to its ``can://`` id (either
|
|
254
|
+
already re-identified on the edge, or resolved through ``sig_to_id``); anything
|
|
255
|
+
neither declared nor listed falls back to an id-keyed :PyExternal ghost rather
|
|
256
|
+
than raising."""
|
|
257
|
+
ext = externals.get(signature)
|
|
258
|
+
if ext is None:
|
|
259
|
+
can_id = sig_to_id.get(signature)
|
|
260
|
+
if can_id is not None:
|
|
261
|
+
ext = externals.get(can_id)
|
|
262
|
+
if ext is None:
|
|
263
|
+
return _sym(can_id)
|
|
264
|
+
elif signature.startswith("can://") and "/@external/" not in signature:
|
|
265
|
+
# An already re-identified declared endpoint (post reidentify_call_graph).
|
|
266
|
+
return _sym(signature)
|
|
267
|
+
if ext is not None:
|
|
268
|
+
return b.node(
|
|
269
|
+
["PySymbol", "PyExternal"],
|
|
270
|
+
"id",
|
|
271
|
+
ext.id or signature,
|
|
272
|
+
prune({"name": ext.name, "module": ext.module}),
|
|
273
|
+
)
|
|
274
|
+
name = signature.rsplit(".", 1)[-1] if "." in signature else signature
|
|
113
275
|
return b.node(
|
|
114
276
|
["PySymbol", "PyExternal"],
|
|
115
|
-
"
|
|
277
|
+
"id",
|
|
116
278
|
signature,
|
|
117
|
-
prune({"name": name
|
|
279
|
+
prune({"name": name}),
|
|
118
280
|
)
|
|
119
281
|
|
|
120
282
|
|
|
@@ -124,18 +286,20 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
124
286
|
|
|
125
287
|
|
|
126
288
|
def _project_module_body(
|
|
127
|
-
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule
|
|
289
|
+
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule,
|
|
290
|
+
externals: dict, sig_to_id: dict, module_id_by_key: dict,
|
|
128
291
|
) -> None:
|
|
129
292
|
for fn in (mod.functions or {}).values():
|
|
130
|
-
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn)
|
|
131
|
-
for cl in (mod.
|
|
132
|
-
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl)
|
|
293
|
+
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id)
|
|
294
|
+
for cl in (mod.types or {}).values():
|
|
295
|
+
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id)
|
|
133
296
|
for v in mod.variables or []:
|
|
134
297
|
_project_variable(b, file_key, mod_ref, file_key, v)
|
|
135
|
-
_project_imports(b, mod_ref, mod)
|
|
298
|
+
_project_imports(b, mod_ref, mod, module_id_by_key)
|
|
136
299
|
|
|
137
300
|
|
|
138
|
-
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule
|
|
301
|
+
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
|
|
302
|
+
module_id_by_key: dict) -> None:
|
|
139
303
|
# At most one PY_IMPORTS edge per (module, target) pair -- mirrors PY_CALLS,
|
|
140
304
|
# which pre-aggregates for the same reason: both writers MERGE edges on
|
|
141
305
|
# (type, from, to) and SET their props, so a second row for the same pair
|
|
@@ -166,10 +330,15 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
|
166
330
|
if im.alias:
|
|
167
331
|
a["aliases"].add(im.alias)
|
|
168
332
|
for key, a in agg.items():
|
|
169
|
-
if a["resolved"] is not None
|
|
170
|
-
|
|
171
|
-
|
|
333
|
+
resolved_id = module_id_by_key.get(a["resolved"]) if a["resolved"] is not None else None
|
|
334
|
+
if resolved_id is not None:
|
|
335
|
+
target = NodeRef("PyModule", "id", resolved_id)
|
|
336
|
+
elif a["resolved"] is None:
|
|
172
337
|
target = b.node(["PyPackage"], "name", key, {})
|
|
338
|
+
else:
|
|
339
|
+
# resolved to a file key that is not in this symbol table (partial
|
|
340
|
+
# run) — keep the module target via its legacy file_key property.
|
|
341
|
+
target = NodeRef("PyModule", "file_key", a["resolved"])
|
|
173
342
|
b.edge(
|
|
174
343
|
"PY_IMPORTS",
|
|
175
344
|
mod_ref,
|
|
@@ -190,31 +359,34 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
|
190
359
|
|
|
191
360
|
|
|
192
361
|
def _project_class(
|
|
193
|
-
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass
|
|
362
|
+
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
|
|
363
|
+
externals: dict, sig_to_id: dict,
|
|
194
364
|
) -> None:
|
|
195
365
|
ref = b.node(
|
|
196
|
-
["PySymbol", "PyClass"], "
|
|
366
|
+
["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key)
|
|
197
367
|
)
|
|
198
368
|
b.edge(parent_rel, parent, ref)
|
|
199
369
|
|
|
200
370
|
for base in cl.base_classes or []:
|
|
201
|
-
|
|
371
|
+
if base:
|
|
372
|
+
b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
|
|
202
373
|
|
|
203
|
-
for m in (cl.
|
|
204
|
-
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m)
|
|
374
|
+
for m in (cl.callables or {}).values():
|
|
375
|
+
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id)
|
|
205
376
|
for a in (cl.attributes or {}).values():
|
|
206
377
|
_project_attribute(b, file_key, ref, cl.signature, a)
|
|
207
|
-
for ic in (cl.
|
|
208
|
-
_project_class(b, file_key, ref, "PY_DECLARES", ic)
|
|
378
|
+
for ic in (cl.types or {}).values():
|
|
379
|
+
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
|
|
209
380
|
|
|
210
381
|
|
|
211
382
|
def _project_callable(
|
|
212
|
-
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable
|
|
383
|
+
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
|
|
384
|
+
externals: dict, sig_to_id: dict,
|
|
213
385
|
) -> None:
|
|
214
386
|
ref = b.node(
|
|
215
387
|
["PySymbol", "PyCallable"],
|
|
216
|
-
"
|
|
217
|
-
c.
|
|
388
|
+
"id",
|
|
389
|
+
c.id,
|
|
218
390
|
_callable_props(c, file_key),
|
|
219
391
|
)
|
|
220
392
|
b.edge(owner_rel, owner, ref)
|
|
@@ -230,14 +402,17 @@ def _project_callable(
|
|
|
230
402
|
cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
|
|
231
403
|
b.edge("PY_HAS_CALLSITE", ref, cs)
|
|
232
404
|
if s.callee_signature:
|
|
233
|
-
b.edge_to_symbol(
|
|
405
|
+
b.edge_to_symbol(
|
|
406
|
+
"PY_RESOLVES_TO", cs,
|
|
407
|
+
_symbol_ref(s.callee_signature, externals, sig_to_id),
|
|
408
|
+
)
|
|
234
409
|
|
|
235
410
|
for v in c.local_variables or []:
|
|
236
411
|
_project_variable(b, file_key, ref, c.signature, v)
|
|
237
|
-
for ic in (c.
|
|
238
|
-
_project_callable(b, file_key, ref, "PY_DECLARES", ic)
|
|
239
|
-
for cl in (c.
|
|
240
|
-
_project_class(b, file_key, ref, "PY_DECLARES", cl)
|
|
412
|
+
for ic in (c.callables or {}).values():
|
|
413
|
+
_project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
|
|
414
|
+
for cl in (c.types or {}).values():
|
|
415
|
+
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id)
|
|
241
416
|
|
|
242
417
|
|
|
243
418
|
def _project_attribute(
|
|
@@ -273,6 +448,8 @@ def _project_decorator(b: RowBuilder, on: NodeRef, decorator: str) -> None:
|
|
|
273
448
|
def _module_props(mod: PyModule, file_key: str) -> Props:
|
|
274
449
|
return prune(
|
|
275
450
|
{
|
|
451
|
+
"id": mod.id,
|
|
452
|
+
"file_key": file_key,
|
|
276
453
|
"module_name": mod.module_name,
|
|
277
454
|
"content_hash": mod.content_hash,
|
|
278
455
|
"last_modified": mod.last_modified,
|
|
@@ -285,8 +462,10 @@ def _module_props(mod: PyModule, file_key: str) -> Props:
|
|
|
285
462
|
def _class_props(cl: PyClass, file_key: str) -> Props:
|
|
286
463
|
return prune(
|
|
287
464
|
{
|
|
465
|
+
"id": cl.id,
|
|
466
|
+
"signature": cl.signature,
|
|
288
467
|
"name": cl.name,
|
|
289
|
-
"code": cl
|
|
468
|
+
"code": getattr(cl, "code", None),
|
|
290
469
|
"base_classes": list(cl.base_classes or []),
|
|
291
470
|
"docstring": _docstring_of(cl.comments),
|
|
292
471
|
"start_line": cl.start_line,
|
|
@@ -299,11 +478,13 @@ def _class_props(cl: PyClass, file_key: str) -> Props:
|
|
|
299
478
|
def _callable_props(c: PyCallable, file_key: str) -> Props:
|
|
300
479
|
return prune(
|
|
301
480
|
{
|
|
481
|
+
"id": c.id,
|
|
482
|
+
"signature": c.signature,
|
|
302
483
|
"name": c.name,
|
|
303
484
|
"path": c.path,
|
|
304
485
|
"return_type": c.return_type,
|
|
305
486
|
"cyclomatic_complexity": c.cyclomatic_complexity,
|
|
306
|
-
"code": c
|
|
487
|
+
"code": getattr(c, "code", None),
|
|
307
488
|
"code_start_line": c.code_start_line,
|
|
308
489
|
"start_line": c.start_line,
|
|
309
490
|
"end_line": c.end_line,
|
|
@@ -368,8 +549,8 @@ def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
|
368
549
|
)
|
|
369
550
|
|
|
370
551
|
|
|
371
|
-
def _call_edge_props(weight: int,
|
|
372
|
-
return prune({"weight": weight, "
|
|
552
|
+
def _call_edge_props(weight: int, prov: List[str]) -> Props:
|
|
553
|
+
return prune({"weight": weight, "prov": list(prov)})
|
|
373
554
|
|
|
374
555
|
|
|
375
556
|
def _docstring_of(comments: Optional[List[PyComment]]) -> Optional[str]:
|
codeanalyzer/neo4j/rows.py
CHANGED
|
@@ -58,6 +58,12 @@ class EdgeRow:
|
|
|
58
58
|
from_ref: NodeRef
|
|
59
59
|
to_ref: NodeRef
|
|
60
60
|
props: Props
|
|
61
|
+
# Optional relationship discriminant: when set, the MERGE is on
|
|
62
|
+
# ``{_k: key}`` so several legitimately-distinct edges of one type may
|
|
63
|
+
# coexist between the same endpoint pair (e.g. per-variable PY_DDG edges,
|
|
64
|
+
# or the true/false PY_CFG_NEXT pair of a conditional). ``None`` keeps the
|
|
65
|
+
# plain endpoint-pair MERGE.
|
|
66
|
+
key: Optional[str] = None
|
|
61
67
|
|
|
62
68
|
|
|
63
69
|
@dataclass
|
|
@@ -105,25 +111,22 @@ class RowBuilder:
|
|
|
105
111
|
self._keys.add((labels[0], value))
|
|
106
112
|
return NodeRef(labels[0], key_prop, value)
|
|
107
113
|
|
|
108
|
-
def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None
|
|
109
|
-
|
|
110
|
-
|
|
114
|
+
def edge(self, type_: str, from_ref: NodeRef, to_ref: NodeRef, props: Optional[Props] = None,
|
|
115
|
+
key: Optional[str] = None) -> None:
|
|
116
|
+
"""An edge whose endpoints are known to exist (both ends emitted this run).
|
|
117
|
+
``key`` sets the relationship discriminant (see :class:`EdgeRow`)."""
|
|
118
|
+
self._edges.append(EdgeRow(type_, from_ref, to_ref, dict(props or {}), key))
|
|
111
119
|
|
|
112
120
|
def edge_to_symbol(
|
|
113
|
-
self, type_: str, from_ref: NodeRef,
|
|
121
|
+
self, type_: str, from_ref: NodeRef, target_ref: NodeRef, props: Optional[Props] = None
|
|
114
122
|
) -> None:
|
|
115
123
|
"""An edge to a ``:PySymbol`` target that may be external/library code not
|
|
116
|
-
present in the graph.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
from_ref,
|
|
123
|
-
NodeRef("PySymbol", "signature", target_signature),
|
|
124
|
-
dict(props or {}),
|
|
125
|
-
)
|
|
126
|
-
)
|
|
124
|
+
present in the graph. The target is an already-resolved :class:`NodeRef`
|
|
125
|
+
(a declared symbol by its can:// id, or a signature-keyed external ghost).
|
|
126
|
+
Deferred and kept only if that ``(label, value)`` was actually emitted as a
|
|
127
|
+
node — so PY_EXTENDS / PY_RESOLVES_TO never dangle (the string fallback lives
|
|
128
|
+
on the source node's props)."""
|
|
129
|
+
self._deferred.append(EdgeRow(type_, from_ref, target_ref, dict(props or {})))
|
|
127
130
|
|
|
128
131
|
def has_key(self, label: str, value: str) -> bool:
|
|
129
132
|
"""Whether a node with this ``(merge_label, value)`` identity was emitted."""
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -35,7 +35,7 @@ from __future__ import annotations
|
|
|
35
35
|
from dataclasses import dataclass, field
|
|
36
36
|
from typing import Dict, List
|
|
37
37
|
|
|
38
|
-
SCHEMA_VERSION = "
|
|
38
|
+
SCHEMA_VERSION = "2.0.0"
|
|
39
39
|
|
|
40
40
|
# PropType ∈ {"string", "integer", "float", "boolean", "string[]", "integer[]"}.
|
|
41
41
|
|
|
@@ -80,8 +80,9 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
80
80
|
NodeLabel(
|
|
81
81
|
"PyModule",
|
|
82
82
|
"PyModule",
|
|
83
|
-
"
|
|
83
|
+
"id",
|
|
84
84
|
{
|
|
85
|
+
"id": "string",
|
|
85
86
|
"file_key": "string",
|
|
86
87
|
"module_name": "string",
|
|
87
88
|
"content_hash": "string",
|
|
@@ -93,8 +94,9 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
93
94
|
NodeLabel(
|
|
94
95
|
"PyClass",
|
|
95
96
|
"PySymbol",
|
|
96
|
-
"
|
|
97
|
+
"id",
|
|
97
98
|
{
|
|
99
|
+
"id": "string",
|
|
98
100
|
"signature": "string",
|
|
99
101
|
"name": "string",
|
|
100
102
|
"code": "string",
|
|
@@ -107,8 +109,9 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
107
109
|
NodeLabel(
|
|
108
110
|
"PyCallable",
|
|
109
111
|
"PySymbol",
|
|
110
|
-
"
|
|
112
|
+
"id",
|
|
111
113
|
{
|
|
114
|
+
"id": "string",
|
|
112
115
|
"signature": "string",
|
|
113
116
|
"name": "string",
|
|
114
117
|
"path": "string",
|
|
@@ -127,8 +130,8 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
127
130
|
NodeLabel(
|
|
128
131
|
"PyExternal",
|
|
129
132
|
"PySymbol",
|
|
130
|
-
"
|
|
131
|
-
{"
|
|
133
|
+
"id",
|
|
134
|
+
{"id": "string", "name": "string", "module": "string"},
|
|
132
135
|
),
|
|
133
136
|
NodeLabel("PyPackage", "PyPackage", "name", {"name": "string"}),
|
|
134
137
|
NodeLabel(
|
|
@@ -186,6 +189,25 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
186
189
|
"_module": "string",
|
|
187
190
|
},
|
|
188
191
|
),
|
|
192
|
+
# Level-3 CPG overlay (present only at -a 3). The dataflow vocabulary is
|
|
193
|
+
# shared cross-language in *shape* (same suffixes, props, semantics) but
|
|
194
|
+
# namespaced per language like every other row family — a multi-language
|
|
195
|
+
# Neo4j database must never mingle one analyzer's dependence edges with
|
|
196
|
+
# another's. `id` = "<signature>#<node_id>"; parameter-passing nodes
|
|
197
|
+
# (formal/actual in/out) ride the same label with `var`/`call_node`.
|
|
198
|
+
NodeLabel(
|
|
199
|
+
"PyCFGNode",
|
|
200
|
+
"PyCFGNode",
|
|
201
|
+
"id",
|
|
202
|
+
{
|
|
203
|
+
"id": "string",
|
|
204
|
+
"kind": "string",
|
|
205
|
+
"var": "string",
|
|
206
|
+
"call_node": "string",
|
|
207
|
+
**_SPAN,
|
|
208
|
+
"_module": "string",
|
|
209
|
+
},
|
|
210
|
+
),
|
|
189
211
|
]
|
|
190
212
|
|
|
191
213
|
_DECL_TARGETS = ["PyClass", "PyCallable"]
|
|
@@ -203,7 +225,7 @@ REL_TYPES: List[RelType] = [
|
|
|
203
225
|
"PY_CALLS",
|
|
204
226
|
["PyCallable", "PyExternal"],
|
|
205
227
|
["PyCallable", "PyExternal"],
|
|
206
|
-
{"weight": "integer", "
|
|
228
|
+
{"weight": "integer", "prov": "string[]"},
|
|
207
229
|
),
|
|
208
230
|
RelType("PY_EXTENDS", ["PyClass"], ["PyClass"]),
|
|
209
231
|
RelType(
|
|
@@ -213,6 +235,20 @@ REL_TYPES: List[RelType] = [
|
|
|
213
235
|
{"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
|
|
214
236
|
),
|
|
215
237
|
RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
|
|
238
|
+
# Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
|
|
239
|
+
# PY_-namespaced so per-language SDK backends can scope their queries.
|
|
240
|
+
RelType("PY_HAS_CFG_NODE", ["PyCallable"], ["PyCFGNode"]),
|
|
241
|
+
# ``_k`` is the relationship-identity discriminant (internal, underscore-
|
|
242
|
+
# prefixed like ``_module``): PY_CFG_NEXT merges per ``kind`` (a conditional's
|
|
243
|
+
# true/false pair), PY_DDG per ``(var, prov)`` (one dependence per variable,
|
|
244
|
+
# and the ssa/points-to split) — a plain endpoint-pair MERGE would collapse
|
|
245
|
+
# legitimately-distinct edges.
|
|
246
|
+
RelType("PY_CFG_NEXT", ["PyCFGNode"], ["PyCFGNode"], {"kind": "string", "_k": "string"}),
|
|
247
|
+
RelType("PY_CDG", ["PyCFGNode"], ["PyCFGNode"]),
|
|
248
|
+
RelType("PY_DDG", ["PyCFGNode"], ["PyCFGNode"], {"var": "string", "prov": "string[]", "_k": "string"}),
|
|
249
|
+
RelType("PY_PARAM_IN", ["PyCFGNode"], ["PyCFGNode"], {"var": "string"}),
|
|
250
|
+
RelType("PY_PARAM_OUT", ["PyCFGNode"], ["PyCFGNode"], {"var": "string"}),
|
|
251
|
+
RelType("PY_SUMMARY", ["PyCFGNode"], ["PyCFGNode"]),
|
|
216
252
|
]
|
|
217
253
|
|
|
218
254
|
|
codeanalyzer/options/options.py
CHANGED
|
@@ -49,6 +49,10 @@ class AnalysisOptions:
|
|
|
49
49
|
neo4j_password: str = "neo4j"
|
|
50
50
|
neo4j_database: Optional[str] = None
|
|
51
51
|
analysis_level: int = 1
|
|
52
|
+
# Level-3 dataflow knobs: which program graphs to emit (csv of
|
|
53
|
+
# cfg|dfg|pdg|sdg) and the access-path k-limit.
|
|
54
|
+
graphs: str = "cfg,dfg,pdg,sdg"
|
|
55
|
+
graph_field_depth: int = 3
|
|
52
56
|
using_ray: bool = False
|
|
53
57
|
rebuild_analysis: bool = False
|
|
54
58
|
skip_tests: bool = True
|
codeanalyzer/schema/__init__.py
CHANGED
|
@@ -2,6 +2,12 @@ from importlib.metadata import version, PackageNotFoundError
|
|
|
2
2
|
from packaging.version import parse as parse_version
|
|
3
3
|
|
|
4
4
|
from .py_schema import (
|
|
5
|
+
Analysis,
|
|
6
|
+
BodyNode,
|
|
7
|
+
CdgEdge,
|
|
8
|
+
CfgEdge,
|
|
9
|
+
DdgEdge,
|
|
10
|
+
ParamEdge,
|
|
5
11
|
PyApplication,
|
|
6
12
|
PyCallable,
|
|
7
13
|
PyCallableParameter,
|
|
@@ -12,9 +18,12 @@ from .py_schema import (
|
|
|
12
18
|
PyImport,
|
|
13
19
|
PyModule,
|
|
14
20
|
PyVariableDeclaration,
|
|
21
|
+
Span,
|
|
22
|
+
SummaryEdge,
|
|
15
23
|
)
|
|
16
24
|
|
|
17
25
|
__all__ = [
|
|
26
|
+
"Analysis",
|
|
18
27
|
"PyApplication",
|
|
19
28
|
"PyExternalSymbol",
|
|
20
29
|
"PyImport",
|
|
@@ -25,6 +34,13 @@ __all__ = [
|
|
|
25
34
|
"PyCallable",
|
|
26
35
|
"PyClassAttribute",
|
|
27
36
|
"PyCallableParameter",
|
|
37
|
+
"Span",
|
|
38
|
+
"BodyNode",
|
|
39
|
+
"CfgEdge",
|
|
40
|
+
"CdgEdge",
|
|
41
|
+
"DdgEdge",
|
|
42
|
+
"SummaryEdge",
|
|
43
|
+
"ParamEdge",
|
|
28
44
|
]
|
|
29
45
|
|
|
30
46
|
try:
|
|
@@ -44,6 +60,7 @@ if not PYDANTIC_V2:
|
|
|
44
60
|
PyClass=PyClass,
|
|
45
61
|
PyModule=PyModule
|
|
46
62
|
)
|
|
63
|
+
Analysis.update_forward_refs(PyApplication=PyApplication)
|
|
47
64
|
|
|
48
65
|
# Compatibility helpers for Pydantic v1/v2
|
|
49
66
|
def model_dump_json(model, **kwargs):
|
|
@@ -55,6 +72,8 @@ def model_dump_json(model, **kwargs):
|
|
|
55
72
|
v1_kwargs = {}
|
|
56
73
|
if 'indent' in kwargs:
|
|
57
74
|
v1_kwargs['indent'] = kwargs['indent']
|
|
75
|
+
if 'exclude_none' in kwargs:
|
|
76
|
+
v1_kwargs['exclude_none'] = kwargs['exclude_none']
|
|
58
77
|
if 'separators' in kwargs:
|
|
59
78
|
# In v1, separators is passed to dumps_kwargs
|
|
60
79
|
v1_kwargs['separators'] = kwargs['separators']
|