codeanalyzer-python 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +95 -123
- codeanalyzer/core.py +21 -45
- codeanalyzer/dataflow/access_paths.py +26 -4
- codeanalyzer/dataflow/builder.py +65 -1
- codeanalyzer/dataflow/identity.py +1 -1
- codeanalyzer/dataflow/pdg.py +7 -2
- codeanalyzer/dataflow/scc.py +1 -1
- codeanalyzer/entrypoints/__init__.py +3 -0
- codeanalyzer/entrypoints/detect.py +124 -0
- codeanalyzer/entrypoints/matching.py +182 -0
- codeanalyzer/entrypoints/pipeline.py +131 -0
- codeanalyzer/entrypoints/rules.py +159 -0
- codeanalyzer/entrypoints/rules.yml +88 -0
- codeanalyzer/neo4j/bolt.py +1 -1
- codeanalyzer/neo4j/project.py +85 -60
- codeanalyzer/neo4j/schema.py +35 -34
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +2 -26
- codeanalyzer/schema/__init__.py +48 -0
- codeanalyzer/schema/l1_body.py +11 -1
- codeanalyzer/schema/l2_callees.py +29 -13
- codeanalyzer/schema/py_schema.py +95 -103
- codeanalyzer/semantic_analysis/call_graph.py +20 -4
- codeanalyzer/semantic_analysis/defuse_linker.py +1499 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +88 -3
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/METADATA +36 -161
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/RECORD +31 -30
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/WHEEL +1 -1
- codeanalyzer/config/__init__.py +0 -3
- codeanalyzer/config/config.py +0 -8
- codeanalyzer/semantic_analysis/pycg/__init__.py +0 -20
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -1115
- codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -23
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -401
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -47,7 +47,7 @@ from codeanalyzer.schema import (
|
|
|
47
47
|
PyModule,
|
|
48
48
|
PyVariableDeclaration,
|
|
49
49
|
)
|
|
50
|
-
from codeanalyzer.schema.py_schema import
|
|
50
|
+
from codeanalyzer.schema.py_schema import PyDecorator
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
@@ -98,7 +98,7 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
98
98
|
|
|
99
99
|
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
100
100
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
101
|
-
_project_program_graphs(b, app)
|
|
101
|
+
_project_program_graphs(b, app, externals, sig_to_id)
|
|
102
102
|
|
|
103
103
|
return b.finish()
|
|
104
104
|
|
|
@@ -109,7 +109,7 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
112
|
-
"""The globally-unique
|
|
112
|
+
"""The globally-unique PyBodyNode merge key for a callable's body node: the
|
|
113
113
|
callable's ``can://`` id joined to its LOCAL body key with a single ``@``.
|
|
114
114
|
The synthetic bookends already carry the leading ``@`` (``"@entry"``/
|
|
115
115
|
``"@exit"``); real statements are bare ``"line:col"`` and gain the ``@``.
|
|
@@ -124,18 +124,20 @@ def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
|
124
124
|
)
|
|
125
125
|
|
|
126
126
|
|
|
127
|
-
def
|
|
128
|
-
return NodeRef("
|
|
127
|
+
def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
128
|
+
return NodeRef("PyBodyNode", "id", _global_ordinal(callable_id, local_key))
|
|
129
129
|
|
|
130
130
|
|
|
131
|
-
def _project_program_graphs(
|
|
131
|
+
def _project_program_graphs(
|
|
132
|
+
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict
|
|
133
|
+
) -> None:
|
|
132
134
|
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
133
135
|
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
134
136
|
|
|
135
|
-
Node label ``
|
|
137
|
+
Node label ``PyBodyNode`` (merge key ``id`` = the GLOBAL ordinal
|
|
136
138
|
``<callable can:// id>@<local body key>`` — identical to the JSON body key
|
|
137
139
|
prefixed with the callable id, so the two projections agree). Edges:
|
|
138
|
-
``
|
|
140
|
+
``PY_HAS_BODY_NODE`` from the owning callable, ``PY_CFG_NEXT`` (prop ``kind``)
|
|
139
141
|
over the CFG, ``PY_CDG`` over control dependence, and ``PY_DDG`` (props
|
|
140
142
|
``var``/``prov``) over data dependence. The vocabulary is cross-language in
|
|
141
143
|
shape but PY_-namespaced like every other row family, so a multi-language
|
|
@@ -149,7 +151,7 @@ def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
|
149
151
|
each callable's transitive pass-throughs (LOCAL ids → global refs), and the
|
|
150
152
|
app-level ``PY_PARAM_IN``/``PY_PARAM_OUT`` edges connect actual↔formal
|
|
151
153
|
vertices across callables (endpoints are already GLOBAL ordinals matching the
|
|
152
|
-
emitted ``
|
|
154
|
+
emitted ``PyBodyNode`` keys). All idempotent under MERGE — no-ops below L4."""
|
|
153
155
|
from codeanalyzer.semantic_analysis.call_graph import _walk_module_callables
|
|
154
156
|
|
|
155
157
|
for file_key, mod in app.symbol_table.items():
|
|
@@ -163,7 +165,7 @@ def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
|
163
165
|
# their owning callsite (``parent``) instead of span lines; both
|
|
164
166
|
# are None on ordinary statement nodes and pruned away there.
|
|
165
167
|
ref = b.node(
|
|
166
|
-
["
|
|
168
|
+
["PyBodyNode"],
|
|
167
169
|
"id",
|
|
168
170
|
_global_ordinal(c.id, local_key),
|
|
169
171
|
prune(
|
|
@@ -173,23 +175,43 @@ def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
|
173
175
|
"end_line": span.end[0] if span else None,
|
|
174
176
|
"var": node.of,
|
|
175
177
|
"call_node": node.parent,
|
|
178
|
+
# Call-site detail (#120). The JSON emits one node per
|
|
179
|
+
# call site; the graph now does too, instead of a
|
|
180
|
+
# separate :PyCallSite under a third id scheme.
|
|
181
|
+
"method_name": node.method_name,
|
|
182
|
+
"receiver_expr": node.receiver_expr,
|
|
183
|
+
"receiver_type": node.receiver_type,
|
|
184
|
+
"return_type": node.return_type,
|
|
185
|
+
"is_constructor_call": node.is_constructor_call,
|
|
186
|
+
"arguments_json": _stringify_if(node.arguments),
|
|
176
187
|
"_module": file_key,
|
|
177
188
|
}
|
|
178
189
|
),
|
|
179
190
|
)
|
|
180
|
-
b.edge("
|
|
191
|
+
b.edge("PY_HAS_BODY_NODE", owner, ref)
|
|
192
|
+
if node.kind == "call" and node.callee:
|
|
193
|
+
# `callee` is ALREADY a resolved can:// id (a declared callable
|
|
194
|
+
# or an @external home), so it must not go through
|
|
195
|
+
# `_symbol_ref`, which expects a dotted signature and would
|
|
196
|
+
# fall back to matching a `signature` property against an id --
|
|
197
|
+
# emitting an edge that matches nothing at load time.
|
|
198
|
+
b.edge(
|
|
199
|
+
"PY_RESOLVES_TO",
|
|
200
|
+
ref,
|
|
201
|
+
_call_endpoint(b, node.callee, externals, sig_to_id),
|
|
202
|
+
)
|
|
181
203
|
for e in c.cfg or []:
|
|
182
204
|
# kind-discriminated: a conditional's true/false pair between one
|
|
183
205
|
# endpoint pair must stay two relationships, not one MERGE.
|
|
184
206
|
b.edge(
|
|
185
207
|
"PY_CFG_NEXT",
|
|
186
|
-
|
|
187
|
-
|
|
208
|
+
_body_ref(c.id, e.src),
|
|
209
|
+
_body_ref(c.id, e.dst),
|
|
188
210
|
{"kind": e.kind},
|
|
189
211
|
key=e.kind,
|
|
190
212
|
)
|
|
191
213
|
for e in c.cdg or []:
|
|
192
|
-
b.edge("PY_CDG",
|
|
214
|
+
b.edge("PY_CDG", _body_ref(c.id, e.src), _body_ref(c.id, e.dst))
|
|
193
215
|
for e in c.ddg or []:
|
|
194
216
|
# (var, prov)-discriminated: the DDG legitimately carries several
|
|
195
217
|
# edges between one statement pair (one per variable, and the
|
|
@@ -197,32 +219,32 @@ def _project_program_graphs(b: RowBuilder, app: PyApplication) -> None:
|
|
|
197
219
|
# them and silently drops dependences.
|
|
198
220
|
b.edge(
|
|
199
221
|
"PY_DDG",
|
|
200
|
-
|
|
201
|
-
|
|
222
|
+
_body_ref(c.id, e.src),
|
|
223
|
+
_body_ref(c.id, e.dst),
|
|
202
224
|
prune({"var": e.var, "prov": list(e.prov) if e.prov else None}),
|
|
203
225
|
key=f"{e.var or ''}|{','.join(e.prov or [])}",
|
|
204
226
|
)
|
|
205
227
|
# L4 intraprocedural summaries (transitive actual_in → actual_out
|
|
206
|
-
# pass-throughs); LOCAL ids resolved to global
|
|
228
|
+
# pass-throughs); LOCAL ids resolved to global PyBodyNode refs.
|
|
207
229
|
for e in c.summary or []:
|
|
208
|
-
b.edge("PY_SUMMARY",
|
|
230
|
+
b.edge("PY_SUMMARY", _body_ref(c.id, e.src), _body_ref(c.id, e.dst))
|
|
209
231
|
|
|
210
232
|
# L4 interprocedural parameter passing, emitted once at the app scope. The
|
|
211
233
|
# endpoints are ALREADY global ordinals (emit_l4 resolved them through the
|
|
212
|
-
# endpoint functions' identity maps), so they land on the very
|
|
234
|
+
# endpoint functions' identity maps), so they land on the very PyBodyNode ids
|
|
213
235
|
# projected above — a formal_in global id equals _global_ordinal(callee.id,
|
|
214
236
|
# "@formal_in:0"). No dangling references.
|
|
215
237
|
for e in app.param_in or []:
|
|
216
238
|
b.edge(
|
|
217
239
|
"PY_PARAM_IN",
|
|
218
|
-
NodeRef("
|
|
219
|
-
NodeRef("
|
|
240
|
+
NodeRef("PyBodyNode", "id", e.src),
|
|
241
|
+
NodeRef("PyBodyNode", "id", e.dst),
|
|
220
242
|
)
|
|
221
243
|
for e in app.param_out or []:
|
|
222
244
|
b.edge(
|
|
223
245
|
"PY_PARAM_OUT",
|
|
224
|
-
NodeRef("
|
|
225
|
-
NodeRef("
|
|
246
|
+
NodeRef("PyBodyNode", "id", e.src),
|
|
247
|
+
NodeRef("PyBodyNode", "id", e.dst),
|
|
226
248
|
)
|
|
227
249
|
|
|
228
250
|
|
|
@@ -369,6 +391,9 @@ def _project_class(
|
|
|
369
391
|
)
|
|
370
392
|
b.edge(parent_rel, parent, ref)
|
|
371
393
|
|
|
394
|
+
for d in cl.decorators or []:
|
|
395
|
+
_project_decorator(b, ref, d)
|
|
396
|
+
|
|
372
397
|
for base in cl.base_classes or []:
|
|
373
398
|
if base:
|
|
374
399
|
b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
|
|
@@ -397,19 +422,6 @@ def _project_callable(
|
|
|
397
422
|
for d in c.decorators or []:
|
|
398
423
|
_project_decorator(b, ref, d)
|
|
399
424
|
|
|
400
|
-
for s in c.call_sites or []:
|
|
401
|
-
# Key off the relative file (a call site lives in its callable's file) so ids stay portable.
|
|
402
|
-
cs_id = (
|
|
403
|
-
f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
|
|
404
|
-
)
|
|
405
|
-
cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
|
|
406
|
-
b.edge("PY_HAS_CALLSITE", ref, cs)
|
|
407
|
-
if s.callee_signature:
|
|
408
|
-
b.edge_to_symbol(
|
|
409
|
-
"PY_RESOLVES_TO", cs,
|
|
410
|
-
_symbol_ref(s.callee_signature, externals, sig_to_id),
|
|
411
|
-
)
|
|
412
|
-
|
|
413
425
|
for v in c.local_variables or []:
|
|
414
426
|
_project_variable(b, file_key, ref, c.signature, v)
|
|
415
427
|
for ic in (c.callables or {}).values():
|
|
@@ -439,9 +451,36 @@ def _project_variable(
|
|
|
439
451
|
b.edge("PY_DECLARES_VAR", owner, ref)
|
|
440
452
|
|
|
441
453
|
|
|
442
|
-
def _project_decorator(b: RowBuilder, on: NodeRef, decorator:
|
|
443
|
-
|
|
444
|
-
|
|
454
|
+
def _project_decorator(b: RowBuilder, on: NodeRef, decorator: PyDecorator) -> None:
|
|
455
|
+
"""Project one decorator application (#128).
|
|
456
|
+
|
|
457
|
+
The merge key is the resolved ``qualified_name`` when Jedi supplies one, so
|
|
458
|
+
``@lru_cache`` and ``@lru_cache(maxsize=128)`` land on one node instead of two,
|
|
459
|
+
and two spellings of one decorator stop being separate nodes. Unresolved
|
|
460
|
+
decorators fall back to the written spelling. Per-application facts (the
|
|
461
|
+
arguments) ride on the relationship, not the shared node -- ``:PyDecorator``
|
|
462
|
+
has no ``_module`` and is never pruned, so anything application-specific on it
|
|
463
|
+
would accumulate across every project in the database.
|
|
464
|
+
"""
|
|
465
|
+
key = decorator.qualified_name or decorator.name
|
|
466
|
+
dec = b.node(
|
|
467
|
+
["PyDecorator"],
|
|
468
|
+
"name",
|
|
469
|
+
key,
|
|
470
|
+
{"name": key, "qualified_name": decorator.qualified_name or ""},
|
|
471
|
+
)
|
|
472
|
+
b.edge(
|
|
473
|
+
"PY_DECORATED_BY",
|
|
474
|
+
on,
|
|
475
|
+
dec,
|
|
476
|
+
{
|
|
477
|
+
"expression": decorator.expression or "",
|
|
478
|
+
"positional_arguments": list(decorator.positional_arguments or []),
|
|
479
|
+
"keyword_arguments_json": json.dumps(
|
|
480
|
+
dict(decorator.keyword_arguments or {}), sort_keys=True
|
|
481
|
+
),
|
|
482
|
+
},
|
|
483
|
+
)
|
|
445
484
|
|
|
446
485
|
|
|
447
486
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -482,10 +521,13 @@ def _class_props(cl: PyClass, file_key: str, source: str) -> Props:
|
|
|
482
521
|
"name": cl.name,
|
|
483
522
|
"code": _span_code(source, cl.span),
|
|
484
523
|
"base_classes": list(cl.base_classes or []),
|
|
524
|
+
"decorators": [d.qualified_name or d.name for d in (cl.decorators or [])],
|
|
485
525
|
"docstring": _docstring_of(cl.comments),
|
|
486
526
|
"start_line": cl.start_line,
|
|
487
527
|
"end_line": cl.end_line,
|
|
488
528
|
"_module": file_key,
|
|
529
|
+
"is_entrypoint": bool(cl.entrypoints),
|
|
530
|
+
"entrypoint_frameworks": sorted({e.framework for e in (cl.entrypoints or [])}),
|
|
489
531
|
}
|
|
490
532
|
)
|
|
491
533
|
|
|
@@ -504,10 +546,13 @@ def _callable_props(c: PyCallable, file_key: str, source: str) -> Props:
|
|
|
504
546
|
"start_line": c.start_line,
|
|
505
547
|
"end_line": c.end_line,
|
|
506
548
|
"docstring": _docstring_of(c.comments),
|
|
507
|
-
"decorators":
|
|
549
|
+
"decorators": [d.qualified_name or d.name for d in (c.decorators or [])],
|
|
550
|
+
"modifiers": list(c.modifiers or []),
|
|
508
551
|
"parameters_json": _stringify_if(c.parameters),
|
|
509
552
|
"accessed_symbols_json": _stringify_if(c.accessed_symbols),
|
|
510
553
|
"_module": file_key,
|
|
554
|
+
"is_entrypoint": bool(c.entrypoints),
|
|
555
|
+
"entrypoint_frameworks": sorted({e.framework for e in (c.entrypoints or [])}),
|
|
511
556
|
}
|
|
512
557
|
)
|
|
513
558
|
|
|
@@ -542,26 +587,6 @@ def _variable_props(v: PyVariableDeclaration, var_id: str, file_key: str) -> Pro
|
|
|
542
587
|
)
|
|
543
588
|
|
|
544
589
|
|
|
545
|
-
def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
546
|
-
cs_id = f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
|
|
547
|
-
return prune(
|
|
548
|
-
{
|
|
549
|
-
"id": cs_id,
|
|
550
|
-
"method_name": s.method_name,
|
|
551
|
-
"receiver_expr": s.receiver_expr,
|
|
552
|
-
"receiver_type": s.receiver_type,
|
|
553
|
-
"argument_types": list(s.argument_types or []),
|
|
554
|
-
"arguments_json": _stringify_if(s.arguments),
|
|
555
|
-
"return_type": s.return_type,
|
|
556
|
-
"callee_signature": s.callee_signature,
|
|
557
|
-
"is_constructor_call": s.is_constructor_call,
|
|
558
|
-
"start_line": s.start_line,
|
|
559
|
-
"start_column": s.start_column,
|
|
560
|
-
"end_line": s.end_line,
|
|
561
|
-
"end_column": s.end_column,
|
|
562
|
-
"_module": file_key,
|
|
563
|
-
}
|
|
564
|
-
)
|
|
565
590
|
|
|
566
591
|
|
|
567
592
|
def _call_edge_props(weight: int, prov: List[str]) -> Props:
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -101,9 +101,12 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
101
101
|
"name": "string",
|
|
102
102
|
"code": "string",
|
|
103
103
|
"base_classes": "string[]",
|
|
104
|
+
"decorators": "string[]",
|
|
104
105
|
"docstring": "string",
|
|
105
106
|
**_SPAN,
|
|
106
107
|
"_module": "string",
|
|
108
|
+
"is_entrypoint": "boolean",
|
|
109
|
+
"entrypoint_frameworks": "string[]",
|
|
107
110
|
},
|
|
108
111
|
),
|
|
109
112
|
NodeLabel(
|
|
@@ -122,9 +125,12 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
122
125
|
**_SPAN,
|
|
123
126
|
"docstring": "string",
|
|
124
127
|
"decorators": "string[]",
|
|
128
|
+
"modifiers": "string[]",
|
|
125
129
|
"parameters_json": "string",
|
|
126
130
|
"accessed_symbols_json": "string",
|
|
127
131
|
"_module": "string",
|
|
132
|
+
"is_entrypoint": "boolean",
|
|
133
|
+
"entrypoint_frameworks": "string[]",
|
|
128
134
|
},
|
|
129
135
|
),
|
|
130
136
|
NodeLabel(
|
|
@@ -138,28 +144,7 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
138
144
|
"PyDecorator",
|
|
139
145
|
"PyDecorator",
|
|
140
146
|
"name",
|
|
141
|
-
{"name": "string"},
|
|
142
|
-
),
|
|
143
|
-
NodeLabel(
|
|
144
|
-
"PyCallSite",
|
|
145
|
-
"PyCallSite",
|
|
146
|
-
"id",
|
|
147
|
-
{
|
|
148
|
-
"id": "string",
|
|
149
|
-
"method_name": "string",
|
|
150
|
-
"receiver_expr": "string",
|
|
151
|
-
"receiver_type": "string",
|
|
152
|
-
"argument_types": "string[]",
|
|
153
|
-
"arguments_json": "string",
|
|
154
|
-
"return_type": "string",
|
|
155
|
-
"callee_signature": "string",
|
|
156
|
-
"is_constructor_call": "boolean",
|
|
157
|
-
"start_line": "integer",
|
|
158
|
-
"start_column": "integer",
|
|
159
|
-
"end_line": "integer",
|
|
160
|
-
"end_column": "integer",
|
|
161
|
-
"_module": "string",
|
|
162
|
-
},
|
|
147
|
+
{"name": "string", "qualified_name": "string"},
|
|
163
148
|
),
|
|
164
149
|
NodeLabel(
|
|
165
150
|
"PyAttribute",
|
|
@@ -196,14 +181,22 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
196
181
|
# another's. `id` = "<signature>#<node_id>"; parameter-passing nodes
|
|
197
182
|
# (formal/actual in/out) ride the same label with `var`/`call_node`.
|
|
198
183
|
NodeLabel(
|
|
199
|
-
"
|
|
200
|
-
"
|
|
184
|
+
"PyBodyNode",
|
|
185
|
+
"PyBodyNode",
|
|
201
186
|
"id",
|
|
202
187
|
{
|
|
203
188
|
"id": "string",
|
|
204
189
|
"kind": "string",
|
|
205
190
|
"var": "string",
|
|
206
191
|
"call_node": "string",
|
|
192
|
+
# Call-site detail (#120): the graph emits one node per call site,
|
|
193
|
+
# matching analysis.json, instead of a separate :PyCallSite.
|
|
194
|
+
"method_name": "string",
|
|
195
|
+
"receiver_expr": "string",
|
|
196
|
+
"receiver_type": "string",
|
|
197
|
+
"return_type": "string",
|
|
198
|
+
"is_constructor_call": "boolean",
|
|
199
|
+
"arguments_json": "string",
|
|
207
200
|
**_SPAN,
|
|
208
201
|
"_module": "string",
|
|
209
202
|
},
|
|
@@ -219,8 +212,7 @@ REL_TYPES: List[RelType] = [
|
|
|
219
212
|
RelType("PY_HAS_METHOD", ["PyClass"], ["PyCallable"]),
|
|
220
213
|
RelType("PY_HAS_ATTRIBUTE", ["PyClass"], ["PyAttribute"]),
|
|
221
214
|
RelType("PY_DECLARES_VAR", ["PyModule", "PyCallable"], ["PyVariable"]),
|
|
222
|
-
RelType("
|
|
223
|
-
RelType("PY_RESOLVES_TO", ["PyCallSite"], ["PyCallable", "PyExternal"]),
|
|
215
|
+
RelType("PY_RESOLVES_TO", ["PyBodyNode"], ["PyCallable", "PyExternal"]),
|
|
224
216
|
RelType(
|
|
225
217
|
"PY_CALLS",
|
|
226
218
|
["PyCallable", "PyExternal"],
|
|
@@ -234,21 +226,30 @@ REL_TYPES: List[RelType] = [
|
|
|
234
226
|
["PyModule", "PyPackage"],
|
|
235
227
|
{"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
|
|
236
228
|
),
|
|
237
|
-
RelType(
|
|
229
|
+
RelType(
|
|
230
|
+
"PY_DECORATED_BY",
|
|
231
|
+
["PyCallable", "PyClass"],
|
|
232
|
+
["PyDecorator"],
|
|
233
|
+
{
|
|
234
|
+
"expression": "string",
|
|
235
|
+
"positional_arguments": "string[]",
|
|
236
|
+
"keyword_arguments_json": "string",
|
|
237
|
+
},
|
|
238
|
+
),
|
|
238
239
|
# Level-3 CPG overlay (-a 3 only): the cross-language dataflow vocabulary,
|
|
239
240
|
# PY_-namespaced so per-language SDK backends can scope their queries.
|
|
240
|
-
RelType("
|
|
241
|
+
RelType("PY_HAS_BODY_NODE", ["PyCallable"], ["PyBodyNode"]),
|
|
241
242
|
# ``_k`` is the relationship-identity discriminant (internal, underscore-
|
|
242
243
|
# prefixed like ``_module``): PY_CFG_NEXT merges per ``kind`` (a conditional's
|
|
243
244
|
# true/false pair), PY_DDG per ``(var, prov)`` (one dependence per variable,
|
|
244
245
|
# and the ssa/points-to split) — a plain endpoint-pair MERGE would collapse
|
|
245
246
|
# legitimately-distinct edges.
|
|
246
|
-
RelType("PY_CFG_NEXT", ["
|
|
247
|
-
RelType("PY_CDG", ["
|
|
248
|
-
RelType("PY_DDG", ["
|
|
249
|
-
RelType("PY_PARAM_IN", ["
|
|
250
|
-
RelType("PY_PARAM_OUT", ["
|
|
251
|
-
RelType("PY_SUMMARY", ["
|
|
247
|
+
RelType("PY_CFG_NEXT", ["PyBodyNode"], ["PyBodyNode"], {"kind": "string", "_k": "string"}),
|
|
248
|
+
RelType("PY_CDG", ["PyBodyNode"], ["PyBodyNode"]),
|
|
249
|
+
RelType("PY_DDG", ["PyBodyNode"], ["PyBodyNode"], {"var": "string", "prov": "string[]", "_k": "string"}),
|
|
250
|
+
RelType("PY_PARAM_IN", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
251
|
+
RelType("PY_PARAM_OUT", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
252
|
+
RelType("PY_SUMMARY", ["PyBodyNode"], ["PyBodyNode"]),
|
|
252
253
|
]
|
|
253
254
|
|
|
254
255
|
|
codeanalyzer/options/__init__.py
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
from .options import AnalysisOptions, EmitTarget
|
|
1
|
+
from .options import AnalysisOptions, EmitTarget
|
|
2
2
|
|
|
3
|
-
__all__ = ["AnalysisOptions", "EmitTarget"
|
|
3
|
+
__all__ = ["AnalysisOptions", "EmitTarget"]
|
codeanalyzer/options/options.py
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Optional
|
|
3
|
+
from typing import Optional, Tuple
|
|
4
4
|
from enum import Enum
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class OutputFormat(str, Enum):
|
|
8
|
-
JSON = "json"
|
|
9
|
-
MSGPACK = "msgpack"
|
|
10
|
-
|
|
11
|
-
|
|
12
7
|
class EmitTarget(str, Enum):
|
|
13
8
|
"""Output target selected by ``--emit``.
|
|
14
9
|
|
|
@@ -23,25 +18,10 @@ class EmitTarget(str, Enum):
|
|
|
23
18
|
SCHEMA = "schema"
|
|
24
19
|
|
|
25
20
|
|
|
26
|
-
class ShardStrategy(str, Enum):
|
|
27
|
-
"""How ``--pycg-shard`` groups files into shards (level 2 only).
|
|
28
|
-
|
|
29
|
-
- ``jedi`` : partition the Jedi module-dependency graph (strongly-
|
|
30
|
-
connected-component condensation + Louvain) so tightly-
|
|
31
|
-
coupled modules co-compute and few call edges are severed
|
|
32
|
-
between shards. Import cycles are never split.
|
|
33
|
-
- ``package`` : legacy one-shard-per-package-directory grouping.
|
|
34
|
-
"""
|
|
35
|
-
|
|
36
|
-
JEDI = "jedi"
|
|
37
|
-
PACKAGE = "package"
|
|
38
|
-
|
|
39
|
-
|
|
40
21
|
@dataclass
|
|
41
22
|
class AnalysisOptions:
|
|
42
23
|
input: Path
|
|
43
24
|
output: Optional[Path] = None
|
|
44
|
-
format: OutputFormat = OutputFormat.JSON
|
|
45
25
|
emit: EmitTarget = EmitTarget.JSON
|
|
46
26
|
app_name: Optional[str] = None
|
|
47
27
|
neo4j_uri: Optional[str] = None
|
|
@@ -61,8 +41,4 @@ class AnalysisOptions:
|
|
|
61
41
|
cache_dir: Optional[Path] = None
|
|
62
42
|
clear_cache: bool = False
|
|
63
43
|
verbosity: int = 0
|
|
64
|
-
|
|
65
|
-
pycg_shard_ceiling: int = 100
|
|
66
|
-
pycg_shard_timeout: int = 120
|
|
67
|
-
pycg_shard_strategy: ShardStrategy = ShardStrategy.JEDI
|
|
68
|
-
pycg_max_iter: int = 50
|
|
44
|
+
entrypoint_rules: Tuple[Path, ...] = ()
|
codeanalyzer/schema/__init__.py
CHANGED
|
@@ -62,6 +62,31 @@ if not PYDANTIC_V2:
|
|
|
62
62
|
)
|
|
63
63
|
Analysis.update_forward_refs(PyApplication=PyApplication)
|
|
64
64
|
|
|
65
|
+
# Fields the analyzer keeps in memory (and in the cache) but never emits.
|
|
66
|
+
# `call_sites` is the internal record `body{}` call nodes are derived from (#120);
|
|
67
|
+
# emitting both shipped the same fact twice under two identity schemes.
|
|
68
|
+
INTERNAL_ONLY_FIELDS = frozenset({"call_sites"})
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def strip_internal_only(data):
|
|
72
|
+
"""Recursively drop `INTERNAL_ONLY_FIELDS` from a dumped payload.
|
|
73
|
+
|
|
74
|
+
Applied at emit time rather than as a field-level Pydantic `exclude`, because
|
|
75
|
+
the analysis cache uses the same serializer -- excluding at the field would
|
|
76
|
+
drop these from the cache as well, and the next warm-cache run would rebuild
|
|
77
|
+
from a payload with no call sites.
|
|
78
|
+
"""
|
|
79
|
+
if isinstance(data, dict):
|
|
80
|
+
return {
|
|
81
|
+
k: strip_internal_only(v)
|
|
82
|
+
for k, v in data.items()
|
|
83
|
+
if k not in INTERNAL_ONLY_FIELDS
|
|
84
|
+
}
|
|
85
|
+
if isinstance(data, list):
|
|
86
|
+
return [strip_internal_only(v) for v in data]
|
|
87
|
+
return data
|
|
88
|
+
|
|
89
|
+
|
|
65
90
|
# Compatibility helpers for Pydantic v1/v2
|
|
66
91
|
def model_dump_json(model, **kwargs):
|
|
67
92
|
"""Compatibility helper for JSON serialization."""
|
|
@@ -79,6 +104,27 @@ def model_dump_json(model, **kwargs):
|
|
|
79
104
|
v1_kwargs['separators'] = kwargs['separators']
|
|
80
105
|
return model.json(**v1_kwargs)
|
|
81
106
|
|
|
107
|
+
def model_dump(model, **kwargs):
|
|
108
|
+
"""Compatibility helper for dict serialization (v2 model_dump / v1 dict).
|
|
109
|
+
|
|
110
|
+
``mode="json"`` (v2) maps to a json round-trip on v1 so both versions
|
|
111
|
+
yield JSON-safe primitives.
|
|
112
|
+
"""
|
|
113
|
+
if PYDANTIC_V2:
|
|
114
|
+
return model.model_dump(**kwargs)
|
|
115
|
+
import json as _json
|
|
116
|
+
mode = kwargs.pop("mode", None)
|
|
117
|
+
v1_kwargs = {k: v for k, v in kwargs.items() if k in ("exclude_none", "exclude")}
|
|
118
|
+
if mode == "json":
|
|
119
|
+
return _json.loads(model.json(**v1_kwargs))
|
|
120
|
+
return model.dict(**v1_kwargs)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def model_copy(model):
|
|
124
|
+
"""Compatibility helper for copying a model (v2 model_copy / v1 copy)."""
|
|
125
|
+
return model.model_copy() if PYDANTIC_V2 else model.copy()
|
|
126
|
+
|
|
127
|
+
|
|
82
128
|
def model_validate_json(model_class, json_data):
|
|
83
129
|
"""Compatibility helper for JSON deserialization."""
|
|
84
130
|
if PYDANTIC_V2:
|
|
@@ -89,5 +135,7 @@ def model_validate_json(model_class, json_data):
|
|
|
89
135
|
__all__.extend([
|
|
90
136
|
"PYDANTIC_V2",
|
|
91
137
|
"model_dump_json",
|
|
138
|
+
"strip_internal_only",
|
|
139
|
+
"INTERNAL_ONLY_FIELDS",
|
|
92
140
|
"model_validate_json"
|
|
93
141
|
])
|
codeanalyzer/schema/l1_body.py
CHANGED
|
@@ -9,7 +9,17 @@ def _do_callable(source: str, c: PyCallable) -> None:
|
|
|
9
9
|
span = Span(start=(cs.start_line, cs.start_column),
|
|
10
10
|
end=(cs.end_line, cs.end_column),
|
|
11
11
|
bytes=byte_offsets(source, cs.start_line, cs.start_column, cs.end_line, cs.end_column)) if source else None
|
|
12
|
-
c.body[key] = BodyNode(
|
|
12
|
+
c.body[key] = BodyNode(
|
|
13
|
+
kind="call",
|
|
14
|
+
span=span,
|
|
15
|
+
callee=None,
|
|
16
|
+
method_name=cs.method_name,
|
|
17
|
+
receiver_expr=cs.receiver_expr,
|
|
18
|
+
receiver_type=cs.receiver_type,
|
|
19
|
+
return_type=cs.return_type,
|
|
20
|
+
is_constructor_call=cs.is_constructor_call,
|
|
21
|
+
arguments=list(cs.arguments or []),
|
|
22
|
+
)
|
|
13
23
|
for ic in (c.callables or {}).values():
|
|
14
24
|
_do_callable(source, ic)
|
|
15
25
|
for icl in (c.types or {}).values():
|
|
@@ -1,36 +1,52 @@
|
|
|
1
1
|
"""L2 refinement: fill each L1 `call` body node's `callee` (null→id) from the
|
|
2
2
|
call site's resolved signature — the one sanctioned value change. A declared
|
|
3
3
|
target becomes its can:// id; an external/library target keeps its dotted
|
|
4
|
-
signature; an unresolved call site leaves `callee` absent.
|
|
4
|
+
signature; an unresolved call site leaves `callee` absent.
|
|
5
|
+
|
|
6
|
+
Two resolution sources feed the backfill: Jedi's `callee_signature` on the
|
|
7
|
+
call site itself, and the defuse linker's returned map (keyed by caller
|
|
8
|
+
signature + "line:col"). The linker's resolutions are deliberately NOT written
|
|
9
|
+
into `callee_signature` — the symbol table round-trips through the analysis
|
|
10
|
+
cache, and a persisted resolution would resurface on a warm run as a Jedi
|
|
11
|
+
edge, silently changing provenance."""
|
|
5
12
|
from __future__ import annotations
|
|
6
13
|
from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable
|
|
7
14
|
|
|
8
15
|
|
|
9
|
-
def _do_callable(c: PyCallable, sig_to_id: dict) -> None:
|
|
16
|
+
def _do_callable(c: PyCallable, sig_to_id: dict, resolutions: dict) -> None:
|
|
10
17
|
for cs in c.call_sites or []:
|
|
11
|
-
if cs.callee_signature is None:
|
|
12
|
-
continue
|
|
13
18
|
key = f"{cs.start_line}:{cs.start_column}"
|
|
19
|
+
jedi_sig = cs.callee_signature
|
|
20
|
+
if jedi_sig and jedi_sig.startswith("typing."):
|
|
21
|
+
# A decorator-typed callable resolved to its annotation, not a
|
|
22
|
+
# target; the linker's resolution (if any) is the real callee.
|
|
23
|
+
jedi_sig = None
|
|
24
|
+
sig = jedi_sig or resolutions.get((c.signature, key))
|
|
25
|
+
if not sig:
|
|
26
|
+
continue
|
|
14
27
|
node = c.body.get(key)
|
|
15
28
|
if node is None or node.kind != "call":
|
|
16
29
|
continue
|
|
17
|
-
node.callee = sig_to_id.get(
|
|
30
|
+
node.callee = sig_to_id.get(sig, sig)
|
|
18
31
|
for ic in (c.callables or {}).values():
|
|
19
|
-
_do_callable(ic, sig_to_id)
|
|
32
|
+
_do_callable(ic, sig_to_id, resolutions)
|
|
20
33
|
for icl in (c.types or {}).values():
|
|
21
|
-
_do_class(icl, sig_to_id)
|
|
34
|
+
_do_class(icl, sig_to_id, resolutions)
|
|
22
35
|
|
|
23
36
|
|
|
24
|
-
def _do_class(cl: PyClass, sig_to_id: dict) -> None:
|
|
37
|
+
def _do_class(cl: PyClass, sig_to_id: dict, resolutions: dict) -> None:
|
|
25
38
|
for m in (cl.callables or {}).values():
|
|
26
|
-
_do_callable(m, sig_to_id)
|
|
39
|
+
_do_callable(m, sig_to_id, resolutions)
|
|
27
40
|
for ic in (cl.types or {}).values():
|
|
28
|
-
_do_class(ic, sig_to_id)
|
|
41
|
+
_do_class(ic, sig_to_id, resolutions)
|
|
29
42
|
|
|
30
43
|
|
|
31
|
-
def backfill_callees(
|
|
44
|
+
def backfill_callees(
|
|
45
|
+
app: PyApplication, sig_to_id: dict, resolutions: dict | None = None
|
|
46
|
+
) -> None:
|
|
47
|
+
resolutions = resolutions or {}
|
|
32
48
|
for mod in app.symbol_table.values():
|
|
33
49
|
for fn in (mod.functions or {}).values():
|
|
34
|
-
_do_callable(fn, sig_to_id)
|
|
50
|
+
_do_callable(fn, sig_to_id, resolutions)
|
|
35
51
|
for cl in (mod.types or {}).values():
|
|
36
|
-
_do_class(cl, sig_to_id)
|
|
52
|
+
_do_class(cl, sig_to_id, resolutions)
|