codeanalyzer-python 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +77 -4
- codeanalyzer/core.py +174 -72
- codeanalyzer/dataflow/__init__.py +35 -0
- codeanalyzer/dataflow/access_paths.py +563 -0
- codeanalyzer/dataflow/alias.py +93 -0
- codeanalyzer/dataflow/builder.py +688 -0
- codeanalyzer/dataflow/cfg.py +605 -0
- codeanalyzer/dataflow/defuse.py +113 -0
- codeanalyzer/dataflow/dominance.py +140 -0
- codeanalyzer/dataflow/identity.py +91 -0
- codeanalyzer/dataflow/pdg.py +100 -0
- codeanalyzer/dataflow/scalpel_oracle.py +269 -0
- codeanalyzer/dataflow/scc.py +91 -0
- codeanalyzer/dataflow/sdg.py +424 -0
- codeanalyzer/dataflow/slicing.py +93 -0
- codeanalyzer/dataflow/summaries.py +217 -0
- codeanalyzer/dataflow/syntactic.py +26 -0
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/bolt.py +19 -4
- codeanalyzer/neo4j/cypher.py +9 -3
- codeanalyzer/neo4j/emit.py +10 -5
- codeanalyzer/neo4j/project.py +307 -60
- codeanalyzer/neo4j/rows.py +18 -15
- codeanalyzer/neo4j/schema.py +297 -15
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/__init__.py +19 -0
- codeanalyzer/schema/assign_ids.py +37 -0
- codeanalyzer/schema/call_graph_ids.py +12 -0
- codeanalyzer/schema/ids.py +23 -0
- codeanalyzer/schema/l1_body.py +29 -0
- codeanalyzer/schema/l2_callees.py +36 -0
- codeanalyzer/schema/py_schema.py +175 -26
- codeanalyzer/semantic_analysis/call_graph.py +24 -27
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +10 -10
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +7 -7
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +103 -20
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/METADATA +233 -43
- codeanalyzer_python-1.0.0.dist-info/RECORD +59 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer_python-0.3.0.dist-info/RECORD +0 -38
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,688 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# Copyright IBM Corporation 2025
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
################################################################################
|
|
16
|
+
|
|
17
|
+
"""The level-3 orchestrator: symbol table + call graph → program graphs.
|
|
18
|
+
|
|
19
|
+
``build_program_graphs`` is the single entry point ``Codeanalyzer.analyze``
|
|
20
|
+
calls at ``-a 3``. It re-parses each module file with the stdlib ``ast`` (the
|
|
21
|
+
same parser the symbol table used), maps every ``PyCallable`` to its def node
|
|
22
|
+
by ``(file, start_line)`` — which is what guarantees graph nodes join back to
|
|
23
|
+
symbol-table signatures — then runs the construction ladder:
|
|
24
|
+
|
|
25
|
+
per callable: CFG → dominance → facts (module-qualified globals)
|
|
26
|
+
whole program: SCC condensation → summary fixpoint → SDG assembly
|
|
27
|
+
|
|
28
|
+
The call graph and Jedi-resolved callsites are frozen oracles: targets are
|
|
29
|
+
looked up, never re-inferred. Callables whose AST cannot be recovered (file
|
|
30
|
+
changed on disk, decorators moving line numbers, generated code) are skipped
|
|
31
|
+
with a warning — their callers still treat them as external pass-through, so
|
|
32
|
+
the result degrades gracefully instead of crashing (contract rule).
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import ast
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Callable, Dict, List, Optional, Set, Tuple
|
|
40
|
+
|
|
41
|
+
from codeanalyzer.dataflow.access_paths import _PathExtractor, _calls_in
|
|
42
|
+
from codeanalyzer.dataflow.alias import TypeBasedAliasOracle
|
|
43
|
+
from codeanalyzer.dataflow.pdg import build_pdg
|
|
44
|
+
from codeanalyzer.dataflow.sdg import ProgramGraphsIR, assemble_sdg
|
|
45
|
+
from codeanalyzer.dataflow.summaries import CallSite, FunctionInfo, compute_summaries
|
|
46
|
+
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
47
|
+
from codeanalyzer.utils import logger
|
|
48
|
+
|
|
49
|
+
DEFAULT_K_LIMIT = 3
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _walk_callables(
|
|
53
|
+
module: PyModule,
|
|
54
|
+
) -> List[Tuple[PyCallable, Tuple[PyCallable, ...]]]:
|
|
55
|
+
"""Every callable in the module with its chain of enclosing callables."""
|
|
56
|
+
out: List[Tuple[PyCallable, Tuple[PyCallable, ...]]] = []
|
|
57
|
+
|
|
58
|
+
def from_callable(c: PyCallable, chain: Tuple[PyCallable, ...]) -> None:
|
|
59
|
+
out.append((c, chain))
|
|
60
|
+
for inner in (c.callables or {}).values():
|
|
61
|
+
from_callable(inner, chain + (c,))
|
|
62
|
+
for cls in (c.types or {}).values():
|
|
63
|
+
from_class(cls, chain + (c,))
|
|
64
|
+
|
|
65
|
+
def from_class(cls: PyClass, chain: Tuple[PyCallable, ...]) -> None:
|
|
66
|
+
for m in (cls.callables or {}).values():
|
|
67
|
+
from_callable(m, chain)
|
|
68
|
+
for inner in (cls.types or {}).values():
|
|
69
|
+
from_class(inner, chain)
|
|
70
|
+
|
|
71
|
+
for fn in (module.functions or {}).values():
|
|
72
|
+
from_callable(fn, ())
|
|
73
|
+
for cls in (module.types or {}).values():
|
|
74
|
+
from_class(cls, ())
|
|
75
|
+
return out
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _locals_of(func: ast.AST) -> Set[str]:
|
|
79
|
+
from codeanalyzer.dataflow.access_paths import _assigned_names, _param_names
|
|
80
|
+
|
|
81
|
+
return set(_param_names(func)) | _assigned_names(func)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _base_types(c: PyCallable) -> Dict[str, Optional[str]]:
|
|
85
|
+
types: Dict[str, Optional[str]] = {}
|
|
86
|
+
for p in c.parameters or []:
|
|
87
|
+
types[p.name] = p.type
|
|
88
|
+
for v in c.local_variables or []:
|
|
89
|
+
types.setdefault(v.name, v.type)
|
|
90
|
+
return types
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _class_index(app: PyApplication) -> Dict[str, PyClass]:
|
|
94
|
+
from codeanalyzer.semantic_analysis.call_graph import iter_classes_in_symbol_table
|
|
95
|
+
|
|
96
|
+
return {c.signature: c for c in iter_classes_in_symbol_table(app.symbol_table)}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _callable_index(app: PyApplication) -> Dict[str, PyCallable]:
|
|
100
|
+
from codeanalyzer.semantic_analysis.call_graph import iter_callables_in_symbol_table
|
|
101
|
+
|
|
102
|
+
return {c.signature: c for c in iter_callables_in_symbol_table(app.symbol_table)}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _match_args(
|
|
106
|
+
call: ast.Call,
|
|
107
|
+
callee: PyCallable,
|
|
108
|
+
extractor: _PathExtractor,
|
|
109
|
+
receiver_path: Optional[str],
|
|
110
|
+
) -> Tuple[Tuple[str, Optional[str]], ...]:
|
|
111
|
+
"""Positional/keyword-match actual access paths to callee param names.
|
|
112
|
+
The receiver (or constructed object) binds the leading self/cls param."""
|
|
113
|
+
params = [p.name for p in (callee.parameters or [])]
|
|
114
|
+
pairs: List[Tuple[str, Optional[str]]] = []
|
|
115
|
+
positional = list(params)
|
|
116
|
+
if params and params[0] in ("self", "cls"):
|
|
117
|
+
if receiver_path is not None:
|
|
118
|
+
pairs.append((params[0], receiver_path))
|
|
119
|
+
positional = params[1:]
|
|
120
|
+
for name, arg in zip(positional, call.args):
|
|
121
|
+
if isinstance(arg, ast.Starred):
|
|
122
|
+
break
|
|
123
|
+
pairs.append((name, extractor.path_of(arg)))
|
|
124
|
+
for kw in call.keywords:
|
|
125
|
+
if kw.arg and kw.arg in params:
|
|
126
|
+
pairs.append((kw.arg, extractor.path_of(kw.value)))
|
|
127
|
+
return tuple(pairs)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def build_function_pdgs(
|
|
131
|
+
app: PyApplication,
|
|
132
|
+
k: int = DEFAULT_K_LIMIT,
|
|
133
|
+
*,
|
|
134
|
+
oracle_factory: Callable[[PyCallable, ast.AST], object],
|
|
135
|
+
) -> Tuple[Dict[str, FunctionInfo], Dict[str, ast.AST]]:
|
|
136
|
+
"""Intraprocedural phase only: one ``FunctionInfo`` (CFG → PDG) per
|
|
137
|
+
callable, keyed by signature, with no SDG/summary/callsite work.
|
|
138
|
+
|
|
139
|
+
``oracle_factory(pycallable, func_ast)`` supplies the may-alias oracle per
|
|
140
|
+
callable — the matched def AST is threaded through so the primary L4 oracle
|
|
141
|
+
(:func:`~codeanalyzer.dataflow.scalpel_oracle.make_alias_oracle`) can build
|
|
142
|
+
Scalpel's SSA from it; ``TypeBasedAliasOracle`` for the plain L4 path and
|
|
143
|
+
``SyntacticOracle`` for L3 simply ignore the AST argument.
|
|
144
|
+
|
|
145
|
+
Returns ``(infos, func_asts)`` rather than bare PDGs so that the L4
|
|
146
|
+
orchestrator (:func:`build_program_graphs`) still has both the
|
|
147
|
+
``FunctionInfo`` records its callsite/summary/SDG phases mutate and the
|
|
148
|
+
matched def nodes its Phase 2 reads. L3 callers just read ``info.pdg`` per
|
|
149
|
+
signature and ignore ``func_asts``.
|
|
150
|
+
"""
|
|
151
|
+
infos: Dict[str, FunctionInfo] = {}
|
|
152
|
+
func_asts: Dict[str, ast.AST] = {}
|
|
153
|
+
|
|
154
|
+
for file_key, module in sorted(app.symbol_table.items()):
|
|
155
|
+
path = Path(module.file_path)
|
|
156
|
+
try:
|
|
157
|
+
tree = ast.parse(path.read_text())
|
|
158
|
+
except (OSError, SyntaxError) as exc:
|
|
159
|
+
logger.warning(f"level 3: skipping {path} (unparseable: {exc})")
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
def_index: Dict[int, ast.AST] = {}
|
|
163
|
+
for node in ast.walk(tree):
|
|
164
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
165
|
+
def_index[node.lineno] = node
|
|
166
|
+
|
|
167
|
+
for pycallable, chain in _walk_callables(module):
|
|
168
|
+
func = def_index.get(pycallable.start_line)
|
|
169
|
+
if func is None or func.name != pycallable.name:
|
|
170
|
+
logger.warning(
|
|
171
|
+
f"level 3: no AST match for {pycallable.signature} "
|
|
172
|
+
f"({path}:{pycallable.start_line}); treated as external"
|
|
173
|
+
)
|
|
174
|
+
continue
|
|
175
|
+
|
|
176
|
+
enclosing_locals: Set[str] = set()
|
|
177
|
+
for enclosing in chain:
|
|
178
|
+
enclosing_ast = def_index.get(enclosing.start_line)
|
|
179
|
+
if enclosing_ast is not None:
|
|
180
|
+
enclosing_locals |= _locals_of(enclosing_ast)
|
|
181
|
+
|
|
182
|
+
oracle = oracle_factory(pycallable, func)
|
|
183
|
+
pdg = build_pdg(
|
|
184
|
+
func,
|
|
185
|
+
enclosing_locals=enclosing_locals,
|
|
186
|
+
oracle=oracle,
|
|
187
|
+
k=k,
|
|
188
|
+
global_qualifier=module.module_name,
|
|
189
|
+
)
|
|
190
|
+
infos[pycallable.signature] = FunctionInfo(
|
|
191
|
+
signature=pycallable.signature, pdg=pdg, oracle=oracle
|
|
192
|
+
)
|
|
193
|
+
func_asts[pycallable.signature] = func
|
|
194
|
+
|
|
195
|
+
return infos, func_asts
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def emit_l3_body(
|
|
199
|
+
app: PyApplication,
|
|
200
|
+
infos: Dict[str, FunctionInfo],
|
|
201
|
+
sig_to_id: Dict[str, str],
|
|
202
|
+
graphs: Set[str],
|
|
203
|
+
) -> None:
|
|
204
|
+
"""Project each callable's syntactic PDG onto the v2 tree at L3.
|
|
205
|
+
|
|
206
|
+
For every callable that produced a ``FunctionInfo`` in
|
|
207
|
+
:func:`build_function_pdgs` (syntactic oracle), this writes onto the
|
|
208
|
+
matching ``PyCallable`` in ``app``'s symbol table:
|
|
209
|
+
|
|
210
|
+
* ``body`` — one node per CFG node, keyed by its LOCAL id (``"@entry"``/
|
|
211
|
+
``"@exit"`` for the synthetic bookends, ``"line:col"`` for real
|
|
212
|
+
statements — the same key format L1 uses). A statement position an L1
|
|
213
|
+
pass already materialized as a ``call`` node lands on the SAME local key,
|
|
214
|
+
so it keeps its ``call`` kind and L2-resolved ``callee`` in place (no
|
|
215
|
+
re-keying, no duplication) and is only given the byte-offset ``span`` L1
|
|
216
|
+
could not compute.
|
|
217
|
+
* ``cfg`` — one ``CfgEdge`` per CFG edge, endpoints as local ids.
|
|
218
|
+
* ``cdg`` — the PDG's control-dependence edges.
|
|
219
|
+
* ``ddg`` — the PDG's syntactic def-use edges, each with ``prov=["ssa"]``
|
|
220
|
+
(no points-to provenance at L3; that is the L4 delta).
|
|
221
|
+
|
|
222
|
+
``graphs`` scopes the edge lists exactly as the dormant
|
|
223
|
+
:func:`to_program_graphs` does: ``cfg`` needs ``"cfg"``; ``cdg`` needs
|
|
224
|
+
``"pdg"``/``"sdg"``; ``ddg`` needs those or ``"dfg"``. ``body`` is always
|
|
225
|
+
populated. Callables absent from ``infos`` (unrecovered AST) are skipped.
|
|
226
|
+
"""
|
|
227
|
+
from codeanalyzer.dataflow.identity import IdentityMap
|
|
228
|
+
from codeanalyzer.schema.py_schema import (
|
|
229
|
+
BodyNode,
|
|
230
|
+
CdgEdge,
|
|
231
|
+
CfgEdge,
|
|
232
|
+
DdgEdge,
|
|
233
|
+
Span,
|
|
234
|
+
byte_offsets,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
want_pdg = bool({"pdg", "sdg"} & graphs)
|
|
238
|
+
want_cfg = "cfg" in graphs
|
|
239
|
+
want_ddg = want_pdg or "dfg" in graphs
|
|
240
|
+
|
|
241
|
+
def _span_of(source: str, node) -> Optional["Span"]:
|
|
242
|
+
if not source or node.start_line < 1:
|
|
243
|
+
return None
|
|
244
|
+
return Span(
|
|
245
|
+
start=(node.start_line, node.start_column),
|
|
246
|
+
end=(node.end_line, node.end_column),
|
|
247
|
+
bytes=byte_offsets(
|
|
248
|
+
source,
|
|
249
|
+
node.start_line,
|
|
250
|
+
node.start_column,
|
|
251
|
+
node.end_line,
|
|
252
|
+
node.end_column,
|
|
253
|
+
),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
for module in app.symbol_table.values():
|
|
257
|
+
source = module.source
|
|
258
|
+
for pycallable, _chain in _walk_callables(module):
|
|
259
|
+
info = infos.get(pycallable.signature)
|
|
260
|
+
if info is None:
|
|
261
|
+
continue
|
|
262
|
+
pdg = info.pdg
|
|
263
|
+
callable_id = sig_to_id.get(pycallable.signature) or pycallable.id
|
|
264
|
+
im = IdentityMap.for_function(callable_id, pdg)
|
|
265
|
+
|
|
266
|
+
for node in pdg.cfg.nodes:
|
|
267
|
+
local = im.local(node.id)
|
|
268
|
+
if node.id == pdg.cfg.entry_id:
|
|
269
|
+
pycallable.body[local] = BodyNode(kind="entry")
|
|
270
|
+
continue
|
|
271
|
+
if node.id == pdg.cfg.exit_id:
|
|
272
|
+
pycallable.body[local] = BodyNode(kind="exit")
|
|
273
|
+
continue
|
|
274
|
+
span = _span_of(source, node)
|
|
275
|
+
# An L1 `call` node was keyed by its LOCAL "line:col"; this CFG
|
|
276
|
+
# node at the same position lands on the SAME key, so keep the
|
|
277
|
+
# node's `call` kind and L2-resolved `callee` in place and just
|
|
278
|
+
# fill any missing span — never re-key or duplicate it.
|
|
279
|
+
existing = pycallable.body.get(local)
|
|
280
|
+
if existing is not None:
|
|
281
|
+
if existing.span is None and span is not None:
|
|
282
|
+
existing.span = span
|
|
283
|
+
continue
|
|
284
|
+
pycallable.body[local] = BodyNode(kind=node.kind, span=span)
|
|
285
|
+
|
|
286
|
+
if want_cfg:
|
|
287
|
+
pycallable.cfg = [
|
|
288
|
+
CfgEdge(
|
|
289
|
+
src=im.local(e.source),
|
|
290
|
+
dst=im.local(e.target),
|
|
291
|
+
kind=e.kind,
|
|
292
|
+
)
|
|
293
|
+
for e in pdg.cfg.edges
|
|
294
|
+
]
|
|
295
|
+
if want_pdg:
|
|
296
|
+
pycallable.cdg = [
|
|
297
|
+
CdgEdge(src=im.local(e.source), dst=im.local(e.target))
|
|
298
|
+
for e in pdg.edges
|
|
299
|
+
if e.type == "CDG"
|
|
300
|
+
]
|
|
301
|
+
if want_ddg:
|
|
302
|
+
pycallable.ddg = [
|
|
303
|
+
DdgEdge(
|
|
304
|
+
src=im.local(e.source),
|
|
305
|
+
dst=im.local(e.target),
|
|
306
|
+
var=e.var,
|
|
307
|
+
prov=["ssa"],
|
|
308
|
+
)
|
|
309
|
+
for e in pdg.edges
|
|
310
|
+
if e.type == "DDG"
|
|
311
|
+
]
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def build_program_graphs(
|
|
315
|
+
app: PyApplication,
|
|
316
|
+
k: int = DEFAULT_K_LIMIT,
|
|
317
|
+
*,
|
|
318
|
+
oracle_factory: Callable[[PyCallable, ast.AST], object] = (
|
|
319
|
+
lambda c, fast: TypeBasedAliasOracle(_base_types(c))
|
|
320
|
+
),
|
|
321
|
+
) -> ProgramGraphsIR:
|
|
322
|
+
"""Build CFG/PDG per callable and the whole-program SDG.
|
|
323
|
+
|
|
324
|
+
``oracle_factory(pycallable, func_ast)`` selects the per-callable may-alias
|
|
325
|
+
oracle. The default is the frozen :class:`TypeBasedAliasOracle` (preserving
|
|
326
|
+
the historical behavior); the L4 path in ``core`` injects
|
|
327
|
+
:func:`~codeanalyzer.dataflow.scalpel_oracle.make_alias_oracle` so Scalpel
|
|
328
|
+
is the primary oracle with the type-based total fallback.
|
|
329
|
+
"""
|
|
330
|
+
class_idx = _class_index(app)
|
|
331
|
+
callable_idx = _callable_index(app)
|
|
332
|
+
|
|
333
|
+
infos, func_asts = build_function_pdgs(app, k, oracle_factory=oracle_factory)
|
|
334
|
+
|
|
335
|
+
# Callsites and nested defs, now that every signature is known.
|
|
336
|
+
for sig, info in infos.items():
|
|
337
|
+
pycallable = callable_idx[sig]
|
|
338
|
+
func = func_asts[sig]
|
|
339
|
+
extractor = _PathExtractor(info.pdg.scope, k)
|
|
340
|
+
|
|
341
|
+
calls_by_pos: Dict[Tuple[int, int], Tuple[int, ast.Call]] = {}
|
|
342
|
+
calls_by_line: Dict[int, Tuple[int, ast.Call]] = {}
|
|
343
|
+
for node in info.pdg.cfg.nodes:
|
|
344
|
+
if node.ast_node is None:
|
|
345
|
+
continue
|
|
346
|
+
for call in _calls_in(node.ast_node):
|
|
347
|
+
pos = (call.lineno, call.col_offset)
|
|
348
|
+
calls_by_pos.setdefault(pos, (node.id, call))
|
|
349
|
+
calls_by_line.setdefault(call.lineno, (node.id, call))
|
|
350
|
+
|
|
351
|
+
for site in pycallable.call_sites or []:
|
|
352
|
+
target = site.callee_signature
|
|
353
|
+
if not target:
|
|
354
|
+
continue
|
|
355
|
+
if target in class_idx and target not in infos:
|
|
356
|
+
target = f"{target}.__init__" # constructor → its initializer
|
|
357
|
+
if target not in infos:
|
|
358
|
+
continue # external or unrecovered: pass-through posture
|
|
359
|
+
|
|
360
|
+
located = calls_by_pos.get((site.start_line, site.start_column))
|
|
361
|
+
if located is None:
|
|
362
|
+
located = calls_by_line.get(site.start_line)
|
|
363
|
+
if located is None:
|
|
364
|
+
continue
|
|
365
|
+
node_id, call = located
|
|
366
|
+
|
|
367
|
+
receiver_path: Optional[str] = None
|
|
368
|
+
if isinstance(call.func, ast.Attribute):
|
|
369
|
+
receiver_path = extractor.path_of(call.func.value)
|
|
370
|
+
elif site.is_constructor_call:
|
|
371
|
+
# p = Box(...) binds the constructed object (self) to p.
|
|
372
|
+
owner = info.pdg.cfg.node_by_id(node_id).ast_node
|
|
373
|
+
if (
|
|
374
|
+
isinstance(owner, ast.Assign)
|
|
375
|
+
and len(owner.targets) == 1
|
|
376
|
+
and isinstance(owner.targets[0], (ast.Name, ast.Attribute))
|
|
377
|
+
):
|
|
378
|
+
receiver_path = extractor.path_of(owner.targets[0])
|
|
379
|
+
|
|
380
|
+
info.call_sites.append(
|
|
381
|
+
CallSite(
|
|
382
|
+
node_id=node_id,
|
|
383
|
+
targets=(target,),
|
|
384
|
+
arg_paths=_match_args(call, callable_idx[target], extractor, receiver_path),
|
|
385
|
+
line=site.start_line,
|
|
386
|
+
)
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
for node in info.pdg.cfg.nodes:
|
|
390
|
+
if isinstance(node.ast_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
391
|
+
nested_sig = f"{sig}.{node.ast_node.name}"
|
|
392
|
+
if nested_sig in infos:
|
|
393
|
+
info.nested_defs.append((node.id, nested_sig))
|
|
394
|
+
|
|
395
|
+
call_edges = [
|
|
396
|
+
(e.src, e.dst)
|
|
397
|
+
for e in app.call_graph
|
|
398
|
+
if e.src in infos and e.dst in infos
|
|
399
|
+
]
|
|
400
|
+
# Callsite resolutions are part of the same oracle (they may include
|
|
401
|
+
# constructor retargets the edge list lacks).
|
|
402
|
+
for sig, info in infos.items():
|
|
403
|
+
for cs in info.call_sites:
|
|
404
|
+
for t in cs.targets:
|
|
405
|
+
call_edges.append((sig, t))
|
|
406
|
+
|
|
407
|
+
summaries = compute_summaries(infos, sorted(set(call_edges)))
|
|
408
|
+
return assemble_sdg(infos, summaries, k)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def emit_l4(
|
|
412
|
+
app: PyApplication,
|
|
413
|
+
ir: ProgramGraphsIR,
|
|
414
|
+
sig_to_id: Dict[str, str],
|
|
415
|
+
) -> None:
|
|
416
|
+
"""Project the interprocedural L4 delta of ``ir`` onto the v2 tree.
|
|
417
|
+
|
|
418
|
+
Layered strictly *on top of* the L3 syntactic overlay (which
|
|
419
|
+
:func:`emit_l3_body` has already written), so L3 ⊆ L4 holds by
|
|
420
|
+
construction — this function only *adds* keys/edges, never rewrites L3's.
|
|
421
|
+
Per callable it emits:
|
|
422
|
+
|
|
423
|
+
* **synthetic param vertices** — each :class:`ParamNode`
|
|
424
|
+
(``formal_in``/``formal_out``/``actual_in``/``actual_out``) becomes a
|
|
425
|
+
``body`` node keyed by its LOCAL id (``@formal_in:<i>``, ``@formal_out``,
|
|
426
|
+
``<callsite-local>/actual_in:<i>``, …), carrying the variable it models in
|
|
427
|
+
``of`` and — for actuals — the owning callsite's local id in ``parent``;
|
|
428
|
+
* **summary edges** — each same-signature ``SUMMARY`` SDG edge (a callee's
|
|
429
|
+
transitive actual_in → actual_out pass-through) lands on the callable's
|
|
430
|
+
``summary`` as a :class:`SummaryEdge` of LOCAL ids;
|
|
431
|
+
* **param_in / param_out** — each cross-function ``PARAM_IN`` / ``PARAM_OUT``
|
|
432
|
+
SDG edge becomes an application-level :class:`ParamEdge` of GLOBAL ids
|
|
433
|
+
(``<callable-id>@<local>``), resolved through the endpoint functions'
|
|
434
|
+
identity maps.
|
|
435
|
+
|
|
436
|
+
``CALL`` SDG edges are dropped — they duplicate the call graph. ``ddg``
|
|
437
|
+
points-to provenance and taint are *not* emitted here (later tasks).
|
|
438
|
+
"""
|
|
439
|
+
from codeanalyzer.dataflow.identity import IdentityMap
|
|
440
|
+
from codeanalyzer.schema.py_schema import BodyNode, ParamEdge, SummaryEdge
|
|
441
|
+
|
|
442
|
+
# L4 emission is additive (it *appends* summary/param edges), so it must
|
|
443
|
+
# first clear any L4 state a reused cache left on these live objects —
|
|
444
|
+
# otherwise repeated ``-a 4`` runs against the same cache_dir would keep
|
|
445
|
+
# growing the lists (1→2→3→…). L3's emit reassigns its lists and is already
|
|
446
|
+
# idempotent; L4 has to reset explicitly. App-scope lists reset once here,
|
|
447
|
+
# before the loop that appends to them; per-callable ``summary`` is reset in
|
|
448
|
+
# the (a) loop below.
|
|
449
|
+
app.param_in = []
|
|
450
|
+
app.param_out = []
|
|
451
|
+
|
|
452
|
+
# Tree callables by signature: these are the live objects in ``app``'s
|
|
453
|
+
# symbol table, so mutating them mutates the emitted tree.
|
|
454
|
+
sig_to_callable: Dict[str, PyCallable] = {}
|
|
455
|
+
for module in app.symbol_table.values():
|
|
456
|
+
for pycallable, _chain in _walk_callables(module):
|
|
457
|
+
sig_to_callable[pycallable.signature] = pycallable
|
|
458
|
+
|
|
459
|
+
# One IdentityMap per function, each folding *that* function's synthetic
|
|
460
|
+
# param vertices, so both intra-function (summary) and cross-function
|
|
461
|
+
# (param_in/param_out) endpoints resolve uniformly by node id.
|
|
462
|
+
ims: Dict[str, IdentityMap] = {}
|
|
463
|
+
for sig, fg in ir.functions.items():
|
|
464
|
+
pycallable = sig_to_callable.get(sig)
|
|
465
|
+
callable_id = sig_to_id.get(sig) or (pycallable.id if pycallable else sig)
|
|
466
|
+
ims[sig] = IdentityMap.for_function(
|
|
467
|
+
callable_id, fg.pdg, param_nodes=fg.param_nodes
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
# (a) synthetic param vertices onto each callable's body.
|
|
471
|
+
for sig, fg in ir.functions.items():
|
|
472
|
+
pycallable = sig_to_callable.get(sig)
|
|
473
|
+
if pycallable is None:
|
|
474
|
+
continue
|
|
475
|
+
# Idempotency under cache reuse: drop L4 state a prior run left on this
|
|
476
|
+
# live callable before re-emitting. ``summary`` is append-built below, so
|
|
477
|
+
# reset it. The param vertices are re-added by keyed assignment (already
|
|
478
|
+
# idempotent), but a code change between runs could leave stale ones — so
|
|
479
|
+
# defensively drop any pre-existing param-kind body nodes first.
|
|
480
|
+
pycallable.summary = []
|
|
481
|
+
for k in [
|
|
482
|
+
k
|
|
483
|
+
for k, n in pycallable.body.items()
|
|
484
|
+
if n.kind in ("formal_in", "formal_out", "actual_in", "actual_out")
|
|
485
|
+
]:
|
|
486
|
+
del pycallable.body[k]
|
|
487
|
+
im = ims[sig]
|
|
488
|
+
for pn in fg.param_nodes:
|
|
489
|
+
parent = im.local(pn.call_node) if pn.call_node is not None else None
|
|
490
|
+
pycallable.body[im.local(pn.id)] = BodyNode(
|
|
491
|
+
kind=pn.kind, of=pn.var, parent=parent
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
# (b/c/d) SDG edges → summary / param_in / param_out; CALL dropped.
|
|
495
|
+
for e in ir.sdg_edges:
|
|
496
|
+
if e.type == "CALL":
|
|
497
|
+
continue
|
|
498
|
+
if e.type == "SUMMARY":
|
|
499
|
+
pycallable = sig_to_callable.get(e.source_sig)
|
|
500
|
+
im = ims.get(e.source_sig)
|
|
501
|
+
if pycallable is None or im is None:
|
|
502
|
+
continue
|
|
503
|
+
pycallable.summary.append(
|
|
504
|
+
SummaryEdge(
|
|
505
|
+
src=im.local(e.source_node),
|
|
506
|
+
dst=im.local(e.target_node),
|
|
507
|
+
)
|
|
508
|
+
)
|
|
509
|
+
elif e.type in ("PARAM_IN", "PARAM_OUT"):
|
|
510
|
+
src_im = ims.get(e.source_sig)
|
|
511
|
+
dst_im = ims.get(e.target_sig)
|
|
512
|
+
if src_im is None or dst_im is None:
|
|
513
|
+
continue
|
|
514
|
+
edge = ParamEdge(
|
|
515
|
+
src=src_im.global_id(e.source_node),
|
|
516
|
+
dst=dst_im.global_id(e.target_node),
|
|
517
|
+
)
|
|
518
|
+
(app.param_in if e.type == "PARAM_IN" else app.param_out).append(edge)
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _ddg_local_set(im, pdg) -> Set[Tuple[str, str, Optional[str]]]:
|
|
522
|
+
"""The DDG edges of ``pdg`` as a set of ``(local_src, local_dst, var)``.
|
|
523
|
+
|
|
524
|
+
Keyed by LOCAL ids (``"line:col"`` / ``"@entry"``…), which are position-
|
|
525
|
+
based and thus stable across oracle choice — so a syntactic-oracle set and
|
|
526
|
+
a real-oracle set are directly comparable through the *same* identity map.
|
|
527
|
+
"""
|
|
528
|
+
return {
|
|
529
|
+
(im.local(e.source), im.local(e.target), e.var)
|
|
530
|
+
for e in pdg.edges
|
|
531
|
+
if e.type == "DDG"
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def emit_ddg_pointsto_delta(
|
|
536
|
+
app: PyApplication,
|
|
537
|
+
syntactic_infos: Dict[str, FunctionInfo],
|
|
538
|
+
ir: ProgramGraphsIR,
|
|
539
|
+
sig_to_id: Dict[str, str],
|
|
540
|
+
) -> None:
|
|
541
|
+
"""Append the semantic ``ddg`` delta at L4: the alias-derived def-use edges
|
|
542
|
+
the real (Scalpel-primary) oracle produces beyond the L3 syntactic
|
|
543
|
+
(name-equality) set, each tagged ``prov=["points-to"]``.
|
|
544
|
+
|
|
545
|
+
Strictly *additive*: :func:`emit_l3_body` has already written the syntactic
|
|
546
|
+
``ssa`` edges, and this function never touches them — it only appends the
|
|
547
|
+
points-to delta. For every signature present in *both* ``syntactic_infos``
|
|
548
|
+
(the L3 syntactic PDGs) and ``ir.functions`` (the real-oracle PDGs):
|
|
549
|
+
|
|
550
|
+
* build one :class:`IdentityMap` from the real PDG — the CFG is
|
|
551
|
+
oracle-independent, so node ids and their ``"line:col"`` locals coincide
|
|
552
|
+
between the two builds, and a single map resolves both sets;
|
|
553
|
+
* ``S`` = the syntactic-oracle DDG set, ``F`` = the real-oracle DDG set,
|
|
554
|
+
both as ``(local_src, local_dst, var)``;
|
|
555
|
+
* for each edge in ``F − S`` (sorted for determinism) whose endpoints exist
|
|
556
|
+
in the callable's ``body`` (defensive — they are CFG nodes), append a
|
|
557
|
+
``DdgEdge(prov=["points-to"])``.
|
|
558
|
+
"""
|
|
559
|
+
from codeanalyzer.dataflow.identity import IdentityMap
|
|
560
|
+
from codeanalyzer.schema.py_schema import DdgEdge
|
|
561
|
+
|
|
562
|
+
# Tree callables by signature: the live objects in ``app``'s symbol table,
|
|
563
|
+
# so appending to their ``ddg`` mutates the emitted tree in place.
|
|
564
|
+
sig_to_callable: Dict[str, PyCallable] = {}
|
|
565
|
+
for module in app.symbol_table.values():
|
|
566
|
+
for pycallable, _chain in _walk_callables(module):
|
|
567
|
+
sig_to_callable[pycallable.signature] = pycallable
|
|
568
|
+
|
|
569
|
+
for sig, fg in ir.functions.items():
|
|
570
|
+
syn = syntactic_infos.get(sig)
|
|
571
|
+
pycallable = sig_to_callable.get(sig)
|
|
572
|
+
if syn is None or pycallable is None:
|
|
573
|
+
continue
|
|
574
|
+
callable_id = sig_to_id.get(sig) or pycallable.id
|
|
575
|
+
im = IdentityMap.for_function(callable_id, fg.pdg)
|
|
576
|
+
|
|
577
|
+
# Idempotency under cache reuse: strip any points-to edges a prior run
|
|
578
|
+
# appended, so this append is idempotent regardless of whether
|
|
579
|
+
# ``emit_l3_body`` reassigned ``ddg`` this run (it only does when the
|
|
580
|
+
# ``--graphs`` selector includes ddg). The ``ssa`` edges are left
|
|
581
|
+
# untouched — they are L3's and this delta is strictly additive over them.
|
|
582
|
+
pycallable.ddg = [e for e in pycallable.ddg if e.prov != ["points-to"]]
|
|
583
|
+
|
|
584
|
+
delta = _ddg_local_set(im, fg.pdg) - _ddg_local_set(im, syn.pdg)
|
|
585
|
+
for src, dst, var in sorted(delta, key=lambda t: (t[0], t[1], t[2] or "")):
|
|
586
|
+
if src not in pycallable.body or dst not in pycallable.body:
|
|
587
|
+
continue
|
|
588
|
+
pycallable.ddg.append(
|
|
589
|
+
DdgEdge(src=src, dst=dst, var=var, prov=["points-to"])
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
VALID_GRAPHS = ("cfg", "dfg", "pdg", "sdg")
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def to_program_graphs(ir: ProgramGraphsIR, graphs: Set[str]):
|
|
597
|
+
"""Project the IR onto the ``program_graphs`` schema section, scoped by
|
|
598
|
+
the ``--graphs`` selector. ``dfg`` emits the PDG's DDG edges only;
|
|
599
|
+
``sdg`` implies the dependence edges it is stitched over."""
|
|
600
|
+
from codeanalyzer.schema.py_schema import (
|
|
601
|
+
PyCFG,
|
|
602
|
+
PyCFGEdge,
|
|
603
|
+
PyFunctionGraphs,
|
|
604
|
+
PyGraphNode,
|
|
605
|
+
PyParamNode,
|
|
606
|
+
PyPDG,
|
|
607
|
+
PyPDGEdge,
|
|
608
|
+
PyProgramGraphs,
|
|
609
|
+
PySDGEdge,
|
|
610
|
+
PySDGEndpoint,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
want_pdg = bool({"pdg", "sdg"} & graphs)
|
|
614
|
+
want_dfg = want_pdg or "dfg" in graphs
|
|
615
|
+
functions: Dict[str, "PyFunctionGraphs"] = {}
|
|
616
|
+
for sig in sorted(ir.functions):
|
|
617
|
+
fg = ir.functions[sig]
|
|
618
|
+
out = PyFunctionGraphs()
|
|
619
|
+
if "cfg" in graphs:
|
|
620
|
+
out.cfg = PyCFG(
|
|
621
|
+
nodes=[
|
|
622
|
+
PyGraphNode(
|
|
623
|
+
id=n.id,
|
|
624
|
+
kind=n.kind,
|
|
625
|
+
start_line=n.start_line,
|
|
626
|
+
end_line=n.end_line,
|
|
627
|
+
start_column=n.start_column,
|
|
628
|
+
end_column=n.end_column,
|
|
629
|
+
)
|
|
630
|
+
for n in fg.pdg.cfg.nodes
|
|
631
|
+
],
|
|
632
|
+
edges=[
|
|
633
|
+
PyCFGEdge(source=e.source, target=e.target, kind=e.kind)
|
|
634
|
+
for e in fg.pdg.cfg.edges
|
|
635
|
+
],
|
|
636
|
+
)
|
|
637
|
+
edges: List["PyPDGEdge"] = []
|
|
638
|
+
if want_pdg:
|
|
639
|
+
edges.extend(
|
|
640
|
+
PyPDGEdge(source=e.source, target=e.target, type="CDG")
|
|
641
|
+
for e in fg.pdg.edges
|
|
642
|
+
if e.type == "CDG"
|
|
643
|
+
)
|
|
644
|
+
if want_dfg:
|
|
645
|
+
edges.extend(
|
|
646
|
+
PyPDGEdge(source=e.source, target=e.target, type="DDG", var=e.var)
|
|
647
|
+
for e in fg.ddg
|
|
648
|
+
)
|
|
649
|
+
edges.extend(
|
|
650
|
+
PyPDGEdge(source=e.source, target=e.target, type=e.type, var=e.var)
|
|
651
|
+
for e in fg.extra_edges
|
|
652
|
+
if e.type == "DDG" or want_pdg
|
|
653
|
+
)
|
|
654
|
+
if edges:
|
|
655
|
+
edges.sort(key=lambda e: (e.source, e.target, e.type, e.var or ""))
|
|
656
|
+
out.pdg = PyPDG(edges=edges)
|
|
657
|
+
if "sdg" in graphs:
|
|
658
|
+
out.param_nodes = [
|
|
659
|
+
PyParamNode(
|
|
660
|
+
id=p.id,
|
|
661
|
+
kind=p.kind,
|
|
662
|
+
var=p.var,
|
|
663
|
+
call_node=p.call_node,
|
|
664
|
+
start_line=p.start_line,
|
|
665
|
+
end_line=p.end_line,
|
|
666
|
+
)
|
|
667
|
+
for p in fg.param_nodes
|
|
668
|
+
]
|
|
669
|
+
functions[sig] = out
|
|
670
|
+
|
|
671
|
+
sdg_edges = []
|
|
672
|
+
if "sdg" in graphs:
|
|
673
|
+
sdg_edges = [
|
|
674
|
+
PySDGEdge(
|
|
675
|
+
source=PySDGEndpoint(signature=e.source_sig, node=e.source_node),
|
|
676
|
+
target=PySDGEndpoint(signature=e.target_sig, node=e.target_node),
|
|
677
|
+
type=e.type,
|
|
678
|
+
var=e.var,
|
|
679
|
+
)
|
|
680
|
+
for e in ir.sdg_edges
|
|
681
|
+
]
|
|
682
|
+
|
|
683
|
+
return PyProgramGraphs(
|
|
684
|
+
schema_version="1.0.0",
|
|
685
|
+
k_limit=ir.k_limit,
|
|
686
|
+
functions=functions,
|
|
687
|
+
sdg_edges=sdg_edges,
|
|
688
|
+
)
|