codeanalyzer-python 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +95 -123
- codeanalyzer/core.py +21 -45
- codeanalyzer/dataflow/access_paths.py +26 -4
- codeanalyzer/dataflow/builder.py +65 -1
- codeanalyzer/dataflow/identity.py +1 -1
- codeanalyzer/dataflow/pdg.py +7 -2
- codeanalyzer/dataflow/scc.py +1 -1
- codeanalyzer/entrypoints/__init__.py +3 -0
- codeanalyzer/entrypoints/detect.py +124 -0
- codeanalyzer/entrypoints/matching.py +182 -0
- codeanalyzer/entrypoints/pipeline.py +131 -0
- codeanalyzer/entrypoints/rules.py +159 -0
- codeanalyzer/entrypoints/rules.yml +88 -0
- codeanalyzer/neo4j/bolt.py +1 -1
- codeanalyzer/neo4j/project.py +85 -60
- codeanalyzer/neo4j/schema.py +35 -34
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +2 -26
- codeanalyzer/schema/__init__.py +48 -0
- codeanalyzer/schema/l1_body.py +11 -1
- codeanalyzer/schema/l2_callees.py +29 -13
- codeanalyzer/schema/py_schema.py +95 -103
- codeanalyzer/semantic_analysis/call_graph.py +20 -4
- codeanalyzer/semantic_analysis/defuse_linker.py +1499 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +88 -3
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/METADATA +36 -161
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/RECORD +31 -30
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/WHEEL +1 -1
- codeanalyzer/config/__init__.py +0 -3
- codeanalyzer/config/config.py +0 -8
- codeanalyzer/semantic_analysis/pycg/__init__.py +0 -20
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -1115
- codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -23
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -401
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,1499 @@
|
|
|
1
|
+
"""Per-callable defuse linker: backfill call edges Jedi could not resolve.
|
|
2
|
+
|
|
3
|
+
The Joern/Fraunhofer CPG pattern applied to this analyzer: Jedi supplies the
|
|
4
|
+
fast base call graph; this pass walks *local* def-use information — lexical
|
|
5
|
+
scopes, alias chains, import bindings — to resolve what Jedi left unresolved.
|
|
6
|
+
Per callable, no global fixpoint, deterministic by construction — see
|
|
7
|
+
``docs/design/specs/2026-08-25-defuse-linker-call-graph-design.md``.
|
|
8
|
+
|
|
9
|
+
Resolution rules (validated edge-for-edge against Joern and Fraunhofer CPG on
|
|
10
|
+
the requests fixture):
|
|
11
|
+
|
|
12
|
+
- **Bare names** resolve through the lexical chain of *function* scopes out to
|
|
13
|
+
the module — class bodies are transparent, and parameters/ordinary
|
|
14
|
+
assignments shadow outer names, both exactly as at runtime. The chain covers
|
|
15
|
+
local ``def``s, alias assignments (``f = handler; f(x)``), ``from m import
|
|
16
|
+
f [as g]`` (cross-module through the symbol table), and finally Python
|
|
17
|
+
builtins (``super()``, ``len()`` → ``builtins.<name>``).
|
|
18
|
+
- **``self.X()`` / ``cls.X()``** resolves against the enclosing class and its
|
|
19
|
+
same-module base chain.
|
|
20
|
+
- **Module-alias receivers** (``import logging; logging.getLogger(...)``)
|
|
21
|
+
resolve through ``import`` bindings — to a declared function when the module
|
|
22
|
+
is in the symbol table, to a dotted external otherwise.
|
|
23
|
+
- **Constructor calls** to imported or builtin classes follow the existing
|
|
24
|
+
conventions: an in-table class becomes ``<class sig>.__init__``; anything
|
|
25
|
+
else stays a dotted external name.
|
|
26
|
+
- **Module- and class-scope call sites and decorator applications** (import
|
|
27
|
+
time work: ``getLogger`` at module level, ``@setupmethod`` in a class body)
|
|
28
|
+
are collected from the AST — the symbol table has no call sites for them —
|
|
29
|
+
and attributed to the module (#131's convention), or to the enclosing
|
|
30
|
+
function for decorators applied inside one.
|
|
31
|
+
|
|
32
|
+
Edges carry ``prov: ["defuse"]`` and merge with Jedi's via
|
|
33
|
+
``call_graph.merge_edges``. Resolutions for real callable sites are
|
|
34
|
+
*returned*, never written into ``PyCallsite.callee_signature``: the symbol
|
|
35
|
+
table round-trips through the analysis cache, and a persisted resolution
|
|
36
|
+
would resurface on the next run as a Jedi edge, silently changing provenance.
|
|
37
|
+
``l2_callees.backfill_callees`` takes the returned map instead.
|
|
38
|
+
"""
|
|
39
|
+
import ast
|
|
40
|
+
import builtins as _py_builtins
|
|
41
|
+
from typing import Dict, List, Optional, Tuple
|
|
42
|
+
|
|
43
|
+
from codeanalyzer.schema.py_schema import PyCallable, PyCallEdge, PyClass, PyModule
|
|
44
|
+
|
|
45
|
+
__all__ = ["defuse_linker_edges"]
|
|
46
|
+
|
|
47
|
+
# (caller signature, "line:col" of the call site) -> resolved callee signature
|
|
48
|
+
Resolutions = Dict[Tuple[str, str], str]
|
|
49
|
+
|
|
50
|
+
_MAX_CHAIN = 16 # assignment-chain hops before giving up (cycle safety net)
|
|
51
|
+
_BUILTINS = frozenset(dir(_py_builtins))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _is_junk_resolution(sig: Optional[str]) -> bool:
|
|
55
|
+
"""Jedi resolutions that name a *type*, not a call target.
|
|
56
|
+
|
|
57
|
+
Decorator-wrapped callables resolve to their wrapper's type —
|
|
58
|
+
``typing.Callable`` for plain decorators, ``functools._lru_cache_wrapper``
|
|
59
|
+
for ``lru_cache`` — which is an annotation, not a callee.
|
|
60
|
+
"""
|
|
61
|
+
return bool(sig) and (
|
|
62
|
+
sig.startswith("typing.")
|
|
63
|
+
or sig.startswith("functools._lru_cache")
|
|
64
|
+
or sig == "builtins.NoneType"
|
|
65
|
+
# descriptor-protocol stamps: a @classmethod/@staticmethod/@property
|
|
66
|
+
# call site resolved to the descriptor's binding machinery, not to
|
|
67
|
+
# the wrapped callable
|
|
68
|
+
or (sig.startswith("builtins.") and sig.endswith((".__get__", ".__set__")))
|
|
69
|
+
or sig.startswith("builtins.property")
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class _Scope:
|
|
74
|
+
"""One *function* scope (or the module scope). Class bodies never get one."""
|
|
75
|
+
|
|
76
|
+
__slots__ = (
|
|
77
|
+
"parent", "funcs", "bindings", "imports", "mod_imports", "blocked",
|
|
78
|
+
"literal_types", "instance_types", "call_assigns", "return_ctors",
|
|
79
|
+
"return_calls", "loopvar_sources",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def __init__(self, parent: Optional["_Scope"]) -> None:
|
|
83
|
+
self.parent = parent
|
|
84
|
+
# bare name -> name-path of a function declared in this scope
|
|
85
|
+
self.funcs: Dict[str, Tuple[str, ...]] = {}
|
|
86
|
+
# bare name -> RHS bare name of a simple alias assignment
|
|
87
|
+
self.bindings: Dict[str, str] = {}
|
|
88
|
+
# bare name -> (module dotted qual, original name) from `from m import f`
|
|
89
|
+
self.imports: Dict[str, Tuple[str, str]] = {}
|
|
90
|
+
# bare name -> dotted module from `import m[.sub] [as alias]`
|
|
91
|
+
self.mod_imports: Dict[str, str] = {}
|
|
92
|
+
# names this scope shadows opaquely (parameters, non-alias assignments)
|
|
93
|
+
self.blocked: set = set()
|
|
94
|
+
# names assigned from literals -> builtin type name ("s = \"\"" -> str)
|
|
95
|
+
self.literal_types: Dict[str, str] = {}
|
|
96
|
+
# names assigned from a constructor call -> class bare name
|
|
97
|
+
# ("jar = RequestsCookieJar()" -> "RequestsCookieJar")
|
|
98
|
+
self.instance_types: Dict[str, str] = {}
|
|
99
|
+
# names assigned from any call -> the call's func expression, for the
|
|
100
|
+
# oracle's return-summary pass ("adapter = self.get_adapter(url)")
|
|
101
|
+
self.call_assigns: Dict[str, ast.expr] = {}
|
|
102
|
+
# bare class names this scope's `return C(...)` statements construct
|
|
103
|
+
self.return_ctors: set = set()
|
|
104
|
+
# func expressions of every `return <call>(...)` in this scope, for
|
|
105
|
+
# chained return summaries (`return self.build_response(...)`)
|
|
106
|
+
self.return_calls: List[ast.expr] = []
|
|
107
|
+
# loop variables drawn from a self-attribute container:
|
|
108
|
+
# `for k, v in self.adapters.items():` -> v: ("elem", "adapters")
|
|
109
|
+
self.loopvar_sources: Dict[str, Tuple[str, str]] = {}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _module_qual(file_key: str) -> str:
|
|
113
|
+
"""Dotted module qual from a symbol-table file key (matches signatures).
|
|
114
|
+
|
|
115
|
+
``requests/api.py`` -> ``requests.api``; a package ``__init__.py`` quals to
|
|
116
|
+
the package itself. File keys are relative POSIX paths by contract.
|
|
117
|
+
"""
|
|
118
|
+
parts = file_key[:-3].split("/") if file_key.endswith(".py") else file_key.split("/")
|
|
119
|
+
if parts and parts[-1] == "__init__":
|
|
120
|
+
parts = parts[:-1]
|
|
121
|
+
return ".".join(parts)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _ModuleFacts:
|
|
125
|
+
"""Everything the walker extracts from one module's AST in a single pass."""
|
|
126
|
+
|
|
127
|
+
__slots__ = (
|
|
128
|
+
"module_scope", "by_def", "toplevel_calls", "decorators", "calls_by_scope",
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def __init__(self) -> None:
|
|
132
|
+
self.module_scope = _Scope(None)
|
|
133
|
+
# (def name, def lineno) -> the scope INSIDE that def
|
|
134
|
+
self.by_def: Dict[Tuple[str, int], _Scope] = {}
|
|
135
|
+
# (kind, node) at module or class scope (import-time calls plus
|
|
136
|
+
# f-string lowerings); kind 'call' or a builtins name
|
|
137
|
+
self.toplevel_calls: List[Tuple[str, ast.AST]] = []
|
|
138
|
+
# (decorator expr, scope it resolves in, function path of the def's
|
|
139
|
+
# container or () for module/class level)
|
|
140
|
+
self.decorators: List[Tuple[ast.expr, _Scope, Tuple[str, ...]]] = []
|
|
141
|
+
# (kind, node) inside each *function* scope, for sites Jedi's
|
|
142
|
+
# extractor never recorded (with-statement context managers, etc.)
|
|
143
|
+
self.calls_by_scope: Dict[int, List[Tuple[str, ast.AST]]] = {}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _collect(tree: ast.Module, module_qual: str) -> _ModuleFacts:
|
|
147
|
+
facts = _ModuleFacts()
|
|
148
|
+
|
|
149
|
+
def record_decorators(node, scope: _Scope, container: Tuple[str, ...]) -> None:
|
|
150
|
+
for dec in node.decorator_list:
|
|
151
|
+
expr = dec.func if isinstance(dec, ast.Call) else dec
|
|
152
|
+
facts.decorators.append((expr, scope, container))
|
|
153
|
+
|
|
154
|
+
def walk(node: ast.AST, scope: _Scope, path: Tuple[str, ...], in_class: bool,
|
|
155
|
+
container: Tuple[str, ...]) -> None:
|
|
156
|
+
for child in ast.iter_child_nodes(node):
|
|
157
|
+
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
158
|
+
child_path = path + (child.name,)
|
|
159
|
+
if not in_class:
|
|
160
|
+
scope.funcs[child.name] = child_path
|
|
161
|
+
record_decorators(child, scope, container)
|
|
162
|
+
inner = _Scope(scope)
|
|
163
|
+
for arg_field in ("args", "posonlyargs", "kwonlyargs"):
|
|
164
|
+
for a in getattr(child.args, arg_field, []) or []:
|
|
165
|
+
inner.blocked.add(a.arg)
|
|
166
|
+
for a in (child.args.vararg, child.args.kwarg):
|
|
167
|
+
if a is not None:
|
|
168
|
+
inner.blocked.add(a.arg)
|
|
169
|
+
facts.by_def[(child.name, child.lineno)] = inner
|
|
170
|
+
facts.calls_by_scope[id(inner)] = list(_scope_expressions(child))
|
|
171
|
+
walk(child, inner, child_path, in_class=False, container=child_path)
|
|
172
|
+
elif isinstance(child, ast.ClassDef):
|
|
173
|
+
record_decorators(child, scope, container)
|
|
174
|
+
# Transparent for bare-name lookup: methods hang off the same
|
|
175
|
+
# enclosing function scope, and class-body names are invisible
|
|
176
|
+
# to them (Python's own rule).
|
|
177
|
+
walk(child, scope, path + (child.name,), in_class=True,
|
|
178
|
+
container=container)
|
|
179
|
+
else:
|
|
180
|
+
if not in_class:
|
|
181
|
+
_record_stmt(child, scope)
|
|
182
|
+
walk(child, scope, path, in_class=in_class, container=container)
|
|
183
|
+
|
|
184
|
+
facts.toplevel_calls = list(_scope_expressions(tree))
|
|
185
|
+
walk(tree, facts.module_scope, (), in_class=False, container=())
|
|
186
|
+
return facts
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _scope_expressions(node: ast.AST):
|
|
190
|
+
"""Yield ``(kind, node)`` for calls and f-string conversions executed in
|
|
191
|
+
*node*'s own scope — descending into class bodies (they execute in the
|
|
192
|
+
enclosing scope) but never into nested ``def`` bodies.
|
|
193
|
+
|
|
194
|
+
Kinds: ``("call", ast.Call)``, ``("repr" | "str" | "ascii" | "format",
|
|
195
|
+
ast.FormattedValue)`` — CPython lowers f-string conversions and format
|
|
196
|
+
specs to ``repr()``/``str()``/``ascii()``/``format()`` calls, and the
|
|
197
|
+
reference CPG tools (Joern, Fraunhofer) emit those edges.
|
|
198
|
+
"""
|
|
199
|
+
root = node
|
|
200
|
+
stack = [node]
|
|
201
|
+
while stack:
|
|
202
|
+
cur = stack.pop()
|
|
203
|
+
if cur is not root and isinstance(
|
|
204
|
+
cur, (ast.FunctionDef, ast.AsyncFunctionDef)
|
|
205
|
+
):
|
|
206
|
+
continue
|
|
207
|
+
if isinstance(cur, ast.Call):
|
|
208
|
+
yield ("call", cur)
|
|
209
|
+
elif isinstance(cur, ast.Compare) and any(
|
|
210
|
+
isinstance(op, (ast.Eq, ast.NotEq)) for op in cur.ops
|
|
211
|
+
):
|
|
212
|
+
# `a == b` / `a != b` dispatches through __eq__/__ne__ at runtime;
|
|
213
|
+
# only self-rooted comparisons are resolvable locally, and the
|
|
214
|
+
# reference CPG tools emit exactly those.
|
|
215
|
+
if isinstance(cur.left, ast.Name) and cur.left.id in ("self", "cls"):
|
|
216
|
+
yield ("selfeq", cur)
|
|
217
|
+
elif isinstance(cur, (ast.For, ast.AsyncFor)):
|
|
218
|
+
# `for x in y:` calls y.__iter__() at runtime; resolvable when
|
|
219
|
+
# y's type is locally known (the reference tools lower this too).
|
|
220
|
+
yield ("iter", cur.iter)
|
|
221
|
+
elif isinstance(cur, ast.comprehension):
|
|
222
|
+
yield ("iter", cur.iter)
|
|
223
|
+
elif isinstance(cur, ast.FormattedValue):
|
|
224
|
+
conv = {114: "repr", 115: "str", 97: "ascii"}.get(cur.conversion)
|
|
225
|
+
if conv is not None:
|
|
226
|
+
yield (conv, cur)
|
|
227
|
+
if cur.format_spec is not None:
|
|
228
|
+
yield ("format", cur)
|
|
229
|
+
stack.extend(reversed(list(ast.iter_child_nodes(cur))))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
_LITERAL_TYPES = {
|
|
233
|
+
ast.List: "list", ast.ListComp: "list",
|
|
234
|
+
ast.Dict: "dict", ast.DictComp: "dict",
|
|
235
|
+
ast.Set: "set", ast.SetComp: "set",
|
|
236
|
+
ast.Tuple: "tuple", ast.JoinedStr: "str",
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _literal_type(value: ast.expr) -> Optional[str]:
|
|
241
|
+
t = _LITERAL_TYPES.get(type(value))
|
|
242
|
+
if t is not None:
|
|
243
|
+
return t
|
|
244
|
+
if isinstance(value, ast.Constant):
|
|
245
|
+
v = value.value
|
|
246
|
+
if isinstance(v, str):
|
|
247
|
+
return "str"
|
|
248
|
+
if isinstance(v, bytes):
|
|
249
|
+
return "bytes"
|
|
250
|
+
if isinstance(v, bool):
|
|
251
|
+
return "bool"
|
|
252
|
+
if isinstance(v, int):
|
|
253
|
+
return "int"
|
|
254
|
+
if isinstance(v, float):
|
|
255
|
+
return "float"
|
|
256
|
+
return None
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _record_stmt(stmt: ast.AST, scope: _Scope) -> None:
|
|
260
|
+
"""Record one statement's contribution to *scope*'s name table."""
|
|
261
|
+
if isinstance(stmt, ast.ImportFrom):
|
|
262
|
+
_record_import_from(stmt, scope)
|
|
263
|
+
elif isinstance(stmt, ast.Import):
|
|
264
|
+
for alias in stmt.names:
|
|
265
|
+
if alias.asname:
|
|
266
|
+
scope.mod_imports[alias.asname] = alias.name
|
|
267
|
+
else:
|
|
268
|
+
# `import a.b` binds only `a`; attribute chains rooted at `a`
|
|
269
|
+
# re-append the rest.
|
|
270
|
+
root = alias.name.split(".", 1)[0]
|
|
271
|
+
scope.mod_imports[root] = root
|
|
272
|
+
elif isinstance(stmt, ast.Assign):
|
|
273
|
+
for tgt in stmt.targets:
|
|
274
|
+
if isinstance(tgt, ast.Name):
|
|
275
|
+
if isinstance(stmt.value, ast.Name):
|
|
276
|
+
scope.bindings[tgt.id] = stmt.value.id
|
|
277
|
+
else:
|
|
278
|
+
scope.blocked.add(tgt.id)
|
|
279
|
+
lt = _literal_type(stmt.value)
|
|
280
|
+
if lt is not None:
|
|
281
|
+
scope.literal_types[tgt.id] = lt
|
|
282
|
+
elif isinstance(stmt.value, ast.Call):
|
|
283
|
+
if isinstance(stmt.value.func, ast.Name):
|
|
284
|
+
scope.instance_types[tgt.id] = stmt.value.func.id
|
|
285
|
+
scope.call_assigns[tgt.id] = stmt.value.func
|
|
286
|
+
elif isinstance(stmt, (ast.For, ast.AsyncFor)):
|
|
287
|
+
attr = _self_container_of(stmt.iter)
|
|
288
|
+
if attr is not None:
|
|
289
|
+
targets = (
|
|
290
|
+
stmt.target.elts if isinstance(stmt.target, ast.Tuple) else [stmt.target]
|
|
291
|
+
)
|
|
292
|
+
# the value position: last element of a tuple target (items()),
|
|
293
|
+
# or the single target (values()/direct iteration)
|
|
294
|
+
val = targets[-1]
|
|
295
|
+
if isinstance(val, ast.Name):
|
|
296
|
+
scope.loopvar_sources[val.id] = ("elem", attr)
|
|
297
|
+
elif isinstance(stmt, ast.Return):
|
|
298
|
+
if stmt.value is not None:
|
|
299
|
+
if isinstance(stmt.value, ast.Call):
|
|
300
|
+
scope.return_calls.append(stmt.value.func)
|
|
301
|
+
if isinstance(stmt.value.func, ast.Name):
|
|
302
|
+
scope.return_ctors.add(stmt.value.func.id)
|
|
303
|
+
elif isinstance(stmt.value, ast.Name):
|
|
304
|
+
# `cj = RequestsCookieJar(); ...; return cj` — resolved when
|
|
305
|
+
# the summary is read, against this scope's ctor-typed locals.
|
|
306
|
+
scope.return_ctors.add("~" + stmt.value.id)
|
|
307
|
+
elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
|
|
308
|
+
if isinstance(stmt.value, ast.Name):
|
|
309
|
+
scope.bindings[stmt.target.id] = stmt.value.id
|
|
310
|
+
elif stmt.value is not None:
|
|
311
|
+
scope.blocked.add(stmt.target.id)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _self_container_of(expr: ast.expr) -> Optional[str]:
|
|
315
|
+
"""``self.X`` / ``self.X.items()`` / ``self.X.values()`` -> ``"X"``."""
|
|
316
|
+
node = expr
|
|
317
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
318
|
+
if node.func.attr not in ("items", "values"):
|
|
319
|
+
return None
|
|
320
|
+
node = node.func.value
|
|
321
|
+
if (
|
|
322
|
+
isinstance(node, ast.Attribute)
|
|
323
|
+
and isinstance(node.value, ast.Name)
|
|
324
|
+
and node.value.id == "self"
|
|
325
|
+
):
|
|
326
|
+
return node.attr
|
|
327
|
+
return None
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _record_import_from(node: ast.ImportFrom, scope: _Scope) -> None:
|
|
331
|
+
"""``from m import f [as g]``; relative spellings resolve at lookup time.
|
|
332
|
+
|
|
333
|
+
The dotted target is stored with the leading-dots convention already
|
|
334
|
+
resolved by the caller via ``module_qual`` — see ``_collect`` usage."""
|
|
335
|
+
# module_qual-relative resolution happens in _resolve_import_target; here
|
|
336
|
+
# the raw (level, module) pair is packed into the stored qual.
|
|
337
|
+
prefix = "." * node.level
|
|
338
|
+
target = prefix + (node.module or "")
|
|
339
|
+
for alias in node.names:
|
|
340
|
+
if alias.name == "*":
|
|
341
|
+
continue
|
|
342
|
+
scope.imports[alias.asname or alias.name] = (target, alias.name)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _absolute_module(spelled: str, module_qual: str) -> Optional[str]:
|
|
346
|
+
"""Resolve a possibly-relative import spelling to a dotted module qual."""
|
|
347
|
+
if not spelled.startswith("."):
|
|
348
|
+
return spelled or None
|
|
349
|
+
level = len(spelled) - len(spelled.lstrip("."))
|
|
350
|
+
rest = spelled[level:]
|
|
351
|
+
base = module_qual.split(".")
|
|
352
|
+
base = base[: len(base) - level]
|
|
353
|
+
parts = base + ([rest] if rest else [])
|
|
354
|
+
joined = ".".join(p for p in parts if p)
|
|
355
|
+
return joined or None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# Typed resolution results
|
|
359
|
+
_LOCAL, _FROM, _MODALIAS, _BUILTIN = "local", "from", "modalias", "builtin"
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _resolve_name(name: str, scope: Optional[_Scope]) -> Optional[Tuple]:
|
|
363
|
+
"""Chase *name* through scopes and alias bindings to a typed target."""
|
|
364
|
+
for _ in range(_MAX_CHAIN):
|
|
365
|
+
s = scope
|
|
366
|
+
while s is not None:
|
|
367
|
+
if name in s.funcs:
|
|
368
|
+
return (_LOCAL, s.funcs[name])
|
|
369
|
+
if name in s.imports:
|
|
370
|
+
return (_FROM,) + s.imports[name]
|
|
371
|
+
if name in s.mod_imports:
|
|
372
|
+
return (_MODALIAS, s.mod_imports[name])
|
|
373
|
+
if name in s.blocked:
|
|
374
|
+
return None
|
|
375
|
+
if name in s.bindings:
|
|
376
|
+
break
|
|
377
|
+
s = s.parent
|
|
378
|
+
if s is None:
|
|
379
|
+
return (_BUILTIN, name) if name in _BUILTINS else None
|
|
380
|
+
# A binding's RHS resolves from the scope the assignment sits in.
|
|
381
|
+
name, scope = s.bindings[name], s
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _signature_for_path(mod: PyModule, path: Tuple[str, ...]) -> Optional[str]:
|
|
386
|
+
"""Navigate the symbol table by name path; return the callable's signature.
|
|
387
|
+
|
|
388
|
+
Paths from the AST walker interleave function AND class names
|
|
389
|
+
(``("Response", "iter_content", "generate")`` for a def nested in a
|
|
390
|
+
method), so each step tries the current container's functions, nested
|
|
391
|
+
callables, and classes.
|
|
392
|
+
"""
|
|
393
|
+
if not path:
|
|
394
|
+
return None
|
|
395
|
+
node = None # PyCallable | PyClass
|
|
396
|
+
container_fns = mod.functions or {}
|
|
397
|
+
container_classes = {c.name: c for c in (mod.types or {}).values()}
|
|
398
|
+
for name in path:
|
|
399
|
+
if node is None:
|
|
400
|
+
node = container_fns.get(name) or container_classes.get(name)
|
|
401
|
+
elif isinstance(node, PyClass):
|
|
402
|
+
node = (node.callables or {}).get(name) or {
|
|
403
|
+
c.name: c for c in (node.types or {}).values()
|
|
404
|
+
}.get(name)
|
|
405
|
+
else:
|
|
406
|
+
node = (node.callables or {}).get(name) or {
|
|
407
|
+
c.name: c for c in (node.types or {}).values()
|
|
408
|
+
}.get(name)
|
|
409
|
+
if node is None:
|
|
410
|
+
return None
|
|
411
|
+
return node.signature if isinstance(node, PyCallable) else None
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _class_in_module(mod: PyModule, name: str) -> Optional[PyClass]:
|
|
415
|
+
for cls in sorted((mod.types or {}).values(), key=lambda c: c.name):
|
|
416
|
+
if cls.name == name:
|
|
417
|
+
return cls
|
|
418
|
+
return None
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _target_signature(
|
|
422
|
+
target: Tuple,
|
|
423
|
+
mod: PyModule,
|
|
424
|
+
module_qual: str,
|
|
425
|
+
by_qual: Dict[str, PyModule],
|
|
426
|
+
attrs: Tuple[str, ...] = (),
|
|
427
|
+
) -> Optional[str]:
|
|
428
|
+
"""Map a typed resolution (plus trailing attributes) to a callee signature.
|
|
429
|
+
|
|
430
|
+
Follows the analyzer's existing conventions: declared functions by their
|
|
431
|
+
symbol-table signature; in-table classes as ``<sig>.__init__`` (a call of
|
|
432
|
+
a class is its constructor); everything else as a dotted external name,
|
|
433
|
+
which the pipeline homes under ``@external`` ids downstream.
|
|
434
|
+
"""
|
|
435
|
+
kind = target[0]
|
|
436
|
+
if kind == _LOCAL:
|
|
437
|
+
if attrs:
|
|
438
|
+
return None
|
|
439
|
+
return _signature_for_path(mod, target[1])
|
|
440
|
+
if kind == _BUILTIN:
|
|
441
|
+
if attrs:
|
|
442
|
+
return None
|
|
443
|
+
return f"builtins.{target[1]}"
|
|
444
|
+
if kind == _FROM:
|
|
445
|
+
spelled, orig = target[1], target[2]
|
|
446
|
+
src_qual = _absolute_module(spelled, module_qual)
|
|
447
|
+
if src_qual is None:
|
|
448
|
+
return None
|
|
449
|
+
src_mod = by_qual.get(src_qual)
|
|
450
|
+
if src_mod is not None and not attrs:
|
|
451
|
+
fn = (src_mod.functions or {}).get(orig)
|
|
452
|
+
if fn is not None:
|
|
453
|
+
return fn.signature
|
|
454
|
+
cls = _class_in_module(src_mod, orig)
|
|
455
|
+
if cls is not None:
|
|
456
|
+
return f"{cls.signature}.__init__"
|
|
457
|
+
return ".".join((src_qual, orig) + attrs)
|
|
458
|
+
if kind == _MODALIAS:
|
|
459
|
+
dotted = target[1]
|
|
460
|
+
if not attrs:
|
|
461
|
+
return None # a bare module reference is not callable
|
|
462
|
+
qual = ".".join((dotted,) + attrs[:-1])
|
|
463
|
+
leaf = attrs[-1]
|
|
464
|
+
src_mod = by_qual.get(qual)
|
|
465
|
+
if src_mod is not None:
|
|
466
|
+
fn = (src_mod.functions or {}).get(leaf)
|
|
467
|
+
if fn is not None:
|
|
468
|
+
return fn.signature
|
|
469
|
+
cls = _class_in_module(src_mod, leaf)
|
|
470
|
+
if cls is not None:
|
|
471
|
+
return f"{cls.signature}.__init__"
|
|
472
|
+
return ".".join((dotted,) + attrs)
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _resolve_expr(
|
|
477
|
+
expr: ast.expr,
|
|
478
|
+
scope: _Scope,
|
|
479
|
+
mod: PyModule,
|
|
480
|
+
module_qual: str,
|
|
481
|
+
by_qual: Dict[str, PyModule],
|
|
482
|
+
) -> Optional[str]:
|
|
483
|
+
"""Resolve a Name / dotted-Attribute expression to a callee signature."""
|
|
484
|
+
attrs: List[str] = []
|
|
485
|
+
node = expr
|
|
486
|
+
while isinstance(node, ast.Attribute):
|
|
487
|
+
attrs.append(node.attr)
|
|
488
|
+
node = node.value
|
|
489
|
+
if not isinstance(node, ast.Name):
|
|
490
|
+
lt = _literal_type(node) if isinstance(node, ast.expr) else None
|
|
491
|
+
if lt is not None and len(attrs) == 1:
|
|
492
|
+
return f"builtins.{lt}.{attrs[0]}"
|
|
493
|
+
return None
|
|
494
|
+
attrs_t = tuple(reversed(attrs))
|
|
495
|
+
target = _resolve_name(node.id, scope)
|
|
496
|
+
if target is None:
|
|
497
|
+
return None
|
|
498
|
+
return _target_signature(target, mod, module_qual, by_qual, attrs_t)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _iter_callables(mod: PyModule):
|
|
502
|
+
"""Every callable in *mod* with its enclosing class (None for functions)."""
|
|
503
|
+
|
|
504
|
+
def from_callable(c: PyCallable, owner: Optional[PyClass]):
|
|
505
|
+
yield c, owner
|
|
506
|
+
for nested in (c.callables or {}).values():
|
|
507
|
+
# A def nested inside a method closes over the method's `self`;
|
|
508
|
+
# inheriting the owner is the right may-call approximation.
|
|
509
|
+
yield from from_callable(nested, owner)
|
|
510
|
+
for cls in (c.types or {}).values():
|
|
511
|
+
yield from from_class(cls)
|
|
512
|
+
|
|
513
|
+
def from_class(cls: PyClass):
|
|
514
|
+
for m in (cls.callables or {}).values():
|
|
515
|
+
yield from from_callable(m, cls)
|
|
516
|
+
for inner in (cls.types or {}).values():
|
|
517
|
+
yield from from_class(inner)
|
|
518
|
+
|
|
519
|
+
for fn in (mod.functions or {}).values():
|
|
520
|
+
yield from from_callable(fn, None)
|
|
521
|
+
for cls in (mod.types or {}).values():
|
|
522
|
+
yield from from_class(cls)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _classes_by_name(mod: PyModule) -> Dict[str, PyClass]:
|
|
526
|
+
"""Every class in *mod* keyed by bare name (first wins, sorted walk)."""
|
|
527
|
+
out: Dict[str, PyClass] = {}
|
|
528
|
+
|
|
529
|
+
def add(cls: PyClass) -> None:
|
|
530
|
+
out.setdefault(cls.name, cls)
|
|
531
|
+
for inner in sorted((cls.types or {}).values(), key=lambda c: c.name):
|
|
532
|
+
add(inner)
|
|
533
|
+
for m in (cls.callables or {}).values():
|
|
534
|
+
for nested_cls in sorted((m.types or {}).values(), key=lambda c: c.name):
|
|
535
|
+
add(nested_cls)
|
|
536
|
+
|
|
537
|
+
for cls in sorted((mod.types or {}).values(), key=lambda c: c.name):
|
|
538
|
+
add(cls)
|
|
539
|
+
return out
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _resolve_self_call(
|
|
543
|
+
method_name: str,
|
|
544
|
+
owner: PyClass,
|
|
545
|
+
classes: Dict[str, PyClass],
|
|
546
|
+
module_scope: Optional[_Scope] = None,
|
|
547
|
+
mod: Optional[PyModule] = None,
|
|
548
|
+
module_qual: str = "",
|
|
549
|
+
by_qual: Optional[Dict[str, PyModule]] = None,
|
|
550
|
+
global_classes: Optional[Dict[str, List[PyClass]]] = None,
|
|
551
|
+
) -> Optional[str]:
|
|
552
|
+
"""Resolve ``self.X()`` against *owner*, its bases, then its subclasses.
|
|
553
|
+
|
|
554
|
+
The base chain is ordinary lookup. The subclass fallback covers the mixin
|
|
555
|
+
pattern: ``SessionRedirectMixin.resolve_redirects`` calls ``self.send``,
|
|
556
|
+
declared only on ``Session(SessionRedirectMixin)`` — every runtime
|
|
557
|
+
``self`` inside the mixin is an instance of a subclass, so a method
|
|
558
|
+
declared by exactly one same-module subclass is the real target (the
|
|
559
|
+
reference CPG tools fabricate an inferred stub on the mixin instead;
|
|
560
|
+
resolving to the declaring subclass is strictly more truthful). Ambiguous
|
|
561
|
+
fan-out (several subclasses declare it) resolves to nothing.
|
|
562
|
+
"""
|
|
563
|
+
seen = set()
|
|
564
|
+
queue = [owner]
|
|
565
|
+
while queue:
|
|
566
|
+
cls = queue.pop(0)
|
|
567
|
+
if id(cls) in seen:
|
|
568
|
+
continue
|
|
569
|
+
seen.add(id(cls))
|
|
570
|
+
target = (cls.callables or {}).get(method_name)
|
|
571
|
+
if target is not None:
|
|
572
|
+
return target.signature
|
|
573
|
+
for base in cls.base_classes or []:
|
|
574
|
+
base_cls = classes.get(base)
|
|
575
|
+
if base_cls is None and global_classes is not None:
|
|
576
|
+
# `class MyCase(TransactionCase)` with the base declared in
|
|
577
|
+
# another module — follow it through the global index when
|
|
578
|
+
# the bare name is unambiguous
|
|
579
|
+
cands = global_classes.get(base.rsplit(".", 1)[-1]) or []
|
|
580
|
+
if len(cands) == 1:
|
|
581
|
+
base_cls = cands[0]
|
|
582
|
+
if base_cls is not None:
|
|
583
|
+
queue.append(base_cls)
|
|
584
|
+
declaring = [
|
|
585
|
+
cls
|
|
586
|
+
for name, cls in sorted(classes.items())
|
|
587
|
+
if owner.name in (cls.base_classes or []) and method_name in (cls.callables or {})
|
|
588
|
+
]
|
|
589
|
+
if len(declaring) == 1:
|
|
590
|
+
return declaring[0].callables[method_name].signature
|
|
591
|
+
|
|
592
|
+
if module_scope is None or by_qual is None or mod is None:
|
|
593
|
+
return None
|
|
594
|
+
|
|
595
|
+
# Class-attribute indirection: `self.response_class(...)` where
|
|
596
|
+
# `response_class = Response` is a class attribute — resolve through the
|
|
597
|
+
# attribute's initializer to the real target (the reference tools stop at
|
|
598
|
+
# the attribute name; the initializer's target is the actual callee).
|
|
599
|
+
stack = [owner]
|
|
600
|
+
visited = set()
|
|
601
|
+
while stack:
|
|
602
|
+
cls = stack.pop(0)
|
|
603
|
+
if id(cls) in visited:
|
|
604
|
+
continue
|
|
605
|
+
visited.add(id(cls))
|
|
606
|
+
attr = (cls.attributes or {}).get(method_name)
|
|
607
|
+
if attr is not None and attr.initializer:
|
|
608
|
+
try:
|
|
609
|
+
expr = ast.parse(attr.initializer.strip(), mode="eval").body
|
|
610
|
+
except SyntaxError:
|
|
611
|
+
expr = None
|
|
612
|
+
if expr is not None:
|
|
613
|
+
if isinstance(expr, ast.Call):
|
|
614
|
+
# `path_type = click.Path(...)` — the attribute holds an
|
|
615
|
+
# instance; a call through it targets that type
|
|
616
|
+
expr = expr.func
|
|
617
|
+
sig = _resolve_expr(expr, module_scope, mod, module_qual, by_qual)
|
|
618
|
+
if sig is None and isinstance(expr, ast.Name):
|
|
619
|
+
target_cls = classes.get(expr.id)
|
|
620
|
+
if target_cls is not None:
|
|
621
|
+
sig = f"{target_cls.signature}.__init__"
|
|
622
|
+
if sig is not None:
|
|
623
|
+
return sig
|
|
624
|
+
for base in cls.base_classes or []:
|
|
625
|
+
base_cls = classes.get(base)
|
|
626
|
+
if base_cls is not None:
|
|
627
|
+
stack.append(base_cls)
|
|
628
|
+
|
|
629
|
+
# Inherited from an imported base: `class FlaskClient(Client)` with
|
|
630
|
+
# `from werkzeug.test import Client` — the method lives on the external
|
|
631
|
+
# base, so name it there instead of fabricating a stub on the subclass.
|
|
632
|
+
stack, visited = [owner], set()
|
|
633
|
+
while stack:
|
|
634
|
+
cls = stack.pop(0)
|
|
635
|
+
if id(cls) in visited:
|
|
636
|
+
continue
|
|
637
|
+
visited.add(id(cls))
|
|
638
|
+
for base in cls.base_classes or []:
|
|
639
|
+
base_cls = classes.get(base)
|
|
640
|
+
if base_cls is not None:
|
|
641
|
+
stack.append(base_cls)
|
|
642
|
+
continue
|
|
643
|
+
if "." in base:
|
|
644
|
+
# dotted spelling (`class X(click.Path)`) — resolve the root
|
|
645
|
+
# through the module scope, append the rest
|
|
646
|
+
root, *restp = base.split(".")
|
|
647
|
+
target = _resolve_name(root, module_scope)
|
|
648
|
+
if target is not None and target[0] in (_MODALIAS, _FROM):
|
|
649
|
+
dotted = _target_signature(
|
|
650
|
+
target, mod, module_qual, by_qual, tuple(restp)
|
|
651
|
+
)
|
|
652
|
+
if dotted:
|
|
653
|
+
return f"{dotted}.{method_name}"
|
|
654
|
+
continue
|
|
655
|
+
target = _resolve_name(base, module_scope)
|
|
656
|
+
if target is None or target[0] == _LOCAL:
|
|
657
|
+
continue
|
|
658
|
+
dotted = _target_signature(target, mod, module_qual, by_qual)
|
|
659
|
+
if dotted:
|
|
660
|
+
return f"{dotted}.{method_name}"
|
|
661
|
+
return None
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _scope_for_callable(
|
|
665
|
+
c: PyCallable, by_def: Dict[Tuple[str, int], _Scope], module_scope: _Scope
|
|
666
|
+
) -> _Scope:
|
|
667
|
+
for line in (c.code_start_line, c.start_line):
|
|
668
|
+
scope = by_def.get((c.name, line))
|
|
669
|
+
if scope is not None:
|
|
670
|
+
return scope
|
|
671
|
+
# Decorated defs: PyCallable lines may point at the first decorator, and
|
|
672
|
+
# odoo-style stacks (`@http.route(...)` spanning many lines) push the
|
|
673
|
+
# `def` far below it — scan the callable's whole span for the def line.
|
|
674
|
+
end = c.end_line if c.end_line and c.end_line > c.start_line else c.start_line + 64
|
|
675
|
+
for line in range(c.start_line + 1, min(end, c.start_line + 256) + 1):
|
|
676
|
+
scope = by_def.get((c.name, line))
|
|
677
|
+
if scope is not None:
|
|
678
|
+
return scope
|
|
679
|
+
return module_scope
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def _receiver_target(
|
|
683
|
+
site_receiver: str,
|
|
684
|
+
method_name: str,
|
|
685
|
+
scope: _Scope,
|
|
686
|
+
mod: PyModule,
|
|
687
|
+
module_qual: str,
|
|
688
|
+
by_qual: Dict[str, PyModule],
|
|
689
|
+
classes: Optional[Dict[str, PyClass]] = None,
|
|
690
|
+
global_classes: Optional[Dict[str, List[PyClass]]] = None,
|
|
691
|
+
) -> Optional[str]:
|
|
692
|
+
"""Resolve ``recv.method()`` when ``recv`` is (rooted at) a module alias."""
|
|
693
|
+
lt = _literal_receiver_type(site_receiver)
|
|
694
|
+
if lt is not None:
|
|
695
|
+
return f"builtins.{lt}.{method_name}"
|
|
696
|
+
parts = tuple(p for p in site_receiver.split(".") if p)
|
|
697
|
+
if not parts:
|
|
698
|
+
return None
|
|
699
|
+
if len(parts) == 1:
|
|
700
|
+
lt = _lookup_literal_type(parts[0], scope)
|
|
701
|
+
if lt is not None:
|
|
702
|
+
return f"builtins.{lt}.{method_name}"
|
|
703
|
+
cls_name = _lookup_instance_type(parts[0], scope)
|
|
704
|
+
if cls_name is not None and classes is not None:
|
|
705
|
+
target_cls = classes.get(cls_name)
|
|
706
|
+
if target_cls is not None:
|
|
707
|
+
sig = _resolve_self_call(
|
|
708
|
+
method_name, target_cls, classes,
|
|
709
|
+
global_classes=global_classes,
|
|
710
|
+
)
|
|
711
|
+
if sig is not None:
|
|
712
|
+
return sig
|
|
713
|
+
target = _resolve_name(parts[0], scope)
|
|
714
|
+
if target is None or target[0] not in (_MODALIAS, _FROM):
|
|
715
|
+
return None
|
|
716
|
+
return _target_signature(
|
|
717
|
+
target, mod, module_qual, by_qual, parts[1:] + (method_name,)
|
|
718
|
+
)
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _literal_receiver_type(receiver_src: str) -> Optional[str]:
|
|
722
|
+
"""Type of a receiver whose source text is itself a literal (``''.join``)."""
|
|
723
|
+
try:
|
|
724
|
+
expr = ast.parse(receiver_src.strip(), mode="eval").body
|
|
725
|
+
except SyntaxError:
|
|
726
|
+
return None
|
|
727
|
+
return _literal_type(expr)
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def _lookup_instance_type(name: str, scope: Optional[_Scope]) -> Optional[str]:
|
|
731
|
+
"""Nearest-scope constructor-assignment type for *name*, if any."""
|
|
732
|
+
s = scope
|
|
733
|
+
while s is not None:
|
|
734
|
+
if name in s.instance_types:
|
|
735
|
+
return s.instance_types[name]
|
|
736
|
+
if (
|
|
737
|
+
name in s.blocked
|
|
738
|
+
or name in s.bindings
|
|
739
|
+
or name in s.funcs
|
|
740
|
+
or name in s.imports
|
|
741
|
+
or name in s.mod_imports
|
|
742
|
+
):
|
|
743
|
+
return None
|
|
744
|
+
s = s.parent
|
|
745
|
+
return None
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def _lookup_literal_type(name: str, scope: Optional[_Scope]) -> Optional[str]:
|
|
749
|
+
"""Nearest scope that binds *name* decides; a literal binding yields a type."""
|
|
750
|
+
s = scope
|
|
751
|
+
while s is not None:
|
|
752
|
+
if name in s.literal_types:
|
|
753
|
+
return s.literal_types[name]
|
|
754
|
+
if (
|
|
755
|
+
name in s.blocked
|
|
756
|
+
or name in s.bindings
|
|
757
|
+
or name in s.funcs
|
|
758
|
+
or name in s.imports
|
|
759
|
+
or name in s.mod_imports
|
|
760
|
+
):
|
|
761
|
+
return None
|
|
762
|
+
s = s.parent
|
|
763
|
+
return None
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _resolve_uncovered_call(
|
|
767
|
+
call: ast.Call,
|
|
768
|
+
scope: _Scope,
|
|
769
|
+
owner: Optional[PyClass],
|
|
770
|
+
classes: Dict[str, PyClass],
|
|
771
|
+
mod: PyModule,
|
|
772
|
+
module_qual: str,
|
|
773
|
+
by_qual: Dict[str, PyModule],
|
|
774
|
+
global_classes: Optional[Dict[str, List[PyClass]]] = None,
|
|
775
|
+
) -> Optional[str]:
|
|
776
|
+
"""Resolve an AST call that has no recorded ``PyCallsite``."""
|
|
777
|
+
func = call.func
|
|
778
|
+
if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
|
|
779
|
+
root = func.value.id
|
|
780
|
+
if root in ("self", "cls") and owner is not None:
|
|
781
|
+
return _resolve_self_call(
|
|
782
|
+
func.attr, owner, classes, scope, mod, module_qual, by_qual,
|
|
783
|
+
global_classes=global_classes,
|
|
784
|
+
)
|
|
785
|
+
lt = _lookup_literal_type(root, scope)
|
|
786
|
+
if lt is not None:
|
|
787
|
+
return f"builtins.{lt}.{func.attr}"
|
|
788
|
+
elif (
|
|
789
|
+
isinstance(func, ast.Attribute)
|
|
790
|
+
and isinstance(func.value, ast.Call)
|
|
791
|
+
and isinstance(func.value.func, ast.Name)
|
|
792
|
+
and func.value.func.id in _BUILTINS
|
|
793
|
+
):
|
|
794
|
+
# method on a builtin temporary: `TypeError(...).with_traceback(tb)`
|
|
795
|
+
return f"builtins.{func.value.func.id}.{func.attr}"
|
|
796
|
+
return _resolve_expr(func, scope, mod, module_qual, by_qual)
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
class _SyntheticSite:
|
|
800
|
+
"""A call the AST shows but Jedi recorded no ``PyCallsite`` for.
|
|
801
|
+
|
|
802
|
+
Shaped like the slice of ``PyCallsite`` the pending loop reads; its
|
|
803
|
+
position never matches an L1 body node, so a resolutions entry for it is
|
|
804
|
+
inert by construction.
|
|
805
|
+
"""
|
|
806
|
+
|
|
807
|
+
__slots__ = ("method_name", "receiver_expr", "start_line", "start_column")
|
|
808
|
+
|
|
809
|
+
def __init__(self, method_name, receiver_expr, line, col):
|
|
810
|
+
self.method_name = method_name
|
|
811
|
+
self.receiver_expr = receiver_expr
|
|
812
|
+
self.start_line = line
|
|
813
|
+
self.start_column = col
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
class _TypeOracle:
|
|
817
|
+
"""One deterministic interprocedural round of receiver typing (#148).
|
|
818
|
+
|
|
819
|
+
Everything is derived from the symbol table plus one AST pass per module,
|
|
820
|
+
computed once and consulted in a strict order — no fixpoint:
|
|
821
|
+
|
|
822
|
+
1. the caller's own parameter ``type`` (Jedi fills these from defaults
|
|
823
|
+
and annotations);
|
|
824
|
+
2. cross-site propagation: for every call site whose callee is already
|
|
825
|
+
resolved (Jedi stamp or the local pass), each positional argument's
|
|
826
|
+
``inferred_type`` votes for the callee parameter's type — a parameter
|
|
827
|
+
with exactly one internal-class candidate is typed;
|
|
828
|
+
3. a return summary of the assigned call (unique ``return C(...)`` /
|
|
829
|
+
``return self.attr`` of a known type inside the target callable);
|
|
830
|
+
4. ``self.attr`` instance-attribute types collected from ``self.X = C()``
|
|
831
|
+
and ``self.X = <literal>`` assignments anywhere in the class.
|
|
832
|
+
|
|
833
|
+
The vocabulary of results is ("class", PyClass) or ("builtin", name).
|
|
834
|
+
"""
|
|
835
|
+
|
|
836
|
+
def __init__(self) -> None:
|
|
837
|
+
self.classes_global: Dict[str, List[PyClass]] = {}
|
|
838
|
+
# bare callable name -> sorted signatures of every internal callable
|
|
839
|
+
# with that name (methods and functions alike) — the name-linked tier
|
|
840
|
+
self.by_name: Dict[str, List[str]] = {}
|
|
841
|
+
self.func_by_sig: Dict[str, PyCallable] = {}
|
|
842
|
+
self.param_names: Dict[str, List[str]] = {}
|
|
843
|
+
self.param_declared: Dict[Tuple[str, str], str] = {}
|
|
844
|
+
self.param_votes: Dict[Tuple[str, int], set] = {}
|
|
845
|
+
self.self_attr: Dict[Tuple[str, str], Tuple[str, str]] = {}
|
|
846
|
+
self.return_class: Dict[str, Optional[str]] = {}
|
|
847
|
+
self.module_classes: Dict[str, Dict[str, PyClass]] = {}
|
|
848
|
+
self.owner_by_sig: Dict[str, Optional[PyClass]] = {}
|
|
849
|
+
# (class sig, attr) -> [(writer method name, value var name)] for
|
|
850
|
+
# `self.attr[key] = value` container writes
|
|
851
|
+
self.elem_writes: Dict[Tuple[str, str], List[Tuple[str, str]]] = {}
|
|
852
|
+
|
|
853
|
+
# -- construction ------------------------------------------------------
|
|
854
|
+
def add_module(self, qual: str, mod: PyModule, tree: ast.Module,
|
|
855
|
+
classes: Dict[str, PyClass]) -> None:
|
|
856
|
+
self.module_classes[qual] = classes
|
|
857
|
+
for name, cls in sorted(classes.items()):
|
|
858
|
+
self.classes_global.setdefault(name, []).append(cls)
|
|
859
|
+
for caller, _owner in _iter_callables(mod):
|
|
860
|
+
self.func_by_sig[caller.signature] = caller
|
|
861
|
+
self.owner_by_sig[caller.signature] = _owner
|
|
862
|
+
self.by_name.setdefault(caller.name, []).append(caller.signature)
|
|
863
|
+
names = [p.name for p in caller.parameters or []]
|
|
864
|
+
self.param_names[caller.signature] = names
|
|
865
|
+
for prm in caller.parameters or []:
|
|
866
|
+
if prm.type and prm.type not in ("None", "NoneType"):
|
|
867
|
+
self.param_declared[(caller.signature, prm.name)] = (
|
|
868
|
+
prm.type.rsplit(".", 1)[-1]
|
|
869
|
+
)
|
|
870
|
+
self._collect_self_attrs(qual, tree, classes)
|
|
871
|
+
|
|
872
|
+
def _collect_self_attrs(self, qual: str, tree: ast.Module,
|
|
873
|
+
classes: Dict[str, PyClass]) -> None:
|
|
874
|
+
for node in ast.walk(tree):
|
|
875
|
+
if not isinstance(node, ast.ClassDef):
|
|
876
|
+
continue
|
|
877
|
+
cls = classes.get(node.name)
|
|
878
|
+
if cls is None:
|
|
879
|
+
continue
|
|
880
|
+
method_stack: Dict[int, str] = {}
|
|
881
|
+
for m in ast.walk(node):
|
|
882
|
+
if isinstance(m, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
883
|
+
for sub in ast.walk(m):
|
|
884
|
+
if not isinstance(sub, ast.Assign):
|
|
885
|
+
continue
|
|
886
|
+
for tgt in sub.targets:
|
|
887
|
+
if (
|
|
888
|
+
isinstance(tgt, ast.Subscript)
|
|
889
|
+
and isinstance(tgt.value, ast.Attribute)
|
|
890
|
+
and isinstance(tgt.value.value, ast.Name)
|
|
891
|
+
and tgt.value.value.id == "self"
|
|
892
|
+
and isinstance(sub.value, ast.Name)
|
|
893
|
+
):
|
|
894
|
+
# self.X[key] = value_name — the writer method
|
|
895
|
+
# and value name; the value's type resolves
|
|
896
|
+
# lazily (parameter votes land later)
|
|
897
|
+
self.elem_writes.setdefault(
|
|
898
|
+
(cls.signature, tgt.value.attr), []
|
|
899
|
+
).append((m.name, sub.value.id))
|
|
900
|
+
for sub in ast.walk(node):
|
|
901
|
+
if not isinstance(sub, ast.Assign):
|
|
902
|
+
continue
|
|
903
|
+
for tgt in sub.targets:
|
|
904
|
+
if not (
|
|
905
|
+
isinstance(tgt, ast.Attribute)
|
|
906
|
+
and isinstance(tgt.value, ast.Name)
|
|
907
|
+
and tgt.value.id == "self"
|
|
908
|
+
):
|
|
909
|
+
continue
|
|
910
|
+
key = (cls.signature, tgt.attr)
|
|
911
|
+
lt = _literal_type(sub.value)
|
|
912
|
+
if lt is not None:
|
|
913
|
+
self.self_attr.setdefault(key, ("builtin", lt))
|
|
914
|
+
elif isinstance(sub.value, ast.Call) and isinstance(
|
|
915
|
+
sub.value.func, ast.Name
|
|
916
|
+
):
|
|
917
|
+
self.self_attr.setdefault(
|
|
918
|
+
key, ("class", sub.value.func.id)
|
|
919
|
+
)
|
|
920
|
+
|
|
921
|
+
def vote(self, callee_sig: str, site) -> None:
|
|
922
|
+
"""One resolved call site's positional argument types vote."""
|
|
923
|
+
for i, arg in enumerate(site.arguments or []):
|
|
924
|
+
t = arg.inferred_type
|
|
925
|
+
if t and t not in ("None", "NoneType"):
|
|
926
|
+
self.param_votes.setdefault((callee_sig, i), set()).add(
|
|
927
|
+
t.rsplit(".", 1)[-1]
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
# -- queries -----------------------------------------------------------
|
|
931
|
+
def _unique_class(self, name: str, prefer_qual: str) -> Optional[PyClass]:
|
|
932
|
+
cands = self.classes_global.get(name) or []
|
|
933
|
+
if len(cands) == 1:
|
|
934
|
+
return cands[0]
|
|
935
|
+
same = [c for c in cands if c.signature.startswith(prefer_qual + ".")]
|
|
936
|
+
return same[0] if len(same) == 1 else None
|
|
937
|
+
|
|
938
|
+
def param_type(self, caller: PyCallable, name: str, qual: str):
|
|
939
|
+
declared = self.param_declared.get((caller.signature, name))
|
|
940
|
+
if declared:
|
|
941
|
+
cls = self._unique_class(declared, qual)
|
|
942
|
+
if cls is not None:
|
|
943
|
+
return ("class", cls)
|
|
944
|
+
if declared.lower() in _BUILTIN_TYPE_NAMES:
|
|
945
|
+
return ("builtin", declared.lower())
|
|
946
|
+
names = self.param_names.get(caller.signature) or []
|
|
947
|
+
if name not in names:
|
|
948
|
+
return None
|
|
949
|
+
idx = names.index(name)
|
|
950
|
+
if names and names[0] in ("self", "cls"):
|
|
951
|
+
idx -= 1
|
|
952
|
+
votes = self.param_votes.get((caller.signature, idx)) or set()
|
|
953
|
+
internal = sorted(
|
|
954
|
+
v for v in votes if self._unique_class(v, qual) is not None
|
|
955
|
+
)
|
|
956
|
+
if len(internal) == 1:
|
|
957
|
+
return ("class", self._unique_class(internal[0], qual))
|
|
958
|
+
return None
|
|
959
|
+
|
|
960
|
+
def returned_class(self, callee: PyCallable, qual: str) -> Optional[PyClass]:
|
|
961
|
+
cached = self.return_class.get(callee.signature, "?")
|
|
962
|
+
if cached != "?":
|
|
963
|
+
return self._unique_class(cached, qual) if cached else None
|
|
964
|
+
self.return_class[callee.signature] = None
|
|
965
|
+
if callee.return_type:
|
|
966
|
+
name = callee.return_type.rsplit(".", 1)[-1]
|
|
967
|
+
if self._unique_class(name, qual) is not None:
|
|
968
|
+
self.return_class[callee.signature] = name
|
|
969
|
+
return self._unique_class(name, qual)
|
|
970
|
+
return None
|
|
971
|
+
|
|
972
|
+
def attr_type(self, owner: Optional[PyClass], attr: str):
|
|
973
|
+
if owner is None:
|
|
974
|
+
return None
|
|
975
|
+
return self.self_attr.get((owner.signature, attr))
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
_BUILTIN_TYPE_NAMES = frozenset(
|
|
979
|
+
{"str", "bytes", "int", "float", "bool", "list", "dict", "set", "tuple"}
|
|
980
|
+
)
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def defuse_linker_edges(
|
|
984
|
+
symbol_table: Dict[str, PyModule],
|
|
985
|
+
) -> Tuple[List[PyCallEdge], Resolutions]:
|
|
986
|
+
"""Derive ``prov=["defuse"]`` call edges from local def-use resolution.
|
|
987
|
+
|
|
988
|
+
Returns ``(edges, resolutions)``; *resolutions* feeds
|
|
989
|
+
``l2_callees.backfill_callees`` so resolved ``call`` body nodes get their
|
|
990
|
+
``callee`` without mutating the cached symbol table. Module-scope and
|
|
991
|
+
decorator edges have no body node and appear only in the edge list.
|
|
992
|
+
"""
|
|
993
|
+
edges: Dict[Tuple[str, str], int] = {}
|
|
994
|
+
resolutions: Resolutions = {}
|
|
995
|
+
by_qual: Dict[str, PyModule] = {
|
|
996
|
+
_module_qual(key): m for key, m in sorted(symbol_table.items())
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
def bump(src: str, dst: str) -> None:
|
|
1000
|
+
edges[(src, dst)] = edges.get((src, dst), 0) + 1
|
|
1001
|
+
|
|
1002
|
+
oracle = _TypeOracle()
|
|
1003
|
+
module_ctx: Dict[str, Tuple[PyModule, _ModuleFacts, Dict[str, PyClass]]] = {}
|
|
1004
|
+
for key, mod in sorted(symbol_table.items()):
|
|
1005
|
+
if not mod.source:
|
|
1006
|
+
continue
|
|
1007
|
+
try:
|
|
1008
|
+
tree = ast.parse(mod.source)
|
|
1009
|
+
except SyntaxError:
|
|
1010
|
+
continue
|
|
1011
|
+
qual = _module_qual(key)
|
|
1012
|
+
facts = _collect(tree, qual)
|
|
1013
|
+
classes = _classes_by_name(mod)
|
|
1014
|
+
oracle.add_module(qual, mod, tree, classes)
|
|
1015
|
+
module_ctx[qual] = (mod, facts, classes)
|
|
1016
|
+
|
|
1017
|
+
# sites whose receiver could not be typed locally — the oracle's round
|
|
1018
|
+
pending: List[Tuple] = []
|
|
1019
|
+
pending_iter: List[Tuple] = []
|
|
1020
|
+
|
|
1021
|
+
for qual, (mod, facts, classes) in sorted(module_ctx.items()):
|
|
1022
|
+
|
|
1023
|
+
# --- function-level call sites (from the symbol table) -------------
|
|
1024
|
+
for caller, owner in _iter_callables(mod):
|
|
1025
|
+
recorded = {
|
|
1026
|
+
(s.start_line, s.start_column) for s in (caller.call_sites or [])
|
|
1027
|
+
}
|
|
1028
|
+
for site in caller.call_sites or []:
|
|
1029
|
+
if site.callee_signature and not _is_junk_resolution(
|
|
1030
|
+
site.callee_signature
|
|
1031
|
+
):
|
|
1032
|
+
oracle.vote(site.callee_signature, site)
|
|
1033
|
+
# A self/cls call whose class chain declares the method:
|
|
1034
|
+
# the declared target holds regardless of what Jedi
|
|
1035
|
+
# stamped (it resolves e.g. odoo's `self._warn(...)` to
|
|
1036
|
+
# stdlib `_warnings.warn`). Additive — Jedi's edge stays.
|
|
1037
|
+
if (
|
|
1038
|
+
site.receiver_expr in ("self", "cls")
|
|
1039
|
+
and owner is not None
|
|
1040
|
+
):
|
|
1041
|
+
declared = _resolve_self_call(
|
|
1042
|
+
site.method_name, owner, classes,
|
|
1043
|
+
global_classes=oracle.classes_global,
|
|
1044
|
+
)
|
|
1045
|
+
if declared and declared != site.callee_signature:
|
|
1046
|
+
bump(caller.signature, declared)
|
|
1047
|
+
sites = [
|
|
1048
|
+
s
|
|
1049
|
+
for s in (caller.call_sites or [])
|
|
1050
|
+
if not s.callee_signature or _is_junk_resolution(s.callee_signature)
|
|
1051
|
+
]
|
|
1052
|
+
scope = _scope_for_callable(caller, facts.by_def, facts.module_scope)
|
|
1053
|
+
for site in sites:
|
|
1054
|
+
sig: Optional[str] = None
|
|
1055
|
+
if not site.receiver_expr:
|
|
1056
|
+
target = _resolve_name(site.method_name, scope)
|
|
1057
|
+
if target is not None:
|
|
1058
|
+
sig = _target_signature(target, mod, qual, by_qual)
|
|
1059
|
+
if sig is None:
|
|
1060
|
+
# bare constructor of a class declared in this module
|
|
1061
|
+
# (or uniquely anywhere): `Frame(...)`
|
|
1062
|
+
ctor_cls = classes.get(site.method_name)
|
|
1063
|
+
if ctor_cls is None:
|
|
1064
|
+
cands = oracle.classes_global.get(site.method_name) or []
|
|
1065
|
+
ctor_cls = cands[0] if len(cands) == 1 else None
|
|
1066
|
+
if ctor_cls is not None:
|
|
1067
|
+
sig = f"{ctor_cls.signature}.__init__"
|
|
1068
|
+
elif site.receiver_expr in ("self", "cls") and owner is not None:
|
|
1069
|
+
sig = _resolve_self_call(
|
|
1070
|
+
site.method_name, owner, classes,
|
|
1071
|
+
facts.module_scope, mod, qual, by_qual,
|
|
1072
|
+
global_classes=oracle.classes_global,
|
|
1073
|
+
)
|
|
1074
|
+
else:
|
|
1075
|
+
sig = _receiver_target(
|
|
1076
|
+
site.receiver_expr, site.method_name, scope, mod, qual,
|
|
1077
|
+
by_qual, classes, global_classes=oracle.classes_global,
|
|
1078
|
+
)
|
|
1079
|
+
if sig is None and site.receiver_type:
|
|
1080
|
+
# Jedi's per-site receiver-type inference names the
|
|
1081
|
+
# receiver's class even when the callee is unresolved.
|
|
1082
|
+
rt = site.receiver_type.rsplit(".", 1)[-1]
|
|
1083
|
+
target_cls = classes.get(rt)
|
|
1084
|
+
if target_cls is not None:
|
|
1085
|
+
sig = _resolve_self_call(
|
|
1086
|
+
site.method_name, target_cls, classes,
|
|
1087
|
+
facts.module_scope, mod, qual, by_qual,
|
|
1088
|
+
global_classes=oracle.classes_global,
|
|
1089
|
+
)
|
|
1090
|
+
if sig is None:
|
|
1091
|
+
pending.append(
|
|
1092
|
+
(caller, owner, site, scope, mod, qual, classes)
|
|
1093
|
+
)
|
|
1094
|
+
continue
|
|
1095
|
+
oracle.vote(sig, site)
|
|
1096
|
+
bump(caller.signature, sig)
|
|
1097
|
+
resolutions[
|
|
1098
|
+
(caller.signature, f"{site.start_line}:{site.start_column}")
|
|
1099
|
+
] = sig
|
|
1100
|
+
|
|
1101
|
+
# Calls Jedi's extractor never recorded as sites at all (with-
|
|
1102
|
+
# statement context managers are the common case). They have no
|
|
1103
|
+
# body node either, so they contribute edges but no resolutions.
|
|
1104
|
+
for kind, node in facts.calls_by_scope.get(id(scope), []):
|
|
1105
|
+
if kind == "selfeq":
|
|
1106
|
+
if owner is not None:
|
|
1107
|
+
sig = _resolve_self_call(
|
|
1108
|
+
"__eq__", owner, classes,
|
|
1109
|
+
facts.module_scope, mod, qual, by_qual,
|
|
1110
|
+
global_classes=oracle.classes_global,
|
|
1111
|
+
)
|
|
1112
|
+
if sig is not None:
|
|
1113
|
+
bump(caller.signature, sig)
|
|
1114
|
+
continue
|
|
1115
|
+
if kind == "iter":
|
|
1116
|
+
if isinstance(node, ast.Name):
|
|
1117
|
+
name = node.id
|
|
1118
|
+
if name in ("self", "cls"):
|
|
1119
|
+
target_cls = owner
|
|
1120
|
+
else:
|
|
1121
|
+
cls_name = _lookup_instance_type(name, scope)
|
|
1122
|
+
target_cls = (
|
|
1123
|
+
classes.get(cls_name) if cls_name else None
|
|
1124
|
+
)
|
|
1125
|
+
sig = (
|
|
1126
|
+
_resolve_self_call(
|
|
1127
|
+
"__iter__", target_cls, classes,
|
|
1128
|
+
facts.module_scope, mod, qual, by_qual,
|
|
1129
|
+
global_classes=oracle.classes_global,
|
|
1130
|
+
)
|
|
1131
|
+
if target_cls is not None
|
|
1132
|
+
else None
|
|
1133
|
+
)
|
|
1134
|
+
if sig is not None:
|
|
1135
|
+
bump(caller.signature, sig)
|
|
1136
|
+
else:
|
|
1137
|
+
pending_iter.append(
|
|
1138
|
+
(caller, owner, name, scope, mod, qual, classes)
|
|
1139
|
+
)
|
|
1140
|
+
else:
|
|
1141
|
+
# attribute chains, calls, subscripts — no local type;
|
|
1142
|
+
# the name-linked tier covers the iteration protocol
|
|
1143
|
+
pending_iter.append(
|
|
1144
|
+
(caller, owner, None, scope, mod, qual, classes)
|
|
1145
|
+
)
|
|
1146
|
+
continue
|
|
1147
|
+
if kind != "call":
|
|
1148
|
+
# f-string conversion/format-spec lowering (repr/str/
|
|
1149
|
+
# ascii/format) — CPython calls these at runtime.
|
|
1150
|
+
bump(caller.signature, f"builtins.{kind}")
|
|
1151
|
+
continue
|
|
1152
|
+
if (node.lineno, node.col_offset) in recorded:
|
|
1153
|
+
continue
|
|
1154
|
+
sig = _resolve_uncovered_call(
|
|
1155
|
+
node, scope, owner, classes, mod, qual, by_qual,
|
|
1156
|
+
global_classes=oracle.classes_global,
|
|
1157
|
+
)
|
|
1158
|
+
if sig is not None:
|
|
1159
|
+
bump(caller.signature, sig)
|
|
1160
|
+
elif isinstance(node.func, ast.Attribute) and isinstance(
|
|
1161
|
+
node.func.value, ast.Name
|
|
1162
|
+
):
|
|
1163
|
+
pending.append(
|
|
1164
|
+
(
|
|
1165
|
+
caller,
|
|
1166
|
+
owner,
|
|
1167
|
+
_SyntheticSite(
|
|
1168
|
+
node.func.attr,
|
|
1169
|
+
node.func.value.id,
|
|
1170
|
+
node.lineno,
|
|
1171
|
+
node.col_offset,
|
|
1172
|
+
),
|
|
1173
|
+
scope,
|
|
1174
|
+
mod,
|
|
1175
|
+
qual,
|
|
1176
|
+
classes,
|
|
1177
|
+
)
|
|
1178
|
+
)
|
|
1179
|
+
|
|
1180
|
+
# --- module/class-scope call sites (from the AST; #131 attribution) -
|
|
1181
|
+
for kind, node in facts.toplevel_calls:
|
|
1182
|
+
if kind == "call":
|
|
1183
|
+
sig = _resolve_expr(node.func, facts.module_scope, mod, qual, by_qual)
|
|
1184
|
+
elif kind in ("selfeq", "iter"):
|
|
1185
|
+
continue
|
|
1186
|
+
else:
|
|
1187
|
+
sig = f"builtins.{kind}"
|
|
1188
|
+
if sig is not None and sig != qual:
|
|
1189
|
+
bump(qual, sig)
|
|
1190
|
+
|
|
1191
|
+
# --- decorator applications ----------------------------------------
|
|
1192
|
+
for expr, scope, container in facts.decorators:
|
|
1193
|
+
sig = _resolve_expr(expr, scope, mod, qual, by_qual)
|
|
1194
|
+
if sig is None:
|
|
1195
|
+
continue
|
|
1196
|
+
src = _signature_for_path(mod, container) if container else None
|
|
1197
|
+
bump(src or qual, sig)
|
|
1198
|
+
|
|
1199
|
+
# ---- interprocedural round (#148 extension): type the receivers the
|
|
1200
|
+
# local pass could not, in a strict deterministic order, then resolve
|
|
1201
|
+
# the method on the typed class. One round, no fixpoint.
|
|
1202
|
+
def _container_elem_type(owner, attr, qual):
|
|
1203
|
+
"""`self.attr[k] = value` writers vote the container's element type."""
|
|
1204
|
+
if owner is None:
|
|
1205
|
+
return None
|
|
1206
|
+
cands = set()
|
|
1207
|
+
for method_name, value_name in oracle.elem_writes.get(
|
|
1208
|
+
(owner.signature, attr), []
|
|
1209
|
+
):
|
|
1210
|
+
writer = (owner.callables or {}).get(method_name)
|
|
1211
|
+
if writer is None:
|
|
1212
|
+
continue
|
|
1213
|
+
t = oracle.param_type(writer, value_name, qual)
|
|
1214
|
+
if t is not None and t[0] == "class":
|
|
1215
|
+
cands.add(t[1].name)
|
|
1216
|
+
return next(iter(cands)) if len(cands) == 1 else None
|
|
1217
|
+
|
|
1218
|
+
_ret_memo: Dict[str, Optional[Tuple]] = {}
|
|
1219
|
+
|
|
1220
|
+
def _returned_summary(callee, depth=0):
|
|
1221
|
+
"""What does *callee* return? -> ("class", PyClass) |
|
|
1222
|
+
("callable", sig) | None. Memoized, cycle-safe, depth-capped —
|
|
1223
|
+
chains through `return self.m(...)` / `return f(...)`.
|
|
1224
|
+
"""
|
|
1225
|
+
if callee is None or depth > 4:
|
|
1226
|
+
return None
|
|
1227
|
+
key = callee.signature
|
|
1228
|
+
if key in _ret_memo:
|
|
1229
|
+
return _ret_memo[key]
|
|
1230
|
+
_ret_memo[key] = None # cycle guard
|
|
1231
|
+
result = None
|
|
1232
|
+
for home_qual, (hmod, hfacts, hclasses) in sorted(module_ctx.items()):
|
|
1233
|
+
if not key.startswith(home_qual + "."):
|
|
1234
|
+
continue
|
|
1235
|
+
cscope = _scope_for_callable(callee, hfacts.by_def, hfacts.module_scope)
|
|
1236
|
+
if cscope is hfacts.module_scope:
|
|
1237
|
+
break
|
|
1238
|
+
names = set()
|
|
1239
|
+
fn_paths = set()
|
|
1240
|
+
for c in sorted(cscope.return_ctors):
|
|
1241
|
+
if c.startswith("~"):
|
|
1242
|
+
nm = c[1:]
|
|
1243
|
+
if nm in cscope.funcs:
|
|
1244
|
+
fn_paths.add(cscope.funcs[nm])
|
|
1245
|
+
continue
|
|
1246
|
+
it = cscope.instance_types.get(nm)
|
|
1247
|
+
if it is None and nm in cscope.loopvar_sources:
|
|
1248
|
+
# loop var drawn from a self container:
|
|
1249
|
+
# `for k, v in self.adapters.items(): ... return v`
|
|
1250
|
+
_, attr = cscope.loopvar_sources[nm]
|
|
1251
|
+
own = oracle.owner_by_sig.get(key)
|
|
1252
|
+
it = _container_elem_type(own, attr, home_qual)
|
|
1253
|
+
if it is not None:
|
|
1254
|
+
names.add(it)
|
|
1255
|
+
else:
|
|
1256
|
+
names.add(c)
|
|
1257
|
+
hits = sorted({n for n in names if n in hclasses})
|
|
1258
|
+
if len(hits) == 1 and len(names) == 1 and not fn_paths:
|
|
1259
|
+
result = ("class", hclasses[hits[0]])
|
|
1260
|
+
break
|
|
1261
|
+
if len(fn_paths) == 1 and not names:
|
|
1262
|
+
sig = _signature_for_path(hmod, next(iter(fn_paths)))
|
|
1263
|
+
if sig:
|
|
1264
|
+
result = ("callable", sig)
|
|
1265
|
+
break
|
|
1266
|
+
if len(cscope.return_calls) == 1 and not names and not fn_paths:
|
|
1267
|
+
fexpr = cscope.return_calls[0]
|
|
1268
|
+
nxt_sig = None
|
|
1269
|
+
if (
|
|
1270
|
+
isinstance(fexpr, ast.Attribute)
|
|
1271
|
+
and isinstance(fexpr.value, ast.Name)
|
|
1272
|
+
and fexpr.value.id in ("self", "cls")
|
|
1273
|
+
):
|
|
1274
|
+
own = oracle.owner_by_sig.get(key)
|
|
1275
|
+
if own is not None:
|
|
1276
|
+
nxt_sig = _resolve_self_call(
|
|
1277
|
+
fexpr.attr, own, hclasses,
|
|
1278
|
+
hfacts.module_scope, hmod, home_qual, by_qual,
|
|
1279
|
+
global_classes=oracle.classes_global,
|
|
1280
|
+
)
|
|
1281
|
+
else:
|
|
1282
|
+
nxt_sig = _resolve_expr(
|
|
1283
|
+
fexpr, cscope, hmod, home_qual, by_qual
|
|
1284
|
+
)
|
|
1285
|
+
nxt = oracle.func_by_sig.get(nxt_sig) if nxt_sig else None
|
|
1286
|
+
if nxt is not None:
|
|
1287
|
+
result = _returned_summary(nxt, depth + 1)
|
|
1288
|
+
break
|
|
1289
|
+
_ret_memo[key] = result
|
|
1290
|
+
return result
|
|
1291
|
+
|
|
1292
|
+
def _returned_ctor_class(callee, qual):
|
|
1293
|
+
r = _returned_summary(callee)
|
|
1294
|
+
return r[1] if r is not None and r[0] == "class" else None
|
|
1295
|
+
|
|
1296
|
+
def _typed_receiver(caller, owner, name, scope, mod, qual, classes):
|
|
1297
|
+
t = oracle.param_type(caller, name, qual)
|
|
1298
|
+
if t is not None:
|
|
1299
|
+
return t
|
|
1300
|
+
s_ = scope
|
|
1301
|
+
while s_ is not None:
|
|
1302
|
+
if name in s_.call_assigns:
|
|
1303
|
+
func = s_.call_assigns[name]
|
|
1304
|
+
callee_sig = None
|
|
1305
|
+
if isinstance(func, ast.Attribute) and isinstance(
|
|
1306
|
+
func.value, ast.Name
|
|
1307
|
+
) and func.value.id in ("self", "cls") and owner is not None:
|
|
1308
|
+
callee_sig = _resolve_self_call(
|
|
1309
|
+
func.attr, owner, classes, module_ctx[qual][1].module_scope,
|
|
1310
|
+
mod, qual, by_qual,
|
|
1311
|
+
global_classes=oracle.classes_global,
|
|
1312
|
+
)
|
|
1313
|
+
else:
|
|
1314
|
+
callee_sig = _resolve_expr(
|
|
1315
|
+
func, s_, mod, qual, by_qual
|
|
1316
|
+
)
|
|
1317
|
+
if callee_sig:
|
|
1318
|
+
callee = oracle.func_by_sig.get(callee_sig)
|
|
1319
|
+
if callee is not None:
|
|
1320
|
+
cls = oracle.returned_class(callee, qual)
|
|
1321
|
+
if cls is None:
|
|
1322
|
+
cls = _returned_ctor_class(callee, qual)
|
|
1323
|
+
if cls is not None:
|
|
1324
|
+
return ("class", cls)
|
|
1325
|
+
break
|
|
1326
|
+
if name in s_.blocked or name in s_.bindings:
|
|
1327
|
+
break
|
|
1328
|
+
s_ = s_.parent
|
|
1329
|
+
return None
|
|
1330
|
+
|
|
1331
|
+
def _method_on(t, method, qual):
|
|
1332
|
+
kind, val = t
|
|
1333
|
+
if kind == "builtin":
|
|
1334
|
+
return f"builtins.{val}.{method}"
|
|
1335
|
+
cls = val
|
|
1336
|
+
home_qual = next(
|
|
1337
|
+
(
|
|
1338
|
+
q
|
|
1339
|
+
for q, cmap in sorted(oracle.module_classes.items())
|
|
1340
|
+
if cmap.get(cls.name) is cls
|
|
1341
|
+
),
|
|
1342
|
+
None,
|
|
1343
|
+
)
|
|
1344
|
+
if home_qual is None or home_qual not in module_ctx:
|
|
1345
|
+
return _resolve_self_call(
|
|
1346
|
+
method, cls, {cls.name: cls},
|
|
1347
|
+
global_classes=oracle.classes_global,
|
|
1348
|
+
)
|
|
1349
|
+
home_mod, home_facts, home_classes = module_ctx[home_qual]
|
|
1350
|
+
return _resolve_self_call(
|
|
1351
|
+
method, cls, home_classes, home_facts.module_scope,
|
|
1352
|
+
home_mod, home_qual, by_qual,
|
|
1353
|
+
global_classes=oracle.classes_global,
|
|
1354
|
+
)
|
|
1355
|
+
|
|
1356
|
+
remaining = pending
|
|
1357
|
+
_MAX_ROUNDS = 8 # monotone: resolutions only grow; cap is a safety net
|
|
1358
|
+
for _round in range(_MAX_ROUNDS):
|
|
1359
|
+
made_progress = False
|
|
1360
|
+
still: List[Tuple] = []
|
|
1361
|
+
for caller, owner, site, scope, mod, qual, classes in remaining:
|
|
1362
|
+
if not (site.receiver_expr or ""):
|
|
1363
|
+
# bare call of a variable holding a returned closure:
|
|
1364
|
+
# `compute = make_compute(...); compute(...)`
|
|
1365
|
+
sig = None
|
|
1366
|
+
s_ = scope
|
|
1367
|
+
while s_ is not None:
|
|
1368
|
+
if site.method_name in s_.call_assigns:
|
|
1369
|
+
fexpr = s_.call_assigns[site.method_name]
|
|
1370
|
+
fsig = None
|
|
1371
|
+
if (
|
|
1372
|
+
isinstance(fexpr, ast.Attribute)
|
|
1373
|
+
and isinstance(fexpr.value, ast.Name)
|
|
1374
|
+
and fexpr.value.id in ("self", "cls")
|
|
1375
|
+
and owner is not None
|
|
1376
|
+
):
|
|
1377
|
+
fsig = _resolve_self_call(
|
|
1378
|
+
fexpr.attr, owner, classes,
|
|
1379
|
+
module_ctx[qual][1].module_scope, mod, qual,
|
|
1380
|
+
by_qual, global_classes=oracle.classes_global,
|
|
1381
|
+
)
|
|
1382
|
+
else:
|
|
1383
|
+
fsig = _resolve_expr(fexpr, s_, mod, qual, by_qual)
|
|
1384
|
+
summ = _returned_summary(oracle.func_by_sig.get(fsig)) if fsig else None
|
|
1385
|
+
if summ is not None and summ[0] == "callable":
|
|
1386
|
+
sig = summ[1]
|
|
1387
|
+
break
|
|
1388
|
+
if (
|
|
1389
|
+
site.method_name in s_.blocked
|
|
1390
|
+
or site.method_name in s_.bindings
|
|
1391
|
+
or site.method_name in s_.funcs
|
|
1392
|
+
or site.method_name in s_.imports
|
|
1393
|
+
or site.method_name in s_.mod_imports
|
|
1394
|
+
):
|
|
1395
|
+
break
|
|
1396
|
+
s_ = s_.parent
|
|
1397
|
+
if sig is not None:
|
|
1398
|
+
oracle.vote(sig, site)
|
|
1399
|
+
bump(caller.signature, sig)
|
|
1400
|
+
resolutions[
|
|
1401
|
+
(caller.signature, f"{site.start_line}:{site.start_column}")
|
|
1402
|
+
] = sig
|
|
1403
|
+
made_progress = True
|
|
1404
|
+
else:
|
|
1405
|
+
still.append((caller, owner, site, scope, mod, qual, classes))
|
|
1406
|
+
continue
|
|
1407
|
+
if (site.receiver_expr or "") in ("self", "cls") and owner is not None:
|
|
1408
|
+
sig = _resolve_self_call(
|
|
1409
|
+
site.method_name, owner, classes,
|
|
1410
|
+
module_ctx[qual][1].module_scope, mod, qual, by_qual,
|
|
1411
|
+
global_classes=oracle.classes_global,
|
|
1412
|
+
)
|
|
1413
|
+
if sig is not None:
|
|
1414
|
+
oracle.vote(sig, site)
|
|
1415
|
+
bump(caller.signature, sig)
|
|
1416
|
+
continue
|
|
1417
|
+
still.append((caller, owner, site, scope, mod, qual, classes))
|
|
1418
|
+
continue
|
|
1419
|
+
recv_txt = (site.receiver_expr or "").strip()
|
|
1420
|
+
root_tok = recv_txt.split("(", 1)[0].split(".", 1)[0].strip()
|
|
1421
|
+
if "(" in recv_txt and root_tok in _BUILTINS:
|
|
1422
|
+
# method on a builtin temporary whose site Jedi recorded with
|
|
1423
|
+
# the call text as the receiver: TypeError(...).with_traceback
|
|
1424
|
+
sig = f"builtins.{root_tok}.{site.method_name}"
|
|
1425
|
+
bump(caller.signature, sig)
|
|
1426
|
+
continue
|
|
1427
|
+
parts = tuple(
|
|
1428
|
+
p_ for p_ in (site.receiver_expr or "").split(".") if p_
|
|
1429
|
+
)
|
|
1430
|
+
t = None
|
|
1431
|
+
if len(parts) == 1:
|
|
1432
|
+
t = _typed_receiver(
|
|
1433
|
+
caller, owner, parts[0], scope, mod, qual, classes
|
|
1434
|
+
)
|
|
1435
|
+
elif len(parts) == 2 and parts[0] in ("self", "cls"):
|
|
1436
|
+
at = oracle.attr_type(owner, parts[1])
|
|
1437
|
+
if at is not None:
|
|
1438
|
+
if at[0] == "class":
|
|
1439
|
+
cls = oracle._unique_class(at[1], qual)
|
|
1440
|
+
t = ("class", cls) if cls is not None else None
|
|
1441
|
+
else:
|
|
1442
|
+
t = at
|
|
1443
|
+
if t is None:
|
|
1444
|
+
still.append((caller, owner, site, scope, mod, qual, classes))
|
|
1445
|
+
continue
|
|
1446
|
+
sig = _method_on(t, site.method_name, qual)
|
|
1447
|
+
if sig is None:
|
|
1448
|
+
# typed, but the type does not declare the method — the type
|
|
1449
|
+
# was a bad vote; fall through to the name-linked tier
|
|
1450
|
+
still.append((caller, owner, site, scope, mod, qual, classes))
|
|
1451
|
+
continue
|
|
1452
|
+
oracle.vote(sig, site)
|
|
1453
|
+
bump(caller.signature, sig)
|
|
1454
|
+
resolutions[
|
|
1455
|
+
(caller.signature, f"{site.start_line}:{site.start_column}")
|
|
1456
|
+
] = sig
|
|
1457
|
+
made_progress = True
|
|
1458
|
+
remaining = still
|
|
1459
|
+
if not made_progress:
|
|
1460
|
+
break
|
|
1461
|
+
|
|
1462
|
+
iter_still: List[Tuple] = []
|
|
1463
|
+
for caller, owner, name, scope, mod, qual, classes in pending_iter:
|
|
1464
|
+
t = (
|
|
1465
|
+
_typed_receiver(caller, owner, name, scope, mod, qual, classes)
|
|
1466
|
+
if name is not None
|
|
1467
|
+
else None
|
|
1468
|
+
)
|
|
1469
|
+
sig = _method_on(t, "__iter__", qual) if t is not None else None
|
|
1470
|
+
if sig is not None:
|
|
1471
|
+
bump(caller.signature, sig)
|
|
1472
|
+
else:
|
|
1473
|
+
iter_still.append((caller, "__iter__"))
|
|
1474
|
+
|
|
1475
|
+
# ---- name-linked tier (CHA-by-name): a receiver no typing tier could
|
|
1476
|
+
# resolve may target any internal callable of that name — the same
|
|
1477
|
+
# over-approximation Joern emits for untyped receivers. Sound may-call;
|
|
1478
|
+
# bounded per site so a common name cannot explode the graph.
|
|
1479
|
+
_FAN_CAP = 1024 # pathology guard only; Joern's widest observed fan is 222
|
|
1480
|
+
for caller, owner, site, scope, mod, qual, classes in remaining:
|
|
1481
|
+
cands = oracle.by_name.get(site.method_name) or []
|
|
1482
|
+
if not (site.receiver_expr or ""):
|
|
1483
|
+
# a bare name can never invoke a method (no receiver at runtime)
|
|
1484
|
+
# — only module-level functions are legal targets
|
|
1485
|
+
cands = [c for c in cands if oracle.owner_by_sig.get(c) is None]
|
|
1486
|
+
for sig in cands[:_FAN_CAP]:
|
|
1487
|
+
if sig != caller.signature:
|
|
1488
|
+
bump(caller.signature, sig)
|
|
1489
|
+
for caller, mname in iter_still:
|
|
1490
|
+
for sig in (oracle.by_name.get(mname) or [])[:_FAN_CAP]:
|
|
1491
|
+
bump(caller.signature, sig)
|
|
1492
|
+
|
|
1493
|
+
return (
|
|
1494
|
+
[
|
|
1495
|
+
PyCallEdge(src=src, dst=dst, weight=n, prov=["defuse"])
|
|
1496
|
+
for (src, dst), n in sorted(edges.items())
|
|
1497
|
+
],
|
|
1498
|
+
resolutions,
|
|
1499
|
+
)
|