cg-code-graph 0.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cg_code_graph-0.10.1.dist-info/METADATA +678 -0
- cg_code_graph-0.10.1.dist-info/RECORD +174 -0
- cg_code_graph-0.10.1.dist-info/WHEEL +5 -0
- cg_code_graph-0.10.1.dist-info/entry_points.txt +3 -0
- cg_code_graph-0.10.1.dist-info/licenses/LICENSE +21 -0
- cg_code_graph-0.10.1.dist-info/top_level.txt +1 -0
- codegraph/__init__.py +2 -0
- codegraph/aitools.py +129 -0
- codegraph/apps.py +76 -0
- codegraph/blindspots.py +428 -0
- codegraph/bridges.py +1701 -0
- codegraph/cli.py +725 -0
- codegraph/concepts.py +362 -0
- codegraph/config.py +559 -0
- codegraph/core/__init__.py +0 -0
- codegraph/core/cache.py +375 -0
- codegraph/core/detect.py +80 -0
- codegraph/core/extractors.py +187 -0
- codegraph/core/fsutil.py +61 -0
- codegraph/core/generated.py +575 -0
- codegraph/core/model.py +174 -0
- codegraph/core/paths.py +175 -0
- codegraph/core/plugin.py +160 -0
- codegraph/core/store.py +80 -0
- codegraph/core/syntax_errors.py +132 -0
- codegraph/coverage.py +928 -0
- codegraph/doctor.py +453 -0
- codegraph/external.py +613 -0
- codegraph/indexer.py +336 -0
- codegraph/link.py +434 -0
- codegraph/lint_async.py +524 -0
- codegraph/mcp_server.py +1303 -0
- codegraph/parity.py +473 -0
- codegraph/parity_structure.py +307 -0
- codegraph/payload.py +321 -0
- codegraph/plans.py +1285 -0
- codegraph/platform_scan.py +643 -0
- codegraph/platforms.py +1369 -0
- codegraph/plugins/__init__.py +0 -0
- codegraph/plugins/cfamily/__init__.py +0 -0
- codegraph/plugins/cfamily/plugin.py +930 -0
- codegraph/plugins/cfamily/syntax.py +881 -0
- codegraph/plugins/dart/__init__.py +0 -0
- codegraph/plugins/dart/bridges.py +345 -0
- codegraph/plugins/dart/extractor/bin/extract.dart +717 -0
- codegraph/plugins/dart/extractor/pubspec.lock +149 -0
- codegraph/plugins/dart/extractor/pubspec.yaml +7 -0
- codegraph/plugins/dart/http.py +904 -0
- codegraph/plugins/dart/models.py +308 -0
- codegraph/plugins/dart/plugin.py +625 -0
- codegraph/plugins/dart/program.py +907 -0
- codegraph/plugins/django/__init__.py +0 -0
- codegraph/plugins/django/extras.py +378 -0
- codegraph/plugins/django/models.py +508 -0
- codegraph/plugins/django/plugin.py +728 -0
- codegraph/plugins/django/schemas.py +339 -0
- codegraph/plugins/django/shapes.py +216 -0
- codegraph/plugins/django/urls.py +603 -0
- codegraph/plugins/express/__init__.py +0 -0
- codegraph/plugins/express/plugin.py +428 -0
- codegraph/plugins/flutter/__init__.py +0 -0
- codegraph/plugins/flutter/plugin.py +538 -0
- codegraph/plugins/kotlin/__init__.py +0 -0
- codegraph/plugins/kotlin/exact.py +457 -0
- codegraph/plugins/kotlin/plugin.py +1961 -0
- codegraph/plugins/kotlin/reparse.py +234 -0
- codegraph/plugins/laravel/__init__.py +0 -0
- codegraph/plugins/laravel/broadcast.py +351 -0
- codegraph/plugins/laravel/plugin.py +863 -0
- codegraph/plugins/laravel/tests.py +262 -0
- codegraph/plugins/laravel/values.py +728 -0
- codegraph/plugins/native/__init__.py +0 -0
- codegraph/plugins/native/gates.py +286 -0
- codegraph/plugins/native/runner.py +183 -0
- codegraph/plugins/native/scipread.py +194 -0
- codegraph/plugins/native/ts.py +54 -0
- codegraph/plugins/nest/__init__.py +0 -0
- codegraph/plugins/nest/plugin.py +654 -0
- codegraph/plugins/nextjs/__init__.py +0 -0
- codegraph/plugins/nextjs/plugin.py +336 -0
- codegraph/plugins/nuxt/__init__.py +0 -0
- codegraph/plugins/nuxt/plugin.py +308 -0
- codegraph/plugins/php/__init__.py +0 -0
- codegraph/plugins/php/extractor/composer.json +5 -0
- codegraph/plugins/php/extractor/composer.lock +76 -0
- codegraph/plugins/php/extractor/extract.php +743 -0
- codegraph/plugins/php/gating.py +573 -0
- codegraph/plugins/php/plugin.py +668 -0
- codegraph/plugins/php/strings.py +197 -0
- codegraph/plugins/python/__init__.py +0 -0
- codegraph/plugins/python/aitools.py +664 -0
- codegraph/plugins/python/external.py +245 -0
- codegraph/plugins/python/fields.py +107 -0
- codegraph/plugins/python/plugin.py +1733 -0
- codegraph/plugins/python/refs.py +485 -0
- codegraph/plugins/python/roots.py +412 -0
- codegraph/plugins/python/socketio.py +210 -0
- codegraph/plugins/python/subproc.py +864 -0
- codegraph/plugins/python/tests.py +1040 -0
- codegraph/plugins/python/values.py +179 -0
- codegraph/plugins/pyweb/__init__.py +0 -0
- codegraph/plugins/pyweb/plugin.py +1334 -0
- codegraph/plugins/pyweb/values.py +68 -0
- codegraph/plugins/rust/__init__.py +0 -0
- codegraph/plugins/rust/cargo.py +226 -0
- codegraph/plugins/rust/plugin.py +980 -0
- codegraph/plugins/rust/syntax.py +678 -0
- codegraph/plugins/scip/__init__.py +0 -0
- codegraph/plugins/scip/importer.py +129 -0
- codegraph/plugins/scip/scip.proto +962 -0
- codegraph/plugins/scip/scip_pb2.py +97 -0
- codegraph/plugins/stubs/__init__.py +0 -0
- codegraph/plugins/stubs/plugins.py +38 -0
- codegraph/plugins/swift/__init__.py +0 -0
- codegraph/plugins/swift/baseurl.py +109 -0
- codegraph/plugins/swift/exact.py +415 -0
- codegraph/plugins/swift/indexstore.py +209 -0
- codegraph/plugins/swift/packages.py +174 -0
- codegraph/plugins/swift/plugin.py +2890 -0
- codegraph/plugins/ts/__init__.py +0 -0
- codegraph/plugins/ts/baseurl.py +185 -0
- codegraph/plugins/ts/extractor/extract.mjs +2652 -0
- codegraph/plugins/ts/extractor/fw.mjs +685 -0
- codegraph/plugins/ts/extractor/package-lock.json +205 -0
- codegraph/plugins/ts/extractor/package.json +9 -0
- codegraph/plugins/ts/plugin.py +480 -0
- codegraph/plugins/tsweb/__init__.py +0 -0
- codegraph/plugins/tsweb/common.py +290 -0
- codegraph/plugins/tsweb/data.py +276 -0
- codegraph/presets/__init__.py +146 -0
- codegraph/presets/c_cpp.yaml +9 -0
- codegraph/presets/common.yaml +66 -0
- codegraph/presets/dart.yaml +9 -0
- codegraph/presets/django-ninja.yaml +15 -0
- codegraph/presets/django.yaml +25 -0
- codegraph/presets/djangorestframework.yaml +17 -0
- codegraph/presets/express.yaml +17 -0
- codegraph/presets/kotlin.yaml +11 -0
- codegraph/presets/laravel.yaml +40 -0
- codegraph/presets/nest.yaml +11 -0
- codegraph/presets/nextjs.yaml +15 -0
- codegraph/presets/nuxt.yaml +9 -0
- codegraph/presets/php.yaml +5 -0
- codegraph/presets/python.yaml +10 -0
- codegraph/presets/rust.yaml +5 -0
- codegraph/presets/swift.yaml +10 -0
- codegraph/presets/typescript.yaml +13 -0
- codegraph/process_runs.py +328 -0
- codegraph/protocols/__init__.py +299 -0
- codegraph/protocols/builtin.py +67 -0
- codegraph/protocols/matchers.py +144 -0
- codegraph/protocols/view.py +334 -0
- codegraph/query.py +2089 -0
- codegraph/realtime.py +260 -0
- codegraph/roundtrip.py +346 -0
- codegraph/routes.py +442 -0
- codegraph/starters.py +218 -0
- codegraph/tests_index.py +117 -0
- codegraph/viz/__init__.py +0 -0
- codegraph/viz/graph.py +369 -0
- codegraph/viz/server.py +198 -0
- codegraph/viz/static/app.css +148 -0
- codegraph/viz/static/app.js +1082 -0
- codegraph/viz/static/index.html +81 -0
- codegraph/viz/static/layered.js +237 -0
- codegraph/viz/static/vendor/VERSIONS.txt +4 -0
- codegraph/viz/static/vendor/cose-base.js +3214 -0
- codegraph/viz/static/vendor/cytoscape-fcose.js +1549 -0
- codegraph/viz/static/vendor/cytoscape.min.js +31 -0
- codegraph/viz/static/vendor/layout-base.js +5230 -0
- codegraph/viz/tools/package-lock.json +303 -0
- codegraph/viz/tools/package.json +7 -0
- codegraph/viz/tools/shoot.mjs +165 -0
- codegraph/xcode.py +251 -0
|
@@ -0,0 +1,1733 @@
|
|
|
1
|
+
"""Python language plugin (stdlib `ast`, no code execution).
|
|
2
|
+
|
|
3
|
+
Parses every .py file once and builds:
|
|
4
|
+
* a module table (dotted names under each source root, detected or configured: see `roots.py`), with import
|
|
5
|
+
resolution (absolute, relative, `import a.b as c`, re-exports through `__init__`, `from x import *`);
|
|
6
|
+
* symbol tables for classes / functions / methods (nested functions and lambdas collapse into their
|
|
7
|
+
enclosing definition) and class bases resolved to local classes or external dotted names
|
|
8
|
+
(`django.db.models.Model`);
|
|
9
|
+
* a flow-insensitive per-function type inference: assignments, annotations (incl. Optional/list),
|
|
10
|
+
`self.x = ...` attributes, `for x in ...`, return annotations; framework plugins extend it with
|
|
11
|
+
`attr_rules` / `call_rules` (e.g. Django `Model.objects.filter()` -> queryset of Model);
|
|
12
|
+
* edges: IMPORTS, CALLS (exact: direct names / self / class instantiation; resolved: inferred
|
|
13
|
+
receiver type; heuristic: unique method name), INSTANTIATES, EXTENDS, CONTAINS, READS_ENV
|
|
14
|
+
(os.environ / os.getenv / django-environ / python-decouple), REFERENCES_FN (functions used as values) and
|
|
15
|
+
CALLS via="collection" (calls through dispatch tables);
|
|
16
|
+
* `script` entry nodes for `__main__` blocks, `pkg/__main__.py` and packaging entry points, plus MCP / click /
|
|
17
|
+
typer registrations (see `refs.py`).
|
|
18
|
+
|
|
19
|
+
Framework plugins (Django) get the `PyProgram` as their context and walk function bodies with
|
|
20
|
+
`prog.infer()` / `prog.resolve_call()`.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import ast
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import time
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any, Callable, Iterator
|
|
31
|
+
|
|
32
|
+
from ...core.model import EXACT, HEURISTIC, RESOLVED
|
|
33
|
+
from ... import presets
|
|
34
|
+
from ...core.fsutil import keep_file
|
|
35
|
+
from ...core.paths import rel_dir, rules as path_rules
|
|
36
|
+
from ...core.plugin import FrameworkPlugin, GraphBuilder, LanguagePlugin, Project, gc_paused
|
|
37
|
+
from .roots import RootPlan, import_names
|
|
38
|
+
|
|
39
|
+
# virtualenvs, caches, build output, static/ and media/ (codegraph/presets/python.yaml; .cg.yaml skip_dirs adjusts them)
|
|
40
|
+
SKIP_DIRS = presets.skip_dirs("python")
|
|
41
|
+
# directories that never hold project code when deciding whether a repository has Python at all
|
|
42
|
+
DETECT_SKIP = presets.skip_dirs("python", "skip_dirs", "detect_skip_dirs")
|
|
43
|
+
MAX_FILE_BYTES = 1_500_000
|
|
44
|
+
# method names too generic for the unique-name fallback
|
|
45
|
+
STOP_METHODS = {"get", "set", "save", "delete", "update", "filter", "all", "items", "keys", "values", "append", "extend",
|
|
46
|
+
"add", "remove", "pop", "clear", "copy", "run", "call", "send", "close", "open", "read", "write", "start",
|
|
47
|
+
"stop", "create", "list", "retrieve", "destroy", "format", "join", "split", "strip", "replace", "encode",
|
|
48
|
+
"decode", "load", "loads", "dump", "dumps", "first", "last", "count", "exists", "order_by", "exclude",
|
|
49
|
+
"next", "iter", "log", "info", "debug", "warning", "error", "exception", "handle", "process", "execute",
|
|
50
|
+
"render", "clean", "validate", "is_valid", "to_representation", "to_internal_value", "as_view", "dispatch",
|
|
51
|
+
"setup", "teardown", "setUp", "tearDown", "main", "lower", "upper", "sort", "index", "insert", "connect",
|
|
52
|
+
"disconnect", "receive", "emit", "apply", "map", "reduce", "flush", "reset", "refresh", "build", "parse",
|
|
53
|
+
"serialize", "deserialize", "post", "put", "patch", "head", "options", "partial_update", "perform_create",
|
|
54
|
+
"perform_update", "perform_destroy", "get_queryset", "get_object", "get_serializer", "get_context_data",
|
|
55
|
+
"form_valid", "form_invalid", "has_permission", "has_object_permission", "authenticate", "init", "submit"}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
ASYNC_WRAPPERS = {"sync_to_async", "async_to_sync", "database_sync_to_async", "run_sync"}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _recv_attr(prog, call, tgt) -> dict:
|
|
62
|
+
"""{"recv": [class ids]} for a call on a receiver whose class inherits the target method, else {}."""
|
|
63
|
+
r = prog.recv.get((id(call.func), id(tgt)))
|
|
64
|
+
return {"recv": list(r)} if r else {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# #78: steps one top-level type inference may take (nested infer calls); beyond it the type is unknown
|
|
68
|
+
INFER_BUDGET = 5000
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class FuncInfo:
|
|
73
|
+
name: str
|
|
74
|
+
qual: str # dotted fqn (module.Class.method / module.func)
|
|
75
|
+
module: "ModInfo"
|
|
76
|
+
node: Any
|
|
77
|
+
cls: "ClassInfo | None" = None
|
|
78
|
+
kind: str = "function" # function | method
|
|
79
|
+
decorators: list = field(default_factory=list)
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def id(self) -> str:
|
|
83
|
+
return f"{self.kind}:{self.qual}"
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def line(self) -> int:
|
|
87
|
+
return self.node.lineno
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def file(self) -> str:
|
|
91
|
+
return self.module.file
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass
|
|
95
|
+
class ClassInfo:
|
|
96
|
+
name: str
|
|
97
|
+
qual: str
|
|
98
|
+
module: "ModInfo"
|
|
99
|
+
node: Any
|
|
100
|
+
bases: list = field(default_factory=list) # ast exprs
|
|
101
|
+
methods: dict = field(default_factory=dict) # name -> FuncInfo
|
|
102
|
+
attrs: dict = field(default_factory=dict) # class-level name -> (value ast, line, annotation ast)
|
|
103
|
+
inner: dict = field(default_factory=dict) # nested classes (Meta, Config) -> ast.ClassDef
|
|
104
|
+
decorators: list = field(default_factory=list)
|
|
105
|
+
self_attrs: dict = field(default_factory=dict) # self.x = value -> [value ast]
|
|
106
|
+
self_attr_lines: dict = field(default_factory=dict) # self.x -> line of its first assignment (#88 field nodes)
|
|
107
|
+
outer: "ClassInfo | None" = None
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def id(self) -> str:
|
|
111
|
+
return f"class:{self.qual}"
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def line(self) -> int:
|
|
115
|
+
return self.node.lineno
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def file(self) -> str:
|
|
119
|
+
return self.module.file
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass
|
|
123
|
+
class ModInfo:
|
|
124
|
+
name: str
|
|
125
|
+
file: str
|
|
126
|
+
tree: Any
|
|
127
|
+
is_pkg: bool = False
|
|
128
|
+
imports: dict = field(default_factory=dict) # local name -> ("mod", dotted) | ("sym", module, name) | ("modprefix", dotted)
|
|
129
|
+
star: list = field(default_factory=list) # modules star-imported
|
|
130
|
+
classes: dict = field(default_factory=dict) # top-level name -> ClassInfo
|
|
131
|
+
funcs: dict = field(default_factory=dict) # top-level name -> FuncInfo
|
|
132
|
+
vars: dict = field(default_factory=dict) # module-level name -> [(value ast, line, annotation)]
|
|
133
|
+
all_classes: list = field(default_factory=list) # incl. nested
|
|
134
|
+
import_nodes: list = field(default_factory=list) # every Import / ImportFrom statement (ast.walk order)
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def id(self) -> str:
|
|
138
|
+
return f"module:{self.name}"
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def package(self) -> str:
|
|
142
|
+
return self.name if self.is_pkg else self.name.rpartition(".")[0]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# --------------------------------------------------------------------------- helpers
|
|
146
|
+
|
|
147
|
+
def dotted(e) -> str | None:
|
|
148
|
+
"""a.b.c for Name/Attribute chains, else None."""
|
|
149
|
+
parts = []
|
|
150
|
+
while isinstance(e, ast.Attribute):
|
|
151
|
+
parts.append(e.attr)
|
|
152
|
+
e = e.value
|
|
153
|
+
if isinstance(e, ast.Name):
|
|
154
|
+
parts.append(e.id)
|
|
155
|
+
return ".".join(reversed(parts))
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def const_str(e) -> str | None:
|
|
160
|
+
if isinstance(e, ast.Constant) and isinstance(e.value, str):
|
|
161
|
+
return e.value
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def kwarg(call: ast.Call, name: str):
|
|
166
|
+
for k in call.keywords:
|
|
167
|
+
if k.arg == name:
|
|
168
|
+
return k.value
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# PEP 758 (Python 3.14): `except A, B:` without parentheses. Older interpreters reject it; the parenthesized form
|
|
173
|
+
# has the same meaning, so a file that fails only on this is re-parsed with the parentheses added (same line numbers).
|
|
174
|
+
PEP758_EXCEPT = re.compile(r"(?m)^(\s*except\*?\s+)([A-Za-z_][\w.]*(?:\s*,\s*[A-Za-z_][\w.]*)+)(\s*:)")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def parse_source(src: str, filename: str = "<unknown>") -> ast.Module:
|
|
178
|
+
try:
|
|
179
|
+
return ast.parse(src, filename=filename)
|
|
180
|
+
except SyntaxError:
|
|
181
|
+
fixed = PEP758_EXCEPT.sub(r"\1(\2)\3", src)
|
|
182
|
+
if fixed == src:
|
|
183
|
+
raise
|
|
184
|
+
return ast.parse(fixed, filename=filename)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def walk_body(node) -> Iterator[ast.AST]:
|
|
188
|
+
"""ast.walk over a def's body including nested defs/lambdas (they collapse into the enclosing def),
|
|
189
|
+
but not into nested classes (those are their own symbols)."""
|
|
190
|
+
stack = list(ast.iter_child_nodes(node))
|
|
191
|
+
while stack:
|
|
192
|
+
n = stack.pop()
|
|
193
|
+
yield n
|
|
194
|
+
if isinstance(n, ast.ClassDef):
|
|
195
|
+
continue
|
|
196
|
+
stack.extend(ast.iter_child_nodes(n))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def ann_base(e) -> Any:
|
|
200
|
+
"""Strip Optional[X] / X | None / Annotated[X, ...] / 'X' strings. Returns (expr, container) where
|
|
201
|
+
container is 'list' for list[X]/List[X]/Sequence[X]/QuerySet[X], else None."""
|
|
202
|
+
if isinstance(e, ast.Constant) and isinstance(e.value, str):
|
|
203
|
+
try:
|
|
204
|
+
e = ast.parse(e.value, mode="eval").body
|
|
205
|
+
except SyntaxError:
|
|
206
|
+
return None, None
|
|
207
|
+
if isinstance(e, ast.BinOp) and isinstance(e.op, ast.BitOr):
|
|
208
|
+
for side in (e.left, e.right):
|
|
209
|
+
if not (isinstance(side, ast.Constant) and side.value is None):
|
|
210
|
+
return ann_base(side)
|
|
211
|
+
if isinstance(e, ast.Subscript):
|
|
212
|
+
head = (dotted(e.value) or "").split(".")[-1]
|
|
213
|
+
sl = e.slice
|
|
214
|
+
if head in ("Optional", "Annotated", "Required", "NotRequired", "Final", "ClassVar", "Type"):
|
|
215
|
+
inner = sl.elts[0] if isinstance(sl, ast.Tuple) else sl
|
|
216
|
+
return ann_base(inner)
|
|
217
|
+
if head == "Union" and isinstance(sl, ast.Tuple):
|
|
218
|
+
non_none = [x for x in sl.elts if not (isinstance(x, ast.Constant) and x.value is None)]
|
|
219
|
+
if len(non_none) == 1:
|
|
220
|
+
return ann_base(non_none[0])
|
|
221
|
+
return None, None
|
|
222
|
+
if head in ("list", "List", "Sequence", "Iterable", "Iterator", "QuerySet", "set", "Set", "tuple", "Tuple",
|
|
223
|
+
"Generator", "AsyncIterator", "AsyncGenerator"):
|
|
224
|
+
inner = sl.elts[0] if isinstance(sl, ast.Tuple) else sl
|
|
225
|
+
b, _ = ann_base(inner)
|
|
226
|
+
return b, "list"
|
|
227
|
+
return None, None
|
|
228
|
+
return e, None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def is_nullable_ann(e) -> bool:
|
|
232
|
+
if isinstance(e, ast.Constant) and isinstance(e.value, str):
|
|
233
|
+
try:
|
|
234
|
+
e = ast.parse(e.value, mode="eval").body
|
|
235
|
+
except SyntaxError:
|
|
236
|
+
return False
|
|
237
|
+
if isinstance(e, ast.BinOp) and isinstance(e.op, ast.BitOr):
|
|
238
|
+
return any(isinstance(s, ast.Constant) and s.value is None for s in (e.left, e.right)) or \
|
|
239
|
+
is_nullable_ann(e.left) or is_nullable_ann(e.right)
|
|
240
|
+
if isinstance(e, ast.Subscript):
|
|
241
|
+
head = (dotted(e.value) or "").split(".")[-1]
|
|
242
|
+
if head == "Optional":
|
|
243
|
+
return True
|
|
244
|
+
if head == "Union":
|
|
245
|
+
sl = e.slice
|
|
246
|
+
return isinstance(sl, ast.Tuple) and any(isinstance(x, ast.Constant) and x.value is None for x in sl.elts)
|
|
247
|
+
if head == "Annotated":
|
|
248
|
+
sl = e.slice
|
|
249
|
+
return is_nullable_ann(sl.elts[0] if isinstance(sl, ast.Tuple) else sl)
|
|
250
|
+
return False
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def ann_text(e) -> str | None:
|
|
254
|
+
if e is None:
|
|
255
|
+
return None
|
|
256
|
+
try:
|
|
257
|
+
return ast.unparse(e)
|
|
258
|
+
except Exception:
|
|
259
|
+
return None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# --------------------------------------------------------------------------- program
|
|
263
|
+
|
|
264
|
+
class PyProgram:
|
|
265
|
+
def __init__(self, project: Project):
|
|
266
|
+
self.project = project
|
|
267
|
+
self.root = project.root
|
|
268
|
+
self.modules: dict[str, ModInfo] = {} # canonical name -> ModInfo
|
|
269
|
+
self.alias: dict[str, ModInfo] = {} # every importable name -> ModInfo
|
|
270
|
+
self.by_file: dict[str, ModInfo] = {}
|
|
271
|
+
self.classes: dict[str, ClassInfo] = {} # qual -> ClassInfo
|
|
272
|
+
self.funcs: dict[str, FuncInfo] = {} # qual -> FuncInfo (functions + methods)
|
|
273
|
+
self.method_index: dict[str, list[FuncInfo]] = {}
|
|
274
|
+
self._elements_active: set = set() # (id(expr), mode) being followed by elements(): cycle guard
|
|
275
|
+
self._collection_steps: int | None = None # steps left for the call site dispatch_targets is resolving
|
|
276
|
+
# (id(call.func), id(target method)) -> receiver class ids of a call landing on a method the receiver's class
|
|
277
|
+
# inherits (`b.run()` with b a B, run defined in Base): edge attrs.recv, so `impact B.run` keeps only calls on
|
|
278
|
+
# receivers that can be a B
|
|
279
|
+
self.recv: dict[tuple[int, int], list[str]] = {}
|
|
280
|
+
self._returns: dict = {} # id(FuncInfo) -> [(return expr, ctx)]
|
|
281
|
+
self.parse_errors: list[dict] = []
|
|
282
|
+
self.attr_rules: list[Callable] = [] # (prog, base_type, attr, ctx) -> type | None
|
|
283
|
+
self.call_rules: list[Callable] = [] # (prog, call, func_type, ctx) -> type | None
|
|
284
|
+
self._var_cache: dict[int, dict] = {}
|
|
285
|
+
self._infer_depth = 0
|
|
286
|
+
# #78: attribute types are memoised per (class, attr); an attribute whose type is being inferred further up
|
|
287
|
+
# the stack is unknown (a cycle: `self.a = self.b.x` / `self.b = self.a.y`), and one top-level inference gets
|
|
288
|
+
# a work budget, so chains of fluent / self-referencing attributes cannot explore exponentially
|
|
289
|
+
self._attr_memo: dict[tuple, object] = {}
|
|
290
|
+
self._attr_busy: set[tuple] = set()
|
|
291
|
+
self._cuts = 0
|
|
292
|
+
self._work = 0
|
|
293
|
+
self.infer_stats = {"attr_cycles": 0, "budget_exhausted": 0}
|
|
294
|
+
|
|
295
|
+
# ---- loading
|
|
296
|
+
def source_roots(self) -> list[Path]:
|
|
297
|
+
"""Directories whose contents are importable by top-level name (after load())."""
|
|
298
|
+
plan = getattr(self, "root_plan", None)
|
|
299
|
+
if plan is None:
|
|
300
|
+
return [self.root]
|
|
301
|
+
return [self.root / r.path if r.path else self.root for r in plan.roots.values()]
|
|
302
|
+
|
|
303
|
+
def configured_roots(self) -> tuple[list[str] | None, str]:
|
|
304
|
+
"""(roots, origin): `--python-root` beats `python.source_roots` in .cg.yaml; (None, "detected") otherwise."""
|
|
305
|
+
opts = self.project.options
|
|
306
|
+
if opts.get("python_roots"):
|
|
307
|
+
return list(opts["python_roots"]), "flag"
|
|
308
|
+
cfg = ((opts.get("config") or {}).get("python") or {}).get("source_roots")
|
|
309
|
+
if cfg:
|
|
310
|
+
return list(cfg), "configured"
|
|
311
|
+
return None, "detected"
|
|
312
|
+
|
|
313
|
+
def files(self) -> list[str]:
|
|
314
|
+
out = []
|
|
315
|
+
self.path_rules = rules = path_rules(self.project, "python")
|
|
316
|
+
for dp, dns, fns in os.walk(self.root):
|
|
317
|
+
rd = rel_dir(self.root, dp)
|
|
318
|
+
dns[:] = rules.prune(rd, dns, dot=True)
|
|
319
|
+
for fn in sorted(fns):
|
|
320
|
+
if fn.endswith(".py"):
|
|
321
|
+
p = os.path.join(dp, fn)
|
|
322
|
+
rel = f"{rd}/{fn}" if rd else fn
|
|
323
|
+
if keep_file(p) and not rules.excluded(rel):
|
|
324
|
+
out.append(rel)
|
|
325
|
+
return out
|
|
326
|
+
|
|
327
|
+
def load(self, skip_migrations=True) -> dict:
|
|
328
|
+
files = self.files()
|
|
329
|
+
n_skipped = 0
|
|
330
|
+
# per-file outcome for coverage (codegraph/coverage.py): which discovered files did not become graph nodes, and why
|
|
331
|
+
self.file_report = rep = {"seen": files, "parse_failed": [], "skipped_oversize": [], "excluded": [], "unmapped": [],
|
|
332
|
+
"syntax_errors": {}}
|
|
333
|
+
parsed: dict[str, tuple] = {}
|
|
334
|
+
for rel in files:
|
|
335
|
+
parts = Path(rel).parts
|
|
336
|
+
if skip_migrations and "migrations" in parts[:-1]:
|
|
337
|
+
n_skipped += 1
|
|
338
|
+
rep["excluded"].append(rel)
|
|
339
|
+
continue
|
|
340
|
+
p = self.root / rel
|
|
341
|
+
try:
|
|
342
|
+
if p.stat().st_size > MAX_FILE_BYTES:
|
|
343
|
+
n_skipped += 1
|
|
344
|
+
rep["skipped_oversize"].append(rel)
|
|
345
|
+
continue
|
|
346
|
+
src = p.read_text(encoding="utf-8", errors="replace")
|
|
347
|
+
tree = parse_source(src, rel)
|
|
348
|
+
except (SyntaxError, ValueError, RecursionError, OSError) as ex:
|
|
349
|
+
self.parse_errors.append({"file": rel, "error": f"{type(ex).__name__}: {getattr(ex, 'msg', str(ex))}",
|
|
350
|
+
"line": getattr(ex, "lineno", None)})
|
|
351
|
+
rep["parse_failed"].append(rel)
|
|
352
|
+
ln = getattr(ex, "lineno", None)
|
|
353
|
+
rep["syntax_errors"][rel] = [[ln, getattr(ex, "end_lineno", None) or ln]] if isinstance(ln, int) and ln > 0 else []
|
|
354
|
+
continue
|
|
355
|
+
parsed[rel] = (tree, [n for n in ast.walk(tree) if isinstance(n, (ast.Import, ast.ImportFrom))])
|
|
356
|
+
configured, origin = self.configured_roots()
|
|
357
|
+
# every discovered file shapes the layout (a namespace package may only hold files that failed to parse)
|
|
358
|
+
plan = RootPlan(self.root, files, import_names(v[1] for v in parsed.values()), configured, origin,
|
|
359
|
+
skip_dirs=self.path_rules.names)
|
|
360
|
+
self.root_plan = plan
|
|
361
|
+
names, unmapped = plan.assign()
|
|
362
|
+
owners: dict = {}
|
|
363
|
+
for rel in unmapped:
|
|
364
|
+
if rel in parsed:
|
|
365
|
+
rep["unmapped"].append(rel) # no importable module path (directory name not an identifier, outside the source roots, name taken)
|
|
366
|
+
for rel, (tree, imps) in parsed.items():
|
|
367
|
+
if rel not in names:
|
|
368
|
+
continue
|
|
369
|
+
name, is_pkg, _, root = names[rel]
|
|
370
|
+
m = ModInfo(name=name, file=rel, tree=tree, is_pkg=is_pkg, import_nodes=imps)
|
|
371
|
+
self.modules[name] = m
|
|
372
|
+
self.by_file[rel] = m
|
|
373
|
+
self.alias[name] = m
|
|
374
|
+
if root is not None:
|
|
375
|
+
owners[root.key] = owners.get(root.key, 0) + 1
|
|
376
|
+
for rel, (name, _, aliases, _) in names.items(): # other importable names, after every canonical one
|
|
377
|
+
m = self.by_file.get(rel)
|
|
378
|
+
for nm in aliases if m else ():
|
|
379
|
+
self.alias.setdefault(nm, m)
|
|
380
|
+
self.roots_report = plan.report(owners)
|
|
381
|
+
for m in self.modules.values():
|
|
382
|
+
self._collect(m)
|
|
383
|
+
return {"files": len(files), "parsed": len(self.modules), "skipped": n_skipped, "parse_errors": len(self.parse_errors)}
|
|
384
|
+
|
|
385
|
+
def _collect(self, m: ModInfo) -> None:
|
|
386
|
+
for st in m.tree.body:
|
|
387
|
+
self._collect_stmt(m, st, top=True)
|
|
388
|
+
top = {id(st) for st in m.tree.body}
|
|
389
|
+
for st in m.import_nodes:
|
|
390
|
+
if id(st) not in top:
|
|
391
|
+
self._import(m, st, local_only=True)
|
|
392
|
+
|
|
393
|
+
def _collect_stmt(self, m: ModInfo, st, top: bool):
|
|
394
|
+
if isinstance(st, (ast.Import, ast.ImportFrom)):
|
|
395
|
+
self._import(m, st)
|
|
396
|
+
elif isinstance(st, ast.ClassDef):
|
|
397
|
+
c = self._class(m, st, None)
|
|
398
|
+
m.classes[st.name] = c
|
|
399
|
+
elif isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
400
|
+
f = FuncInfo(st.name, f"{m.name}.{st.name}", m, st, decorators=st.decorator_list)
|
|
401
|
+
m.funcs[st.name] = f
|
|
402
|
+
self.funcs[f.qual] = f
|
|
403
|
+
elif isinstance(st, ast.Assign):
|
|
404
|
+
for t in st.targets:
|
|
405
|
+
for nm in self._target_names(t):
|
|
406
|
+
m.vars.setdefault(nm, []).append((st.value, st.lineno, None))
|
|
407
|
+
elif isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Name):
|
|
408
|
+
m.vars.setdefault(st.target.id, []).append((st.value, st.lineno, st.annotation))
|
|
409
|
+
elif isinstance(st, (ast.If, ast.Try, ast.With)) or (hasattr(ast, "TryStar") and isinstance(st, getattr(ast, "TryStar"))):
|
|
410
|
+
for sub in ast.iter_child_nodes(st):
|
|
411
|
+
if isinstance(sub, ast.stmt):
|
|
412
|
+
self._collect_stmt(m, sub, top)
|
|
413
|
+
elif isinstance(sub, ast.ExceptHandler):
|
|
414
|
+
for s2 in sub.body:
|
|
415
|
+
self._collect_stmt(m, s2, top)
|
|
416
|
+
|
|
417
|
+
@staticmethod
|
|
418
|
+
def _target_names(t) -> list[str]:
|
|
419
|
+
if isinstance(t, ast.Name):
|
|
420
|
+
return [t.id]
|
|
421
|
+
if isinstance(t, (ast.Tuple, ast.List)):
|
|
422
|
+
return [n for e in t.elts for n in PyProgram._target_names(e)]
|
|
423
|
+
return []
|
|
424
|
+
|
|
425
|
+
def _class(self, m: ModInfo, node: ast.ClassDef, outer: ClassInfo | None) -> ClassInfo:
|
|
426
|
+
qual = f"{outer.qual}.{node.name}" if outer else f"{m.name}.{node.name}"
|
|
427
|
+
c = ClassInfo(node.name, qual, m, node, bases=list(node.bases), decorators=node.decorator_list, outer=outer)
|
|
428
|
+
self.classes[qual] = c
|
|
429
|
+
m.all_classes.append(c)
|
|
430
|
+
for st in node.body:
|
|
431
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
432
|
+
f = FuncInfo(st.name, f"{qual}.{st.name}", m, st, cls=c, kind="method", decorators=st.decorator_list)
|
|
433
|
+
c.methods[st.name] = f
|
|
434
|
+
self.funcs[f.qual] = f
|
|
435
|
+
self.method_index.setdefault(st.name, []).append(f)
|
|
436
|
+
for sub in walk_body(st):
|
|
437
|
+
if isinstance(sub, (ast.Assign, ast.AnnAssign)):
|
|
438
|
+
tg = sub.targets if isinstance(sub, ast.Assign) else [sub.target]
|
|
439
|
+
for t in tg:
|
|
440
|
+
if isinstance(t, ast.Attribute) and isinstance(t.value, ast.Name) and t.value.id == "self":
|
|
441
|
+
c.self_attrs.setdefault(t.attr, []).append(
|
|
442
|
+
(sub.value, getattr(sub, "annotation", None), f))
|
|
443
|
+
c.self_attr_lines.setdefault(t.attr, sub.lineno)
|
|
444
|
+
elif isinstance(st, ast.ClassDef):
|
|
445
|
+
if st.name in ("Meta", "Config", "Media", "Params", "Input", "Output"):
|
|
446
|
+
c.inner[st.name] = st
|
|
447
|
+
ic = self._class(m, st, c)
|
|
448
|
+
c.attrs.setdefault(st.name, (None, st.lineno, None))
|
|
449
|
+
c.inner.setdefault(st.name, st)
|
|
450
|
+
c.inner_infos = getattr(c, "inner_infos", {})
|
|
451
|
+
c.inner_infos[st.name] = ic
|
|
452
|
+
elif isinstance(st, ast.Assign):
|
|
453
|
+
for t in st.targets:
|
|
454
|
+
for nm in self._target_names(t):
|
|
455
|
+
c.attrs[nm] = (st.value, st.lineno, None)
|
|
456
|
+
elif isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Name):
|
|
457
|
+
c.attrs[st.target.id] = (st.value, st.lineno, st.annotation)
|
|
458
|
+
return c
|
|
459
|
+
|
|
460
|
+
def _import(self, m: ModInfo, st, local_only=False):
|
|
461
|
+
target = m.imports
|
|
462
|
+
if isinstance(st, ast.Import):
|
|
463
|
+
for a in st.names:
|
|
464
|
+
if a.asname:
|
|
465
|
+
target.setdefault(a.asname, ("mod", a.name)) if local_only else target.__setitem__(a.asname, ("mod", a.name))
|
|
466
|
+
else:
|
|
467
|
+
head = a.name.split(".")[0]
|
|
468
|
+
if not local_only or head not in target:
|
|
469
|
+
target[head] = ("modprefix", head)
|
|
470
|
+
else:
|
|
471
|
+
base = self._abs_from(m, st.module, st.level)
|
|
472
|
+
if base is None:
|
|
473
|
+
return
|
|
474
|
+
for a in st.names:
|
|
475
|
+
if a.name == "*":
|
|
476
|
+
if base not in m.star:
|
|
477
|
+
m.star.append(base)
|
|
478
|
+
continue
|
|
479
|
+
nm = a.asname or a.name
|
|
480
|
+
if local_only and nm in target:
|
|
481
|
+
continue
|
|
482
|
+
target[nm] = ("sym", base, a.name)
|
|
483
|
+
|
|
484
|
+
def _abs_from(self, m: ModInfo, mod: str | None, level: int) -> str | None:
|
|
485
|
+
if not level:
|
|
486
|
+
return mod
|
|
487
|
+
pkg = m.package.split(".") if m.package else []
|
|
488
|
+
if level > 1:
|
|
489
|
+
pkg = pkg[:len(pkg) - (level - 1)] if level - 1 <= len(pkg) else []
|
|
490
|
+
base = ".".join(pkg)
|
|
491
|
+
if mod:
|
|
492
|
+
base = f"{base}.{mod}" if base else mod
|
|
493
|
+
return base
|
|
494
|
+
|
|
495
|
+
# ---- symbol resolution
|
|
496
|
+
def module(self, name: str) -> ModInfo | None:
|
|
497
|
+
return self.alias.get(name)
|
|
498
|
+
|
|
499
|
+
def lookup(self, modname: str, name: str, depth=0):
|
|
500
|
+
"""Resolve `name` in module `modname`. Returns a type tuple: ("type", ClassInfo) | ("func", FuncInfo) |
|
|
501
|
+
("mod", ModInfo) | ("var", ModInfo, name) | ("ext", dotted) | None."""
|
|
502
|
+
if depth > 8:
|
|
503
|
+
return None
|
|
504
|
+
m = self.module(modname)
|
|
505
|
+
if m is None:
|
|
506
|
+
sub = self.module(f"{modname}.{name}")
|
|
507
|
+
if sub:
|
|
508
|
+
return ("mod", sub)
|
|
509
|
+
return ("ext", f"{modname}.{name}")
|
|
510
|
+
if name in m.classes:
|
|
511
|
+
return ("type", m.classes[name])
|
|
512
|
+
if name in m.funcs:
|
|
513
|
+
return ("func", m.funcs[name])
|
|
514
|
+
if name in m.imports:
|
|
515
|
+
return self._resolve_import(m.imports[name], depth + 1)
|
|
516
|
+
if name in m.vars:
|
|
517
|
+
return ("var", m, name)
|
|
518
|
+
sub = self.module(f"{m.name}.{name}")
|
|
519
|
+
if sub:
|
|
520
|
+
return ("mod", sub)
|
|
521
|
+
for s in m.star:
|
|
522
|
+
r = self.lookup(s, name, depth + 1)
|
|
523
|
+
if r and not (r[0] == "ext" and self.module(s) is not None):
|
|
524
|
+
return r
|
|
525
|
+
return None
|
|
526
|
+
|
|
527
|
+
def _resolve_import(self, imp, depth=0):
|
|
528
|
+
if imp[0] == "mod":
|
|
529
|
+
m = self.module(imp[1])
|
|
530
|
+
return ("mod", m) if m else ("ext", imp[1])
|
|
531
|
+
if imp[0] == "modprefix":
|
|
532
|
+
m = self.module(imp[1])
|
|
533
|
+
return ("mod", m) if m else ("ext", imp[1])
|
|
534
|
+
_, base, name = imp
|
|
535
|
+
sub = self.module(f"{base}.{name}")
|
|
536
|
+
if sub and (self.module(base) is None or name not in self.module(base).classes and name not in self.module(base).funcs):
|
|
537
|
+
return ("mod", sub)
|
|
538
|
+
return self.lookup(base, name, depth)
|
|
539
|
+
|
|
540
|
+
def resolve_name(self, m: ModInfo, name: str):
|
|
541
|
+
if name in m.classes:
|
|
542
|
+
return ("type", m.classes[name])
|
|
543
|
+
if name in m.funcs:
|
|
544
|
+
return ("func", m.funcs[name])
|
|
545
|
+
if name in m.imports:
|
|
546
|
+
return self._resolve_import(m.imports[name])
|
|
547
|
+
if name in m.vars:
|
|
548
|
+
return ("var", m, name)
|
|
549
|
+
for s in m.star:
|
|
550
|
+
r = self.lookup(s, name)
|
|
551
|
+
if r:
|
|
552
|
+
return r
|
|
553
|
+
if name in BUILTINS:
|
|
554
|
+
return ("ext", f"builtins.{name}")
|
|
555
|
+
return None
|
|
556
|
+
|
|
557
|
+
def member(self, t, attr: str, ctx=None):
|
|
558
|
+
"""Type of `<t>.attr`."""
|
|
559
|
+
if t is None:
|
|
560
|
+
return None
|
|
561
|
+
for rule in self.attr_rules:
|
|
562
|
+
r = rule(self, t, attr, ctx)
|
|
563
|
+
if r is not None:
|
|
564
|
+
return r
|
|
565
|
+
k = t[0]
|
|
566
|
+
if k == "mod":
|
|
567
|
+
return self.lookup(t[1].name, attr)
|
|
568
|
+
if k == "ext":
|
|
569
|
+
sub = self.module(f"{t[1]}.{attr}")
|
|
570
|
+
return ("mod", sub) if sub else ("ext", f"{t[1]}.{attr}")
|
|
571
|
+
if k == "var":
|
|
572
|
+
return self.member(self.var_type(t[1], t[2]), attr, ctx)
|
|
573
|
+
if k in ("type", "inst"):
|
|
574
|
+
c = t[1]
|
|
575
|
+
f = self.find_method(c, attr)
|
|
576
|
+
if f:
|
|
577
|
+
return ("bound", f, t)
|
|
578
|
+
v = self.class_attr(c, attr)
|
|
579
|
+
if v is not None:
|
|
580
|
+
return v
|
|
581
|
+
if k == "inst":
|
|
582
|
+
sa = self.self_attr_type(c, attr)
|
|
583
|
+
if sa is not None:
|
|
584
|
+
return sa
|
|
585
|
+
return None
|
|
586
|
+
|
|
587
|
+
def clear_memo(self) -> None:
|
|
588
|
+
"""Forget memoised attribute types (framework hooks registered after load() can change them)."""
|
|
589
|
+
self._attr_memo.clear()
|
|
590
|
+
|
|
591
|
+
def _memo_attr(self, kind: str, fn, c: ClassInfo, attr: str, depth: int):
|
|
592
|
+
key = (kind, id(c), attr)
|
|
593
|
+
if key in self._attr_memo:
|
|
594
|
+
return self._attr_memo[key]
|
|
595
|
+
if key in self._attr_busy:
|
|
596
|
+
self._cuts += 1
|
|
597
|
+
self.infer_stats["attr_cycles"] += 1
|
|
598
|
+
return None
|
|
599
|
+
self._attr_busy.add(key)
|
|
600
|
+
cuts = self._cuts
|
|
601
|
+
try:
|
|
602
|
+
r = fn(c, attr, depth)
|
|
603
|
+
finally:
|
|
604
|
+
self._attr_busy.discard(key)
|
|
605
|
+
if self._cuts == cuts and self._work <= INFER_BUDGET: # complete answer: no cycle cut, no budget stop
|
|
606
|
+
self._attr_memo[key] = r
|
|
607
|
+
return r
|
|
608
|
+
|
|
609
|
+
def class_attr(self, c: ClassInfo, attr: str, depth=0):
|
|
610
|
+
return self._memo_attr("c", self._class_attr, c, attr, depth)
|
|
611
|
+
|
|
612
|
+
def self_attr_type(self, c: ClassInfo, attr: str, depth=0):
|
|
613
|
+
return self._memo_attr("s", self._self_attr_type, c, attr, depth)
|
|
614
|
+
|
|
615
|
+
def _class_attr(self, c: ClassInfo, attr: str, depth=0):
|
|
616
|
+
if depth > 10:
|
|
617
|
+
return None
|
|
618
|
+
if attr in c.attrs:
|
|
619
|
+
val, _, ann = c.attrs[attr]
|
|
620
|
+
if hasattr(c, "inner_infos") and attr in c.inner_infos:
|
|
621
|
+
return ("type", c.inner_infos[attr])
|
|
622
|
+
if ann is not None:
|
|
623
|
+
t = self.ann_type(c.module, ann)
|
|
624
|
+
if t:
|
|
625
|
+
return t
|
|
626
|
+
if val is not None:
|
|
627
|
+
return self.infer(val, Ctx(c.module, None, c))
|
|
628
|
+
return None
|
|
629
|
+
for b in self.bases(c):
|
|
630
|
+
if b[0] == "type":
|
|
631
|
+
r = self.class_attr(b[1], attr, depth + 1)
|
|
632
|
+
if r is not None:
|
|
633
|
+
return r
|
|
634
|
+
return None
|
|
635
|
+
|
|
636
|
+
def _self_attr_type(self, c: ClassInfo, attr: str, depth=0):
|
|
637
|
+
if depth > 10:
|
|
638
|
+
return None
|
|
639
|
+
for val, ann, f in c.self_attrs.get(attr, []):
|
|
640
|
+
if ann is not None:
|
|
641
|
+
t = self.ann_type(c.module, ann)
|
|
642
|
+
if t:
|
|
643
|
+
return t
|
|
644
|
+
if val is not None:
|
|
645
|
+
t = self.infer(val, Ctx(c.module, f, c))
|
|
646
|
+
if t:
|
|
647
|
+
return t
|
|
648
|
+
for b in self.bases(c):
|
|
649
|
+
if b[0] == "type":
|
|
650
|
+
r = self.self_attr_type(b[1], attr, depth + 1)
|
|
651
|
+
if r is not None:
|
|
652
|
+
return r
|
|
653
|
+
return None
|
|
654
|
+
|
|
655
|
+
def bases(self, c: ClassInfo) -> list:
|
|
656
|
+
cached = getattr(c, "_bases", None)
|
|
657
|
+
if cached is not None:
|
|
658
|
+
return cached
|
|
659
|
+
c._bases = []
|
|
660
|
+
out = []
|
|
661
|
+
for b in c.bases:
|
|
662
|
+
if isinstance(b, ast.Subscript): # Generic[...] / Model[T]
|
|
663
|
+
b = b.value
|
|
664
|
+
t = self.infer(b, Ctx(c.module, None, c.outer))
|
|
665
|
+
if t and t[0] == "var":
|
|
666
|
+
t = self.var_type(t[1], t[2])
|
|
667
|
+
if t:
|
|
668
|
+
out.append(t)
|
|
669
|
+
c._bases = out
|
|
670
|
+
return out
|
|
671
|
+
|
|
672
|
+
def mro(self, c: ClassInfo, depth=0) -> list:
|
|
673
|
+
"""Local classes and external dotted names in (approximate) MRO order."""
|
|
674
|
+
out, seen = [], set()
|
|
675
|
+
|
|
676
|
+
def rec(x, d):
|
|
677
|
+
if d > 15:
|
|
678
|
+
return
|
|
679
|
+
for b in self.bases(x):
|
|
680
|
+
key = b[1].qual if b[0] == "type" else str(b[1])
|
|
681
|
+
if key in seen:
|
|
682
|
+
continue
|
|
683
|
+
seen.add(key)
|
|
684
|
+
out.append(b)
|
|
685
|
+
if b[0] == "type":
|
|
686
|
+
rec(b[1], d + 1)
|
|
687
|
+
rec(c, 0)
|
|
688
|
+
return out
|
|
689
|
+
|
|
690
|
+
def ext_bases(self, c: ClassInfo) -> set[str]:
|
|
691
|
+
return {b[1] for b in self.mro(c) if b[0] == "ext"}
|
|
692
|
+
|
|
693
|
+
def lineage(self, c: ClassInfo) -> list[str]:
|
|
694
|
+
"""Dotted names of the class's ancestors: external bases plus local ancestor quals (so indexing a
|
|
695
|
+
framework's own source, where `ninja.Router` is a local class, still recognises its subclasses)."""
|
|
696
|
+
cached = getattr(c, "_lineage", None)
|
|
697
|
+
if cached is None:
|
|
698
|
+
cached = [b[1] if b[0] == "ext" else b[1].qual for b in self.mro(c) if b[0] in ("ext", "type")]
|
|
699
|
+
c._lineage = cached
|
|
700
|
+
return cached
|
|
701
|
+
|
|
702
|
+
def subclass_of(self, c: ClassInfo, *names: str, include_self: bool = False) -> bool:
|
|
703
|
+
"""True when an ancestor's dotted name equals / ends with one of `names` (e.g. 'models.Model', 'Schema')."""
|
|
704
|
+
cands = self.lineage(c) + ([c.qual] if include_self else [])
|
|
705
|
+
for b in cands:
|
|
706
|
+
for n in names:
|
|
707
|
+
if b == n or b.endswith("." + n):
|
|
708
|
+
return True
|
|
709
|
+
return False
|
|
710
|
+
|
|
711
|
+
def find_method(self, c: ClassInfo, name: str, depth=0) -> FuncInfo | None:
|
|
712
|
+
if name in c.methods:
|
|
713
|
+
return c.methods[name]
|
|
714
|
+
if depth > 15:
|
|
715
|
+
return None
|
|
716
|
+
for b in self.bases(c):
|
|
717
|
+
if b[0] == "type":
|
|
718
|
+
r = self.find_method(b[1], name, depth + 1)
|
|
719
|
+
if r:
|
|
720
|
+
return r
|
|
721
|
+
return None
|
|
722
|
+
|
|
723
|
+
def var_type(self, m: ModInfo, name: str):
|
|
724
|
+
key = (id(m), name)
|
|
725
|
+
if key in self._var_cache:
|
|
726
|
+
return self._var_cache[key]
|
|
727
|
+
self._var_cache[key] = None
|
|
728
|
+
t = None
|
|
729
|
+
for val, _, ann in m.vars.get(name, []):
|
|
730
|
+
if ann is not None:
|
|
731
|
+
t = self.ann_type(m, ann)
|
|
732
|
+
if t is None and val is not None:
|
|
733
|
+
t = self.infer(val, Ctx(m, None, None))
|
|
734
|
+
if t:
|
|
735
|
+
break
|
|
736
|
+
self._var_cache[key] = t
|
|
737
|
+
return t
|
|
738
|
+
|
|
739
|
+
def var_value(self, m: ModInfo, name: str):
|
|
740
|
+
vs = m.vars.get(name) or []
|
|
741
|
+
return vs[-1][0] if vs else None
|
|
742
|
+
|
|
743
|
+
def ann_type(self, m: ModInfo, ann):
|
|
744
|
+
e, cont = ann_base(ann)
|
|
745
|
+
if e is None:
|
|
746
|
+
return None
|
|
747
|
+
t = self.infer(e, Ctx(m, None, None))
|
|
748
|
+
if t and t[0] == "type":
|
|
749
|
+
t = ("inst", t[1])
|
|
750
|
+
elif t and t[0] == "ext":
|
|
751
|
+
t = ("einst", t[1])
|
|
752
|
+
else:
|
|
753
|
+
return None
|
|
754
|
+
return ("list", t) if cont == "list" else t
|
|
755
|
+
|
|
756
|
+
# ---- inference
|
|
757
|
+
def local_vars(self, ctx: "Ctx") -> dict:
|
|
758
|
+
"""name -> list of (value ast | None, annotation ast | None, kind) for the current def."""
|
|
759
|
+
f = ctx.func
|
|
760
|
+
if f is None:
|
|
761
|
+
return {}
|
|
762
|
+
key = id(f.node)
|
|
763
|
+
if key in self._var_cache:
|
|
764
|
+
return self._var_cache[key]
|
|
765
|
+
lv: dict[str, list] = {}
|
|
766
|
+
args = f.node.args
|
|
767
|
+
allargs = list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)
|
|
768
|
+
for i, a in enumerate(allargs):
|
|
769
|
+
lv.setdefault(a.arg, []).append((None, a.annotation, "param"))
|
|
770
|
+
for sub in walk_body(f.node):
|
|
771
|
+
if isinstance(sub, ast.Assign):
|
|
772
|
+
for t in sub.targets:
|
|
773
|
+
if isinstance(t, ast.Name):
|
|
774
|
+
lv.setdefault(t.id, []).append((sub.value, None, "assign"))
|
|
775
|
+
elif isinstance(t, ast.Tuple) and isinstance(sub.value, ast.Call):
|
|
776
|
+
for i, el in enumerate(t.elts):
|
|
777
|
+
if isinstance(el, ast.Name):
|
|
778
|
+
lv.setdefault(el.id, []).append((sub.value, None, f"unpack{i}"))
|
|
779
|
+
elif isinstance(sub, ast.AnnAssign) and isinstance(sub.target, ast.Name):
|
|
780
|
+
lv.setdefault(sub.target.id, []).append((sub.value, sub.annotation, "assign"))
|
|
781
|
+
elif isinstance(sub, (ast.For, ast.AsyncFor, ast.comprehension)) and isinstance(sub.target, ast.Name):
|
|
782
|
+
lv.setdefault(sub.target.id, []).append((sub.iter, None, "iter"))
|
|
783
|
+
elif isinstance(sub, (ast.For, ast.AsyncFor, ast.comprehension)) and isinstance(sub.target, ast.Tuple):
|
|
784
|
+
for i, el in enumerate(sub.target.elts): # for name, fn in TABLE / TABLE.items()
|
|
785
|
+
if isinstance(el, ast.Name):
|
|
786
|
+
lv.setdefault(el.id, []).append((sub.iter, None, f"iterunpack{i}"))
|
|
787
|
+
elif isinstance(sub, (ast.With, ast.AsyncWith)):
|
|
788
|
+
for it in sub.items:
|
|
789
|
+
if isinstance(it.optional_vars, ast.Name):
|
|
790
|
+
lv.setdefault(it.optional_vars.id, []).append((it.context_expr, None, "with"))
|
|
791
|
+
elif isinstance(sub, ast.NamedExpr) and isinstance(sub.target, ast.Name):
|
|
792
|
+
lv.setdefault(sub.target.id, []).append((sub.value, None, "assign"))
|
|
793
|
+
self._var_cache[key] = lv
|
|
794
|
+
return lv
|
|
795
|
+
|
|
796
|
+
def func_imports(self, f: FuncInfo) -> dict:
|
|
797
|
+
"""Names bound by import statements inside a def (incl. its nested defs): local name -> import tuple."""
|
|
798
|
+
cached = getattr(f, "_imports", None)
|
|
799
|
+
if cached is not None:
|
|
800
|
+
return cached
|
|
801
|
+
out: dict = {}
|
|
802
|
+
m = f.module
|
|
803
|
+
local = getattr(m, "_local_imports", None)
|
|
804
|
+
if local is None:
|
|
805
|
+
top = {id(st) for st in m.tree.body}
|
|
806
|
+
local = m._local_imports = [st for st in m.import_nodes if id(st) not in top]
|
|
807
|
+
lo, hi = f.node.lineno, getattr(f.node, "end_lineno", None) or f.node.lineno
|
|
808
|
+
for sub in local:
|
|
809
|
+
if not lo <= sub.lineno <= hi:
|
|
810
|
+
continue
|
|
811
|
+
if isinstance(sub, ast.Import):
|
|
812
|
+
for a in sub.names:
|
|
813
|
+
if a.asname:
|
|
814
|
+
out[a.asname] = ("mod", a.name)
|
|
815
|
+
else:
|
|
816
|
+
out.setdefault(a.name.split(".")[0], ("modprefix", a.name.split(".")[0]))
|
|
817
|
+
elif isinstance(sub, ast.ImportFrom):
|
|
818
|
+
base = self._abs_from(f.module, sub.module, sub.level)
|
|
819
|
+
if base is None:
|
|
820
|
+
continue
|
|
821
|
+
for a in sub.names:
|
|
822
|
+
if a.name != "*":
|
|
823
|
+
out[a.asname or a.name] = ("sym", base, a.name)
|
|
824
|
+
f._imports = out
|
|
825
|
+
return out
|
|
826
|
+
|
|
827
|
+
def infer(self, e, ctx: "Ctx"):
|
|
828
|
+
if e is None or self._infer_depth > 40:
|
|
829
|
+
return None
|
|
830
|
+
if self._infer_depth == 0:
|
|
831
|
+
self._work = 0
|
|
832
|
+
self._work += 1
|
|
833
|
+
if self._work > INFER_BUDGET:
|
|
834
|
+
if self._work == INFER_BUDGET + 1:
|
|
835
|
+
self.infer_stats["budget_exhausted"] += 1
|
|
836
|
+
return None
|
|
837
|
+
self._infer_depth += 1
|
|
838
|
+
try:
|
|
839
|
+
return self._infer(e, ctx)
|
|
840
|
+
finally:
|
|
841
|
+
self._infer_depth -= 1
|
|
842
|
+
|
|
843
|
+
def _infer(self, e, ctx: "Ctx"):
|
|
844
|
+
if isinstance(e, ast.Name):
|
|
845
|
+
n = e.id
|
|
846
|
+
if ctx.func is not None:
|
|
847
|
+
if n == "self" and ctx.cls is not None:
|
|
848
|
+
return ("inst", ctx.cls)
|
|
849
|
+
if n == "cls" and ctx.cls is not None:
|
|
850
|
+
return ("type", ctx.cls)
|
|
851
|
+
lv = self.local_vars(ctx)
|
|
852
|
+
if n in lv:
|
|
853
|
+
key = (id(ctx.func.node), n)
|
|
854
|
+
if key in ctx.visiting:
|
|
855
|
+
return None
|
|
856
|
+
ctx.visiting.add(key)
|
|
857
|
+
try:
|
|
858
|
+
for val, ann, kind in lv[n]:
|
|
859
|
+
t = None
|
|
860
|
+
if ann is not None:
|
|
861
|
+
t = self.ann_type(ctx.mod, ann)
|
|
862
|
+
if t is None and val is not None and not kind.startswith("iterunpack"):
|
|
863
|
+
t = self.infer(val, ctx)
|
|
864
|
+
if t is not None and kind == "iter":
|
|
865
|
+
t = t[1] if t[0] == "list" else self.iter_type(t)
|
|
866
|
+
elif t is not None and kind.startswith("unpack"):
|
|
867
|
+
t = self.unpack_type(t, int(kind[6:]))
|
|
868
|
+
if t is not None:
|
|
869
|
+
return t
|
|
870
|
+
finally:
|
|
871
|
+
ctx.visiting.discard(key)
|
|
872
|
+
return None
|
|
873
|
+
li = self.func_imports(ctx.func)
|
|
874
|
+
if n in li: # `from . import routes as R` inside this def: the def's own binding, not another def's
|
|
875
|
+
return self._resolve_import(li[n])
|
|
876
|
+
if ctx.cls is not None and ctx.func is None and n in ctx.cls.attrs:
|
|
877
|
+
return self.class_attr(ctx.cls, n)
|
|
878
|
+
if ctx.cls is not None and ctx.func is None and n in ctx.cls.methods: # class body: `TABLE = {"go": start}`
|
|
879
|
+
return ("func", ctx.cls.methods[n])
|
|
880
|
+
r = self.resolve_name(ctx.mod, n)
|
|
881
|
+
if r and r[0] == "var":
|
|
882
|
+
vt = self.var_type(r[1], r[2])
|
|
883
|
+
return vt if vt is not None else r
|
|
884
|
+
return r
|
|
885
|
+
if isinstance(e, ast.Attribute):
|
|
886
|
+
base = self.infer(e.value, ctx)
|
|
887
|
+
return self.member(base, e.attr, ctx)
|
|
888
|
+
if isinstance(e, ast.Call):
|
|
889
|
+
ft = self.infer(e.func, ctx)
|
|
890
|
+
for rule in self.call_rules:
|
|
891
|
+
r = rule(self, e, ft, ctx)
|
|
892
|
+
if r is not None:
|
|
893
|
+
return r
|
|
894
|
+
if ft is None:
|
|
895
|
+
if isinstance(e.func, ast.Name) and e.func.id == "super" and ctx.cls is not None:
|
|
896
|
+
return ("super", ctx.cls)
|
|
897
|
+
return None
|
|
898
|
+
k = ft[0]
|
|
899
|
+
if k == "type":
|
|
900
|
+
return ("inst", ft[1])
|
|
901
|
+
if k == "ext":
|
|
902
|
+
return ("einst", ft[1])
|
|
903
|
+
if k in ("func", "bound"):
|
|
904
|
+
f = ft[1]
|
|
905
|
+
ann = f.node.returns
|
|
906
|
+
if ann is not None:
|
|
907
|
+
t = self.ann_type(f.module, ann)
|
|
908
|
+
if t:
|
|
909
|
+
return t
|
|
910
|
+
if ft[0] == "bound" and isinstance(f.node, ast.FunctionDef) and any(
|
|
911
|
+
dotted(d) == "classmethod" for d in f.decorators):
|
|
912
|
+
return ("inst", ft[2][1]) if f.name in ("create", "build", "from_dict", "new") else None
|
|
913
|
+
if ann is None:
|
|
914
|
+
return self.returned_instance(f)
|
|
915
|
+
return None
|
|
916
|
+
return None
|
|
917
|
+
if isinstance(e, ast.Await):
|
|
918
|
+
return self.infer(e.value, ctx)
|
|
919
|
+
if isinstance(e, (ast.List, ast.ListComp)) and isinstance(e, ast.ListComp):
|
|
920
|
+
t = self.infer(e.elt, ctx)
|
|
921
|
+
return ("list", t) if t else None
|
|
922
|
+
if isinstance(e, ast.Subscript):
|
|
923
|
+
t = self.infer(e.value, ctx)
|
|
924
|
+
if t and t[0] == "list":
|
|
925
|
+
return t[1]
|
|
926
|
+
if t and t[0] == "qs":
|
|
927
|
+
return ("inst", t[1]) if not isinstance(e.slice, ast.Slice) else t
|
|
928
|
+
return None
|
|
929
|
+
if isinstance(e, ast.IfExp):
|
|
930
|
+
return self.infer(e.body, ctx) or self.infer(e.orelse, ctx)
|
|
931
|
+
if isinstance(e, ast.BoolOp):
|
|
932
|
+
for v in e.values:
|
|
933
|
+
t = self.infer(v, ctx)
|
|
934
|
+
if t:
|
|
935
|
+
return t
|
|
936
|
+
return None
|
|
937
|
+
|
|
938
|
+
def iter_type(self, t):
|
|
939
|
+
if t and t[0] == "qs":
|
|
940
|
+
return ("inst", t[1])
|
|
941
|
+
return None
|
|
942
|
+
|
|
943
|
+
def unpack_type(self, t, i):
|
|
944
|
+
if t and t[0] == "tuple":
|
|
945
|
+
return t[1][i] if i < len(t[1]) else None
|
|
946
|
+
return None
|
|
947
|
+
|
|
948
|
+
# ---- call resolution
|
|
949
|
+
def resolve_call(self, call: ast.Call, ctx: "Ctx"):
|
|
950
|
+
"""-> list of (target FuncInfo|ClassInfo, confidence, via)"""
|
|
951
|
+
fn = call.func
|
|
952
|
+
out = []
|
|
953
|
+
if isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Call) and isinstance(fn.value.func, ast.Name) \
|
|
954
|
+
and fn.value.func.id == "super" and ctx.cls is not None:
|
|
955
|
+
for b in self.bases(ctx.cls):
|
|
956
|
+
if b[0] == "type":
|
|
957
|
+
f = self.find_method(b[1], fn.attr)
|
|
958
|
+
if f:
|
|
959
|
+
return [(f, EXACT, "super")]
|
|
960
|
+
return []
|
|
961
|
+
# sync_to_async(f)(...), database_sync_to_async(f)(...), async_to_sync(f)(...): calls f
|
|
962
|
+
if isinstance(fn, ast.Call) and fn.args and (
|
|
963
|
+
(isinstance(fn.func, ast.Name) and fn.func.id in ASYNC_WRAPPERS) or
|
|
964
|
+
(isinstance(fn.func, ast.Attribute) and fn.func.attr in ASYNC_WRAPPERS)) \
|
|
965
|
+
and not isinstance(fn.args[0], ast.Lambda):
|
|
966
|
+
inner = ast.Call(func=fn.args[0], args=call.args, keywords=call.keywords)
|
|
967
|
+
ast.copy_location(inner, call)
|
|
968
|
+
return [(tgt, conf, via or "async_wrapper") for tgt, conf, via in self.resolve_call(inner, ctx)]
|
|
969
|
+
t = self.infer(fn, ctx)
|
|
970
|
+
if t is not None:
|
|
971
|
+
if t[0] == "func":
|
|
972
|
+
return [(t[1], EXACT, None)]
|
|
973
|
+
if t[0] == "bound":
|
|
974
|
+
recv = t[2]
|
|
975
|
+
conf = EXACT if isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Name) and \
|
|
976
|
+
fn.value.id in ("self", "cls") else RESOLVED
|
|
977
|
+
if isinstance(fn, ast.Attribute):
|
|
978
|
+
vt = self.infer(fn.value, ctx)
|
|
979
|
+
if vt and vt[0] == "type" and isinstance(fn.value, (ast.Name, ast.Attribute)):
|
|
980
|
+
conf = EXACT
|
|
981
|
+
out.append((t[1], conf, None))
|
|
982
|
+
if recv[0] in ("inst", "type") and isinstance(recv[1], ClassInfo) and t[1].cls is not None \
|
|
983
|
+
and recv[1] is not t[1].cls:
|
|
984
|
+
self.recv[(id(fn), id(t[1]))] = [recv[1].id]
|
|
985
|
+
# dispatch to overrides in subclasses (self.m() in a base class)
|
|
986
|
+
if recv[0] == "inst" and isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Name) and fn.value.id == "self":
|
|
987
|
+
for sub in self.subclasses(recv[1]):
|
|
988
|
+
if fn.attr in sub.methods:
|
|
989
|
+
out.append((sub.methods[fn.attr], RESOLVED, "override"))
|
|
990
|
+
return out
|
|
991
|
+
if t[0] == "type":
|
|
992
|
+
return [(t[1], EXACT, "instantiate")]
|
|
993
|
+
return []
|
|
994
|
+
if isinstance(fn, (ast.Name, ast.Attribute, ast.Subscript, ast.Call)):
|
|
995
|
+
out = self.dispatch_targets(fn, ctx)
|
|
996
|
+
if out:
|
|
997
|
+
return out
|
|
998
|
+
if isinstance(fn, ast.Attribute):
|
|
999
|
+
cands = self.method_index.get(fn.attr) or []
|
|
1000
|
+
if len(cands) == 1 and fn.attr not in STOP_METHODS and not fn.attr.startswith("__"):
|
|
1001
|
+
vt = self.infer(fn.value, ctx)
|
|
1002
|
+
if vt is None:
|
|
1003
|
+
return [(cands[0], HEURISTIC, "unique-method-name")]
|
|
1004
|
+
return []
|
|
1005
|
+
|
|
1006
|
+
# ---- calls through a collection (dispatch tables, plugin lists, callback registries)
|
|
1007
|
+
def dispatch_targets(self, fn, ctx: "Ctx") -> list:
|
|
1008
|
+
"""Callee drawn from a collection of known functions / classes / instances: `for check in CHECKS: check(x)`,
|
|
1009
|
+
`HANDLERS[kind](x)`, `HANDLERS.get(kind)(x)`, `for p in PLUGINS: p.index()`. -> [(target, RESOLVED, "collection")]
|
|
1010
|
+
Following the collection back to its literals takes at most COLLECTION_STEPS steps per call site."""
|
|
1011
|
+
if self._collection_steps is None:
|
|
1012
|
+
self._collection_steps = COLLECTION_STEPS
|
|
1013
|
+
try:
|
|
1014
|
+
return self.dispatch_targets(fn, ctx)
|
|
1015
|
+
finally:
|
|
1016
|
+
self._collection_steps = None
|
|
1017
|
+
out, seen = [], set()
|
|
1018
|
+
|
|
1019
|
+
def add(tgt):
|
|
1020
|
+
if tgt is not None and id(tgt) not in seen:
|
|
1021
|
+
seen.add(id(tgt))
|
|
1022
|
+
out.append((tgt, RESOLVED, "collection"))
|
|
1023
|
+
if isinstance(fn, ast.Attribute):
|
|
1024
|
+
if isinstance(fn.value, ast.Name) and fn.value.id in ("self", "cls"):
|
|
1025
|
+
return []
|
|
1026
|
+
for e, ectx in self.drawn_from(fn.value, ctx):
|
|
1027
|
+
t = self.infer(e, ectx)
|
|
1028
|
+
if t and t[0] in ("inst", "type"):
|
|
1029
|
+
m = self.find_method(t[1], fn.attr)
|
|
1030
|
+
add(m)
|
|
1031
|
+
if m is not None and m.cls is not None and isinstance(t[1], ClassInfo):
|
|
1032
|
+
rs = self.recv.setdefault((id(fn), id(m)), [])
|
|
1033
|
+
if t[1].id not in rs:
|
|
1034
|
+
rs.append(t[1].id)
|
|
1035
|
+
return out
|
|
1036
|
+
from .refs import is_property
|
|
1037
|
+
for e, ectx in self.drawn_from(fn, ctx):
|
|
1038
|
+
t = self.infer(e, ectx)
|
|
1039
|
+
if t and t[0] in ("func", "bound") and not (t[0] == "bound" and is_property(t[1])):
|
|
1040
|
+
add(t[1])
|
|
1041
|
+
elif t and t[0] == "type":
|
|
1042
|
+
add(t[1])
|
|
1043
|
+
return out
|
|
1044
|
+
|
|
1045
|
+
def drawn_from(self, e, ctx: "Ctx", depth=0) -> list:
|
|
1046
|
+
"""Element expressions `e` may hold when it is taken out of a collection: a loop variable over it, an item
|
|
1047
|
+
lookup (`T[k]`, `T.get(k)`) or a local assigned from one. [] when `e` is not drawn from a known collection."""
|
|
1048
|
+
if depth > 4:
|
|
1049
|
+
return []
|
|
1050
|
+
if isinstance(e, ast.Subscript) and not isinstance(e.slice, ast.Slice):
|
|
1051
|
+
return self.elements(e.value, ctx, "values")
|
|
1052
|
+
if isinstance(e, ast.Call) and isinstance(e.func, ast.Attribute) and e.func.attr in ("get", "pop", "setdefault") \
|
|
1053
|
+
and e.args:
|
|
1054
|
+
out = self.elements(e.func.value, ctx, "values")
|
|
1055
|
+
if out and len(e.args) > 1:
|
|
1056
|
+
out = out + [(e.args[1], ctx)]
|
|
1057
|
+
return out
|
|
1058
|
+
if not isinstance(e, ast.Name) or ctx.func is None:
|
|
1059
|
+
return []
|
|
1060
|
+
lv = self.local_vars(ctx)
|
|
1061
|
+
out = []
|
|
1062
|
+
for val, _ann, kind in lv.get(e.id, ()):
|
|
1063
|
+
if val is None:
|
|
1064
|
+
continue
|
|
1065
|
+
if kind == "iter":
|
|
1066
|
+
out += self.elements(val, ctx, "iter")
|
|
1067
|
+
elif kind.startswith("iterunpack"):
|
|
1068
|
+
i = int(kind[10:])
|
|
1069
|
+
if isinstance(val, ast.Call) and isinstance(val.func, ast.Attribute) and val.func.attr == "items" \
|
|
1070
|
+
and not val.args:
|
|
1071
|
+
out += self.elements(val.func.value, ctx, "keys" if i == 0 else "values")
|
|
1072
|
+
continue
|
|
1073
|
+
for x, xctx in self.elements(val, ctx, "iter"):
|
|
1074
|
+
if isinstance(x, (ast.Tuple, ast.List)) and i < len(x.elts):
|
|
1075
|
+
out.append((x.elts[i], xctx))
|
|
1076
|
+
elif kind == "assign":
|
|
1077
|
+
out += self.drawn_from(val, ctx, depth + 1)
|
|
1078
|
+
return out
|
|
1079
|
+
|
|
1080
|
+
def elements(self, e, ctx: "Ctx", mode="iter", depth=0) -> list:
|
|
1081
|
+
"""(expr, ctx) for each element of collection `e`: list / tuple / set / dict literals (mode iter|keys -> dict
|
|
1082
|
+
keys, values -> dict values) behind names, module constants, class attributes, `*spread`, `+`,
|
|
1083
|
+
list()/tuple()/sorted(), `copy.copy()` / `copy.deepcopy()`, filtering comprehensions, the return value of a
|
|
1084
|
+
project function and a constant key of a dict / tuple it returns (`setup()["plugins"]`); module-level
|
|
1085
|
+
`.append()` / `.extend()` / `[k] = v` / `.update()` / `.add()` count too. Each hop (a local name, a copy, a
|
|
1086
|
+
filter, a returned value) costs one level of depth; a hop back to an expression already being followed (`x = x
|
|
1087
|
+
+ [...]`) is a cycle and adds nothing."""
|
|
1088
|
+
if e is None or depth > COLLECTION_DEPTH or not self._collection_step():
|
|
1089
|
+
return []
|
|
1090
|
+
key = (id(e), mode)
|
|
1091
|
+
active = self._elements_active
|
|
1092
|
+
if key in active:
|
|
1093
|
+
return []
|
|
1094
|
+
active.add(key)
|
|
1095
|
+
try:
|
|
1096
|
+
return self._elements(e, ctx, mode, depth)
|
|
1097
|
+
finally:
|
|
1098
|
+
active.discard(key)
|
|
1099
|
+
|
|
1100
|
+
def _collection_step(self) -> bool:
|
|
1101
|
+
"""Spend one step of the current call site's budget (unlimited outside dispatch_targets)."""
|
|
1102
|
+
if self._collection_steps is None:
|
|
1103
|
+
return True
|
|
1104
|
+
self._collection_steps -= 1
|
|
1105
|
+
return self._collection_steps >= 0
|
|
1106
|
+
|
|
1107
|
+
def _elements(self, e, ctx: "Ctx", mode: str, depth: int) -> list:
|
|
1108
|
+
if isinstance(e, (ast.List, ast.Tuple, ast.Set)):
|
|
1109
|
+
out = []
|
|
1110
|
+
for x in e.elts:
|
|
1111
|
+
if isinstance(x, ast.Starred):
|
|
1112
|
+
out += self.elements(x.value, ctx, mode, depth + 1)
|
|
1113
|
+
else:
|
|
1114
|
+
out.append((x, ctx))
|
|
1115
|
+
return out
|
|
1116
|
+
if isinstance(e, ast.Dict):
|
|
1117
|
+
out = []
|
|
1118
|
+
for k, v in zip(e.keys, e.values):
|
|
1119
|
+
if k is None: # {**OTHER}
|
|
1120
|
+
out += self.elements(v, ctx, mode, depth + 1)
|
|
1121
|
+
else:
|
|
1122
|
+
out.append((v if mode == "values" else k, ctx))
|
|
1123
|
+
return out
|
|
1124
|
+
if isinstance(e, ast.BinOp) and isinstance(e.op, (ast.Add, ast.BitOr)):
|
|
1125
|
+
return self.elements(e.left, ctx, mode, depth + 1) + self.elements(e.right, ctx, mode, depth + 1)
|
|
1126
|
+
if isinstance(e, ast.IfExp):
|
|
1127
|
+
return self.elements(e.body, ctx, mode, depth + 1) + self.elements(e.orelse, ctx, mode, depth + 1)
|
|
1128
|
+
if isinstance(e, (ast.ListComp, ast.GeneratorExp, ast.SetComp)) and len(e.generators) == 1 \
|
|
1129
|
+
and isinstance(e.elt, ast.Name) and isinstance(e.generators[0].target, ast.Name) \
|
|
1130
|
+
and e.generators[0].target.id == e.elt.id:
|
|
1131
|
+
return self.elements(e.generators[0].iter, ctx, mode, depth + 1)
|
|
1132
|
+
if isinstance(e, ast.Call):
|
|
1133
|
+
fn = e.func
|
|
1134
|
+
if isinstance(fn, ast.Name) and fn.id in ("list", "tuple", "set", "frozenset", "sorted", "reversed") and e.args:
|
|
1135
|
+
return self.elements(e.args[0], ctx, mode, depth + 1)
|
|
1136
|
+
if isinstance(fn, ast.Name) and fn.id == "dict" and e.args:
|
|
1137
|
+
return self.elements(e.args[0], ctx, mode, depth + 1)
|
|
1138
|
+
if isinstance(fn, ast.Attribute) and fn.attr in ("values", "keys") and not e.args:
|
|
1139
|
+
return self.elements(fn.value, ctx, "values" if fn.attr == "values" else "keys", depth + 1)
|
|
1140
|
+
if dotted(fn) in ("itertools.chain", "chain"):
|
|
1141
|
+
return [x for a in e.args for x in self.elements(a, ctx, mode, depth + 1)]
|
|
1142
|
+
if dotted(fn) in COPY_FUNCS and e.args: # a copy holds the same kinds of elements
|
|
1143
|
+
return self.elements(e.args[0], ctx, mode, depth + 1)
|
|
1144
|
+
if isinstance(fn, ast.Attribute) and fn.attr == "copy" and not e.args: # PLUGINS.copy()
|
|
1145
|
+
return self.elements(fn.value, ctx, mode, depth + 1)
|
|
1146
|
+
return [x for r, rctx in self.returned(e, ctx) for x in self.elements(r, rctx, mode, depth + 1)]
|
|
1147
|
+
if isinstance(e, ast.Subscript) and isinstance(e.slice, ast.Constant): # plan["plugins"], pair[0]
|
|
1148
|
+
out = []
|
|
1149
|
+
for d, dctx in self.values_of(e.value, ctx, depth + 1):
|
|
1150
|
+
if isinstance(d, ast.Dict):
|
|
1151
|
+
for k, v in zip(d.keys, d.values):
|
|
1152
|
+
if isinstance(k, ast.Constant) and k.value == e.slice.value:
|
|
1153
|
+
out += self.elements(v, dctx, mode, depth + 1)
|
|
1154
|
+
elif isinstance(d, (ast.Tuple, ast.List)) and isinstance(e.slice.value, int) \
|
|
1155
|
+
and -len(d.elts) <= e.slice.value < len(d.elts) \
|
|
1156
|
+
and not any(isinstance(x, ast.Starred) for x in d.elts):
|
|
1157
|
+
out += self.elements(d.elts[e.slice.value], dctx, mode, depth + 1)
|
|
1158
|
+
return out
|
|
1159
|
+
if isinstance(e, ast.Name):
|
|
1160
|
+
if ctx.func is not None:
|
|
1161
|
+
lv = self.local_vars(ctx)
|
|
1162
|
+
if e.id in lv:
|
|
1163
|
+
out = []
|
|
1164
|
+
for val, _ann, kind in lv[e.id]:
|
|
1165
|
+
if kind == "assign" and val is not None:
|
|
1166
|
+
out += self.elements(val, ctx, mode, depth + 1)
|
|
1167
|
+
return out
|
|
1168
|
+
r = self.resolve_name(ctx.mod, e.id)
|
|
1169
|
+
if r and r[0] == "var":
|
|
1170
|
+
return self.module_elements(r[1], r[2], mode, depth)
|
|
1171
|
+
return []
|
|
1172
|
+
if isinstance(e, ast.Attribute):
|
|
1173
|
+
base = self.infer(e.value, ctx)
|
|
1174
|
+
if base and base[0] == "mod":
|
|
1175
|
+
if e.attr in base[1].vars:
|
|
1176
|
+
return self.module_elements(base[1], e.attr, mode, depth)
|
|
1177
|
+
r = self.lookup(base[1].name, e.attr)
|
|
1178
|
+
if r and r[0] == "var":
|
|
1179
|
+
return self.module_elements(r[1], r[2], mode, depth)
|
|
1180
|
+
elif base and base[0] in ("inst", "type"):
|
|
1181
|
+
c = base[1]
|
|
1182
|
+
for k in [c] + [b[1] for b in self.mro(c) if b[0] == "type"]:
|
|
1183
|
+
if e.attr in k.attrs and k.attrs[e.attr][0] is not None:
|
|
1184
|
+
return self.elements(k.attrs[e.attr][0], Ctx(k.module, None, k), mode, depth + 1)
|
|
1185
|
+
return []
|
|
1186
|
+
return []
|
|
1187
|
+
|
|
1188
|
+
def returned_instance(self, f: "FuncInfo"):
|
|
1189
|
+
"""("inst", C) when every `return` of the unannotated project function f returns an instance of the same
|
|
1190
|
+
project class C (`def make(): return B()`; `return None` aside), so `v = make(); v.run()` has a receiver type
|
|
1191
|
+
(#96). Not for generators; memoised; a function whose return type is being inferred further up is unknown."""
|
|
1192
|
+
key = ("ret", id(f))
|
|
1193
|
+
if key in self._attr_memo:
|
|
1194
|
+
return self._attr_memo[key]
|
|
1195
|
+
if key in self._attr_busy:
|
|
1196
|
+
return None
|
|
1197
|
+
self._attr_busy.add(key)
|
|
1198
|
+
try:
|
|
1199
|
+
got, fctx, stack = None, Ctx(f.module, f, f.cls), list(f.node.body)
|
|
1200
|
+
while stack:
|
|
1201
|
+
n = stack.pop()
|
|
1202
|
+
if isinstance(n, (ast.Yield, ast.YieldFrom)):
|
|
1203
|
+
got = False
|
|
1204
|
+
break
|
|
1205
|
+
if isinstance(n, ast.Return):
|
|
1206
|
+
v = n.value
|
|
1207
|
+
if v is None or (isinstance(v, ast.Constant) and v.value is None):
|
|
1208
|
+
continue
|
|
1209
|
+
leaves, todo = [], [v] # both arms of `A() if x else B()` / `a or B()`
|
|
1210
|
+
while todo:
|
|
1211
|
+
x = todo.pop()
|
|
1212
|
+
if isinstance(x, ast.IfExp):
|
|
1213
|
+
todo += [x.body, x.orelse]
|
|
1214
|
+
elif isinstance(x, ast.BoolOp):
|
|
1215
|
+
todo += x.values
|
|
1216
|
+
elif not (isinstance(x, ast.Constant) and x.value is None):
|
|
1217
|
+
leaves.append(x)
|
|
1218
|
+
for x in leaves:
|
|
1219
|
+
t = self.infer(x, fctx)
|
|
1220
|
+
if not t or t[0] != "inst" or not isinstance(t[1], ClassInfo) or (got and got[1] is not t[1]):
|
|
1221
|
+
got = False
|
|
1222
|
+
break
|
|
1223
|
+
got = t
|
|
1224
|
+
if got is False:
|
|
1225
|
+
break
|
|
1226
|
+
elif not isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)):
|
|
1227
|
+
stack.extend(ast.iter_child_nodes(n))
|
|
1228
|
+
r = ("inst", got[1]) if got else None
|
|
1229
|
+
finally:
|
|
1230
|
+
self._attr_busy.discard(key)
|
|
1231
|
+
if self._work <= INFER_BUDGET:
|
|
1232
|
+
self._attr_memo[key] = r
|
|
1233
|
+
return r
|
|
1234
|
+
|
|
1235
|
+
def returned(self, call, ctx: "Ctx") -> list:
|
|
1236
|
+
"""(expr, callee ctx) for each `return <expr>` of the project function `call` calls (not a class), so a
|
|
1237
|
+
collection a helper builds and returns keeps its elements: `plan = setup(); for p in plan["plugins"]: ...`."""
|
|
1238
|
+
t = self.infer(call.func, ctx)
|
|
1239
|
+
if not t or t[0] not in ("func", "bound") or not isinstance(t[1], FuncInfo):
|
|
1240
|
+
return []
|
|
1241
|
+
f = t[1]
|
|
1242
|
+
got = self._returns.get(id(f))
|
|
1243
|
+
if got is None:
|
|
1244
|
+
fctx = Ctx(f.module, f, f.cls)
|
|
1245
|
+
got, stack = [], list(f.node.body)
|
|
1246
|
+
while stack:
|
|
1247
|
+
n = stack.pop()
|
|
1248
|
+
if isinstance(n, ast.Return):
|
|
1249
|
+
if n.value is not None:
|
|
1250
|
+
got.append((n.value, fctx))
|
|
1251
|
+
elif not isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)):
|
|
1252
|
+
stack.extend(ast.iter_child_nodes(n))
|
|
1253
|
+
self._returns[id(f)] = got
|
|
1254
|
+
return got
|
|
1255
|
+
|
|
1256
|
+
def values_of(self, e, ctx: "Ctx", depth: int) -> list:
|
|
1257
|
+
"""(expr, ctx) for the values `e` may hold: through local and module names and the returns of a called
|
|
1258
|
+
project function; `e` itself otherwise. A value that leads back to itself (recursive helpers) adds nothing."""
|
|
1259
|
+
if depth > COLLECTION_DEPTH or not self._collection_step():
|
|
1260
|
+
return []
|
|
1261
|
+
key = (id(e), "value")
|
|
1262
|
+
active = self._elements_active
|
|
1263
|
+
if key in active:
|
|
1264
|
+
return []
|
|
1265
|
+
active.add(key)
|
|
1266
|
+
try:
|
|
1267
|
+
return self._values_of(e, ctx, depth)
|
|
1268
|
+
finally:
|
|
1269
|
+
active.discard(key)
|
|
1270
|
+
|
|
1271
|
+
def _values_of(self, e, ctx: "Ctx", depth: int) -> list:
|
|
1272
|
+
if isinstance(e, ast.Name):
|
|
1273
|
+
if ctx.func is not None:
|
|
1274
|
+
lv = self.local_vars(ctx)
|
|
1275
|
+
if e.id in lv:
|
|
1276
|
+
return [x for val, _ann, kind in lv[e.id] if kind == "assign" and val is not None
|
|
1277
|
+
for x in self.values_of(val, ctx, depth + 1)]
|
|
1278
|
+
r = self.resolve_name(ctx.mod, e.id)
|
|
1279
|
+
if r and r[0] == "var":
|
|
1280
|
+
mctx = Ctx(r[1], None, None)
|
|
1281
|
+
return [x for val, _ln, _ann in r[1].vars.get(r[2], ()) for x in self.values_of(val, mctx, depth + 1)]
|
|
1282
|
+
return []
|
|
1283
|
+
if isinstance(e, ast.Call):
|
|
1284
|
+
return [x for r, rctx in self.returned(e, ctx) for x in self.values_of(r, rctx, depth + 1)]
|
|
1285
|
+
return [(e, ctx)]
|
|
1286
|
+
|
|
1287
|
+
def module_elements(self, m: ModInfo, name: str, mode: str, depth: int) -> list:
|
|
1288
|
+
mctx = Ctx(m, None, None)
|
|
1289
|
+
out = []
|
|
1290
|
+
for val, _line, _ann in m.vars.get(name, ()):
|
|
1291
|
+
out += self.elements(val, mctx, mode, depth + 1)
|
|
1292
|
+
for val, how in self.mutations(m).get(name, ()):
|
|
1293
|
+
if how == "item":
|
|
1294
|
+
out += [(val, mctx)] if mode == "values" else []
|
|
1295
|
+
elif how == "one":
|
|
1296
|
+
out.append((val, mctx))
|
|
1297
|
+
else:
|
|
1298
|
+
out += self.elements(val, mctx, mode, depth + 1)
|
|
1299
|
+
return out
|
|
1300
|
+
|
|
1301
|
+
def mutations(self, m: ModInfo) -> dict:
|
|
1302
|
+
"""Module-level `NAME.append(x)` / `.add(x)` / `.extend(xs)` / `.update({...})` / `NAME[k] = x`."""
|
|
1303
|
+
cached = getattr(m, "_mutations", None)
|
|
1304
|
+
if cached is not None:
|
|
1305
|
+
return cached
|
|
1306
|
+
out: dict = {}
|
|
1307
|
+
stack = list(m.tree.body)
|
|
1308
|
+
while stack:
|
|
1309
|
+
st = stack.pop()
|
|
1310
|
+
if isinstance(st, (ast.If, ast.Try, ast.With, ast.For)):
|
|
1311
|
+
stack.extend(x for x in ast.iter_child_nodes(st) if isinstance(x, ast.stmt))
|
|
1312
|
+
for h in getattr(st, "handlers", ()):
|
|
1313
|
+
stack.extend(h.body)
|
|
1314
|
+
continue
|
|
1315
|
+
if isinstance(st, ast.Expr) and isinstance(st.value, ast.Call) and isinstance(st.value.func, ast.Attribute) \
|
|
1316
|
+
and isinstance(st.value.func.value, ast.Name) and st.value.args and st.value.func.value.id in m.vars:
|
|
1317
|
+
nm, attr = st.value.func.value.id, st.value.func.attr
|
|
1318
|
+
if attr in ("append", "add"):
|
|
1319
|
+
out.setdefault(nm, []).append((st.value.args[0], "one"))
|
|
1320
|
+
elif attr in ("extend", "update"):
|
|
1321
|
+
out.setdefault(nm, []).append((st.value.args[0], "many"))
|
|
1322
|
+
elif isinstance(st, ast.Assign):
|
|
1323
|
+
for t in st.targets:
|
|
1324
|
+
if isinstance(t, ast.Subscript) and isinstance(t.value, ast.Name) and t.value.id in m.vars:
|
|
1325
|
+
out.setdefault(t.value.id, []).append((st.value, "item"))
|
|
1326
|
+
m._mutations = out
|
|
1327
|
+
return out
|
|
1328
|
+
|
|
1329
|
+
def subclasses(self, c: ClassInfo) -> list[ClassInfo]:
|
|
1330
|
+
if not hasattr(self, "_subs"):
|
|
1331
|
+
self._subs = {}
|
|
1332
|
+
for x in self.classes.values():
|
|
1333
|
+
for b in self.mro(x):
|
|
1334
|
+
if b[0] == "type":
|
|
1335
|
+
self._subs.setdefault(b[1].qual, []).append(x)
|
|
1336
|
+
return self._subs.get(c.qual, [])
|
|
1337
|
+
|
|
1338
|
+
def all_defs(self) -> Iterator[FuncInfo]:
|
|
1339
|
+
yield from self.funcs.values()
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
@dataclass
|
|
1343
|
+
class Ctx:
|
|
1344
|
+
mod: ModInfo
|
|
1345
|
+
func: FuncInfo | None
|
|
1346
|
+
cls: ClassInfo | None
|
|
1347
|
+
visiting: set = field(default_factory=set)
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
COPY_FUNCS = {"copy.copy", "copy.deepcopy", "copy", "deepcopy"}
|
|
1351
|
+
# hops elements() / values_of() follow from a collection use back to its literal: `for fw in fws` -> a filter ->
|
|
1352
|
+
# `plan["plugins"]` -> the helper's return -> another filter -> a copy -> the module constant takes about ten
|
|
1353
|
+
COLLECTION_DEPTH = 16
|
|
1354
|
+
# steps (elements() / values_of() calls) one call site may spend: chains through helpers with many returns branch,
|
|
1355
|
+
# and the budget keeps that linear; the deepest chain in the tests and OSS validation takes a few hundred
|
|
1356
|
+
COLLECTION_STEPS = 2000
|
|
1357
|
+
BUILTINS = {"len", "str", "int", "float", "dict", "list", "set", "tuple", "print", "isinstance", "getattr", "setattr",
|
|
1358
|
+
"hasattr", "super", "open", "range", "enumerate", "zip", "map", "filter", "sorted", "min", "max", "sum",
|
|
1359
|
+
"any", "all", "bool", "type", "repr", "iter", "next", "round", "abs", "format", "id", "vars", "callable",
|
|
1360
|
+
"Exception", "ValueError", "KeyError", "TypeError", "RuntimeError", "NotImplementedError", "object",
|
|
1361
|
+
"property", "staticmethod", "classmethod", "bytes", "frozenset", "reversed", "divmod", "hash", "input"}
|
|
1362
|
+
|
|
1363
|
+
ENV_FUNCS = {"os.getenv", "os.environ.get", "os.environ.setdefault", "os.environ.pop", "decouple.config",
|
|
1364
|
+
"environ.Env", "os.environ.__getitem__"}
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
def module_of(path: str | None) -> str | None:
|
|
1368
|
+
if not path:
|
|
1369
|
+
return None
|
|
1370
|
+
return "/".join(path.split("/")[:-1]) or None
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
class PythonPlugin(LanguagePlugin):
|
|
1374
|
+
name = "python"
|
|
1375
|
+
|
|
1376
|
+
def __init__(self):
|
|
1377
|
+
self.program: PyProgram | None = None
|
|
1378
|
+
self.tests = None
|
|
1379
|
+
|
|
1380
|
+
def detect(self, project: Project) -> bool:
|
|
1381
|
+
"""A project marker at the root, or a .py file anywhere outside dependency / build / cache directories."""
|
|
1382
|
+
if any(project.exists(m) for m in ("pyproject.toml", "setup.py", "setup.cfg", "requirements.txt", "manage.py", "Pipfile")):
|
|
1383
|
+
return True
|
|
1384
|
+
rules = path_rules(project, "python", base=DETECT_SKIP)
|
|
1385
|
+
for dp, dns, fns in os.walk(project.root):
|
|
1386
|
+
rd = rel_dir(project.root, dp)
|
|
1387
|
+
if any(fn.endswith(".py") and not rules.excluded(f"{rd}/{fn}" if rd else fn) for fn in fns):
|
|
1388
|
+
return True
|
|
1389
|
+
dns[:] = rules.prune(rd, dns, dot=True)
|
|
1390
|
+
return False
|
|
1391
|
+
|
|
1392
|
+
def index(self, project: Project, builder: GraphBuilder, frameworks: list[FrameworkPlugin]) -> dict:
|
|
1393
|
+
t0 = time.time()
|
|
1394
|
+
prog = PyProgram(project)
|
|
1395
|
+
self.program = prog
|
|
1396
|
+
st = prog.load(skip_migrations=not project.options.get("python_include_migrations"))
|
|
1397
|
+
self.file_report = prog.file_report
|
|
1398
|
+
from .tests import PyTests
|
|
1399
|
+
self.tests = PyTests(prog, builder)
|
|
1400
|
+
n_alias = self.tests.rootdir_aliases()
|
|
1401
|
+
for fw in frameworks:
|
|
1402
|
+
fw.register_hooks(prog)
|
|
1403
|
+
prog.clear_memo()
|
|
1404
|
+
b = builder
|
|
1405
|
+
for m in prog.modules.values():
|
|
1406
|
+
b.add_node("module", m.name, name=m.name, fqn=m.name, file=m.file, line=1, module=module_of(m.file), lang="python")
|
|
1407
|
+
for c in prog.classes.values():
|
|
1408
|
+
b.add_node("class", c.qual, name=c.name, fqn=c.qual, file=c.file, line=c.line,
|
|
1409
|
+
end_line=getattr(c.node, "end_lineno", None), module=module_of(c.file), lang="python",
|
|
1410
|
+
doc=ast.get_docstring(c.node))
|
|
1411
|
+
owner = c.outer.id if c.outer else c.module.id
|
|
1412
|
+
b.add_edge(owner, c.id, "CONTAINS", c.file, c.line, EXACT)
|
|
1413
|
+
for f in prog.funcs.values():
|
|
1414
|
+
b.add_node(f.kind, f.qual, name=f.name, fqn=f.qual, file=f.file, line=f.line,
|
|
1415
|
+
end_line=getattr(f.node, "end_lineno", None), module=module_of(f.file), lang="python",
|
|
1416
|
+
doc=ast.get_docstring(f.node),
|
|
1417
|
+
attrs={"decorators": [ann_text(d) for d in f.decorators]} if f.decorators else None)
|
|
1418
|
+
b.add_edge(f.cls.id if f.cls else f.module.id, f.id, "CONTAINS", f.file, f.line, EXACT)
|
|
1419
|
+
# inheritance
|
|
1420
|
+
n_ext = 0
|
|
1421
|
+
for c in prog.classes.values():
|
|
1422
|
+
for bt in prog.bases(c):
|
|
1423
|
+
if bt[0] == "type":
|
|
1424
|
+
b.add_edge(c.id, bt[1].id, "EXTENDS", c.file, c.line, EXACT)
|
|
1425
|
+
n_ext += 1
|
|
1426
|
+
elif bt[0] == "ext":
|
|
1427
|
+
b.nodes[c.id].attrs.setdefault("ext_bases", []).append(bt[1])
|
|
1428
|
+
# overrides: base method -> child override (dispatch)
|
|
1429
|
+
for name, f in c.methods.items():
|
|
1430
|
+
for bt in prog.mro(c):
|
|
1431
|
+
if bt[0] == "type" and name in bt[1].methods and not name.startswith("__"):
|
|
1432
|
+
b.add_edge(bt[1].methods[name].id, f.id, "OVERRIDDEN_BY", f.file, f.line, RESOLVED)
|
|
1433
|
+
break
|
|
1434
|
+
# imports
|
|
1435
|
+
n_imp = 0
|
|
1436
|
+
for m in prog.modules.values():
|
|
1437
|
+
for st_ in m.import_nodes:
|
|
1438
|
+
if isinstance(st_, ast.Import):
|
|
1439
|
+
for a in st_.names:
|
|
1440
|
+
tm = prog.module(a.name)
|
|
1441
|
+
if tm:
|
|
1442
|
+
b.add_edge(m.id, tm.id, "IMPORTS", m.file, st_.lineno, EXACT); n_imp += 1
|
|
1443
|
+
elif isinstance(st_, ast.ImportFrom):
|
|
1444
|
+
base = prog._abs_from(m, st_.module, st_.level)
|
|
1445
|
+
tm = prog.module(base) if base else None
|
|
1446
|
+
for a in st_.names:
|
|
1447
|
+
sub = prog.module(f"{base}.{a.name}") if base else None
|
|
1448
|
+
if sub:
|
|
1449
|
+
b.add_edge(m.id, sub.id, "IMPORTS", m.file, st_.lineno, EXACT); n_imp += 1
|
|
1450
|
+
elif tm:
|
|
1451
|
+
b.add_edge(m.id, tm.id, "IMPORTS", m.file, st_.lineno, EXACT); n_imp += 1
|
|
1452
|
+
# calls + env reads
|
|
1453
|
+
conf_ct = {EXACT: 0, RESOLVED: 0, HEURISTIC: 0}
|
|
1454
|
+
n_env = 0
|
|
1455
|
+
from .refs import RefPass
|
|
1456
|
+
rp = RefPass(prog, b, Ctx)
|
|
1457
|
+
for f in list(prog.funcs.values()):
|
|
1458
|
+
ctx = Ctx(f.module, f, f.cls)
|
|
1459
|
+
# one walk per def: function references (refs.py) + its calls, in walk_body order
|
|
1460
|
+
calls: list = []
|
|
1461
|
+
body = {id(x) for x in f.node.body}
|
|
1462
|
+
rp.scan(f.id, [(c, f.node, id(c) in body) for c in ast.iter_child_nodes(f.node)], ctx, f.file, False, calls)
|
|
1463
|
+
for sub in calls:
|
|
1464
|
+
for tgt, conf, via in prog.resolve_call(sub, ctx):
|
|
1465
|
+
if isinstance(tgt, ClassInfo):
|
|
1466
|
+
b.add_edge(f.id, tgt.id, "INSTANTIATES", f.file, sub.lineno, conf)
|
|
1467
|
+
init = prog.find_method(tgt, "__init__")
|
|
1468
|
+
if init:
|
|
1469
|
+
b.add_edge(f.id, init.id, "CALLS", f.file, sub.lineno, conf, via="constructor")
|
|
1470
|
+
else:
|
|
1471
|
+
b.add_edge(f.id, tgt.id, "CALLS", f.file, sub.lineno, conf, **({"via": via} if via else {}),
|
|
1472
|
+
**_recv_attr(prog, sub, tgt))
|
|
1473
|
+
conf_ct[conf] += 1
|
|
1474
|
+
n_env += self.env_reads(prog, b, f.id, f.node, ctx)
|
|
1475
|
+
for m in prog.modules.values():
|
|
1476
|
+
n_env += self.env_reads(prog, b, m.id, m.tree, Ctx(m, None, None), top_only=True)
|
|
1477
|
+
# stored attributes -> field:<Class>.<attr> nodes with READS_PROP / WRITES_PROP (fields.py, #88)
|
|
1478
|
+
from .fields import index as fields_index
|
|
1479
|
+
fld_st = fields_index(prog, b, Ctx, walk_body)
|
|
1480
|
+
refs = self.references_and_entries(prog, b, conf_ct, rp)
|
|
1481
|
+
from .values import Values
|
|
1482
|
+
val_st = Values(prog, b).run()
|
|
1483
|
+
# programs started in a subprocess (`python -m pkg.cli`, console scripts) -> their entry points (subproc.py)
|
|
1484
|
+
from .subproc import Subprocesses
|
|
1485
|
+
sub_st = Subprocesses(prog, b, Ctx, FuncInfo, ClassInfo, dotted, walk_body, rp,
|
|
1486
|
+
root=project.root).link()
|
|
1487
|
+
if sub_st:
|
|
1488
|
+
refs["subprocess"] = sub_st
|
|
1489
|
+
# python-socketio / Flask-SocketIO events -> endpoint:socketio:<namespace>#<event> (protocol model, #31)
|
|
1490
|
+
from .socketio import index as socketio_index
|
|
1491
|
+
sio_st = socketio_index(prog, b, walk_body)
|
|
1492
|
+
if sio_st:
|
|
1493
|
+
refs["socketio"] = sio_st
|
|
1494
|
+
# settings dicts / URLs -> external-system facts (codegraph/external.py, #40)
|
|
1495
|
+
from .external import index as ext_index
|
|
1496
|
+
if (n_ext := ext_index(prog, b, walk_body)):
|
|
1497
|
+
refs["external_settings"] = n_ext
|
|
1498
|
+
# LLM tools, MCP servers / clients, agents -> endpoint:llm_tool / mcp_* (protocol model, #66)
|
|
1499
|
+
from .aitools import index as ai_index
|
|
1500
|
+
ai_st = ai_index(prog, b, walk_body, Ctx)
|
|
1501
|
+
if ai_st:
|
|
1502
|
+
refs["ai_tools"] = ai_st
|
|
1503
|
+
# pytest / unittest: test code marked attrs.test, test cases, fixtures, HTTP test requests (tests.py)
|
|
1504
|
+
with gc_paused():
|
|
1505
|
+
tests_st = self.tests.index()
|
|
1506
|
+
if tests_st and n_alias:
|
|
1507
|
+
tests_st["rootdir_import_names"] = n_alias
|
|
1508
|
+
st.update({"classes": len(prog.classes), "functions": sum(1 for f in prog.funcs.values() if f.kind == "function"),
|
|
1509
|
+
"methods": sum(1 for f in prog.funcs.values() if f.kind == "method"), "imports": n_imp,
|
|
1510
|
+
"extends": n_ext, "calls": conf_ct, "env_reads": n_env, **refs, "values": val_st,
|
|
1511
|
+
**fld_st,
|
|
1512
|
+
**({"tests": tests_st} if tests_st else {}),
|
|
1513
|
+
"parse_error_files": [e["file"] for e in prog.parse_errors[:20]],
|
|
1514
|
+
"roots_mode": prog.root_plan.mode, "source_roots": prog.roots_report,
|
|
1515
|
+
**({"roots_warnings": prog.root_plan.warnings} if prog.root_plan.warnings else {}),
|
|
1516
|
+
**({"roots_ambiguous": {"count": len(prog.root_plan.ambiguous), "samples": prog.root_plan.ambiguous[:5]}}
|
|
1517
|
+
if prog.root_plan.ambiguous else {}),
|
|
1518
|
+
**({"module_name_collisions": {"count": prog.root_plan.n_collisions,
|
|
1519
|
+
"path_named": prog.root_plan.n_requalified,
|
|
1520
|
+
"samples": prog.root_plan.collisions[:5]}}
|
|
1521
|
+
if prog.root_plan.n_collisions else {}),
|
|
1522
|
+
# #78: inference cut short (attribute type cycles, per-expression work budget) -> those types unknown
|
|
1523
|
+
**({"inference_limits": dict(prog.infer_stats)} if any(prog.infer_stats.values()) else {}),
|
|
1524
|
+
"seconds": round(time.time() - t0, 2)})
|
|
1525
|
+
return st
|
|
1526
|
+
|
|
1527
|
+
def link_test_requests(self, b, st: dict) -> None:
|
|
1528
|
+
"""The HTTP requests of test code, matched to the routes the framework plugins added (runs after them)."""
|
|
1529
|
+
tests = getattr(self, "tests", None)
|
|
1530
|
+
if tests is not None and tests.pending:
|
|
1531
|
+
http = tests.match_http()
|
|
1532
|
+
if http:
|
|
1533
|
+
st.setdefault("tests", {})["http"] = http
|
|
1534
|
+
|
|
1535
|
+
def after_frameworks(self, b, st: dict) -> None:
|
|
1536
|
+
"""A function a framework plugin wired up (Django route -> view, signal -> receiver, task) keeps that precise
|
|
1537
|
+
edge; the module-level reference to it (`path("x/", views.x)`, `@router.get(...)`) is the same registration
|
|
1538
|
+
seen syntactically and is dropped."""
|
|
1539
|
+
code = {"function", "method", "module", "class", "script", "field", "test"}
|
|
1540
|
+
wired = set()
|
|
1541
|
+
for e in b.edges.values():
|
|
1542
|
+
if e.kind in ("CONTAINS", "REFERENCES_FN") or e.kind.startswith("TEST_"):
|
|
1543
|
+
continue
|
|
1544
|
+
n = b.nodes.get(e.src)
|
|
1545
|
+
if n is not None and n.lang == "python" and n.kind not in code:
|
|
1546
|
+
wired.add(e.dst)
|
|
1547
|
+
drop = [k for k, e in b.edges.items() if e.kind == "REFERENCES_FN" and e.src.startswith("module:") and e.dst in wired]
|
|
1548
|
+
for k in drop:
|
|
1549
|
+
del b.edges[k]
|
|
1550
|
+
if drop:
|
|
1551
|
+
st["references_superseded"] = len(drop)
|
|
1552
|
+
|
|
1553
|
+
@staticmethod
|
|
1554
|
+
def _call_edges(prog: PyProgram, b, src: str, file: str, stmts, ctx: Ctx, conf_ct: dict) -> None:
|
|
1555
|
+
"""CALLS / INSTANTIATES from `src` for the calls in module-level statements (a script body)."""
|
|
1556
|
+
for st in stmts:
|
|
1557
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
1558
|
+
continue
|
|
1559
|
+
for sub in ast.walk(st):
|
|
1560
|
+
if not isinstance(sub, ast.Call):
|
|
1561
|
+
continue
|
|
1562
|
+
for tgt, conf, via in prog.resolve_call(sub, ctx):
|
|
1563
|
+
if isinstance(tgt, ClassInfo):
|
|
1564
|
+
b.add_edge(src, tgt.id, "INSTANTIATES", file, sub.lineno, conf)
|
|
1565
|
+
init = prog.find_method(tgt, "__init__")
|
|
1566
|
+
if init:
|
|
1567
|
+
b.add_edge(src, init.id, "CALLS", file, sub.lineno, conf, via="constructor")
|
|
1568
|
+
else:
|
|
1569
|
+
b.add_edge(src, tgt.id, "CALLS", file, sub.lineno, conf, **({"via": via} if via else {}),
|
|
1570
|
+
**_recv_attr(prog, sub, tgt))
|
|
1571
|
+
conf_ct[conf] += 1
|
|
1572
|
+
|
|
1573
|
+
def references_and_entries(self, prog: PyProgram, b, conf_ct: dict, rp) -> dict:
|
|
1574
|
+
"""Function references (REFERENCES_FN), decorator calls, `__main__` blocks / `__main__.py` / packaging entry
|
|
1575
|
+
points as `script` entry nodes, framework registrations (MCP tools, click / typer commands). See refs.py."""
|
|
1576
|
+
from .refs import is_main_guard, packaging_entry_points
|
|
1577
|
+
for m in prog.modules.values():
|
|
1578
|
+
mctx = Ctx(m, None, None)
|
|
1579
|
+
body = list(m.tree.body)
|
|
1580
|
+
if m.name == "__main__" or m.name.endswith(".__main__"):
|
|
1581
|
+
pkg = m.name.rpartition(".")[0]
|
|
1582
|
+
label = f"python -m {pkg}" if pkg else f"python {m.file}"
|
|
1583
|
+
sid = b.add_node("script", m.name, name=label, fqn=label,
|
|
1584
|
+
file=m.file, line=1, module=module_of(m.file), lang="python", entry_kind="main",
|
|
1585
|
+
attrs={"via": "__main__.py", "module": m.name})
|
|
1586
|
+
b.add_edge(m.id, sid, "CONTAINS", m.file, 1, EXACT)
|
|
1587
|
+
self._call_edges(prog, b, sid, m.file, body, mctx, conf_ct)
|
|
1588
|
+
rp.scan(sid, [(st, m.tree) for st in body], mctx, m.file, True)
|
|
1589
|
+
rp.object_calls(sid, body, mctx, m.file)
|
|
1590
|
+
rp.stats["main_blocks"] += 1
|
|
1591
|
+
continue
|
|
1592
|
+
mains = [st for st in body if isinstance(st, ast.If) and is_main_guard(st.test)]
|
|
1593
|
+
rest = [st for st in body if not any(st is x for x in mains)] + [x for st in mains for x in st.orelse]
|
|
1594
|
+
rp.scan(m.id, [(st, m.tree) for st in rest], mctx, m.file, True)
|
|
1595
|
+
for st in mains:
|
|
1596
|
+
name = f"python -m {m.name}" if "." in m.name else f"python {m.file}"
|
|
1597
|
+
sid = b.add_node("script", m.name, name=name, fqn=name, file=m.file, line=st.lineno,
|
|
1598
|
+
module=module_of(m.file), lang="python", entry_kind="main",
|
|
1599
|
+
attrs={"via": "__main__ block", "module": m.name})
|
|
1600
|
+
b.add_edge(m.id, sid, "CONTAINS", m.file, st.lineno, EXACT)
|
|
1601
|
+
self._call_edges(prog, b, sid, m.file, st.body, mctx, conf_ct)
|
|
1602
|
+
rp.scan(sid, [(x, st) for x in st.body], mctx, m.file, True)
|
|
1603
|
+
rp.object_calls(sid, st.body, mctx, m.file)
|
|
1604
|
+
rp.stats["main_blocks"] += 1
|
|
1605
|
+
# packaging entry points (console / GUI scripts, plugin groups), resolved through the source roots
|
|
1606
|
+
plan = prog.root_plan
|
|
1607
|
+
try:
|
|
1608
|
+
projects = [""] + plan._nested_projects()
|
|
1609
|
+
except Exception: # noqa: BLE001
|
|
1610
|
+
projects = [""]
|
|
1611
|
+
for ep in packaging_entry_points(prog.root, projects):
|
|
1612
|
+
self._entry_point(prog, b, rp, ep)
|
|
1613
|
+
rp.finish()
|
|
1614
|
+
st = rp.stats
|
|
1615
|
+
out = {"references": st["references"], "decorator_calls": st["decorator_calls"],
|
|
1616
|
+
"script_entries": {"main_blocks": st["main_blocks"], "packaging": st["entry_points"]}}
|
|
1617
|
+
if st["registrations"]:
|
|
1618
|
+
out["registrations"] = st["registrations"]
|
|
1619
|
+
if st["entry_points_unresolved"]:
|
|
1620
|
+
out["entry_points_unresolved"] = {"count": len(st["entry_points_unresolved"]),
|
|
1621
|
+
"samples": st["entry_points_unresolved"][:5]}
|
|
1622
|
+
return out
|
|
1623
|
+
|
|
1624
|
+
@staticmethod
|
|
1625
|
+
def _entry_point(prog: PyProgram, b, rp, ep: dict) -> None:
|
|
1626
|
+
from .refs import SCRIPT_GROUPS
|
|
1627
|
+
group, name = ep["group"], ep["name"]
|
|
1628
|
+
target = ep["target"].split("[", 1)[0].strip()
|
|
1629
|
+
modname, _, attr = target.partition(":")
|
|
1630
|
+
m = prog.module(modname.strip())
|
|
1631
|
+
if m is None:
|
|
1632
|
+
rp.stats["entry_points_unresolved"].append(f"{ep['declared_in']}: {name} = {ep['target']}")
|
|
1633
|
+
return
|
|
1634
|
+
obj = ("mod", m)
|
|
1635
|
+
for i, part in enumerate(p for p in attr.strip().split(".") if p):
|
|
1636
|
+
obj = prog.lookup(m.name, part) if i == 0 else prog.member(obj, part)
|
|
1637
|
+
if obj is None:
|
|
1638
|
+
break
|
|
1639
|
+
if obj is None:
|
|
1640
|
+
rp.stats["entry_points_unresolved"].append(f"{ep['declared_in']}: {name} = {ep['target']}")
|
|
1641
|
+
return
|
|
1642
|
+
key = f"{group}:{name}"
|
|
1643
|
+
if f"script:{key}" in b.nodes and ep["project"]:
|
|
1644
|
+
key = f"{group}:{ep['project']}/{name}"
|
|
1645
|
+
is_script = group in SCRIPT_GROUPS
|
|
1646
|
+
label = f"{SCRIPT_GROUPS[group]} {name}" if is_script else f"{group} entry point {name}"
|
|
1647
|
+
sid = b.add_node("script", key, name=label, fqn=label, file=ep["file"], line=ep["line"], module=module_of(ep["file"]),
|
|
1648
|
+
lang="python", entry_kind="main" if is_script else "public_api",
|
|
1649
|
+
attrs={"via": "packaging entry point", "group": group, "script": name, "target": ep["target"],
|
|
1650
|
+
"declared_in": ep["declared_in"]})
|
|
1651
|
+
rp.stats["entry_points"] += 1
|
|
1652
|
+
f, ln = ep["file"], ep["line"]
|
|
1653
|
+
if obj[0] in ("func", "bound"):
|
|
1654
|
+
b.add_edge(sid, obj[1].id, "CALLS", f, ln, EXACT, via="entry_point")
|
|
1655
|
+
n = b.nodes.get(obj[1].id)
|
|
1656
|
+
if n is not None:
|
|
1657
|
+
if n.attrs is None:
|
|
1658
|
+
n.attrs = {}
|
|
1659
|
+
n.attrs.setdefault("scripts", [])
|
|
1660
|
+
if name not in n.attrs["scripts"]:
|
|
1661
|
+
n.attrs["scripts"].append(name)
|
|
1662
|
+
elif obj[0] == "type":
|
|
1663
|
+
c = obj[1]
|
|
1664
|
+
b.add_edge(sid, c.id, "INSTANTIATES", f, ln, EXACT)
|
|
1665
|
+
init = prog.find_method(c, "__init__")
|
|
1666
|
+
if init:
|
|
1667
|
+
b.add_edge(sid, init.id, "CALLS", f, ln, EXACT, via="entry_point")
|
|
1668
|
+
for mn, mf in sorted(c.methods.items()):
|
|
1669
|
+
if not mn.startswith("_"):
|
|
1670
|
+
rp.ref(sid, mf, "entry point", f, ln)
|
|
1671
|
+
elif obj[0] == "mod":
|
|
1672
|
+
for fn_name, fn in sorted(obj[1].funcs.items()):
|
|
1673
|
+
if not fn_name.startswith("_"):
|
|
1674
|
+
rp.ref(sid, fn, "entry point", f, ln)
|
|
1675
|
+
elif obj[0] == "var":
|
|
1676
|
+
for fn in rp.var_regs.get((obj[1].name, obj[2]), []):
|
|
1677
|
+
rp.ref(sid, fn, "entry point", f, ln, via="registered command")
|
|
1678
|
+
b.nodes[sid].attrs["target_kind"] = "object"
|
|
1679
|
+
|
|
1680
|
+
@staticmethod
|
|
1681
|
+
def env_key(prog: PyProgram, call: ast.Call, ctx: Ctx) -> tuple[str, str] | None:
|
|
1682
|
+
"""(key, via) when `call` reads an environment variable."""
|
|
1683
|
+
fn = call.func
|
|
1684
|
+
d = dotted(fn)
|
|
1685
|
+
if not call.args:
|
|
1686
|
+
return None
|
|
1687
|
+
key = const_str(call.args[0])
|
|
1688
|
+
if key is None:
|
|
1689
|
+
return None
|
|
1690
|
+
if isinstance(fn, ast.Attribute):
|
|
1691
|
+
base = prog.infer(fn.value, ctx)
|
|
1692
|
+
bname = base[1] if base and base[0] in ("ext", "einst") else None
|
|
1693
|
+
if bname in ("os.environ",) and fn.attr in ("get", "setdefault", "pop"):
|
|
1694
|
+
return key, f"os.environ.{fn.attr}"
|
|
1695
|
+
if bname in ("environ.Env",) and base[0] == "einst":
|
|
1696
|
+
return key, f"env.{fn.attr}"
|
|
1697
|
+
if base and base[0] == "einst" and bname and bname.endswith("environ.Env"):
|
|
1698
|
+
return key, f"env.{fn.attr}"
|
|
1699
|
+
if d and d.endswith("environ.get") and d.split(".")[0] == "os":
|
|
1700
|
+
return key, "os.environ.get"
|
|
1701
|
+
t = prog.infer(fn, ctx)
|
|
1702
|
+
if t and t[0] == "ext":
|
|
1703
|
+
if t[1] in ("os.getenv", "os.environ.get", "decouple.config", "os.environ.setdefault"):
|
|
1704
|
+
return key, t[1]
|
|
1705
|
+
if t and t[0] == "einst" and t[1].endswith("environ.Env"):
|
|
1706
|
+
return key, "env()"
|
|
1707
|
+
return None
|
|
1708
|
+
|
|
1709
|
+
def env_reads(self, prog, b, src, node, ctx, top_only=False) -> int:
|
|
1710
|
+
n = 0
|
|
1711
|
+
it = node.body if top_only else None
|
|
1712
|
+
nodes = []
|
|
1713
|
+
if top_only:
|
|
1714
|
+
for st in it:
|
|
1715
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
1716
|
+
continue
|
|
1717
|
+
nodes.extend(ast.walk(st))
|
|
1718
|
+
else:
|
|
1719
|
+
nodes = walk_body(node)
|
|
1720
|
+
for sub in nodes:
|
|
1721
|
+
if isinstance(sub, ast.Call):
|
|
1722
|
+
r = self.env_key(prog, sub, ctx)
|
|
1723
|
+
if r:
|
|
1724
|
+
b.add_edge(src, b.add_node("env", r[0], lang="env"), "READS_ENV", ctx.mod.file, sub.lineno, EXACT, via=r[1])
|
|
1725
|
+
n += 1
|
|
1726
|
+
elif isinstance(sub, ast.Subscript) and isinstance(sub.ctx, ast.Load):
|
|
1727
|
+
k = const_str(sub.slice)
|
|
1728
|
+
if k:
|
|
1729
|
+
t = prog.infer(sub.value, ctx)
|
|
1730
|
+
if t and t[0] == "ext" and t[1] == "os.environ":
|
|
1731
|
+
b.add_edge(src, b.add_node("env", k, lang="env"), "READS_ENV", ctx.mod.file, sub.lineno, EXACT, via="os.environ[]")
|
|
1732
|
+
n += 1
|
|
1733
|
+
return n
|