featmap 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of featmap might be problematic. Click here for more details.
- featmap-0.1.0.dist-info/METADATA +129 -0
- featmap-0.1.0.dist-info/RECORD +130 -0
- featmap-0.1.0.dist-info/WHEEL +5 -0
- featmap-0.1.0.dist-info/entry_points.txt +2 -0
- featmap-0.1.0.dist-info/licenses/LICENSE +24 -0
- featmap-0.1.0.dist-info/licenses/fmap/vendor/typescript/LICENSE.txt +55 -0
- featmap-0.1.0.dist-info/licenses/fmap/vendor/typescript/ThirdPartyNoticeText.txt +193 -0
- featmap-0.1.0.dist-info/top_level.txt +1 -0
- fmap/__init__.py +1 -0
- fmap/__main__.py +3 -0
- fmap/axum.py +485 -0
- fmap/cache.py +25 -0
- fmap/cli.py +907 -0
- fmap/goext/go.mod +3 -0
- fmap/goext/main.go +547 -0
- fmap/goext.py +43 -0
- fmap/graph.py +920 -0
- fmap/impact.py +171 -0
- fmap/index.py +66 -0
- fmap/pyx.py +1133 -0
- fmap/routes.py +163 -0
- fmap/rsx.py +381 -0
- fmap/scip.py +80 -0
- fmap/trace_site.py +63 -0
- fmap/tsx.cjs +1258 -0
- fmap/tsx.py +98 -0
- fmap/vendor/typescript/LICENSE.txt +55 -0
- fmap/vendor/typescript/ThirdPartyNoticeText.txt +193 -0
- fmap/vendor/typescript/lib/lib.d.ts +22 -0
- fmap/vendor/typescript/lib/lib.decorators.d.ts +384 -0
- fmap/vendor/typescript/lib/lib.decorators.legacy.d.ts +22 -0
- fmap/vendor/typescript/lib/lib.dom.asynciterable.d.ts +41 -0
- fmap/vendor/typescript/lib/lib.dom.d.ts +39429 -0
- fmap/vendor/typescript/lib/lib.dom.iterable.d.ts +571 -0
- fmap/vendor/typescript/lib/lib.es2015.collection.d.ts +147 -0
- fmap/vendor/typescript/lib/lib.es2015.core.d.ts +597 -0
- fmap/vendor/typescript/lib/lib.es2015.d.ts +28 -0
- fmap/vendor/typescript/lib/lib.es2015.generator.d.ts +77 -0
- fmap/vendor/typescript/lib/lib.es2015.iterable.d.ts +605 -0
- fmap/vendor/typescript/lib/lib.es2015.promise.d.ts +81 -0
- fmap/vendor/typescript/lib/lib.es2015.proxy.d.ts +128 -0
- fmap/vendor/typescript/lib/lib.es2015.reflect.d.ts +144 -0
- fmap/vendor/typescript/lib/lib.es2015.symbol.d.ts +46 -0
- fmap/vendor/typescript/lib/lib.es2015.symbol.wellknown.d.ts +326 -0
- fmap/vendor/typescript/lib/lib.es2016.array.include.d.ts +116 -0
- fmap/vendor/typescript/lib/lib.es2016.d.ts +21 -0
- fmap/vendor/typescript/lib/lib.es2016.full.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.es2016.intl.d.ts +31 -0
- fmap/vendor/typescript/lib/lib.es2017.arraybuffer.d.ts +21 -0
- fmap/vendor/typescript/lib/lib.es2017.d.ts +26 -0
- fmap/vendor/typescript/lib/lib.es2017.date.d.ts +31 -0
- fmap/vendor/typescript/lib/lib.es2017.full.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.es2017.intl.d.ts +44 -0
- fmap/vendor/typescript/lib/lib.es2017.object.d.ts +49 -0
- fmap/vendor/typescript/lib/lib.es2017.sharedmemory.d.ts +135 -0
- fmap/vendor/typescript/lib/lib.es2017.string.d.ts +45 -0
- fmap/vendor/typescript/lib/lib.es2017.typedarrays.d.ts +53 -0
- fmap/vendor/typescript/lib/lib.es2018.asyncgenerator.d.ts +77 -0
- fmap/vendor/typescript/lib/lib.es2018.asynciterable.d.ts +53 -0
- fmap/vendor/typescript/lib/lib.es2018.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2018.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2018.intl.d.ts +83 -0
- fmap/vendor/typescript/lib/lib.es2018.promise.d.ts +30 -0
- fmap/vendor/typescript/lib/lib.es2018.regexp.d.ts +37 -0
- fmap/vendor/typescript/lib/lib.es2019.array.d.ts +79 -0
- fmap/vendor/typescript/lib/lib.es2019.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2019.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2019.intl.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.es2019.object.d.ts +33 -0
- fmap/vendor/typescript/lib/lib.es2019.string.d.ts +37 -0
- fmap/vendor/typescript/lib/lib.es2019.symbol.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2020.bigint.d.ts +765 -0
- fmap/vendor/typescript/lib/lib.es2020.d.ts +27 -0
- fmap/vendor/typescript/lib/lib.es2020.date.d.ts +42 -0
- fmap/vendor/typescript/lib/lib.es2020.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2020.intl.d.ts +474 -0
- fmap/vendor/typescript/lib/lib.es2020.number.d.ts +28 -0
- fmap/vendor/typescript/lib/lib.es2020.promise.d.ts +47 -0
- fmap/vendor/typescript/lib/lib.es2020.sharedmemory.d.ts +99 -0
- fmap/vendor/typescript/lib/lib.es2020.string.d.ts +44 -0
- fmap/vendor/typescript/lib/lib.es2020.symbol.wellknown.d.ts +41 -0
- fmap/vendor/typescript/lib/lib.es2021.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.es2021.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2021.intl.d.ts +166 -0
- fmap/vendor/typescript/lib/lib.es2021.promise.d.ts +48 -0
- fmap/vendor/typescript/lib/lib.es2021.string.d.ts +33 -0
- fmap/vendor/typescript/lib/lib.es2021.weakref.d.ts +78 -0
- fmap/vendor/typescript/lib/lib.es2022.array.d.ts +121 -0
- fmap/vendor/typescript/lib/lib.es2022.d.ts +25 -0
- fmap/vendor/typescript/lib/lib.es2022.error.d.ts +75 -0
- fmap/vendor/typescript/lib/lib.es2022.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2022.intl.d.ts +145 -0
- fmap/vendor/typescript/lib/lib.es2022.object.d.ts +26 -0
- fmap/vendor/typescript/lib/lib.es2022.regexp.d.ts +39 -0
- fmap/vendor/typescript/lib/lib.es2022.string.d.ts +25 -0
- fmap/vendor/typescript/lib/lib.es2023.array.d.ts +924 -0
- fmap/vendor/typescript/lib/lib.es2023.collection.d.ts +21 -0
- fmap/vendor/typescript/lib/lib.es2023.d.ts +22 -0
- fmap/vendor/typescript/lib/lib.es2023.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2023.intl.d.ts +56 -0
- fmap/vendor/typescript/lib/lib.es2024.arraybuffer.d.ts +65 -0
- fmap/vendor/typescript/lib/lib.es2024.collection.d.ts +29 -0
- fmap/vendor/typescript/lib/lib.es2024.d.ts +26 -0
- fmap/vendor/typescript/lib/lib.es2024.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.es2024.object.d.ts +29 -0
- fmap/vendor/typescript/lib/lib.es2024.promise.d.ts +35 -0
- fmap/vendor/typescript/lib/lib.es2024.regexp.d.ts +25 -0
- fmap/vendor/typescript/lib/lib.es2024.sharedmemory.d.ts +68 -0
- fmap/vendor/typescript/lib/lib.es2024.string.d.ts +29 -0
- fmap/vendor/typescript/lib/lib.es5.d.ts +4601 -0
- fmap/vendor/typescript/lib/lib.es6.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.esnext.array.d.ts +35 -0
- fmap/vendor/typescript/lib/lib.esnext.collection.d.ts +96 -0
- fmap/vendor/typescript/lib/lib.esnext.d.ts +29 -0
- fmap/vendor/typescript/lib/lib.esnext.decorators.d.ts +28 -0
- fmap/vendor/typescript/lib/lib.esnext.disposable.d.ts +193 -0
- fmap/vendor/typescript/lib/lib.esnext.error.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.esnext.float16.d.ts +445 -0
- fmap/vendor/typescript/lib/lib.esnext.full.d.ts +24 -0
- fmap/vendor/typescript/lib/lib.esnext.intl.d.ts +21 -0
- fmap/vendor/typescript/lib/lib.esnext.iterator.d.ts +148 -0
- fmap/vendor/typescript/lib/lib.esnext.promise.d.ts +34 -0
- fmap/vendor/typescript/lib/lib.esnext.sharedmemory.d.ts +25 -0
- fmap/vendor/typescript/lib/lib.scripthost.d.ts +322 -0
- fmap/vendor/typescript/lib/lib.webworker.asynciterable.d.ts +41 -0
- fmap/vendor/typescript/lib/lib.webworker.d.ts +13150 -0
- fmap/vendor/typescript/lib/lib.webworker.importscripts.d.ts +23 -0
- fmap/vendor/typescript/lib/lib.webworker.iterable.d.ts +340 -0
- fmap/vendor/typescript/lib/typescript.js +200276 -0
- fmap/vendor/typescript/package.json +120 -0
fmap/pyx.py
ADDED
|
@@ -0,0 +1,1133 @@
|
|
|
1
|
+
"""Python extractor: stdlib ast -> nodes, edges, entries (spec §6.1).
|
|
2
|
+
|
|
3
|
+
Resolution is exact: an edge is written only when the target is provable.
|
|
4
|
+
Everything else becomes a typed `?` edge or is dropped as provably external.
|
|
5
|
+
"""
|
|
6
|
+
import ast
|
|
7
|
+
import builtins
|
|
8
|
+
import fnmatch
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import sysconfig
|
|
13
|
+
from functools import lru_cache
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
BUILTINS = set(dir(builtins))
|
|
17
|
+
ROUTE_METHODS = {"get", "post", "put", "delete", "patch", "options", "head", "api_route", "websocket"}
|
|
18
|
+
FLAG_RE = re.compile(r"^--[A-Za-z0-9][A-Za-z0-9_-]*$")
|
|
19
|
+
SUBPROCESS = {
|
|
20
|
+
"subprocess.run", "subprocess.Popen", "subprocess.call", "subprocess.check_call",
|
|
21
|
+
"subprocess.check_output", "subprocess.getoutput", "subprocess.getstatusoutput",
|
|
22
|
+
"os.system", "os.popen", "asyncio.create_subprocess_exec", "asyncio.create_subprocess_shell",
|
|
23
|
+
}
|
|
24
|
+
DYNAMIC = {
|
|
25
|
+
"importlib.import_module", "importlib.util.spec_from_file_location",
|
|
26
|
+
"builtins.__import__", "builtins.exec", "builtins.eval",
|
|
27
|
+
}
|
|
28
|
+
# calling the result of these is dynamic dispatch, not an external call
|
|
29
|
+
DYNAMIC_RESULT = {"builtins.getattr", "importlib.import_module", "builtins.__import__"}
|
|
30
|
+
# values of these types have no project methods: `seen: dict = {}; seen.get(k)` is an external call, not `method?`
|
|
31
|
+
CONTAINERS = {f"builtins.{n}": f"builtins.{n}" for n in ("dict", "list", "set", "frozenset", "tuple", "str", "bytes")}
|
|
32
|
+
CONTAINERS.update({f"typing.{a}": f"builtins.{n}" for a, n in
|
|
33
|
+
(("Dict", "dict"), ("List", "list"), ("Set", "set"), ("FrozenSet", "frozenset"), ("Tuple", "tuple"))})
|
|
34
|
+
LITERALS = {ast.Dict: "dict", ast.DictComp: "dict", ast.List: "list", ast.ListComp: "list",
|
|
35
|
+
ast.Set: "set", ast.SetComp: "set", ast.Tuple: "tuple", ast.JoinedStr: "str"}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# shortcut: the stdlib server/handler families by name; a third-party socketserver subclass outside these is not followed
|
|
39
|
+
SERVERS = {f"socketserver.{n}" for n in (
|
|
40
|
+
"BaseServer", "TCPServer", "UDPServer", "UnixStreamServer", "UnixDatagramServer", "ThreadingTCPServer",
|
|
41
|
+
"ThreadingUDPServer", "ForkingTCPServer", "ForkingUDPServer", "ThreadingUnixStreamServer",
|
|
42
|
+
"ThreadingUnixDatagramServer", "ForkingUnixStreamServer", "ForkingUnixDatagramServer")} | {
|
|
43
|
+
"http.server.HTTPServer", "http.server.ThreadingHTTPServer"}
|
|
44
|
+
HANDLERS = {"socketserver.BaseRequestHandler", "socketserver.StreamRequestHandler",
|
|
45
|
+
"socketserver.DatagramRequestHandler", "http.server.BaseHTTPRequestHandler",
|
|
46
|
+
"http.server.SimpleHTTPRequestHandler", "http.server.CGIHTTPRequestHandler"}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def route_path(call):
|
|
50
|
+
"""The literal "/..." path a route-shaped call takes (first argument or path=), else None."""
|
|
51
|
+
if not (isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute)
|
|
52
|
+
and (call.func.attr in ROUTE_METHODS or call.func.attr in ("route", "add_api_route"))):
|
|
53
|
+
return None
|
|
54
|
+
a = call.args[0] if call.args else next((k.value for k in call.keywords if k.arg == "path"), None)
|
|
55
|
+
return a.value if isinstance(a, ast.Constant) and isinstance(a.value, str) and a.value.startswith("/") else None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def is_hook(name):
|
|
59
|
+
"""Request hooks the server calls; other overrides (version_string, log_message) stay uncertain."""
|
|
60
|
+
return name.startswith("do_") or name in ("handle", "setup", "finish")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@lru_cache(maxsize=None)
|
|
64
|
+
def stdlib_methods(name, seen=()):
|
|
65
|
+
"""Read stdlib declarations without importing any target code."""
|
|
66
|
+
if name in seen:
|
|
67
|
+
return None
|
|
68
|
+
module, _, class_name = name.rpartition(".")
|
|
69
|
+
if module == "builtins":
|
|
70
|
+
cls = vars(builtins).get(class_name)
|
|
71
|
+
return set().union(*(vars(base) for base in cls.__mro__)) if isinstance(cls, type) else None
|
|
72
|
+
if module.split(".")[0] not in sys.stdlib_module_names:
|
|
73
|
+
return None
|
|
74
|
+
path = Path(sysconfig.get_path("stdlib")) / module.replace(".", "/")
|
|
75
|
+
path = path.with_suffix(".py") if path.with_suffix(".py").is_file() else path / "__init__.py"
|
|
76
|
+
try:
|
|
77
|
+
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
78
|
+
except (OSError, SyntaxError, UnicodeError):
|
|
79
|
+
return None # Missing source is uncertainty, never evidence that a method is absent.
|
|
80
|
+
classes = {n.name: n for n in tree.body if isinstance(n, ast.ClassDef)}
|
|
81
|
+
package = module if path.name == "__init__.py" else module.rpartition(".")[0]
|
|
82
|
+
imports = {}
|
|
83
|
+
for node in tree.body:
|
|
84
|
+
if isinstance(node, ast.Import):
|
|
85
|
+
imports.update((a.asname or a.name, a.name) for a in node.names)
|
|
86
|
+
elif isinstance(node, ast.ImportFrom):
|
|
87
|
+
source = node.module
|
|
88
|
+
if node.level: # `from .case import TestCase` in unittest/__init__.py
|
|
89
|
+
base = package.rsplit(".", node.level - 1)[0] if node.level > 1 else package
|
|
90
|
+
source = f"{base}.{node.module}" if node.module else base
|
|
91
|
+
imports.update((a.asname or a.name, f"{source}.{a.name}") for a in node.names)
|
|
92
|
+
declaration = classes.get(class_name)
|
|
93
|
+
if declaration is None: # a package re-export: read the defining module
|
|
94
|
+
return stdlib_methods(imports[class_name], seen + (name,)) if class_name in imports else None
|
|
95
|
+
methods = set(vars(object)) | {n.name for n in declaration.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))}
|
|
96
|
+
for base in declaration.bases:
|
|
97
|
+
parts = text(base).split(".")
|
|
98
|
+
target = module + "." + parts[0] if parts[0] in classes else imports.get(parts[0], "builtins." + parts[0])
|
|
99
|
+
inherited = stdlib_methods(".".join([target, *parts[1:]]), seen + (name,))
|
|
100
|
+
if inherited is None:
|
|
101
|
+
return None
|
|
102
|
+
methods.update(inherited)
|
|
103
|
+
return methods
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def is_subprocess(name):
|
|
107
|
+
return name in SUBPROCESS or name.startswith(("os.exec", "os.spawn"))
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def module_name(path):
|
|
111
|
+
p = path[:-3]
|
|
112
|
+
if p.endswith("/__init__") or p == "__init__":
|
|
113
|
+
p = p[: -len("__init__")].rstrip("/")
|
|
114
|
+
return p.replace("/", ".")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
TEST_PATH = re.compile(r"(^|/)(tests?|__tests__|e2e)/|\.(test|spec)\.[^/]+$|(^|/)test_[^/]+\.py$")
|
|
118
|
+
PROJECT_FILES = ("pyproject.toml", "setup.py", "setup.cfg")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def package_roots(root, sources):
|
|
122
|
+
"""Subdirectories Python puts on sys.path for their own code: a project file's directory, and its src/."""
|
|
123
|
+
dirs = {"/".join(p.split("/")[:i]) for p in sources for i in range(1, p.count("/") + 1)}
|
|
124
|
+
has_project = lambda d: any((root / d / f).is_file() for f in PROJECT_FILES)
|
|
125
|
+
out = [d for d in dirs if has_project(d)]
|
|
126
|
+
out += [d for d in dirs if d.rsplit("/", 1)[-1] == "src" and has_project(d.rpartition("/")[0])]
|
|
127
|
+
return sorted(set(out))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def optional_inner(ann):
|
|
131
|
+
"""`X | None`, `Optional[X]`, `Union[X, None]` -> X; anything else unchanged."""
|
|
132
|
+
none = lambda x: isinstance(x, ast.Constant) and x.value is None
|
|
133
|
+
if isinstance(ann, ast.BinOp) and isinstance(ann.op, ast.BitOr):
|
|
134
|
+
if none(ann.right):
|
|
135
|
+
return ann.left
|
|
136
|
+
if none(ann.left):
|
|
137
|
+
return ann.right
|
|
138
|
+
if isinstance(ann, ast.Subscript):
|
|
139
|
+
name = text(ann.value).rsplit(".", 1)[-1]
|
|
140
|
+
if name == "Optional":
|
|
141
|
+
return ann.slice
|
|
142
|
+
if name == "Union" and isinstance(ann.slice, ast.Tuple):
|
|
143
|
+
rest = [x for x in ann.slice.elts if not none(x)]
|
|
144
|
+
return rest[0] if len(rest) == 1 else ann
|
|
145
|
+
return ann
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def text(node):
|
|
149
|
+
return " ".join(ast.unparse(node).split())
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def expr_match(pattern, callee, full):
|
|
153
|
+
"""`(*)` / `*` in a rule pattern match anything; whitespace is normalized."""
|
|
154
|
+
rx = re.escape(" ".join(pattern.split())).replace(r"\(\*\)", r"\(.*\)").replace(r"\*", ".*")
|
|
155
|
+
return re.fullmatch(rx, callee, re.S) is not None or re.fullmatch(rx, full, re.S) is not None
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def path_match(pattern, path):
|
|
159
|
+
return path == pattern or fnmatch.fnmatchcase(path, pattern)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class Project:
|
|
163
|
+
def __init__(self, sources, config, root=None):
|
|
164
|
+
self.root = Path(root or ".").resolve()
|
|
165
|
+
self.dynamic_cache = {}
|
|
166
|
+
self.sources = sources # path -> source text
|
|
167
|
+
self.config = config
|
|
168
|
+
self.trees = {}
|
|
169
|
+
self.mods = {} # module name -> path
|
|
170
|
+
self.path_mod = {}
|
|
171
|
+
self.syms = {} # module -> name -> binding
|
|
172
|
+
self.nodes = {} # id -> dict
|
|
173
|
+
self.astid = {} # id(ast node) -> node id
|
|
174
|
+
self.fn_ast = {} # function node id -> its def
|
|
175
|
+
self.classes = {} # class id -> info
|
|
176
|
+
self.methods_by_name = {} # method name -> [ids]
|
|
177
|
+
self.edges = []
|
|
178
|
+
self.entries = []
|
|
179
|
+
self.mounts = []
|
|
180
|
+
self.served = {} # handler class id -> [(function that builds the server, at)]
|
|
181
|
+
self.linked = set() # served classes whose hooks are already linked
|
|
182
|
+
self.hooked = set() # hook methods some server is proven to call
|
|
183
|
+
self.errors = []
|
|
184
|
+
for path in sorted(sources):
|
|
185
|
+
mod = module_name(path)
|
|
186
|
+
self.mods[mod] = path
|
|
187
|
+
self.path_mod[path] = mod
|
|
188
|
+
self.roots = package_roots(self.root, sources)
|
|
189
|
+
|
|
190
|
+
# ---------- pass 1: definitions and bindings ----------
|
|
191
|
+
def parse(self):
|
|
192
|
+
for path in sorted(self.sources):
|
|
193
|
+
try:
|
|
194
|
+
tree = ast.parse(self.sources[path], filename=path)
|
|
195
|
+
except SyntaxError as e:
|
|
196
|
+
raise SystemExit(f"fmap: cannot parse {path}:{e.lineno}: {e.msg}")
|
|
197
|
+
self.trees[path] = tree
|
|
198
|
+
mod = self.path_mod[path]
|
|
199
|
+
self.syms[mod] = {}
|
|
200
|
+
last = max((getattr(n, "end_lineno", 1) or 1) for n in ast.walk(tree) if hasattr(n, "lineno")) if tree.body else 1
|
|
201
|
+
self.add_node(f"{path}:<module>", "module", path, 1, last)
|
|
202
|
+
self.collect(tree.body, "", mod, path, None, top=True)
|
|
203
|
+
for cid, info in self.classes.items():
|
|
204
|
+
for name, mid in info["methods"].items():
|
|
205
|
+
self.methods_by_name.setdefault(name, []).append(mid)
|
|
206
|
+
|
|
207
|
+
def add_node(self, nid, kind, path, line, end):
|
|
208
|
+
base, n = nid, 2
|
|
209
|
+
while nid in self.nodes:
|
|
210
|
+
nid = f"{base}#{n}"
|
|
211
|
+
n += 1
|
|
212
|
+
self.nodes[nid] = {"kind": kind, "file": path, "line": line, "end_line": end}
|
|
213
|
+
return nid
|
|
214
|
+
|
|
215
|
+
def collect(self, body, prefix, mod, path, cls, top):
|
|
216
|
+
for st in body:
|
|
217
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
218
|
+
nid = self.add_node(f"{path}:{prefix}{st.name}", "method" if cls else "function",
|
|
219
|
+
path, st.lineno, st.end_lineno)
|
|
220
|
+
self.astid[id(st)] = nid
|
|
221
|
+
self.fn_ast[nid] = st
|
|
222
|
+
if cls:
|
|
223
|
+
self.classes[cls]["methods"][st.name] = nid
|
|
224
|
+
if top:
|
|
225
|
+
self.syms[mod][st.name] = ("node", nid)
|
|
226
|
+
self.collect(st.body, f"{prefix}{st.name}.", mod, path, None, False)
|
|
227
|
+
elif isinstance(st, ast.ClassDef):
|
|
228
|
+
nid = self.add_node(f"{path}:{prefix}{st.name}", "class", path, st.lineno, st.end_lineno)
|
|
229
|
+
self.astid[id(st)] = nid
|
|
230
|
+
self.classes[nid] = {"bases": st.bases, "mod": mod, "methods": {}, "attrs": {}, "node": st}
|
|
231
|
+
if top:
|
|
232
|
+
self.syms[mod][st.name] = ("node", nid)
|
|
233
|
+
self.collect(st.body, f"{prefix}{st.name}.", mod, path, nid, False)
|
|
234
|
+
elif any(hasattr(st, f) for f in ("body", "orelse", "finalbody", "handlers", "cases")):
|
|
235
|
+
# compound statements (if/try/with/for/while/match, async variants) keep the scope
|
|
236
|
+
for field in ("body", "orelse", "finalbody"):
|
|
237
|
+
self.collect(getattr(st, field, []), prefix, mod, path, cls, top)
|
|
238
|
+
for h in getattr(st, "handlers", []) + getattr(st, "cases", []):
|
|
239
|
+
self.collect(h.body, prefix, mod, path, cls, top)
|
|
240
|
+
elif top and isinstance(st, ast.Import):
|
|
241
|
+
for a in st.names:
|
|
242
|
+
target = self.abs_module(mod, path, a.name)
|
|
243
|
+
if a.asname:
|
|
244
|
+
self.syms[mod][a.asname] = ("mod", target)
|
|
245
|
+
else:
|
|
246
|
+
head = a.name.split(".")[0]
|
|
247
|
+
self.syms[mod][head] = ("mod", self.abs_module(mod, path, head))
|
|
248
|
+
elif top and isinstance(st, ast.ImportFrom):
|
|
249
|
+
base = self.from_module(mod, path, st)
|
|
250
|
+
for a in st.names:
|
|
251
|
+
if a.name != "*":
|
|
252
|
+
self.syms[mod][a.asname or a.name] = ("imp", base, a.name)
|
|
253
|
+
elif top and isinstance(st, (ast.Assign, ast.AnnAssign)):
|
|
254
|
+
targets = st.targets if isinstance(st, ast.Assign) else [st.target]
|
|
255
|
+
for t in targets:
|
|
256
|
+
if isinstance(t, ast.Name):
|
|
257
|
+
self.syms[mod][t.id] = ("var", f"{path}:{t.id}", st.value)
|
|
258
|
+
|
|
259
|
+
def is_proj_mod(self, m):
|
|
260
|
+
return m in self.mods or any(k.startswith(m + ".") for k in self.mods)
|
|
261
|
+
|
|
262
|
+
def abs_module(self, mod, path, name):
|
|
263
|
+
if self.is_proj_mod(name):
|
|
264
|
+
return name
|
|
265
|
+
# script directory is on sys.path when a script runs: tests/x.py can `import probes`
|
|
266
|
+
d = path.rsplit("/", 1)[0].replace("/", ".") if "/" in path else ""
|
|
267
|
+
if d and self.is_proj_mod(f"{d}.{name}"):
|
|
268
|
+
return f"{d}.{name}"
|
|
269
|
+
# a package root (backend/ with pyproject.toml, or its src/) is on sys.path: `import app.main`
|
|
270
|
+
hits = [r for r in self.roots if self.is_proj_mod(f"{r.replace('/', '.')}.{name}")]
|
|
271
|
+
own = [r for r in hits if path.startswith(r + "/")]
|
|
272
|
+
pick = max(own, key=len) if own else (hits[0] if len(hits) == 1 else None) # several foreign roots: no guess
|
|
273
|
+
return f"{pick.replace('/', '.')}.{name}" if pick else name
|
|
274
|
+
|
|
275
|
+
def from_module(self, mod, path, st):
|
|
276
|
+
if st.level == 0:
|
|
277
|
+
return self.abs_module(mod, path, st.module or "")
|
|
278
|
+
parts = mod.split(".")
|
|
279
|
+
if not path.endswith("__init__.py"):
|
|
280
|
+
parts = parts[:-1]
|
|
281
|
+
if st.level > 1:
|
|
282
|
+
parts = parts[: len(parts) - (st.level - 1)]
|
|
283
|
+
base = ".".join(parts)
|
|
284
|
+
return f"{base}.{st.module}" if st.module and base else (st.module or base)
|
|
285
|
+
|
|
286
|
+
# ---------- resolution ----------
|
|
287
|
+
def resolve(self, mod, name, seen=()):
|
|
288
|
+
b = self.syms.get(mod, {}).get(name)
|
|
289
|
+
if b is None:
|
|
290
|
+
return ("ext", f"builtins.{name}") if name in BUILTINS else None
|
|
291
|
+
return self.binding(b, mod, seen)
|
|
292
|
+
|
|
293
|
+
def binding(self, b, mod, seen=()):
|
|
294
|
+
if b[0] == "var":
|
|
295
|
+
key = (mod, b[1])
|
|
296
|
+
if key not in self.dynamic_cache:
|
|
297
|
+
self.dynamic_cache[key] = None # recursion guard for alias cycles
|
|
298
|
+
t = self.importlib_result(b[2], None, mod) if isinstance(b[2], ast.Call) else None
|
|
299
|
+
self.dynamic_cache[key] = t if t and t[0] in ("mod", "spec") else None
|
|
300
|
+
return self.dynamic_cache[key] or b
|
|
301
|
+
if b[0] == "node":
|
|
302
|
+
return b
|
|
303
|
+
if b[0] == "mod":
|
|
304
|
+
return ("mod", b[1]) if self.is_proj_mod(b[1]) else ("ext", b[1])
|
|
305
|
+
m, n = b[1], b[2]
|
|
306
|
+
if self.is_proj_mod(f"{m}.{n}"):
|
|
307
|
+
return ("mod", f"{m}.{n}")
|
|
308
|
+
if m in self.mods:
|
|
309
|
+
if (m, n) in seen:
|
|
310
|
+
return None
|
|
311
|
+
return self.resolve(m, n, seen + ((m, n),))
|
|
312
|
+
if self.is_proj_mod(m):
|
|
313
|
+
return None
|
|
314
|
+
return ("ext", f"{m}.{n}")
|
|
315
|
+
|
|
316
|
+
def module_attr(self, m, attr):
|
|
317
|
+
if self.is_proj_mod(f"{m}.{attr}"):
|
|
318
|
+
return ("mod", f"{m}.{attr}")
|
|
319
|
+
if m in self.mods:
|
|
320
|
+
return self.resolve(m, attr)
|
|
321
|
+
return None
|
|
322
|
+
|
|
323
|
+
def lookup(self, cid, name, seen=None):
|
|
324
|
+
"""Method/attr on a project class through its bases. 'ext' if a base is external."""
|
|
325
|
+
seen = seen or set()
|
|
326
|
+
if cid in seen:
|
|
327
|
+
return None
|
|
328
|
+
seen.add(cid)
|
|
329
|
+
info = self.classes[cid]
|
|
330
|
+
if name in info["methods"]:
|
|
331
|
+
return ("node", info["methods"][name])
|
|
332
|
+
if name in info["attrs"]:
|
|
333
|
+
return info["attrs"][name]
|
|
334
|
+
result = None
|
|
335
|
+
for b in info["bases"]:
|
|
336
|
+
t = self.typeof(b, None, info["mod"])
|
|
337
|
+
if t and t[0] == "node" and t[1] in self.classes:
|
|
338
|
+
r = self.lookup(t[1], name, seen)
|
|
339
|
+
if r:
|
|
340
|
+
return r
|
|
341
|
+
elif not (isinstance(b, ast.Name) and b.id == "object"):
|
|
342
|
+
result = ("ext", "?") # external or unknown base may define it
|
|
343
|
+
return result
|
|
344
|
+
|
|
345
|
+
def bases(self, cid):
|
|
346
|
+
info = self.classes[cid]
|
|
347
|
+
out = []
|
|
348
|
+
for b in info["bases"]:
|
|
349
|
+
t = self.typeof(b, None, info["mod"])
|
|
350
|
+
if t and t[0] == "node" and t[1] in self.classes:
|
|
351
|
+
out.append(t[1])
|
|
352
|
+
return out
|
|
353
|
+
|
|
354
|
+
def typeof(self, e, frame, mod):
|
|
355
|
+
"""Returns ('node', id) | ('inst', class_id) | ('mod', m) | ('ext', name) |
|
|
356
|
+
('var', id, value) | ('param',) | None (unknown)."""
|
|
357
|
+
if isinstance(e, ast.Name):
|
|
358
|
+
f = frame
|
|
359
|
+
while f is not None:
|
|
360
|
+
if e.id in f.defs:
|
|
361
|
+
return ("node", f.defs[e.id])
|
|
362
|
+
if e.id in f.params:
|
|
363
|
+
return f.params[e.id]
|
|
364
|
+
if e.id in f.locals:
|
|
365
|
+
return f.local_type(e.id, self, mod)
|
|
366
|
+
f = f.parent
|
|
367
|
+
return self.resolve(mod, e.id)
|
|
368
|
+
if isinstance(e, ast.Attribute):
|
|
369
|
+
t = self.typeof(e.value, frame, mod)
|
|
370
|
+
if t is None or t[0] == "param":
|
|
371
|
+
return None
|
|
372
|
+
if t[0] == "ext":
|
|
373
|
+
return ("ext", f"{t[1]}.{e.attr}")
|
|
374
|
+
if t[0] == "mod":
|
|
375
|
+
return self.module_attr(t[1], e.attr)
|
|
376
|
+
if t[0] in ("inst", "node") and t[1] in self.classes:
|
|
377
|
+
return self.lookup(t[1], e.attr)
|
|
378
|
+
return None
|
|
379
|
+
if isinstance(e, ast.IfExp):
|
|
380
|
+
a, b = self.typeof(e.body, frame, mod), self.typeof(e.orelse, frame, mod)
|
|
381
|
+
if a == b:
|
|
382
|
+
return a
|
|
383
|
+
if a and b and a[0] == b[0] == "ext":
|
|
384
|
+
return ("ext", "?")
|
|
385
|
+
return None
|
|
386
|
+
if isinstance(e, ast.Call):
|
|
387
|
+
imported = self.importlib_result(e, frame, mod)
|
|
388
|
+
if imported:
|
|
389
|
+
return imported
|
|
390
|
+
t = self.typeof(e.func, frame, mod)
|
|
391
|
+
if t and t[0] == "node" and t[1] in self.classes:
|
|
392
|
+
return ("inst", t[1])
|
|
393
|
+
if t and t[0] == "ext" and t[1] in CONTAINERS: # dict(...), list(...): a builtin value
|
|
394
|
+
return ("ext", CONTAINERS[t[1]])
|
|
395
|
+
fn = self.fn_ast.get(t[1]) if t and t[0] == "node" else None
|
|
396
|
+
if fn is not None and fn.returns is not None: # `def make() -> Provider`
|
|
397
|
+
return self.inst_of(fn.returns, None, self.path_mod[self.nodes[t[1]]["file"]])
|
|
398
|
+
return None
|
|
399
|
+
if isinstance(e, ast.BoolOp) and isinstance(e.op, ast.Or): # `given or make()`: one type, or unknown
|
|
400
|
+
values = [v for v in e.values if not (isinstance(v, ast.Constant) and v.value is None)]
|
|
401
|
+
kinds = {self.typeof(v, frame, mod) for v in values}
|
|
402
|
+
if len(kinds) == 1:
|
|
403
|
+
return kinds.pop()
|
|
404
|
+
# `provider or makeDefault()`: the annotated parameter declares the type (Protocol), the default fills it
|
|
405
|
+
first = values[0] if values else None
|
|
406
|
+
f = frame
|
|
407
|
+
while isinstance(first, ast.Name) and f is not None and first.id not in f.params:
|
|
408
|
+
f = f.parent
|
|
409
|
+
declared = f.params.get(first.id) if isinstance(first, ast.Name) and f is not None else None
|
|
410
|
+
return declared if declared and declared[0] == "inst" else None
|
|
411
|
+
if type(e) in LITERALS:
|
|
412
|
+
return ("ext", f"builtins.{LITERALS[type(e)]}")
|
|
413
|
+
if isinstance(e, ast.Constant) and isinstance(e.value, str):
|
|
414
|
+
# string annotations: "Cls"
|
|
415
|
+
try:
|
|
416
|
+
return self.typeof(ast.parse(e.value, mode="eval").body, frame, mod)
|
|
417
|
+
except SyntaxError:
|
|
418
|
+
return None
|
|
419
|
+
return None
|
|
420
|
+
|
|
421
|
+
def static_path(self, e, frame, mod, seen=()):
|
|
422
|
+
"""Evaluate only literal path operations; never execute project code."""
|
|
423
|
+
if isinstance(e, ast.Constant) and isinstance(e.value, str):
|
|
424
|
+
return e.value
|
|
425
|
+
if isinstance(e, ast.Name):
|
|
426
|
+
if e.id in seen:
|
|
427
|
+
return None
|
|
428
|
+
f = frame
|
|
429
|
+
while f is not None:
|
|
430
|
+
if e.id in f.params or e.id in f.defs:
|
|
431
|
+
return None
|
|
432
|
+
if e.id in f.locals:
|
|
433
|
+
values = f.locals[e.id]
|
|
434
|
+
return self.static_path(values[0], f, mod, seen + (e.id,)) if len(values) == 1 else None
|
|
435
|
+
f = f.parent
|
|
436
|
+
if e.id == "__file__":
|
|
437
|
+
return str(self.root / self.mods[mod])
|
|
438
|
+
b = self.syms.get(mod, {}).get(e.id)
|
|
439
|
+
if b and b[0] == "var":
|
|
440
|
+
return self.static_path(b[2], None, mod, seen + (e.id,))
|
|
441
|
+
if isinstance(e, ast.Attribute) and e.attr == "parent":
|
|
442
|
+
value = self.static_path(e.value, frame, mod, seen)
|
|
443
|
+
return value.parent if isinstance(value, Path) else None
|
|
444
|
+
if isinstance(e, ast.BinOp) and isinstance(e.op, ast.Div):
|
|
445
|
+
left = self.static_path(e.left, frame, mod, seen)
|
|
446
|
+
right = self.static_path(e.right, frame, mod, seen)
|
|
447
|
+
return left / right if isinstance(left, Path) and isinstance(right, (str, Path)) else None
|
|
448
|
+
if isinstance(e, ast.Call) and not e.keywords:
|
|
449
|
+
return self.static_path_call(e, frame, mod, seen)
|
|
450
|
+
return None
|
|
451
|
+
|
|
452
|
+
def static_path_call(self, e, frame, mod, seen):
|
|
453
|
+
values = [self.static_path(a, frame, mod, seen) for a in e.args]
|
|
454
|
+
if any(v is None for v in values):
|
|
455
|
+
return None
|
|
456
|
+
t = self.typeof(e.func, frame, mod)
|
|
457
|
+
name = t[1] if t and t[0] == "ext" else None
|
|
458
|
+
if name == "builtins.str" and len(values) == 1:
|
|
459
|
+
return str(values[0])
|
|
460
|
+
if name == "pathlib.Path" and len(values) == 1:
|
|
461
|
+
return Path(values[0])
|
|
462
|
+
if name == "os.path.dirname" and len(values) == 1:
|
|
463
|
+
return os.path.dirname(values[0])
|
|
464
|
+
if name == "os.path.join" and values:
|
|
465
|
+
return os.path.join(*values)
|
|
466
|
+
if isinstance(e.func, ast.Attribute) and e.func.attr == "resolve" and not values:
|
|
467
|
+
value = self.static_path(e.func.value, frame, mod, seen)
|
|
468
|
+
if isinstance(value, Path):
|
|
469
|
+
return (self.root / value).resolve()
|
|
470
|
+
return None
|
|
471
|
+
|
|
472
|
+
def importlib_result(self, e, frame, mod):
|
|
473
|
+
t = self.typeof(e.func, frame, mod)
|
|
474
|
+
name = t[1] if t and t[0] == "ext" else None
|
|
475
|
+
if name == "importlib.import_module" and e.args:
|
|
476
|
+
arg = e.args[0]
|
|
477
|
+
if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and arg.value in self.mods:
|
|
478
|
+
return ("mod", arg.value)
|
|
479
|
+
elif name == "importlib.util.spec_from_file_location" and len(e.args) >= 2:
|
|
480
|
+
try:
|
|
481
|
+
value = self.static_path(e.args[1], frame, mod)
|
|
482
|
+
if value is not None:
|
|
483
|
+
path = (self.root / value).resolve().relative_to(self.root).as_posix()
|
|
484
|
+
if path in self.path_mod:
|
|
485
|
+
return ("spec", self.path_mod[path])
|
|
486
|
+
except (ValueError, OSError, RuntimeError):
|
|
487
|
+
return None # An invalid or outside-project path is still dynamic.
|
|
488
|
+
elif name == "importlib.util.module_from_spec" and len(e.args) == 1:
|
|
489
|
+
spec = self.typeof(e.args[0], frame, mod)
|
|
490
|
+
if spec and spec[0] == "spec":
|
|
491
|
+
return ("mod", spec[1])
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
def process_target(self, node, frame, mod, name):
|
|
495
|
+
if name not in {"subprocess.run", "subprocess.Popen", "subprocess.call",
|
|
496
|
+
"subprocess.check_call", "subprocess.check_output"}:
|
|
497
|
+
return None
|
|
498
|
+
if any(k.arg in ("shell", "executable") and not (
|
|
499
|
+
k.arg == "shell" and isinstance(k.value, ast.Constant) and k.value.value is False) for k in node.keywords):
|
|
500
|
+
return None
|
|
501
|
+
command = node.args[0] if node.args else next((k.value for k in node.keywords if k.arg == "args"), None)
|
|
502
|
+
if not isinstance(command, (ast.List, ast.Tuple)) or not command.elts:
|
|
503
|
+
return None
|
|
504
|
+
executable = self.static_path(command.elts[0], frame, mod)
|
|
505
|
+
executable_type = self.typeof(command.elts[0], frame, mod)
|
|
506
|
+
is_python = executable_type == ("ext", "sys.executable") or (
|
|
507
|
+
isinstance(executable, (str, Path)) and re.fullmatch(r"python(?:[0-9]+(?:\.[0-9]+)*)?", Path(executable).name))
|
|
508
|
+
index = 1 if is_python else 0
|
|
509
|
+
if len(command.elts) <= index:
|
|
510
|
+
return None
|
|
511
|
+
argument = command.elts[index]
|
|
512
|
+
value = self.static_path(argument, frame, mod)
|
|
513
|
+
if value is None:
|
|
514
|
+
return None
|
|
515
|
+
cwd = next((k.value for k in node.keywords if k.arg == "cwd"), None)
|
|
516
|
+
if cwd is not None:
|
|
517
|
+
cwd = self.static_path(cwd, frame, mod)
|
|
518
|
+
if cwd is None:
|
|
519
|
+
return None
|
|
520
|
+
try:
|
|
521
|
+
path = (self.root / (cwd or "") / value).resolve().relative_to(self.root).as_posix()
|
|
522
|
+
except (ValueError, OSError, RuntimeError):
|
|
523
|
+
return None
|
|
524
|
+
return path if path in self.path_mod else None
|
|
525
|
+
|
|
526
|
+
def inst_of(self, ann, frame, mod):
|
|
527
|
+
if isinstance(ann, ast.Constant) and isinstance(ann.value, str): # "Provider | None"
|
|
528
|
+
try:
|
|
529
|
+
ann = ast.parse(ann.value, mode="eval").body
|
|
530
|
+
except SyntaxError:
|
|
531
|
+
return None
|
|
532
|
+
ann = optional_inner(ann)
|
|
533
|
+
t = self.typeof(ann, frame, mod) if ann is not None else None
|
|
534
|
+
if t and t[0] == "node" and t[1] in self.classes:
|
|
535
|
+
return ("inst", t[1])
|
|
536
|
+
if isinstance(ann, ast.Subscript): # dict[str, int], List[Row]
|
|
537
|
+
t = self.typeof(ann.value, frame, mod)
|
|
538
|
+
if t and t[0] == "ext" and t[1] in CONTAINERS:
|
|
539
|
+
return ("ext", CONTAINERS[t[1]])
|
|
540
|
+
return None
|
|
541
|
+
|
|
542
|
+
# ---------- pass 2: edges ----------
|
|
543
|
+
def walk(self):
|
|
544
|
+
for cid, info in self.classes.items():
|
|
545
|
+
self.class_attrs(cid, info)
|
|
546
|
+
for path in sorted(self.trees):
|
|
547
|
+
mod = self.path_mod[path]
|
|
548
|
+
v = Visitor(self, path, mod)
|
|
549
|
+
v.frame = Frame(f"{path}:<module>", None)
|
|
550
|
+
for st in self.trees[path].body:
|
|
551
|
+
v.visit(st)
|
|
552
|
+
v.finish()
|
|
553
|
+
self.impl_edges()
|
|
554
|
+
self.framework_callbacks()
|
|
555
|
+
|
|
556
|
+
def class_attrs(self, cid, info):
|
|
557
|
+
mod = info["mod"]
|
|
558
|
+
for st in info["node"].body:
|
|
559
|
+
if isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Name):
|
|
560
|
+
t = self.inst_of(st.annotation, None, mod)
|
|
561
|
+
if t:
|
|
562
|
+
info["attrs"][st.target.id] = t
|
|
563
|
+
seen = {}
|
|
564
|
+
for st in ast.walk(info["node"]):
|
|
565
|
+
if isinstance(st, (ast.Assign, ast.AnnAssign)):
|
|
566
|
+
targets = st.targets if isinstance(st, ast.Assign) else [st.target]
|
|
567
|
+
for t in targets:
|
|
568
|
+
if isinstance(t, ast.Attribute) and isinstance(t.value, ast.Name) and t.value.id == "self":
|
|
569
|
+
ty = None
|
|
570
|
+
if isinstance(st, ast.AnnAssign):
|
|
571
|
+
ty = self.inst_of(st.annotation, None, mod)
|
|
572
|
+
if ty is None and st.value is not None and isinstance(st.value, ast.Call):
|
|
573
|
+
ty = self.typeof(st.value, None, mod)
|
|
574
|
+
seen.setdefault(t.attr, set()).add(ty)
|
|
575
|
+
for attr, types in seen.items():
|
|
576
|
+
if attr not in info["attrs"] and len(types) == 1 and None not in types:
|
|
577
|
+
info["attrs"][attr] = types.pop()
|
|
578
|
+
|
|
579
|
+
def impl_edges(self):
|
|
580
|
+
done = set()
|
|
581
|
+
|
|
582
|
+
def impl(bm, mid, **extra):
|
|
583
|
+
# app code reaches a test double only through the test that builds it
|
|
584
|
+
if (bm, mid) in done or (TEST_PATH.search(self.nodes[mid]["file"]) and not TEST_PATH.search(self.nodes[bm]["file"])):
|
|
585
|
+
return
|
|
586
|
+
done.add((bm, mid))
|
|
587
|
+
n = self.nodes[mid]
|
|
588
|
+
self.edge(bm, mid, "impl", f"{n['file']}:{n['line']}", **extra)
|
|
589
|
+
|
|
590
|
+
for cid in self.classes:
|
|
591
|
+
for name, mid in self.classes[cid]["methods"].items():
|
|
592
|
+
todo, seen = list(self.bases(cid)), set()
|
|
593
|
+
while todo:
|
|
594
|
+
b = todo.pop()
|
|
595
|
+
if b in seen:
|
|
596
|
+
continue
|
|
597
|
+
seen.add(b)
|
|
598
|
+
if name in self.classes[b]["methods"]:
|
|
599
|
+
impl(self.classes[b]["methods"][name], mid)
|
|
600
|
+
else:
|
|
601
|
+
todo.extend(self.bases(b))
|
|
602
|
+
# typing.Protocol is structural: a class defining every protocol method implements it without inheriting
|
|
603
|
+
protocols = {cid: set(info["methods"]) for cid, info in self.classes.items()
|
|
604
|
+
if self.ext_bases(cid) & {"typing.Protocol", "typing_extensions.Protocol"} and info["methods"]}
|
|
605
|
+
for pid, names in protocols.items():
|
|
606
|
+
for cid in self.classes:
|
|
607
|
+
if cid == pid or cid in protocols:
|
|
608
|
+
continue
|
|
609
|
+
found = {n: self.lookup(cid, n) for n in names}
|
|
610
|
+
if all(t and t[0] == "node" for t in found.values()):
|
|
611
|
+
for n, t in found.items():
|
|
612
|
+
if t[1] in self.classes[cid]["methods"].values():
|
|
613
|
+
impl(self.classes[pid]["methods"][n], t[1], structural=True)
|
|
614
|
+
|
|
615
|
+
def ext_bases(self, cid):
|
|
616
|
+
"""External base names reachable from a project class through its project bases."""
|
|
617
|
+
todo, seen, out = [cid], set(), set()
|
|
618
|
+
while todo:
|
|
619
|
+
current = todo.pop()
|
|
620
|
+
if current in seen:
|
|
621
|
+
continue
|
|
622
|
+
seen.add(current)
|
|
623
|
+
info = self.classes[current]
|
|
624
|
+
for base in info["bases"]:
|
|
625
|
+
binding = self.typeof(base, None, info["mod"])
|
|
626
|
+
if binding and binding[0] == "node" and binding[1] in self.classes:
|
|
627
|
+
todo.append(binding[1])
|
|
628
|
+
elif binding and binding[0] == "ext":
|
|
629
|
+
out.add(binding[1])
|
|
630
|
+
return out
|
|
631
|
+
|
|
632
|
+
def framework_callbacks(self):
|
|
633
|
+
for cid, info in self.classes.items():
|
|
634
|
+
todo, seen = [cid], set()
|
|
635
|
+
while todo:
|
|
636
|
+
current = todo.pop()
|
|
637
|
+
if current in seen:
|
|
638
|
+
continue
|
|
639
|
+
seen.add(current)
|
|
640
|
+
base_info = self.classes[current]
|
|
641
|
+
for base in base_info["bases"]:
|
|
642
|
+
binding = self.typeof(base, None, base_info["mod"])
|
|
643
|
+
if binding and binding[0] == "node" and binding[1] in self.classes:
|
|
644
|
+
todo.append(binding[1])
|
|
645
|
+
elif binding and binding[0] == "ext" and binding[1] != "builtins.object":
|
|
646
|
+
self.external_callbacks(info, binding[1], cid)
|
|
647
|
+
# a hook a server is proven to call (served class or an inherited base) is no longer uncertain;
|
|
648
|
+
# nor is a constructor the project itself calls (`raise ResearchDatasetError(...)`)
|
|
649
|
+
constructed = {e["to"] for e in self.edges if e["kind"] == "call" and e["to"].endswith(".__init__")}
|
|
650
|
+
self.edges = [e for e in self.edges if not (e["kind"] == "?callback" and e.get("framework")
|
|
651
|
+
and (e["from"] in self.hooked or e["from"] in constructed))]
|
|
652
|
+
|
|
653
|
+
def mro(self, cid, stack=()):
|
|
654
|
+
"""C3 linearization; an external base is one opaque ("ext", name) entry holding all its stdlib methods."""
|
|
655
|
+
if isinstance(cid, tuple) or cid in stack:
|
|
656
|
+
return [cid]
|
|
657
|
+
klass = self.classes[cid]
|
|
658
|
+
bases = []
|
|
659
|
+
for b in (self.typeof(x, None, klass["mod"]) for x in klass["bases"]):
|
|
660
|
+
if b and b[0] == "node" and b[1] in self.classes:
|
|
661
|
+
bases.append(b[1])
|
|
662
|
+
elif b and b[0] == "ext" and b[1] != "builtins.object":
|
|
663
|
+
bases.append(("ext", b[1]))
|
|
664
|
+
seqs, out = [self.mro(b, stack + (cid,)) for b in bases] + [bases], [cid]
|
|
665
|
+
while True:
|
|
666
|
+
seqs = [s for s in seqs if s]
|
|
667
|
+
if not seqs:
|
|
668
|
+
return out
|
|
669
|
+
head = next((s[0] for s in seqs if not any(s[0] in t[1:] for t in seqs)), None)
|
|
670
|
+
if head is None: # inconsistent hierarchy (Python would raise): keep the remaining order
|
|
671
|
+
return out + [c for s in seqs for c in s if c not in out]
|
|
672
|
+
out.append(head)
|
|
673
|
+
seqs = [s[1:] if s[0] == head else s for s in seqs]
|
|
674
|
+
|
|
675
|
+
def external_callbacks(self, info, base, cid=None):
|
|
676
|
+
methods = stdlib_methods(base)
|
|
677
|
+
if base in HANDLERS and self.served.get(cid) and cid not in self.linked:
|
|
678
|
+
# the server handed this class calls its request hooks, inherited ones too (socketserver contract)
|
|
679
|
+
self.linked.add(cid)
|
|
680
|
+
hooks = {}
|
|
681
|
+
for current in self.mro(cid): # Python's lookup order: the first definition wins
|
|
682
|
+
if isinstance(current, tuple): # a stdlib definition shadows later project mixins
|
|
683
|
+
for name in stdlib_methods(current[1]) or ():
|
|
684
|
+
if is_hook(name):
|
|
685
|
+
hooks.setdefault(name, None)
|
|
686
|
+
continue
|
|
687
|
+
for name, mid in self.classes[current]["methods"].items():
|
|
688
|
+
if is_hook(name):
|
|
689
|
+
hooks.setdefault(name, mid)
|
|
690
|
+
hooks = {name: mid for name, mid in hooks.items() if mid}
|
|
691
|
+
self.hooked |= set(hooks.values())
|
|
692
|
+
for mid in hooks.values():
|
|
693
|
+
for frm, at in self.served[cid]:
|
|
694
|
+
self.edge(frm, mid, "ref", at, framework=True, reason=f"{base} given to a socketserver server")
|
|
695
|
+
for name, mid in info["methods"].items():
|
|
696
|
+
# http.server dispatches do_<HTTP method> by name; the base has no do_GET declaration.
|
|
697
|
+
convention = base == "http.server.BaseHTTPRequestHandler" and name.startswith("do_")
|
|
698
|
+
if methods is not None and name not in methods and not convention:
|
|
699
|
+
continue
|
|
700
|
+
node = self.nodes[mid]
|
|
701
|
+
if base in HANDLERS and self.served.get(cid) and is_hook(name):
|
|
702
|
+
continue
|
|
703
|
+
self.edge(mid, "?", "?callback", f"{node['file']}:{node['line']}",
|
|
704
|
+
expr=f"{base}.{name}", framework=True,
|
|
705
|
+
reason="external override" if methods is not None else "possible external override")
|
|
706
|
+
|
|
707
|
+
def edge(self, frm, to, kind, at, **extra):
|
|
708
|
+
e = {"from": frm, "to": to, "kind": kind, "at": at}
|
|
709
|
+
e.update(extra)
|
|
710
|
+
self.edges.append(e)
|
|
711
|
+
|
|
712
|
+
def dispatch_targets(self, rule):
|
|
713
|
+
"""Targets of a [[dispatch]] rule: explicit list or a module-level literal."""
|
|
714
|
+
if "targets" in rule:
|
|
715
|
+
return [t for t in rule["targets"] if t in self.nodes]
|
|
716
|
+
return self._resolve_targets_from(rule.get("targets_from", ""))[0]
|
|
717
|
+
|
|
718
|
+
def targets_from_errors(self, rule):
|
|
719
|
+
src = rule.get("targets_from")
|
|
720
|
+
if not src:
|
|
721
|
+
return []
|
|
722
|
+
return self._resolve_targets_from(src)[1]
|
|
723
|
+
|
|
724
|
+
def _resolve_targets_from(self, src):
|
|
725
|
+
path, _, name = src.partition(":")
|
|
726
|
+
mod = self.path_mod.get(path)
|
|
727
|
+
b = self.syms.get(mod, {}).get(name) if mod else None
|
|
728
|
+
if not b or b[0] != "var" or not isinstance(b[2], (ast.Dict, ast.List, ast.Tuple, ast.Set)):
|
|
729
|
+
return [], [f"targets_from {src}: no module-level dict/list literal"]
|
|
730
|
+
values = b[2].values if isinstance(b[2], ast.Dict) else b[2].elts
|
|
731
|
+
out, errs = [], []
|
|
732
|
+
for v in values:
|
|
733
|
+
t = self.typeof(v, None, mod)
|
|
734
|
+
if t and t[0] == "node" and self.nodes[t[1]]["kind"] in ("function", "method"):
|
|
735
|
+
out.append(t[1])
|
|
736
|
+
else:
|
|
737
|
+
errs.append(f"targets_from {src}: value `{text(v)}` is not a project function")
|
|
738
|
+
return out, errs
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
class Frame:
|
|
742
|
+
def __init__(self, nid, parent, cls=None):
|
|
743
|
+
self.id = nid
|
|
744
|
+
self.parent = parent
|
|
745
|
+
self.cls = cls
|
|
746
|
+
self.defs = {}
|
|
747
|
+
self.params = {}
|
|
748
|
+
self.locals = {} # name -> [value exprs] (None = unknown binding)
|
|
749
|
+
self.cache = {}
|
|
750
|
+
self.consts = set() # ruled process consts referenced here
|
|
751
|
+
self.pending = [] # subprocess calls waiting for const coverage
|
|
752
|
+
|
|
753
|
+
def local_type(self, name, project, mod):
|
|
754
|
+
if name in self.cache:
|
|
755
|
+
return self.cache[name]
|
|
756
|
+
self.cache[name] = None # recursion guard
|
|
757
|
+
types = set()
|
|
758
|
+
for v in self.locals[name]:
|
|
759
|
+
if v is None:
|
|
760
|
+
types.add(None)
|
|
761
|
+
elif isinstance(v, tuple):
|
|
762
|
+
types.add(v)
|
|
763
|
+
elif isinstance(v, ast.Lambda):
|
|
764
|
+
types.add(("lambda",))
|
|
765
|
+
else:
|
|
766
|
+
t = project.typeof(v, self, mod)
|
|
767
|
+
types.add(t if t and t[0] in ("node", "inst", "mod", "ext", "spec") else None)
|
|
768
|
+
result = types.pop() if len(types) == 1 else None
|
|
769
|
+
self.cache[name] = result
|
|
770
|
+
return result
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def bound_names(fn):
|
|
774
|
+
"""Names bound inside a function body, not descending into nested scopes."""
|
|
775
|
+
out = {}
|
|
776
|
+
|
|
777
|
+
def add(name, value):
|
|
778
|
+
out.setdefault(name, []).append(value)
|
|
779
|
+
|
|
780
|
+
def targets(t, value):
|
|
781
|
+
if isinstance(t, ast.Name):
|
|
782
|
+
add(t.id, value)
|
|
783
|
+
elif isinstance(t, (ast.Tuple, ast.List)):
|
|
784
|
+
for x in t.elts:
|
|
785
|
+
targets(x, None)
|
|
786
|
+
elif isinstance(t, ast.Starred):
|
|
787
|
+
targets(t.value, None)
|
|
788
|
+
|
|
789
|
+
stack = list(fn.body)
|
|
790
|
+
while stack:
|
|
791
|
+
n = stack.pop()
|
|
792
|
+
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
793
|
+
add(n.name, None) # handled by defs; marks shadowing
|
|
794
|
+
continue
|
|
795
|
+
if isinstance(n, ast.Lambda):
|
|
796
|
+
continue
|
|
797
|
+
if isinstance(n, ast.Assign):
|
|
798
|
+
for t in n.targets:
|
|
799
|
+
targets(t, n.value if isinstance(t, ast.Name) else None)
|
|
800
|
+
elif isinstance(n, ast.AnnAssign):
|
|
801
|
+
targets(n.target, ("ann", n.annotation))
|
|
802
|
+
elif isinstance(n, (ast.AugAssign,)):
|
|
803
|
+
targets(n.target, None)
|
|
804
|
+
elif isinstance(n, (ast.For, ast.AsyncFor, ast.comprehension)):
|
|
805
|
+
targets(n.target, None)
|
|
806
|
+
elif isinstance(n, ast.withitem) and n.optional_vars is not None:
|
|
807
|
+
targets(n.optional_vars, None)
|
|
808
|
+
elif isinstance(n, ast.ExceptHandler) and n.name:
|
|
809
|
+
add(n.name, None)
|
|
810
|
+
elif isinstance(n, ast.NamedExpr):
|
|
811
|
+
targets(n.target, n.value)
|
|
812
|
+
elif isinstance(n, ast.Import):
|
|
813
|
+
for a in n.names:
|
|
814
|
+
if a.asname:
|
|
815
|
+
add(a.asname, ("import", a.name))
|
|
816
|
+
else:
|
|
817
|
+
add(a.name.split(".")[0], ("import", a.name.split(".")[0]))
|
|
818
|
+
elif isinstance(n, ast.ImportFrom):
|
|
819
|
+
for a in n.names:
|
|
820
|
+
if a.name != "*":
|
|
821
|
+
add(a.asname or a.name, ("from", n, a.name))
|
|
822
|
+
elif isinstance(n, (ast.Global, ast.Nonlocal)):
|
|
823
|
+
for name in n.names:
|
|
824
|
+
out.pop(name, None)
|
|
825
|
+
continue
|
|
826
|
+
stack.extend(ast.iter_child_nodes(n))
|
|
827
|
+
return out
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
class Visitor(ast.NodeVisitor):
|
|
831
|
+
def __init__(self, project, path, mod):
|
|
832
|
+
self.p = project
|
|
833
|
+
self.path = path
|
|
834
|
+
self.mod = mod
|
|
835
|
+
cfg = project.config
|
|
836
|
+
self.dispatch = [r for r in cfg.get("dispatch", []) if path_match(r.get("at", ""), path)]
|
|
837
|
+
self.process_rules = [r for r in cfg.get("process", []) if "at" in r and path_match(r["at"], path)]
|
|
838
|
+
self.process_consts = {r["const"]: r["target"] for r in cfg.get("process", []) if "const" in r}
|
|
839
|
+
|
|
840
|
+
def at(self, node):
|
|
841
|
+
return f"{self.path}:{node.lineno}"
|
|
842
|
+
|
|
843
|
+
def finish(self):
|
|
844
|
+
self.flush(self.frame)
|
|
845
|
+
|
|
846
|
+
def flush(self, f):
|
|
847
|
+
for call, callee, full in f.pending:
|
|
848
|
+
if not f.consts:
|
|
849
|
+
self.p.edge(f.id, "?", "unresolved", self.at(call), expr=callee, full=full)
|
|
850
|
+
f.pending = []
|
|
851
|
+
|
|
852
|
+
# scopes
|
|
853
|
+
def fastapi(self, e):
|
|
854
|
+
"""(var id, own literal prefix) if e is a module-level FastAPI()/APIRouter() variable."""
|
|
855
|
+
t = self.typeof(e)
|
|
856
|
+
if not t or t[0] != "var" or not isinstance(t[2], ast.Call):
|
|
857
|
+
return None
|
|
858
|
+
c = self.p.typeof(t[2].func, None, self.p.path_mod[t[1].rsplit(":", 1)[0]]) # in the defining module
|
|
859
|
+
if not (c and c[0] == "ext" and c[1].startswith("fastapi.") and c[1].rsplit(".", 1)[-1] in ("FastAPI", "APIRouter")):
|
|
860
|
+
return None
|
|
861
|
+
pre = next((k.value for k in t[2].keywords if k.arg == "prefix"), None)
|
|
862
|
+
return t[1], (pre.value if isinstance(pre, ast.Constant) and isinstance(pre.value, str) else "")
|
|
863
|
+
|
|
864
|
+
def visit_FunctionDef(self, node):
|
|
865
|
+
for d in node.decorator_list:
|
|
866
|
+
app = (self.fastapi(d.func.value) if isinstance(d, ast.Call) and isinstance(d.func, ast.Attribute)
|
|
867
|
+
and d.func.attr in ROUTE_METHODS else None)
|
|
868
|
+
if app:
|
|
869
|
+
a = d.args[0] if d.args else next((k.value for k in d.keywords if k.arg == "path"), None)
|
|
870
|
+
if isinstance(a, ast.Constant) and isinstance(a.value, str):
|
|
871
|
+
method = d.func.attr.upper()
|
|
872
|
+
self.p.entries.append({"kind": "route", "value": app[1].rstrip("/") + a.value if app[1] else a.value,
|
|
873
|
+
"method": method, "app": app[0],
|
|
874
|
+
"node": self.p.astid[id(node)], "at": self.at(d)})
|
|
875
|
+
else:
|
|
876
|
+
self.p.edge(self.frame.id, "?", "?route", self.at(d), expr=text(d.func))
|
|
877
|
+
elif route_path(d):
|
|
878
|
+
# safety net (spec §10.1): a route-shaped decorator on an app we cannot prove is marked, never dropped
|
|
879
|
+
self.p.edge(self.frame.id, "?", "?route", self.at(d), expr=text(d.func), handler=self.p.astid[id(node)],
|
|
880
|
+
path=route_path(d), method=d.func.attr.upper() if d.func.attr in ROUTE_METHODS else None)
|
|
881
|
+
self.visit(d)
|
|
882
|
+
for d in node.args.defaults + [d for d in node.args.kw_defaults if d is not None]:
|
|
883
|
+
self.visit(d)
|
|
884
|
+
nid = self.p.astid[id(node)]
|
|
885
|
+
if node.name and self.frame and not isinstance(self.frame, ClassFrame):
|
|
886
|
+
self.frame.defs.setdefault(node.name, nid)
|
|
887
|
+
parent = self.frame
|
|
888
|
+
cls = parent.cls if isinstance(parent, ClassFrame) else None
|
|
889
|
+
f = Frame(nid, self.outer_function_frame(parent), cls)
|
|
890
|
+
args = node.args.posonlyargs + node.args.args + node.args.kwonlyargs
|
|
891
|
+
static = any(isinstance(d, ast.Name) and d.id == "staticmethod" for d in node.decorator_list)
|
|
892
|
+
classm = any(isinstance(d, ast.Name) and d.id == "classmethod" for d in node.decorator_list)
|
|
893
|
+
for i, a in enumerate(args):
|
|
894
|
+
if i == 0 and cls and not static:
|
|
895
|
+
f.params[a.arg] = ("node", cls) if classm else ("inst", cls)
|
|
896
|
+
else:
|
|
897
|
+
f.params[a.arg] = self.p.inst_of(a.annotation, parent, self.mod) or ("param",)
|
|
898
|
+
for a in (node.args.vararg, node.args.kwarg):
|
|
899
|
+
if a:
|
|
900
|
+
f.params[a.arg] = ("param",)
|
|
901
|
+
for name, values in bound_names(node).items():
|
|
902
|
+
if name in f.params:
|
|
903
|
+
continue
|
|
904
|
+
vals = []
|
|
905
|
+
for v in values:
|
|
906
|
+
if isinstance(v, tuple) and v[0] == "ann":
|
|
907
|
+
v = self.p.inst_of(v[1], parent, self.mod)
|
|
908
|
+
elif isinstance(v, tuple) and v[0] == "import":
|
|
909
|
+
v = self.p.binding(("mod", self.p.abs_module(self.mod, self.path, v[1])), self.mod)
|
|
910
|
+
elif isinstance(v, tuple) and v[0] == "from":
|
|
911
|
+
v = self.p.binding(("imp", self.p.from_module(self.mod, self.path, v[1]), v[2]), self.mod)
|
|
912
|
+
vals.append(v)
|
|
913
|
+
f.locals[name] = vals
|
|
914
|
+
for st in node.body:
|
|
915
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
916
|
+
f.defs[st.name] = self.p.astid[id(st)]
|
|
917
|
+
f.locals.pop(st.name, None)
|
|
918
|
+
self.frame = f
|
|
919
|
+
for st in node.body:
|
|
920
|
+
self.visit(st)
|
|
921
|
+
self.flush(f)
|
|
922
|
+
self.frame = parent
|
|
923
|
+
|
|
924
|
+
visit_AsyncFunctionDef = visit_FunctionDef
|
|
925
|
+
|
|
926
|
+
def outer_function_frame(self, f):
|
|
927
|
+
while isinstance(f, ClassFrame):
|
|
928
|
+
f = f.parent
|
|
929
|
+
return None if (f is not None and f.id.endswith(":<module>")) else f
|
|
930
|
+
|
|
931
|
+
def visit_ClassDef(self, node):
|
|
932
|
+
for d in node.decorator_list + node.bases + [k.value for k in node.keywords]:
|
|
933
|
+
self.visit(d)
|
|
934
|
+
nid = self.p.astid[id(node)]
|
|
935
|
+
parent = self.frame
|
|
936
|
+
cf = ClassFrame(nid, parent, nid)
|
|
937
|
+
self.frame = cf
|
|
938
|
+
for st in node.body:
|
|
939
|
+
self.visit(st)
|
|
940
|
+
self.flush(cf)
|
|
941
|
+
self.frame = parent
|
|
942
|
+
|
|
943
|
+
def visit_Lambda(self, node):
|
|
944
|
+
self.visit(node.body)
|
|
945
|
+
|
|
946
|
+
# references
|
|
947
|
+
def visit_Name(self, node):
|
|
948
|
+
if isinstance(node.ctx, ast.Load):
|
|
949
|
+
self.reference(node)
|
|
950
|
+
|
|
951
|
+
def visit_Attribute(self, node):
|
|
952
|
+
if isinstance(node.ctx, ast.Load):
|
|
953
|
+
self.reference(node)
|
|
954
|
+
self.visit(node.value)
|
|
955
|
+
|
|
956
|
+
def scope(self):
|
|
957
|
+
f = self.frame
|
|
958
|
+
return f if not isinstance(f, ClassFrame) else None
|
|
959
|
+
|
|
960
|
+
def typeof(self, e):
|
|
961
|
+
return self.p.typeof(e, self.scope(), self.mod)
|
|
962
|
+
|
|
963
|
+
def reference(self, node):
|
|
964
|
+
t = self.typeof(node)
|
|
965
|
+
if not t:
|
|
966
|
+
return
|
|
967
|
+
if t[0] == "node" and self.p.nodes[t[1]]["kind"] in ("function", "method"):
|
|
968
|
+
if not isinstance(node, ast.Name) or t[1] != self.frame.id:
|
|
969
|
+
self.p.edge(self.frame.id, t[1], "ref", self.at(node))
|
|
970
|
+
elif t[0] == "var" and t[1] in self.process_consts:
|
|
971
|
+
self.frame.consts.add(t[1])
|
|
972
|
+
target = self.process_consts[t[1]]
|
|
973
|
+
self.p.edge(self.frame.id, f"{target}:<module>", "process", self.at(node), src="rule",
|
|
974
|
+
target_file=target)
|
|
975
|
+
|
|
976
|
+
def visit_Compare(self, node):
|
|
977
|
+
vals = [node.left] + node.comparators
|
|
978
|
+
for v in vals:
|
|
979
|
+
items = v.elts if isinstance(v, (ast.Tuple, ast.List, ast.Set)) else [v]
|
|
980
|
+
for x in items:
|
|
981
|
+
if isinstance(x, ast.Constant) and isinstance(x.value, str) and FLAG_RE.match(x.value):
|
|
982
|
+
self.entry("flag", x.value, x)
|
|
983
|
+
self.generic_visit(node)
|
|
984
|
+
|
|
985
|
+
def entry(self, kind, value, node):
|
|
986
|
+
self.p.entries.append({"kind": kind, "value": value, "node": self.frame.id, "at": self.at(node)})
|
|
987
|
+
|
|
988
|
+
# calls
|
|
989
|
+
def visit_Call(self, node):
|
|
990
|
+
self.call(node)
|
|
991
|
+
f = node.func
|
|
992
|
+
if isinstance(f, ast.Attribute):
|
|
993
|
+
self.visit(f.value)
|
|
994
|
+
elif not isinstance(f, ast.Name):
|
|
995
|
+
self.visit(f)
|
|
996
|
+
for a in node.args:
|
|
997
|
+
self.visit(a)
|
|
998
|
+
for k in node.keywords:
|
|
999
|
+
self.visit(k.value)
|
|
1000
|
+
|
|
1001
|
+
def super_call(self, f, me, at):
|
|
1002
|
+
"""`super().m()` in a method: the next class in the MRO that defines m. True when settled."""
|
|
1003
|
+
if not (isinstance(f, ast.Attribute) and isinstance(f.value, ast.Call) and isinstance(f.value.func, ast.Name)
|
|
1004
|
+
and f.value.func.id == "super" and not f.value.args and self.frame.cls in self.p.classes):
|
|
1005
|
+
return False
|
|
1006
|
+
chain = self.p.mro(self.frame.cls)
|
|
1007
|
+
for c in chain: # an unresolved base could define m: stay uncertain
|
|
1008
|
+
if not isinstance(c, tuple) and any(self.p.typeof(b, None, self.p.classes[c]["mod"]) is None
|
|
1009
|
+
for b in self.p.classes[c]["bases"]):
|
|
1010
|
+
return False
|
|
1011
|
+
for c in chain[1:]:
|
|
1012
|
+
if isinstance(c, tuple):
|
|
1013
|
+
return True # an external base answers it (ValueError.__init__)
|
|
1014
|
+
mid = self.p.classes[c]["methods"].get(f.attr)
|
|
1015
|
+
if mid:
|
|
1016
|
+
self.p.edge(me, mid, "call", at)
|
|
1017
|
+
return True
|
|
1018
|
+
return f.attr in vars(object) # object.__init__ and friends
|
|
1019
|
+
|
|
1020
|
+
def server_handler(self, node, t, me, at):
|
|
1021
|
+
"""`SomeServer(addr, Handler)`: record which function hands a project handler class to a stdlib server."""
|
|
1022
|
+
if not t or not (t[0] == "ext" and t[1] in SERVERS
|
|
1023
|
+
or t[0] == "node" and t[1] in self.p.classes and self.p.ext_bases(t[1]) & SERVERS):
|
|
1024
|
+
return
|
|
1025
|
+
arg = node.args[1] if len(node.args) > 1 else next(
|
|
1026
|
+
(k.value for k in node.keywords if k.arg == "RequestHandlerClass"), None)
|
|
1027
|
+
h = self.typeof(arg) if arg is not None else None
|
|
1028
|
+
if h and h[0] == "node" and h[1] in self.p.classes:
|
|
1029
|
+
self.p.served.setdefault(h[1], []).append((me, at))
|
|
1030
|
+
|
|
1031
|
+
def call(self, node):
|
|
1032
|
+
f = node.func
|
|
1033
|
+
callee, full = text(f), text(node)
|
|
1034
|
+
at = self.at(node)
|
|
1035
|
+
me = self.frame.id
|
|
1036
|
+
if isinstance(f, ast.Attribute) and f.attr == "include_router" and node.args:
|
|
1037
|
+
parent, child = self.fastapi(f.value), self.fastapi(node.args[0])
|
|
1038
|
+
if parent and child:
|
|
1039
|
+
pre = next((k.value for k in node.keywords if k.arg == "prefix"), None)
|
|
1040
|
+
if pre is None or (isinstance(pre, ast.Constant) and isinstance(pre.value, str)):
|
|
1041
|
+
self.p.mounts.append({"parent": parent[0], "child": child[0], "prefix": pre.value if pre else "", "at": at})
|
|
1042
|
+
else:
|
|
1043
|
+
self.p.edge(me, "?", "?route", at, expr=callee)
|
|
1044
|
+
if isinstance(f, ast.Attribute) and f.attr == "add_api_route" and route_path(node):
|
|
1045
|
+
ep = node.args[1] if len(node.args) > 1 else next((k.value for k in node.keywords if k.arg == "endpoint"), None)
|
|
1046
|
+
t = self.typeof(ep) if ep is not None else None
|
|
1047
|
+
methods = next((k.value for k in node.keywords if k.arg == "methods"), None)
|
|
1048
|
+
verbs = [m.value.upper() for m in getattr(methods, "elts", []) if isinstance(m, ast.Constant) and isinstance(m.value, str)]
|
|
1049
|
+
self.p.edge(me, "?", "?route", at, expr=callee, path=route_path(node),
|
|
1050
|
+
**({"handler": t[1]} if t and t[0] == "node" else {}), **({"method": verbs[0]} if len(verbs) == 1 else {}))
|
|
1051
|
+
if isinstance(f, ast.Attribute) and f.attr == "add_argument":
|
|
1052
|
+
for a in node.args:
|
|
1053
|
+
if isinstance(a, ast.Constant) and isinstance(a.value, str) and FLAG_RE.match(a.value):
|
|
1054
|
+
self.entry("flag", a.value, a)
|
|
1055
|
+
for r in self.dispatch:
|
|
1056
|
+
if expr_match(r.get("expr", ""), callee, full):
|
|
1057
|
+
for target in self.p.dispatch_targets(r):
|
|
1058
|
+
self.p.edge(me, target, "dispatch", at, src="rule")
|
|
1059
|
+
return
|
|
1060
|
+
if self.super_call(f, me, at):
|
|
1061
|
+
return
|
|
1062
|
+
t = self.typeof(f)
|
|
1063
|
+
self.server_handler(node, t, me, at)
|
|
1064
|
+
if t and t[0] == "node":
|
|
1065
|
+
target = t[1]
|
|
1066
|
+
if target in self.p.classes:
|
|
1067
|
+
init = self.p.lookup(target, "__init__")
|
|
1068
|
+
target = init[1] if init and init[0] == "node" else target
|
|
1069
|
+
self.p.edge(me, target, "call", at)
|
|
1070
|
+
elif t and t[0] == "ext":
|
|
1071
|
+
name = t[1]
|
|
1072
|
+
if is_subprocess(name):
|
|
1073
|
+
for r in self.process_rules:
|
|
1074
|
+
if expr_match(r.get("expr", ""), callee, full):
|
|
1075
|
+
self.p.edge(me, f"{r['target']}:<module>", "process", at, src="rule",
|
|
1076
|
+
target_file=r["target"])
|
|
1077
|
+
return
|
|
1078
|
+
target = self.p.process_target(node, self.scope(), self.mod, name)
|
|
1079
|
+
if target:
|
|
1080
|
+
self.p.edge(me, f"{target}:<module>", "process", at)
|
|
1081
|
+
return
|
|
1082
|
+
self.frame.pending.append((node, callee, full))
|
|
1083
|
+
elif name in DYNAMIC and self.p.importlib_result(node, self.scope(), self.mod) is None:
|
|
1084
|
+
self.p.edge(me, "?", "unresolved", at, expr=callee, full=full)
|
|
1085
|
+
elif t and t[0] == "lambda":
|
|
1086
|
+
# shortcut: calls inside a lambda are attributed to the frame defining it, upgrade if lambdas get their own nodes
|
|
1087
|
+
return
|
|
1088
|
+
elif t and t[0] == "param":
|
|
1089
|
+
self.p.edge(me, "?", "?callback", at, expr=callee)
|
|
1090
|
+
elif t and t[0] in ("mod", "inst", "var"):
|
|
1091
|
+
if t[0] == "var" and isinstance(t[2], ast.Lambda):
|
|
1092
|
+
return # shortcut: module-level lambda body is attributed to <module>
|
|
1093
|
+
if t[0] == "var" and t[2] is not None: # module-level alias: handler = some_function
|
|
1094
|
+
v = self.p.typeof(t[2], None, self.p.path_mod[t[1].split(":")[0]])
|
|
1095
|
+
if v and v[0] == "node" and self.p.nodes[v[1]]["kind"] in ("function", "method"):
|
|
1096
|
+
self.p.edge(me, v[1], "call", at)
|
|
1097
|
+
return
|
|
1098
|
+
if v and v[0] == "ext":
|
|
1099
|
+
return # alias of an external function (`prep = extmod.prepare`): provably external
|
|
1100
|
+
if t[0] == "inst":
|
|
1101
|
+
c = self.p.lookup(t[1], "__call__")
|
|
1102
|
+
if c and c[0] == "node":
|
|
1103
|
+
self.p.edge(me, c[1], "call", at)
|
|
1104
|
+
return
|
|
1105
|
+
if c and c[0] == "ext":
|
|
1106
|
+
return
|
|
1107
|
+
self.p.edge(me, "?", "unresolved", at, expr=callee, full=full)
|
|
1108
|
+
elif isinstance(f, ast.Attribute):
|
|
1109
|
+
cands = self.p.methods_by_name.get(f.attr)
|
|
1110
|
+
if cands and not TEST_PATH.search(self.path): # app code reaches a test double only through the test itself
|
|
1111
|
+
cands = [c for c in cands if not TEST_PATH.search(self.p.nodes[c]["file"])]
|
|
1112
|
+
if cands:
|
|
1113
|
+
self.p.edge(me, "?", "method?", at, expr=callee, method=f.attr, candidates=sorted(cands))
|
|
1114
|
+
elif isinstance(f, ast.Call):
|
|
1115
|
+
inner = self.typeof(f.func)
|
|
1116
|
+
if inner and inner[0] == "ext" and inner[1] not in DYNAMIC_RESULT:
|
|
1117
|
+
return
|
|
1118
|
+
self.p.edge(me, "?", "unresolved", at, expr=callee, full=full)
|
|
1119
|
+
else:
|
|
1120
|
+
self.p.edge(me, "?", "unresolved", at, expr=callee, full=full)
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
class ClassFrame(Frame):
|
|
1124
|
+
pass
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def extract(sources, config, root=None):
|
|
1128
|
+
p = Project(sources, config, root)
|
|
1129
|
+
p.parse()
|
|
1130
|
+
p.walk()
|
|
1131
|
+
for r in config.get("dispatch", []):
|
|
1132
|
+
p.errors += p.targets_from_errors(r)
|
|
1133
|
+
return p
|