diffcone 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffcone/__init__.py +13 -0
- diffcone/cache.py +528 -0
- diffcone/check.py +399 -0
- diffcone/classify.py +255 -0
- diffcone/cli.py +886 -0
- diffcone/collect.py +846 -0
- diffcone/cython.py +757 -0
- diffcone/declarations.py +111 -0
- diffcone/discovery/__init__.py +121 -0
- diffcone/discovery/asv_static.py +494 -0
- diffcone/discovery/common.py +130 -0
- diffcone/discovery/pytest_static.py +2680 -0
- diffcone/evidence.py +738 -0
- diffcone/evidence_plan.py +1487 -0
- diffcone/execution.py +1681 -0
- diffcone/indexer/__init__.py +48 -0
- diffcone/indexer/core.py +286 -0
- diffcone/indexer/definitions.py +292 -0
- diffcone/indexer/dynamics.py +406 -0
- diffcone/indexer/facts.py +373 -0
- diffcone/indexer/literals.py +388 -0
- diffcone/indexer/references.py +895 -0
- diffcone/indexer/resolver.py +1001 -0
- diffcone/indexer/scopes.py +276 -0
- diffcone/indexer/state.py +83 -0
- diffcone/indexer/symbols.py +441 -0
- diffcone/indexer/syntax.py +173 -0
- diffcone/manifest.py +166 -0
- diffcone/model.py +191 -0
- diffcone/planner.py +1453 -0
- diffcone/report.py +256 -0
- diffcone/selection.py +133 -0
- diffcone/snapshot.py +739 -0
- diffcone/testing.py +200 -0
- diffcone-0.1.0.dist-info/METADATA +133 -0
- diffcone-0.1.0.dist-info/RECORD +39 -0
- diffcone-0.1.0.dist-info/WHEEL +4 -0
- diffcone-0.1.0.dist-info/entry_points.txt +3 -0
- diffcone-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Source index and dependency resolver.
|
|
2
|
+
|
|
3
|
+
Parses every Python module of a snapshot, assigns stable symbol identities,
|
|
4
|
+
hashes bodies and definitions, and resolves the statically resolvable subset
|
|
5
|
+
of references into explicit dependency edges. Everything that cannot be
|
|
6
|
+
resolved is recorded as an :class:`UnresolvedReference`, never dropped.
|
|
7
|
+
|
|
8
|
+
Supported subset (see internal/design.md):
|
|
9
|
+
|
|
10
|
+
* module-level functions, classes, methods (and nested classes) as symbols;
|
|
11
|
+
* bare names and dotted attribute chains rooted at a module-level definition,
|
|
12
|
+
an import alias, ``self``/``cls`` inside a method, or a star import;
|
|
13
|
+
* ``import``/``from ... import`` (absolute and relative) within source roots;
|
|
14
|
+
* ``importlib.import_module`` / ``getattr`` with literal arguments (a
|
|
15
|
+
literal table's keys, values or elements included: ``D[k]``, ``D.values()``,
|
|
16
|
+
``for k, v in D.items()``), with parameters whose call sites pass literals,
|
|
17
|
+
and with instance attributes that ``__init__`` binds to such values
|
|
18
|
+
(``getattr(x, self.name)``);
|
|
19
|
+
* attribute lookup on classes through their in-scope MRO (``self.m`` for an
|
|
20
|
+
inherited ``m``, ``Sub.m``, ``super().m``).
|
|
21
|
+
|
|
22
|
+
Deliberately unsupported: type inference, dynamic dispatch on unknown
|
|
23
|
+
receivers, instance attributes written outside ``__init__`` or reflectively,
|
|
24
|
+
decorators that rewrite call targets.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from diffcone.indexer.core import Indexer, build_index
|
|
28
|
+
from diffcone.indexer.scopes import resolve_relative_module
|
|
29
|
+
from diffcone.indexer.syntax import (
|
|
30
|
+
DEF_NODES,
|
|
31
|
+
FUNC_NODES,
|
|
32
|
+
decode_source,
|
|
33
|
+
hash_nodes,
|
|
34
|
+
hash_scope_body,
|
|
35
|
+
iter_scope_statements,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"DEF_NODES",
|
|
40
|
+
"FUNC_NODES",
|
|
41
|
+
"Indexer",
|
|
42
|
+
"build_index",
|
|
43
|
+
"decode_source",
|
|
44
|
+
"hash_nodes",
|
|
45
|
+
"hash_scope_body",
|
|
46
|
+
"iter_scope_statements",
|
|
47
|
+
"resolve_relative_module",
|
|
48
|
+
]
|
diffcone/indexer/core.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""The indexer: parses a snapshot's modules (pass 1, cacheable per module),
|
|
2
|
+
then resolves references into edges (pass 2). The work is layered, each
|
|
3
|
+
layer a subclass of the one below: state, symbols (pass 1), resolver
|
|
4
|
+
(pass 2), dynamics (bounds after pass 2), and here the pipeline."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
import json
|
|
10
|
+
from collections import defaultdict
|
|
11
|
+
|
|
12
|
+
from diffcone.indexer.dynamics import DynamicBounds
|
|
13
|
+
from diffcone.indexer.facts import (
|
|
14
|
+
_facts_to_dict,
|
|
15
|
+
_Output,
|
|
16
|
+
_output_from_dict,
|
|
17
|
+
_output_to_dict,
|
|
18
|
+
_tuples,
|
|
19
|
+
)
|
|
20
|
+
from diffcone.indexer.literals import INDEXED
|
|
21
|
+
from diffcone.indexer.scopes import ClassScope, ImportBinding, ModuleScope
|
|
22
|
+
from diffcone.indexer.syntax import _digest, decode_source
|
|
23
|
+
from diffcone.model import REFERENCES, Edge, SourceIndex, Symbol
|
|
24
|
+
from diffcone.snapshot import Snapshot, child_modules, module_name_for
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Indexer(DynamicBounds):
|
|
28
|
+
"""Builds a SourceIndex from a snapshot: ``Indexer(snapshot).build()``."""
|
|
29
|
+
|
|
30
|
+
def build(self) -> SourceIndex:
|
|
31
|
+
cache = self.module_cache
|
|
32
|
+
facts: dict[str, dict | None] = {}
|
|
33
|
+
candidates = [
|
|
34
|
+
(path, module)
|
|
35
|
+
for path in sorted(self.snapshot.files)
|
|
36
|
+
if (module := module_name_for(path, self.snapshot.source_roots)) is not None
|
|
37
|
+
]
|
|
38
|
+
# Submodule names per package: a top-level binding of that name gets
|
|
39
|
+
# a distinct identity (member_symbol_id).
|
|
40
|
+
self._children = child_modules(self.snapshot)
|
|
41
|
+
keys: dict[str, str] = {} # path -> cache key; every record is loaded in one query
|
|
42
|
+
if cache is not None:
|
|
43
|
+
keys = {p: cache.key(m, p, self.snapshot.files[p]) for p, m in candidates}
|
|
44
|
+
loaded = cache.load_facts(list(keys.values())) if cache is not None else {}
|
|
45
|
+
for path, module in candidates:
|
|
46
|
+
if module in self.scopes:
|
|
47
|
+
other = self.scopes[module].path
|
|
48
|
+
self._error(path, f"module {module!r} is also defined by {other}")
|
|
49
|
+
continue
|
|
50
|
+
record = loaded.get(keys[path]) if path in keys else None
|
|
51
|
+
if record is None:
|
|
52
|
+
tree = self._parse(path, module)
|
|
53
|
+
if tree is None:
|
|
54
|
+
continue
|
|
55
|
+
else:
|
|
56
|
+
tree = None
|
|
57
|
+
scope = ModuleScope(
|
|
58
|
+
name=module, path=path, is_package=path.endswith("__init__.py"), tree=tree
|
|
59
|
+
)
|
|
60
|
+
scope.cache_key = keys.get(path)
|
|
61
|
+
self.scopes[module] = scope
|
|
62
|
+
self.index.modules.add(module)
|
|
63
|
+
facts[module] = record
|
|
64
|
+
new_facts: dict[str, dict] = {}
|
|
65
|
+
new_resolved: dict[str, dict] = {}
|
|
66
|
+
for module in self.index.modules:
|
|
67
|
+
parts = module.split(".")
|
|
68
|
+
for i in range(1, len(parts) + 1):
|
|
69
|
+
self._module_prefixes.add(".".join(parts[:i]))
|
|
70
|
+
for module in sorted(self.scopes):
|
|
71
|
+
scope = self.scopes[module]
|
|
72
|
+
record = facts[module]
|
|
73
|
+
if record is not None:
|
|
74
|
+
# A record whose symbols collide with an earlier module's (an
|
|
75
|
+
# error either way) is not applied, so the errors come out
|
|
76
|
+
# exactly as without a cache; a malformed record is a miss.
|
|
77
|
+
try:
|
|
78
|
+
applied = self._apply_facts(scope, record)
|
|
79
|
+
except (KeyError, TypeError, ValueError, AttributeError):
|
|
80
|
+
applied = False
|
|
81
|
+
if applied:
|
|
82
|
+
continue
|
|
83
|
+
scope.tree = self._parse(scope.path, module)
|
|
84
|
+
if scope.tree is None: # pragma: no cover - same content parsed before
|
|
85
|
+
del self.scopes[module]
|
|
86
|
+
self.index.modules.discard(module)
|
|
87
|
+
continue
|
|
88
|
+
self._added_symbols, self._added_classes, self._collided = [], [], False
|
|
89
|
+
self.out = _Output()
|
|
90
|
+
try:
|
|
91
|
+
self._index_module(scope)
|
|
92
|
+
finally:
|
|
93
|
+
captured, self.out = self.out, self._global
|
|
94
|
+
self._global.merge(captured)
|
|
95
|
+
if cache is not None:
|
|
96
|
+
record = _facts_to_dict(
|
|
97
|
+
scope, self._added_symbols, self._added_classes, captured.edges
|
|
98
|
+
)
|
|
99
|
+
scope.env_digest = record["env"]
|
|
100
|
+
if not self._collided and scope.cache_key is not None:
|
|
101
|
+
new_facts[scope.cache_key] = record
|
|
102
|
+
self._unbind_mutated_variables()
|
|
103
|
+
for class_id in sorted(self.class_scopes):
|
|
104
|
+
self._ensure_bases(class_id)
|
|
105
|
+
self._bases_final = True # MROs may be memoised from here on
|
|
106
|
+
self._build_descendants()
|
|
107
|
+
self._external_callers()
|
|
108
|
+
fingerprint = self._environment_fingerprint() if cache is not None else ""
|
|
109
|
+
loaded = cache.load_resolved(list(keys.values()), fingerprint) if cache else {}
|
|
110
|
+
for module in sorted(self.scopes):
|
|
111
|
+
scope = self.scopes[module]
|
|
112
|
+
key = scope.cache_key
|
|
113
|
+
out: _Output | None = None
|
|
114
|
+
if key is not None and key in loaded:
|
|
115
|
+
try:
|
|
116
|
+
out = _output_from_dict(loaded[key], self.scopes)
|
|
117
|
+
except (KeyError, TypeError, ValueError, AttributeError):
|
|
118
|
+
out = None # malformed record: resolve as on a miss
|
|
119
|
+
if out is None:
|
|
120
|
+
if scope.tree is None:
|
|
121
|
+
self._load_tree(scope)
|
|
122
|
+
self.out = _Output()
|
|
123
|
+
try:
|
|
124
|
+
self._resolve_module(scope)
|
|
125
|
+
finally:
|
|
126
|
+
out, self.out = self.out, self._global
|
|
127
|
+
if key is not None:
|
|
128
|
+
new_resolved[key] = _output_to_dict(out)
|
|
129
|
+
self._global.merge(out)
|
|
130
|
+
self._resolve_param_dynamics()
|
|
131
|
+
self._registrations()
|
|
132
|
+
# Classes whose instances (or the class itself) are handed to someone
|
|
133
|
+
# else: whoever holds one may read any attribute off it by a name
|
|
134
|
+
# nothing resolves, so holding it depends on its members.
|
|
135
|
+
returns = self._global.returns
|
|
136
|
+
self.index.escaped_classes = {
|
|
137
|
+
cls
|
|
138
|
+
for sites in self._global.call_sites.values()
|
|
139
|
+
for site in sites
|
|
140
|
+
for cls in [
|
|
141
|
+
*site.positional_classes,
|
|
142
|
+
*site.keyword_classes.values(),
|
|
143
|
+
# A factory's class counts as handed on too: every return of
|
|
144
|
+
# that function yields it, so that is what the callee holds.
|
|
145
|
+
*(c for f in site.positional_sources if f for c in returns.get(f, ())),
|
|
146
|
+
*(c for f in site.keyword_sources.values() if f for c in returns.get(f, ())),
|
|
147
|
+
]
|
|
148
|
+
if cls is not None
|
|
149
|
+
}
|
|
150
|
+
if cache is not None and (new_facts or new_resolved):
|
|
151
|
+
cache.store(new_facts, new_resolved, fingerprint, list(keys.values()))
|
|
152
|
+
return self.index
|
|
153
|
+
|
|
154
|
+
def _registrations(self) -> None:
|
|
155
|
+
"""Edges to code a decorator or a base class may keep and call later:
|
|
156
|
+
from the decorator's receiver (``show`` holds what
|
|
157
|
+
``@show.register(int)`` registers, ``app`` what ``@app.command``
|
|
158
|
+
does), from each variable a decorator function writes into (the
|
|
159
|
+
``REG`` a ``@register`` fills), and from a base class whose
|
|
160
|
+
``__init_subclass__`` runs for its in-scope subclasses. Computed over
|
|
161
|
+
the whole index on every build, as the class model's other global
|
|
162
|
+
edges are."""
|
|
163
|
+
writers: dict[str, set[str]] = defaultdict(set)
|
|
164
|
+
for e in self.index.edges:
|
|
165
|
+
if e.detail == "mutated_by":
|
|
166
|
+
writers[e.target].add(e.source)
|
|
167
|
+
for decorated, decorator, receiver in sorted(self._global.decorations):
|
|
168
|
+
if receiver:
|
|
169
|
+
self.index.edges.add(Edge(receiver, decorated, REFERENCES, "registers"))
|
|
170
|
+
for variable in sorted(writers.get(decorator, ())):
|
|
171
|
+
self.index.edges.add(Edge(variable, decorated, REFERENCES, "registers"))
|
|
172
|
+
for cls, bases in sorted(self.index.class_bases.items()):
|
|
173
|
+
stack, seen = list(bases), set(bases)
|
|
174
|
+
while stack:
|
|
175
|
+
base = stack.pop()
|
|
176
|
+
if f"{base}.__init_subclass__" in self.index.symbols:
|
|
177
|
+
self.index.edges.add(Edge(base, cls, REFERENCES, "registers"))
|
|
178
|
+
for up in self.index.class_bases.get(base, ()):
|
|
179
|
+
if up not in seen:
|
|
180
|
+
seen.add(up)
|
|
181
|
+
stack.append(up)
|
|
182
|
+
|
|
183
|
+
def _parse(self, path: str, module: str) -> ast.Module | None:
|
|
184
|
+
try:
|
|
185
|
+
source = decode_source(self.snapshot.files[path])
|
|
186
|
+
return ast.parse(source, filename=path)
|
|
187
|
+
except (SyntaxError, UnicodeDecodeError, ValueError) as exc:
|
|
188
|
+
self._error(path, f"cannot parse: {exc}")
|
|
189
|
+
self.index.failed_modules.add(module)
|
|
190
|
+
return None
|
|
191
|
+
|
|
192
|
+
def _load_tree(self, scope: ModuleScope) -> None:
|
|
193
|
+
"""Parse a module whose facts came from the cache but which must be
|
|
194
|
+
resolved again (its file or the environment changed)."""
|
|
195
|
+
tree = self._parse(scope.path, scope.name)
|
|
196
|
+
if tree is None: # pragma: no cover - the same content parsed before
|
|
197
|
+
tree = ast.Module(body=[], type_ignores=[])
|
|
198
|
+
scope.tree = tree
|
|
199
|
+
_, _, variable_stmts = self._module_statements(scope, register_imports=False)
|
|
200
|
+
scope.variable_stmts = {n: s for n, s in variable_stmts.items() if n in scope.variables}
|
|
201
|
+
|
|
202
|
+
def _apply_facts(self, scope: ModuleScope, record: dict) -> bool:
|
|
203
|
+
"""Install a cached facts record. Everything is built before anything
|
|
204
|
+
is stored, so a malformed record (which raises) or one whose symbols
|
|
205
|
+
collide with an earlier module's (returns False) leaves no trace."""
|
|
206
|
+
imports = {k: ImportBinding(m, a) for k, (m, a) in record["imports"].items()}
|
|
207
|
+
alt_imports = {
|
|
208
|
+
k: [ImportBinding(m, a) for m, a in v] for k, v in record["alt_imports"].items()
|
|
209
|
+
}
|
|
210
|
+
star_imports = list(record["star_imports"])
|
|
211
|
+
bindings = set(record["bindings"])
|
|
212
|
+
members = dict(record["members"])
|
|
213
|
+
variables = dict(record["variables"])
|
|
214
|
+
literal_names = {k: _tuples(v) for k, v in record["literal_names"].items()}
|
|
215
|
+
mutations = frozenset(record["mutations"])
|
|
216
|
+
env_digest = str(record["env"])
|
|
217
|
+
symbols: list[Symbol] = []
|
|
218
|
+
for data in record["symbols"]:
|
|
219
|
+
data = dict(data)
|
|
220
|
+
data["line_ranges"] = tuple(tuple(r) for r in data["line_ranges"])
|
|
221
|
+
data["imports"] = tuple(data["imports"])
|
|
222
|
+
data["import_layout"] = tuple(data["import_layout"])
|
|
223
|
+
symbols.append(Symbol(**data))
|
|
224
|
+
classes: dict[str, ClassScope] = {}
|
|
225
|
+
for data in record["classes"]:
|
|
226
|
+
enclosing = classes[data["enclosing"]] if data["enclosing"] else None
|
|
227
|
+
classes[data["id"]] = ClassScope(
|
|
228
|
+
id=str(data["id"]),
|
|
229
|
+
module=scope,
|
|
230
|
+
enclosing=enclosing,
|
|
231
|
+
members=dict(data["members"]),
|
|
232
|
+
bindings=set(data["bindings"]),
|
|
233
|
+
base_chains=[list(c) if c is not None else None for c in data["base_chains"]],
|
|
234
|
+
base_names=[list(c) if c is not None else None for c in data["base_names"]],
|
|
235
|
+
plain=bool(data["plain"]),
|
|
236
|
+
)
|
|
237
|
+
edges = {Edge(*e) for e in record["edges"]}
|
|
238
|
+
if any(s.id in self.index.symbols for s in symbols):
|
|
239
|
+
return False
|
|
240
|
+
# Top-level identities depend on which submodules exist: a record
|
|
241
|
+
# made before a shadowed submodule was added (or removed) is stale.
|
|
242
|
+
for name, symbol_id in [*members.items(), *variables.items()]:
|
|
243
|
+
if symbol_id != self._member_id(scope.name, name):
|
|
244
|
+
return False
|
|
245
|
+
scope.imports, scope.star_imports, scope.bindings = imports, star_imports, bindings
|
|
246
|
+
scope.alt_imports = alt_imports
|
|
247
|
+
scope.members, scope.variables, scope.literal_names = members, variables, literal_names
|
|
248
|
+
scope.mutations = mutations
|
|
249
|
+
scope.env_digest = env_digest
|
|
250
|
+
for symbol in symbols:
|
|
251
|
+
self._add_symbol(symbol)
|
|
252
|
+
self.class_scopes.update(classes)
|
|
253
|
+
self.out.edges |= edges
|
|
254
|
+
return True
|
|
255
|
+
|
|
256
|
+
def _unbind_mutated_variables(self) -> None:
|
|
257
|
+
"""A module-level container that any module mutates in place is not
|
|
258
|
+
the literal it was assigned: unbind it everywhere (its own module and
|
|
259
|
+
the modules that import it), so names drawn from it stay dynamic."""
|
|
260
|
+
mutated: set[tuple[str, str]] = set()
|
|
261
|
+
for scope in self.scopes.values():
|
|
262
|
+
for name in scope.mutations:
|
|
263
|
+
if name in scope.variables:
|
|
264
|
+
mutated.add((scope.name, name))
|
|
265
|
+
binding = scope.imports.get(name)
|
|
266
|
+
if binding is not None and binding.attr is not None:
|
|
267
|
+
mutated.add((binding.module, binding.attr))
|
|
268
|
+
for module, name in mutated:
|
|
269
|
+
target = self.scopes.get(module)
|
|
270
|
+
if target is not None and name in target.literal_names:
|
|
271
|
+
target.literal_names[name] = target.literal_names[name + INDEXED] = None
|
|
272
|
+
|
|
273
|
+
def _environment_fingerprint(self) -> str:
|
|
274
|
+
"""Digest of everything a module's resolution reads from other
|
|
275
|
+
modules: their observable facts plus every class's resolved bases,
|
|
276
|
+
and the root specs (whether ``__name__`` is a module's runtime name)."""
|
|
277
|
+
material = [
|
|
278
|
+
sorted(self.snapshot.source_roots),
|
|
279
|
+
[[m, self.scopes[m].env_digest] for m in sorted(self.scopes)],
|
|
280
|
+
[[c, cs.bases, cs.complete] for c, cs in sorted(self.class_scopes.items())],
|
|
281
|
+
]
|
|
282
|
+
return _digest(json.dumps(material))
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def build_index(snapshot: Snapshot, module_cache=None) -> SourceIndex:
|
|
286
|
+
return Indexer(snapshot, module_cache=module_cache).build()
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"""Facts about definitions the indexer hashes and classifies: line spans,
|
|
2
|
+
decorators, class attributes, variable statements, inert definitions."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import ast
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
|
|
9
|
+
from diffcone.indexer.literals import _collect_store_names
|
|
10
|
+
from diffcone.indexer.scopes import ModuleScope, VariableStatement, _absolute_module
|
|
11
|
+
from diffcone.indexer.syntax import DEF_NODES, _digest, iter_scope_statements
|
|
12
|
+
from diffcone.model import CLASS_STATEMENT, OPAQUE_ATTRIBUTE
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _canonical_imports(scope: ModuleScope) -> set[str]:
|
|
16
|
+
"""One string per imported binding, independent of statement grouping or
|
|
17
|
+
order: adding a name to ``from m import (a, b)`` adds one entry."""
|
|
18
|
+
out: set[str] = set()
|
|
19
|
+
for node in scope.import_nodes:
|
|
20
|
+
if isinstance(node, ast.Import):
|
|
21
|
+
for alias in node.names:
|
|
22
|
+
out.add(f"import {alias.name}" + (f" as {alias.asname}" if alias.asname else ""))
|
|
23
|
+
elif isinstance(node, ast.ImportFrom):
|
|
24
|
+
base = _absolute_module(scope, node.module, node.level)
|
|
25
|
+
for alias in node.names:
|
|
26
|
+
entry = f"from {base} import {alias.name}"
|
|
27
|
+
out.add(entry + (f" as {alias.asname}" if alias.asname else ""))
|
|
28
|
+
return out
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _import_layout(scope: ModuleScope) -> tuple[str, ...]:
|
|
32
|
+
"""Each module-level import binding, in source order, prefixed with the
|
|
33
|
+
blocks it sits in (``if <test>``, ``try``, ``except``, ``else``, ...):
|
|
34
|
+
what runs at import depends on where an import is, not only on which
|
|
35
|
+
names are bound."""
|
|
36
|
+
assert scope.tree is not None
|
|
37
|
+
out: list[str] = []
|
|
38
|
+
|
|
39
|
+
def walk(body: list[ast.stmt], context: str) -> None:
|
|
40
|
+
# Ordinary statements before an import in its block: an import moved
|
|
41
|
+
# across ``os.environ[...] = ...`` or ``sys.path.insert`` changes what
|
|
42
|
+
# it runs with. Counted, not hashed: editing them is a body change.
|
|
43
|
+
before = 0
|
|
44
|
+
for stmt in body:
|
|
45
|
+
if isinstance(stmt, DEF_NODES):
|
|
46
|
+
continue
|
|
47
|
+
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
|
|
48
|
+
for entry in _import_entries(scope, stmt):
|
|
49
|
+
out.append(f"{context}#{before}|{entry}")
|
|
50
|
+
continue
|
|
51
|
+
before += 1
|
|
52
|
+
label = type(stmt).__name__
|
|
53
|
+
if isinstance(stmt, (ast.If, ast.While)):
|
|
54
|
+
label += f"({ast.unparse(stmt.test)})"
|
|
55
|
+
elif isinstance(stmt, (ast.With, ast.AsyncWith)):
|
|
56
|
+
label += f"({', '.join(ast.unparse(i) for i in stmt.items)})"
|
|
57
|
+
elif isinstance(stmt, (ast.For, ast.AsyncFor)):
|
|
58
|
+
label += f"({ast.unparse(stmt.target)} in {ast.unparse(stmt.iter)})"
|
|
59
|
+
elif isinstance(stmt, ast.Match):
|
|
60
|
+
label += f"({ast.unparse(stmt.subject)})"
|
|
61
|
+
for attr in ("body", "orelse", "finalbody"):
|
|
62
|
+
child = getattr(stmt, attr, None)
|
|
63
|
+
if isinstance(child, list) and child:
|
|
64
|
+
walk(child, f"{context}/{label}.{attr}")
|
|
65
|
+
for i, handler in enumerate(getattr(stmt, "handlers", []) or []):
|
|
66
|
+
walk(handler.body, f"{context}/{label}.except{i}")
|
|
67
|
+
for i, case in enumerate(getattr(stmt, "cases", []) or []):
|
|
68
|
+
walk(case.body, f"{context}/{label}.case{i}")
|
|
69
|
+
|
|
70
|
+
walk(scope.tree.body, "")
|
|
71
|
+
return tuple(out)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _import_entries(scope: ModuleScope, node: ast.Import | ast.ImportFrom) -> list[str]:
|
|
75
|
+
if isinstance(node, ast.Import):
|
|
76
|
+
return [f"import {a.name}" + (f" as {a.asname}" if a.asname else "") for a in node.names]
|
|
77
|
+
base = _absolute_module(scope, node.module, node.level)
|
|
78
|
+
return [
|
|
79
|
+
f"from {base} import {a.name}" + (f" as {a.asname}" if a.asname else "") for a in node.names
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _class_attributes(node: ast.ClassDef) -> dict[str, str]:
|
|
84
|
+
"""Each name a class body binds by plain assignment, with a hash of the
|
|
85
|
+
statements binding it; every other non-definition statement (a loop, a
|
|
86
|
+
call, a ``del``, a conditional binding) is hashed under OPAQUE_ATTRIBUTE,
|
|
87
|
+
and the class statement itself (bases, keywords, decorators) under
|
|
88
|
+
CLASS_STATEMENT.
|
|
89
|
+
The docstring is not an attribute here: it has its own hash."""
|
|
90
|
+
parts: dict[str, list[str]] = defaultdict(list)
|
|
91
|
+
body = node.body
|
|
92
|
+
if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
|
|
93
|
+
if isinstance(body[0].value.value, str):
|
|
94
|
+
body = body[1:]
|
|
95
|
+
for stmt in body:
|
|
96
|
+
if isinstance(stmt, DEF_NODES):
|
|
97
|
+
continue
|
|
98
|
+
dumped = ast.dump(stmt)
|
|
99
|
+
names: list[str] = []
|
|
100
|
+
if isinstance(stmt, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
|
101
|
+
targets = stmt.targets if isinstance(stmt, ast.Assign) else [stmt.target]
|
|
102
|
+
for target in targets:
|
|
103
|
+
elements = target.elts if isinstance(target, (ast.Tuple, ast.List)) else [target]
|
|
104
|
+
plain = [e.id for e in elements if isinstance(e, ast.Name)]
|
|
105
|
+
if len(plain) != len(elements):
|
|
106
|
+
names = []
|
|
107
|
+
break
|
|
108
|
+
names += plain
|
|
109
|
+
for name in names or [OPAQUE_ATTRIBUTE]:
|
|
110
|
+
parts[name].append(dumped)
|
|
111
|
+
parts[CLASS_STATEMENT] = [
|
|
112
|
+
ast.dump(n) for n in [*node.bases, *node.keywords, *node.decorator_list]
|
|
113
|
+
]
|
|
114
|
+
return {name: _digest("\n".join(dumps)) for name, dumps in sorted(parts.items())}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _start_line(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> int:
|
|
118
|
+
"""First line of a definition including its decorators, which belong to
|
|
119
|
+
the definition (they are part of its definition hash)."""
|
|
120
|
+
return min([node.lineno, *(d.lineno for d in node.decorator_list)])
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _variable_statements(scope: ModuleScope, body: list[ast.stmt]) -> dict[str, VariableStatement]:
|
|
124
|
+
"""Top-level ``NAME = <expr>`` / ``NAME: T = <expr>`` statements whose name
|
|
125
|
+
is bound exactly once in the module: candidates for variable symbols.
|
|
126
|
+
Names bound any other way too (in a loop, by unpacking, inside a block)
|
|
127
|
+
stay on the module symbol."""
|
|
128
|
+
assert scope.tree is not None
|
|
129
|
+
simple: dict[str, list[VariableStatement]] = defaultdict(list)
|
|
130
|
+
for stmt in body:
|
|
131
|
+
if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1:
|
|
132
|
+
target = stmt.targets[0]
|
|
133
|
+
elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None:
|
|
134
|
+
target = stmt.target
|
|
135
|
+
else:
|
|
136
|
+
continue
|
|
137
|
+
if isinstance(target, ast.Name):
|
|
138
|
+
simple[target.id].append(stmt)
|
|
139
|
+
simple_stmts = {id(s) for stmts in simple.values() for s in stmts}
|
|
140
|
+
other_bound: set[str] = set()
|
|
141
|
+
for stmt in iter_scope_statements(scope.tree.body):
|
|
142
|
+
if isinstance(stmt, DEF_NODES + (ast.Import, ast.ImportFrom)):
|
|
143
|
+
continue
|
|
144
|
+
if id(stmt) in simple_stmts:
|
|
145
|
+
continue
|
|
146
|
+
other_bound |= _collect_store_names(stmt)
|
|
147
|
+
return {
|
|
148
|
+
name: stmts[0]
|
|
149
|
+
for name, stmts in simple.items()
|
|
150
|
+
if len(stmts) == 1 and name not in other_bound and name not in scope.imports
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _end_line(node: ast.stmt | ast.Module) -> int:
|
|
155
|
+
"""Last source line of a definition; a Module has no position of its own."""
|
|
156
|
+
if isinstance(node, ast.Module):
|
|
157
|
+
return node.body[-1].end_lineno or 1 if node.body else 1
|
|
158
|
+
return node.end_lineno or node.lineno
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _has_decorator(node: ast.FunctionDef | ast.AsyncFunctionDef, name: str) -> bool:
|
|
162
|
+
for dec in node.decorator_list:
|
|
163
|
+
if isinstance(dec, ast.Name) and dec.id == name:
|
|
164
|
+
return True
|
|
165
|
+
if isinstance(dec, ast.Attribute) and dec.attr == name:
|
|
166
|
+
return True
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# Bases outside the source roots that only add structure and never call
|
|
171
|
+
# a subclass's methods.
|
|
172
|
+
_STRUCTURAL_BASES = frozenset(
|
|
173
|
+
{
|
|
174
|
+
"abc.ABC",
|
|
175
|
+
"typing.Generic",
|
|
176
|
+
"typing.Protocol",
|
|
177
|
+
"typing.NamedTuple",
|
|
178
|
+
"typing.TypedDict",
|
|
179
|
+
"typing_extensions.Generic",
|
|
180
|
+
"typing_extensions.Protocol",
|
|
181
|
+
"typing_extensions.NamedTuple",
|
|
182
|
+
"typing_extensions.TypedDict",
|
|
183
|
+
}
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# Reached explicitly: construction and class creation record their own edges.
|
|
188
|
+
_EXPLICIT_SPECIAL_METHODS = frozenset({"__init__", "__new__", "__init_subclass__"})
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _is_special_method(name: str) -> bool:
|
|
192
|
+
return (
|
|
193
|
+
len(name) > 4
|
|
194
|
+
and name.startswith("__")
|
|
195
|
+
and name.endswith("__")
|
|
196
|
+
and name not in _EXPLICIT_SPECIAL_METHODS
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _annotated_args(args: ast.arguments) -> list[ast.arg]:
|
|
201
|
+
"""The parameters that carry an annotation, in signature order."""
|
|
202
|
+
params = [*args.posonlyargs, *args.args, *args.kwonlyargs, args.vararg, args.kwarg]
|
|
203
|
+
return [a for a in params if a is not None and a.annotation is not None]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# Decorators that run no code with the function beyond returning it (or
|
|
207
|
+
# recording it in typing's overload registry), by canonical name.
|
|
208
|
+
_INERT_DECORATORS = frozenset(
|
|
209
|
+
f"{module}.{name}"
|
|
210
|
+
for module in ("typing", "typing_extensions")
|
|
211
|
+
for name in ("overload", "override", "final")
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _is_inert_decorator(node: ast.expr, scope: ModuleScope) -> bool:
|
|
216
|
+
"""Only the ``typing``/``typing_extensions`` decorators, resolved through
|
|
217
|
+
the module's imports: a project decorator named ``final`` may do anything."""
|
|
218
|
+
parts = _flatten_chain(node)
|
|
219
|
+
if parts is None:
|
|
220
|
+
return False
|
|
221
|
+
binding = scope.imports.get(parts[0])
|
|
222
|
+
if binding is None:
|
|
223
|
+
return False
|
|
224
|
+
base = binding.module if binding.attr is None else f"{binding.module}.{binding.attr}"
|
|
225
|
+
return ".".join([base, *parts[1:]]) in _INERT_DECORATORS
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _is_literal(node: ast.expr) -> bool:
|
|
229
|
+
if isinstance(node, ast.Constant):
|
|
230
|
+
return True
|
|
231
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
|
|
232
|
+
return _is_literal(node.operand)
|
|
233
|
+
if isinstance(node, (ast.Tuple, ast.List, ast.Set)):
|
|
234
|
+
return all(_is_literal(e) for e in node.elts)
|
|
235
|
+
if isinstance(node, ast.Dict):
|
|
236
|
+
return all(k is not None and _is_literal(k) for k in node.keys) and all(
|
|
237
|
+
_is_literal(v) for v in node.values
|
|
238
|
+
)
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _inert_def(node: ast.FunctionDef | ast.AsyncFunctionDef, scope: ModuleScope) -> bool:
|
|
243
|
+
"""Whether executing the ``def`` runs no code beyond binding the name:
|
|
244
|
+
inert decorators and literal defaults (annotations are checked apart)."""
|
|
245
|
+
defaults = [*node.args.defaults, *(d for d in node.args.kw_defaults if d is not None)]
|
|
246
|
+
return all(_is_inert_decorator(d, scope) for d in node.decorator_list) and all(
|
|
247
|
+
_is_literal(d) for d in defaults
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _has_annotations(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
|
|
252
|
+
return node.returns is not None or any(
|
|
253
|
+
isinstance(a, ast.arg) and a.annotation is not None for a in ast.walk(node.args)
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _future_annotations(scope: ModuleScope) -> bool:
|
|
258
|
+
binding = scope.imports.get("annotations")
|
|
259
|
+
return binding is not None and binding.module == "__future__"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _is_staticmethod(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
|
|
263
|
+
return _has_decorator(node, "staticmethod")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# Defining one of these changes what ``self.<attr>`` reads or writes.
|
|
267
|
+
_ATTRIBUTE_HOOKS = frozenset({"__setattr__", "__delattr__", "__getattr__", "__getattribute__"})
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _rebound_names(node: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]:
|
|
271
|
+
"""Every name the function's body (nested scopes included, which only
|
|
272
|
+
over-approximates) binds, deletes or imports."""
|
|
273
|
+
names: set[str] = set()
|
|
274
|
+
for inner in ast.walk(node):
|
|
275
|
+
if isinstance(inner, ast.Name) and not isinstance(inner.ctx, ast.Load):
|
|
276
|
+
names.add(inner.id)
|
|
277
|
+
elif isinstance(inner, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)) and inner.name:
|
|
278
|
+
names.add(inner.name)
|
|
279
|
+
elif isinstance(inner, (ast.Import, ast.ImportFrom)):
|
|
280
|
+
names |= {(a.asname or a.name).split(".")[0] for a in inner.names}
|
|
281
|
+
return names
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _flatten_chain(node: ast.expr) -> list[str] | None:
|
|
285
|
+
parts: list[str] = []
|
|
286
|
+
while isinstance(node, ast.Attribute):
|
|
287
|
+
parts.append(node.attr)
|
|
288
|
+
node = node.value
|
|
289
|
+
if isinstance(node, ast.Name):
|
|
290
|
+
parts.append(node.id)
|
|
291
|
+
return list(reversed(parts))
|
|
292
|
+
return None
|