blockpath 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- blockpath/__init__.py +22 -0
- blockpath/analysis.py +329 -0
- blockpath/callgraph.py +298 -0
- blockpath/check.py +33 -0
- blockpath/cli.py +144 -0
- blockpath/collect.py +206 -0
- blockpath/config.py +75 -0
- blockpath/model.py +112 -0
- blockpath/oracle.py +143 -0
- blockpath/oracle.yaml +222 -0
- blockpath/py.typed +0 -0
- blockpath/report.py +140 -0
- blockpath/resolve.py +192 -0
- blockpath/runtime/__init__.py +22 -0
- blockpath/runtime/monitor.py +157 -0
- blockpath/runtime/predicate.py +39 -0
- blockpath/runtime/verify.py +177 -0
- blockpath/runtime/watchdog.py +117 -0
- blockpath/symbols.py +284 -0
- blockpath-0.1.0.dist-info/METADATA +180 -0
- blockpath-0.1.0.dist-info/RECORD +24 -0
- blockpath-0.1.0.dist-info/WHEEL +4 -0
- blockpath-0.1.0.dist-info/entry_points.txt +2 -0
- blockpath-0.1.0.dist-info/licenses/LICENSE +21 -0
blockpath/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""blockpath: find blocking calls reachable from a coroutine, and print the whole path.
|
|
2
|
+
|
|
3
|
+
Public API is deliberately tiny: run ``check`` over a project root, read the ``Finding`` objects
|
|
4
|
+
it returns. The command-line interface in ``blockpath.cli`` is a thin wrapper over this.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from blockpath.check import check
|
|
10
|
+
from blockpath.config import Config
|
|
11
|
+
from blockpath.model import CheckResult, Code, Finding, Frame, Position, Severity
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CheckResult",
|
|
15
|
+
"Code",
|
|
16
|
+
"Config",
|
|
17
|
+
"Finding",
|
|
18
|
+
"Frame",
|
|
19
|
+
"Position",
|
|
20
|
+
"Severity",
|
|
21
|
+
"check",
|
|
22
|
+
]
|
blockpath/analysis.py
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
"""Layer 6: three breadth-first searches, and deliberately nothing cleverer.
|
|
2
|
+
|
|
3
|
+
The diagnostic is the *intersection* of two independent reachability questions, and that
|
|
4
|
+
intersection is the whole reason the tool can be quiet:
|
|
5
|
+
|
|
6
|
+
1. **Backward** from every blocking leaf: which functions can reach a blocking call at all?
|
|
7
|
+
Formally ``blocking(v) := oracle(v) or exists edge v->w with blocking(w)`` -- a boolean join
|
|
8
|
+
on a lattice of height two, which is exactly backward reachability, so one reverse BFS
|
|
9
|
+
computes it. Cycles are absorbed by a visited set.
|
|
10
|
+
2. **Forward** from every async entry point: which functions can a coroutine reach?
|
|
11
|
+
3. **Forward again**, restricted to the intersection and carrying parent pointers, to recover the
|
|
12
|
+
witness: the actual chain of call sites from the coroutine down to the blocking call.
|
|
13
|
+
|
|
14
|
+
Without step 2 the tool would shout about ``time.sleep`` in a synchronous CLI script in the same
|
|
15
|
+
repository and be switched off within a day. Without step 1 the search would wander through
|
|
16
|
+
every clean subtree a coroutine can reach.
|
|
17
|
+
|
|
18
|
+
No Tarjan, no SCC condensation, no fixpoint iteration: naive iteration is O(V*E), a reverse BFS
|
|
19
|
+
is O(V+E) and three times simpler, and the condensation buys nothing for a height-two join
|
|
20
|
+
(ADR 0002). Only CALL edges are traversed; a function merely passed as a value is not chased
|
|
21
|
+
(ADR 0007).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from collections import deque
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from functools import lru_cache
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
from blockpath.callgraph import CallGraph, Edge, EdgeKind, FuncNode
|
|
32
|
+
from blockpath.config import Config
|
|
33
|
+
from blockpath.model import Code, Finding, Frame, Position, Severity
|
|
34
|
+
from blockpath.oracle import Oracle
|
|
35
|
+
from blockpath.resolve import External, Internal
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@lru_cache(maxsize=256)
|
|
39
|
+
def _lines(path: Path) -> tuple[str, ...]:
|
|
40
|
+
try:
|
|
41
|
+
return tuple(path.read_text(encoding="utf-8").splitlines())
|
|
42
|
+
except (OSError, UnicodeDecodeError):
|
|
43
|
+
return ()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _source_line(position: Position) -> str:
|
|
47
|
+
lines = _lines(position.path)
|
|
48
|
+
if 1 <= position.line <= len(lines):
|
|
49
|
+
return lines[position.line - 1].strip()
|
|
50
|
+
return ""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True, slots=True)
|
|
54
|
+
class _BlockingLeaf:
|
|
55
|
+
"""A call site that blocks: the function containing it, and what it calls."""
|
|
56
|
+
|
|
57
|
+
owner: str
|
|
58
|
+
edge: Edge
|
|
59
|
+
dotted: str
|
|
60
|
+
category: str
|
|
61
|
+
severity: Severity
|
|
62
|
+
hint_note: str
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _is_entry(node: FuncNode, config: Config) -> str:
|
|
66
|
+
"""Why this node is an async entry point, or "" if it is not one."""
|
|
67
|
+
if node.is_async:
|
|
68
|
+
return "async def"
|
|
69
|
+
for decorator in node.decorators:
|
|
70
|
+
for configured in config.entry_decorators:
|
|
71
|
+
if decorator == configured or decorator.endswith(f".{configured}"):
|
|
72
|
+
return f"@{decorator}"
|
|
73
|
+
return ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def analyse(graph: CallGraph, oracle: Oracle, config: Config) -> tuple[Finding, ...]:
|
|
77
|
+
"""Find every blocking call a coroutine can reach, with the path that gets there."""
|
|
78
|
+
call_edges = [e for e in graph.edges if e.kind is EdgeKind.CALL]
|
|
79
|
+
|
|
80
|
+
# --- the blocking leaves: external calls the oracle says stall the loop -------------------
|
|
81
|
+
leaves: list[_BlockingLeaf] = []
|
|
82
|
+
for edge in call_edges:
|
|
83
|
+
if not isinstance(edge.target, External):
|
|
84
|
+
continue
|
|
85
|
+
entry = oracle.blocks(edge.target.dotted, config.enabled_tiers)
|
|
86
|
+
if entry is None:
|
|
87
|
+
continue
|
|
88
|
+
leaves.append(
|
|
89
|
+
_BlockingLeaf(
|
|
90
|
+
owner=edge.source,
|
|
91
|
+
edge=edge,
|
|
92
|
+
dotted=entry.name,
|
|
93
|
+
category=entry.category,
|
|
94
|
+
severity=entry.severity,
|
|
95
|
+
hint_note=entry.note,
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# --- 1. backward BFS: which functions can reach a blocking call --------------------------
|
|
100
|
+
callers: dict[str, set[str]] = {}
|
|
101
|
+
outgoing: dict[str, list[tuple[str, Edge]]] = {}
|
|
102
|
+
for edge in call_edges:
|
|
103
|
+
if isinstance(edge.target, Internal) and edge.target.qualname in graph.nodes:
|
|
104
|
+
callers.setdefault(edge.target.qualname, set()).add(edge.source)
|
|
105
|
+
outgoing.setdefault(edge.source, []).append((edge.target.qualname, edge))
|
|
106
|
+
|
|
107
|
+
blocking: set[str] = set()
|
|
108
|
+
queue: deque[str] = deque()
|
|
109
|
+
for leaf in leaves:
|
|
110
|
+
if leaf.owner not in blocking:
|
|
111
|
+
blocking.add(leaf.owner)
|
|
112
|
+
queue.append(leaf.owner)
|
|
113
|
+
while queue:
|
|
114
|
+
current = queue.popleft()
|
|
115
|
+
for caller in callers.get(current, ()):
|
|
116
|
+
if caller not in blocking:
|
|
117
|
+
blocking.add(caller)
|
|
118
|
+
queue.append(caller)
|
|
119
|
+
|
|
120
|
+
# --- 2 and 3. forward BFS from the entry points, restricted to the intersection ----------
|
|
121
|
+
# Restricting to `blocking` is what makes the recovered path a *witness*: every hop on it is
|
|
122
|
+
# itself on a route to a blocking call, so the trace never wanders into a clean subtree.
|
|
123
|
+
entry_reason: dict[str, str] = {}
|
|
124
|
+
for qualname, node in graph.nodes.items():
|
|
125
|
+
reason = _is_entry(node, config)
|
|
126
|
+
if reason:
|
|
127
|
+
entry_reason[qualname] = reason
|
|
128
|
+
|
|
129
|
+
parent: dict[str, tuple[str, Edge]] = {}
|
|
130
|
+
reachable: set[str] = set()
|
|
131
|
+
frontier: deque[str] = deque()
|
|
132
|
+
for qualname in entry_reason:
|
|
133
|
+
if qualname in blocking and qualname not in reachable:
|
|
134
|
+
reachable.add(qualname)
|
|
135
|
+
frontier.append(qualname)
|
|
136
|
+
while frontier:
|
|
137
|
+
current = frontier.popleft()
|
|
138
|
+
for nxt, edge in outgoing.get(current, ()):
|
|
139
|
+
# every node on a path to a blocking owner is itself in `blocking` (the backward BFS
|
|
140
|
+
# put it there), so this restriction prunes clean subtrees without losing a witness
|
|
141
|
+
if nxt in blocking and nxt not in reachable:
|
|
142
|
+
reachable.add(nxt)
|
|
143
|
+
parent[nxt] = (current, edge)
|
|
144
|
+
frontier.append(nxt)
|
|
145
|
+
|
|
146
|
+
# --- build the findings -------------------------------------------------------------------
|
|
147
|
+
findings: list[Finding] = []
|
|
148
|
+
for leaf in leaves:
|
|
149
|
+
if leaf.owner not in reachable:
|
|
150
|
+
continue
|
|
151
|
+
witness = _witness(leaf, parent, entry_reason, graph)
|
|
152
|
+
if witness is None:
|
|
153
|
+
continue
|
|
154
|
+
findings.append(
|
|
155
|
+
Finding(
|
|
156
|
+
code=Code.BLK001,
|
|
157
|
+
severity=leaf.severity,
|
|
158
|
+
callee=leaf.dotted,
|
|
159
|
+
category=leaf.category,
|
|
160
|
+
witness=witness,
|
|
161
|
+
hint=_hint(witness),
|
|
162
|
+
)
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
# Plain reachability from the coroutines, unrestricted by `blocking`. The three searches
|
|
166
|
+
# above answer "where does a block hide"; this one only scopes the two diagnostics below to
|
|
167
|
+
# code a coroutine can actually get to.
|
|
168
|
+
from_coroutines: set[str] = set(entry_reason)
|
|
169
|
+
plain: deque[str] = deque(from_coroutines)
|
|
170
|
+
while plain:
|
|
171
|
+
current = plain.popleft()
|
|
172
|
+
for nxt, _edge in outgoing.get(current, ()):
|
|
173
|
+
if nxt not in from_coroutines:
|
|
174
|
+
from_coroutines.add(nxt)
|
|
175
|
+
plain.append(nxt)
|
|
176
|
+
|
|
177
|
+
findings.extend(_blk002(call_edges, oracle, entry_reason, from_coroutines))
|
|
178
|
+
if config.strict:
|
|
179
|
+
findings.extend(_blk003(graph.edges, oracle, entry_reason, from_coroutines, config))
|
|
180
|
+
findings.sort(key=lambda f: (str(f.leaf.position.path), f.leaf.position.line, f.callee))
|
|
181
|
+
return tuple(findings)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _witness(
|
|
185
|
+
leaf: _BlockingLeaf,
|
|
186
|
+
parent: dict[str, tuple[str, Edge]],
|
|
187
|
+
entry_reason: dict[str, str],
|
|
188
|
+
graph: CallGraph,
|
|
189
|
+
) -> tuple[Frame, ...] | None:
|
|
190
|
+
"""Walk parent pointers from the blocking call back to the entry, then read the chain down.
|
|
191
|
+
|
|
192
|
+
Each frame is a *call site*: the function it names, and the position within that function
|
|
193
|
+
where control passes to the next frame. The last frame is the blocking call itself.
|
|
194
|
+
"""
|
|
195
|
+
chain: list[tuple[str, Edge]] = [] # (function, the edge by which it calls the next hop)
|
|
196
|
+
current = leaf.owner
|
|
197
|
+
guard = 0
|
|
198
|
+
while current not in entry_reason:
|
|
199
|
+
step = parent.get(current)
|
|
200
|
+
if step is None:
|
|
201
|
+
return None # not actually reachable from an entry
|
|
202
|
+
previous, edge = step
|
|
203
|
+
chain.append((previous, edge))
|
|
204
|
+
current = previous
|
|
205
|
+
guard += 1
|
|
206
|
+
if guard > len(graph.nodes) + 1:
|
|
207
|
+
return None # defensive: a malformed parent chain must not spin
|
|
208
|
+
chain.reverse()
|
|
209
|
+
|
|
210
|
+
frames: list[Frame] = []
|
|
211
|
+
for index, (qualname, edge) in enumerate(chain):
|
|
212
|
+
frames.append(
|
|
213
|
+
Frame(
|
|
214
|
+
qualname=qualname,
|
|
215
|
+
position=edge.position,
|
|
216
|
+
source_line=_source_line(edge.position),
|
|
217
|
+
entry_reason=entry_reason.get(qualname, "") if index == 0 else "",
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
frames.append(
|
|
221
|
+
Frame(
|
|
222
|
+
qualname=leaf.owner,
|
|
223
|
+
position=leaf.edge.position,
|
|
224
|
+
source_line=_source_line(leaf.edge.position),
|
|
225
|
+
entry_reason=entry_reason.get(leaf.owner, "") if not frames else "",
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
return tuple(frames)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _hint(witness: tuple[Frame, ...]) -> str:
|
|
232
|
+
"""A remediation naming the first hop off the coroutine, which is where to_thread belongs."""
|
|
233
|
+
if len(witness) < 2:
|
|
234
|
+
return "await asyncio.to_thread(...) # move the blocking call off the loop"
|
|
235
|
+
handoff = witness[1].qualname.rsplit(".", 1)[-1]
|
|
236
|
+
return f"await asyncio.to_thread({handoff}, ...)"
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _blk003(
|
|
240
|
+
edges: list[Edge],
|
|
241
|
+
oracle: Oracle,
|
|
242
|
+
entry_reason: dict[str, str],
|
|
243
|
+
from_coroutines: set[str],
|
|
244
|
+
config: Config,
|
|
245
|
+
) -> list[Finding]:
|
|
246
|
+
"""Strict mode: a blocking callable handed to a callee we cannot follow.
|
|
247
|
+
|
|
248
|
+
``dispatch(time.sleep)`` -- we can see a blocking function escaping into something the
|
|
249
|
+
resolver could not pin down, so it may well be invoked on the loop. We cannot prove it, which
|
|
250
|
+
is exactly why this is silent by default and only surfaces under ``--strict``. References
|
|
251
|
+
consumed by a known thread-offload primitive are excluded: those provably run elsewhere.
|
|
252
|
+
"""
|
|
253
|
+
out: list[Finding] = []
|
|
254
|
+
for edge in edges:
|
|
255
|
+
if edge.kind is not EdgeKind.REF or not isinstance(edge.target, External):
|
|
256
|
+
continue
|
|
257
|
+
entry = oracle.blocks(edge.target.dotted, config.enabled_tiers)
|
|
258
|
+
if entry is None or edge.source not in from_coroutines:
|
|
259
|
+
continue
|
|
260
|
+
if oracle.offload_for(edge.consumer) is not None:
|
|
261
|
+
continue # handed to a thread on purpose: correct code
|
|
262
|
+
frame = Frame(
|
|
263
|
+
qualname=edge.source,
|
|
264
|
+
position=edge.position,
|
|
265
|
+
source_line=_source_line(edge.position),
|
|
266
|
+
entry_reason=entry_reason.get(edge.source, ""),
|
|
267
|
+
)
|
|
268
|
+
out.append(
|
|
269
|
+
Finding(
|
|
270
|
+
code=Code.BLK003,
|
|
271
|
+
severity=entry.severity,
|
|
272
|
+
callee=entry.name,
|
|
273
|
+
category=entry.category,
|
|
274
|
+
witness=(frame,),
|
|
275
|
+
hint=(
|
|
276
|
+
f"{entry.name} is passed to {edge.consumer or 'an unresolved callee'}; "
|
|
277
|
+
"blockpath cannot tell whether it runs on the loop"
|
|
278
|
+
),
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
return out
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _blk002(
|
|
285
|
+
call_edges: list[Edge],
|
|
286
|
+
oracle: Oracle,
|
|
287
|
+
entry_reason: dict[str, str],
|
|
288
|
+
from_coroutines: set[str],
|
|
289
|
+
) -> list[Finding]:
|
|
290
|
+
"""``run_in_executor(None, fetch())`` -- the callable slot was called, not passed.
|
|
291
|
+
|
|
292
|
+
The parentheses run ``fetch`` on the loop before the executor ever sees it, which is the
|
|
293
|
+
exact inverse of the intended fix. Reported only inside a coroutine, which is the only place
|
|
294
|
+
the offload primitives are used, so the diagnostic cannot fire on ordinary sync code.
|
|
295
|
+
"""
|
|
296
|
+
out: list[Finding] = []
|
|
297
|
+
for edge in call_edges:
|
|
298
|
+
if oracle.blk002(edge.consumer, edge.arg_index) is None:
|
|
299
|
+
continue
|
|
300
|
+
if edge.source not in from_coroutines:
|
|
301
|
+
continue
|
|
302
|
+
called = (
|
|
303
|
+
edge.target.qualname
|
|
304
|
+
if isinstance(edge.target, Internal)
|
|
305
|
+
else edge.target.dotted
|
|
306
|
+
if isinstance(edge.target, External)
|
|
307
|
+
else edge.source_text
|
|
308
|
+
)
|
|
309
|
+
frame = Frame(
|
|
310
|
+
qualname=edge.source,
|
|
311
|
+
position=edge.position,
|
|
312
|
+
source_line=_source_line(edge.position),
|
|
313
|
+
entry_reason=entry_reason.get(edge.source, ""),
|
|
314
|
+
)
|
|
315
|
+
short = called.rsplit(".", 1)[-1]
|
|
316
|
+
out.append(
|
|
317
|
+
Finding(
|
|
318
|
+
code=Code.BLK002,
|
|
319
|
+
severity=Severity.ERROR,
|
|
320
|
+
callee=short,
|
|
321
|
+
category="offload",
|
|
322
|
+
witness=(frame,),
|
|
323
|
+
hint=f"pass the function, do not call it: {edge.consumer}(..., {short})",
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
return out
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
__all__ = ["analyse"]
|
blockpath/callgraph.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Layer 4: the call graph -- who calls whom, and who merely *hands off* whom.
|
|
2
|
+
|
|
3
|
+
A node is every ``def`` / ``async def`` / ``lambda`` (nested ones too, so a closure is a real
|
|
4
|
+
node, which is what makes offloading fall out of the graph structure). Two kinds of edge:
|
|
5
|
+
|
|
6
|
+
* **CALL** -- ``f(...)`` executes ``f`` in the caller's frame, on the caller's thread. This is
|
|
7
|
+
the edge reachability follows.
|
|
8
|
+
* **REF** -- ``f`` is passed as a *value*: ``run_in_executor(None, f)``, ``sorted(xs, key=f)``,
|
|
9
|
+
``loop.call_soon(f)``. Whether that reference ever runs, and on which thread, depends entirely
|
|
10
|
+
on what received it -- so a REF edge records its *consumer* (the callee it was an argument to)
|
|
11
|
+
and its argument position, and the oracle (layer 5) decides which REF edges to cut (a thread
|
|
12
|
+
offload) and which to keep (a synchronous ``key=`` / ``call_soon`` that runs on the loop).
|
|
13
|
+
|
|
14
|
+
The single pair of parentheses that flips a bug into correct code lives here: ``to_thread(f)``
|
|
15
|
+
is a REF (offloaded), ``to_thread(f())`` is a CALL of ``f`` in the coroutine (a real block, the
|
|
16
|
+
BLK002 shape). The graph records both faithfully; the oracle reads the difference.
|
|
17
|
+
|
|
18
|
+
Only calls *by name* are resolved. An attribute call on a value (``self.session.get()``) is not
|
|
19
|
+
resolved in v1 (ADR 0003); every call site is tallied by how it resolved, so the price of that
|
|
20
|
+
cut is a measured number, not a hand-wave.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import ast
|
|
26
|
+
from collections import Counter
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from enum import Enum
|
|
29
|
+
|
|
30
|
+
from blockpath.collect import Collection, Module
|
|
31
|
+
from blockpath.model import Position
|
|
32
|
+
from blockpath.resolve import External, Internal, Resolver, Target, Unresolved
|
|
33
|
+
from blockpath.symbols import Analysis, ModuleScopes, ModuleSymbols, ScopeNode
|
|
34
|
+
|
|
35
|
+
MODULE_OWNER = "<module>"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class EdgeKind(Enum):
|
|
39
|
+
CALL = "call"
|
|
40
|
+
REF = "ref"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class FuncNode:
|
|
45
|
+
qualname: str
|
|
46
|
+
position: Position
|
|
47
|
+
is_async: bool
|
|
48
|
+
is_lambda: bool
|
|
49
|
+
#: dotted decorator expressions as written (``router.message`` for ``@router.message()``), so
|
|
50
|
+
#: a framework can mark a handler as an async entry point even when it is a plain ``def``
|
|
51
|
+
decorators: tuple[str, ...] = ()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True, slots=True)
|
|
55
|
+
class Edge:
|
|
56
|
+
"""One call-graph edge. ``target`` is the resolver's verdict, kept whole so the analysis can
|
|
57
|
+
treat Internal / External / Unresolved differently."""
|
|
58
|
+
|
|
59
|
+
source: str
|
|
60
|
+
target: Target
|
|
61
|
+
kind: EdgeKind
|
|
62
|
+
position: Position
|
|
63
|
+
source_text: str
|
|
64
|
+
#: if this call or reference is itself an argument to another call, the dotted callee of that
|
|
65
|
+
#: enclosing call (best effort, e.g. "loop.run_in_executor", "asyncio.to_thread"); else None
|
|
66
|
+
consumer: str | None = None
|
|
67
|
+
#: positional index of this argument within the consumer call, or None if a keyword/not-an-arg
|
|
68
|
+
arg_index: int | None = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(slots=True)
|
|
72
|
+
class ModuleGraph:
|
|
73
|
+
nodes: dict[str, FuncNode]
|
|
74
|
+
edges: list[Edge]
|
|
75
|
+
#: how every call site resolved: by_name / attribute / dynamic / to a lambda / ...
|
|
76
|
+
resolution: Counter[str] = field(default_factory=Counter)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(slots=True)
|
|
80
|
+
class CallGraph:
|
|
81
|
+
nodes: dict[str, FuncNode]
|
|
82
|
+
edges: list[Edge]
|
|
83
|
+
resolution: Counter[str]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _dotted_of(expr: ast.expr) -> str | None:
|
|
87
|
+
parts: list[str] = []
|
|
88
|
+
cur: ast.expr = expr
|
|
89
|
+
while isinstance(cur, ast.Attribute):
|
|
90
|
+
parts.append(cur.attr)
|
|
91
|
+
cur = cur.value
|
|
92
|
+
if not isinstance(cur, ast.Name):
|
|
93
|
+
return None
|
|
94
|
+
parts.append(cur.id)
|
|
95
|
+
return ".".join(reversed(parts))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class _ModuleBuilder:
|
|
99
|
+
"""Walks one module's AST, creating nodes and edges. Owners are tracked as AST nodes so the
|
|
100
|
+
right scope table can be consulted for each call."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, module: Module, analysis: Analysis, resolver: Resolver) -> None:
|
|
103
|
+
self._module = module
|
|
104
|
+
self._symbols: ModuleSymbols = analysis.symbols
|
|
105
|
+
self._scopes: ModuleScopes = analysis.scopes
|
|
106
|
+
self._resolver = resolver
|
|
107
|
+
self._lambda_counter = 0
|
|
108
|
+
self.nodes: dict[str, FuncNode] = {}
|
|
109
|
+
self.edges: list[Edge] = []
|
|
110
|
+
self.resolution: Counter[str] = Counter()
|
|
111
|
+
|
|
112
|
+
def build(self) -> ModuleGraph:
|
|
113
|
+
for stmt in self._module.tree.body:
|
|
114
|
+
self._visit(
|
|
115
|
+
stmt,
|
|
116
|
+
owner=MODULE_OWNER,
|
|
117
|
+
owner_node=None,
|
|
118
|
+
prefix=self._module.qualname,
|
|
119
|
+
ctx="module",
|
|
120
|
+
)
|
|
121
|
+
return ModuleGraph(self.nodes, self.edges, self.resolution)
|
|
122
|
+
|
|
123
|
+
def _add_node(self, qual: str, node: ScopeNode, is_lambda: bool) -> None:
|
|
124
|
+
decorators: tuple[str, ...] = ()
|
|
125
|
+
if not isinstance(node, ast.Lambda):
|
|
126
|
+
names = []
|
|
127
|
+
for dec in node.decorator_list:
|
|
128
|
+
expr = dec.func if isinstance(dec, ast.Call) else dec
|
|
129
|
+
dotted = _dotted_of(expr)
|
|
130
|
+
if dotted:
|
|
131
|
+
names.append(dotted)
|
|
132
|
+
decorators = tuple(names)
|
|
133
|
+
self.nodes[qual] = FuncNode(
|
|
134
|
+
qualname=qual,
|
|
135
|
+
position=Position(self._module.path, node.lineno, node.col_offset),
|
|
136
|
+
is_async=isinstance(node, ast.AsyncFunctionDef),
|
|
137
|
+
is_lambda=is_lambda,
|
|
138
|
+
decorators=decorators,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def _pos(self, node: ast.expr) -> Position:
|
|
142
|
+
return Position(self._module.path, node.lineno, node.col_offset)
|
|
143
|
+
|
|
144
|
+
def _visit(
|
|
145
|
+
self,
|
|
146
|
+
node: ast.AST,
|
|
147
|
+
owner: str,
|
|
148
|
+
owner_node: ScopeNode | None,
|
|
149
|
+
prefix: str,
|
|
150
|
+
ctx: str,
|
|
151
|
+
arg_of: tuple[str | None, int | None] | None = None,
|
|
152
|
+
) -> None:
|
|
153
|
+
"""``arg_of`` is set when ``node`` is itself an argument to an enclosing call: (consumer
|
|
154
|
+
dotted callee, positional index or None for a keyword). It tags the edge this node makes
|
|
155
|
+
so the oracle can read the offload / BLK002 context, and it is consumed at one level."""
|
|
156
|
+
match node:
|
|
157
|
+
case ast.FunctionDef() | ast.AsyncFunctionDef():
|
|
158
|
+
for dec in node.decorator_list:
|
|
159
|
+
self._visit(dec, owner, owner_node, prefix, ctx)
|
|
160
|
+
for dflt in [*node.args.defaults, *(d for d in node.args.kw_defaults if d)]:
|
|
161
|
+
self._visit(dflt, owner, owner_node, prefix, ctx)
|
|
162
|
+
qual = (
|
|
163
|
+
f"{prefix}.{node.name}"
|
|
164
|
+
if ctx in ("module", "class")
|
|
165
|
+
else f"{owner}.<def:{node.name}@{node.lineno}:{node.col_offset}>"
|
|
166
|
+
)
|
|
167
|
+
self._add_node(qual, node, is_lambda=False)
|
|
168
|
+
for stmt in node.body:
|
|
169
|
+
self._visit(stmt, qual, node, qual, "function")
|
|
170
|
+
|
|
171
|
+
case ast.Lambda():
|
|
172
|
+
for dflt in [*node.args.defaults, *(d for d in node.args.kw_defaults if d)]:
|
|
173
|
+
self._visit(dflt, owner, owner_node, prefix, ctx)
|
|
174
|
+
self._lambda_counter += 1
|
|
175
|
+
qual = f"{owner}.<lambda@{node.lineno}:{node.col_offset}#{self._lambda_counter}>"
|
|
176
|
+
self._add_node(qual, node, is_lambda=True)
|
|
177
|
+
self._visit(node.body, qual, node, qual, "function")
|
|
178
|
+
|
|
179
|
+
case ast.ClassDef():
|
|
180
|
+
for dec in node.decorator_list:
|
|
181
|
+
self._visit(dec, owner, owner_node, prefix, ctx)
|
|
182
|
+
for base in [*node.bases, *(kw.value for kw in node.keywords)]:
|
|
183
|
+
self._visit(base, owner, owner_node, prefix, ctx)
|
|
184
|
+
cprefix = (
|
|
185
|
+
f"{prefix}.{node.name}"
|
|
186
|
+
if ctx in ("module", "class")
|
|
187
|
+
else f"{owner}.<class:{node.name}@{node.lineno}>"
|
|
188
|
+
)
|
|
189
|
+
for stmt in node.body:
|
|
190
|
+
self._visit(stmt, owner, owner_node, cprefix, "class")
|
|
191
|
+
|
|
192
|
+
case ast.Call():
|
|
193
|
+
self._emit_call(node, owner, owner_node, arg_of)
|
|
194
|
+
# the call's own arguments are visited with THIS call as their consumer, so a
|
|
195
|
+
# reference or a nested call in argument position is tagged; the callee and the
|
|
196
|
+
# rest get no argument context.
|
|
197
|
+
self._visit(node.func, owner, owner_node, prefix, ctx)
|
|
198
|
+
consumer = _dotted_of(node.func)
|
|
199
|
+
for index, arg in enumerate(node.args):
|
|
200
|
+
self._visit(arg, owner, owner_node, prefix, ctx, arg_of=(consumer, index))
|
|
201
|
+
for kw in node.keywords:
|
|
202
|
+
self._visit(kw.value, owner, owner_node, prefix, ctx, arg_of=(consumer, None))
|
|
203
|
+
|
|
204
|
+
case ast.Name() | ast.Attribute() if arg_of is not None:
|
|
205
|
+
# a function reference passed as an argument: a REF edge (passed, not called)
|
|
206
|
+
self._emit_ref(node, owner, owner_node, arg_of)
|
|
207
|
+
|
|
208
|
+
case _:
|
|
209
|
+
for child in ast.iter_child_nodes(node):
|
|
210
|
+
self._visit(child, owner, owner_node, prefix, ctx)
|
|
211
|
+
|
|
212
|
+
def _emit_call(
|
|
213
|
+
self,
|
|
214
|
+
call: ast.Call,
|
|
215
|
+
owner: str,
|
|
216
|
+
owner_node: ScopeNode | None,
|
|
217
|
+
arg_of: tuple[str | None, int | None] | None,
|
|
218
|
+
) -> None:
|
|
219
|
+
scope = self._scopes.info(owner_node) if owner_node is not None else None
|
|
220
|
+
target = self._resolver.resolve_callee(self._symbols, call.func, scope)
|
|
221
|
+
self._tally(call.func, target)
|
|
222
|
+
consumer, arg_index = arg_of if arg_of is not None else (None, None)
|
|
223
|
+
self.edges.append(
|
|
224
|
+
Edge(
|
|
225
|
+
source=owner,
|
|
226
|
+
target=target,
|
|
227
|
+
kind=EdgeKind.CALL,
|
|
228
|
+
position=self._pos(call),
|
|
229
|
+
source_text=_unparse(call),
|
|
230
|
+
consumer=consumer,
|
|
231
|
+
arg_index=arg_index,
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
def _emit_ref(
|
|
236
|
+
self,
|
|
237
|
+
ref: ast.expr,
|
|
238
|
+
owner: str,
|
|
239
|
+
owner_node: ScopeNode | None,
|
|
240
|
+
arg_of: tuple[str | None, int | None],
|
|
241
|
+
) -> None:
|
|
242
|
+
scope = self._scopes.info(owner_node) if owner_node is not None else None
|
|
243
|
+
target = self._resolver.resolve_callee(self._symbols, ref, scope)
|
|
244
|
+
if isinstance(target, Internal | External):
|
|
245
|
+
consumer, arg_index = arg_of
|
|
246
|
+
self.edges.append(
|
|
247
|
+
Edge(
|
|
248
|
+
source=owner,
|
|
249
|
+
target=target,
|
|
250
|
+
kind=EdgeKind.REF,
|
|
251
|
+
position=self._pos(ref),
|
|
252
|
+
source_text=_unparse(ref),
|
|
253
|
+
consumer=consumer,
|
|
254
|
+
arg_index=arg_index,
|
|
255
|
+
)
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
def _tally(self, callee: ast.expr, target: Target) -> None:
|
|
259
|
+
self.resolution["total"] += 1
|
|
260
|
+
if isinstance(callee, ast.Name):
|
|
261
|
+
shape = "by_name"
|
|
262
|
+
elif isinstance(callee, ast.Attribute):
|
|
263
|
+
shape = "attribute"
|
|
264
|
+
else:
|
|
265
|
+
shape = "dynamic"
|
|
266
|
+
self.resolution[f"shape_{shape}"] += 1
|
|
267
|
+
match target:
|
|
268
|
+
case Internal():
|
|
269
|
+
self.resolution["resolved_internal"] += 1
|
|
270
|
+
case External():
|
|
271
|
+
self.resolution["resolved_external"] += 1
|
|
272
|
+
case Unresolved(reason=reason):
|
|
273
|
+
self.resolution[f"unresolved_{reason}"] += 1
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _unparse(node: ast.expr) -> str:
|
|
277
|
+
try:
|
|
278
|
+
return ast.unparse(node)
|
|
279
|
+
except (ValueError, RecursionError):
|
|
280
|
+
return "<unparseable>"
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def build_call_graph(
|
|
284
|
+
collection: Collection, analyses: dict[str, Analysis], resolver: Resolver
|
|
285
|
+
) -> CallGraph:
|
|
286
|
+
"""Assemble the whole-project call graph from every module's nodes and edges."""
|
|
287
|
+
nodes: dict[str, FuncNode] = {}
|
|
288
|
+
edges: list[Edge] = []
|
|
289
|
+
resolution: Counter[str] = Counter()
|
|
290
|
+
for module in collection.modules:
|
|
291
|
+
analysis = analyses.get(module.qualname)
|
|
292
|
+
if analysis is None:
|
|
293
|
+
continue
|
|
294
|
+
graph = _ModuleBuilder(module, analysis, resolver).build()
|
|
295
|
+
nodes.update(graph.nodes)
|
|
296
|
+
edges.extend(graph.edges)
|
|
297
|
+
resolution.update(graph.resolution)
|
|
298
|
+
return CallGraph(nodes=nodes, edges=edges, resolution=resolution)
|
blockpath/check.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""The top-level entry point: analyse a project tree, return its findings.
|
|
2
|
+
|
|
3
|
+
Six layers, in order: walk and parse (collect), name tables and scopes (symbols), resolve names
|
|
4
|
+
to functions (resolve), build the call graph (callgraph), decide what blocks (oracle), and
|
|
5
|
+
intersect the two reachabilities to produce witnesses (analysis).
|
|
6
|
+
|
|
7
|
+
The project's own code is never imported or executed. A project that needs a database to import
|
|
8
|
+
is analysed exactly like any other.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from blockpath.analysis import analyse
|
|
16
|
+
from blockpath.callgraph import build_call_graph
|
|
17
|
+
from blockpath.collect import collect
|
|
18
|
+
from blockpath.config import DEFAULT, Config
|
|
19
|
+
from blockpath.model import CheckResult
|
|
20
|
+
from blockpath.oracle import Oracle
|
|
21
|
+
from blockpath.resolve import Resolver
|
|
22
|
+
from blockpath.symbols import analyze
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check(root: Path, config: Config = DEFAULT, oracle: Oracle | None = None) -> CheckResult:
|
|
26
|
+
"""Statically analyse the project rooted at ``root`` and return its findings."""
|
|
27
|
+
collection = collect(root)
|
|
28
|
+
analyses = {module.qualname: analyze(module) for module in collection.modules}
|
|
29
|
+
symbols = {qualname: item.symbols for qualname, item in analyses.items()}
|
|
30
|
+
resolver = Resolver(collection, symbols, max_depth=config.max_import_depth)
|
|
31
|
+
graph = build_call_graph(collection, analyses, resolver)
|
|
32
|
+
findings = analyse(graph, oracle or Oracle.load(), config)
|
|
33
|
+
return CheckResult(findings=findings, unparsed=collection.unparsed)
|