stacklatch 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stacklatch/__init__.py +2 -0
- stacklatch/cli.py +76 -0
- stacklatch/core.py +386 -0
- stacklatch/v21.py +273 -0
- stacklatch/v22.py +483 -0
- stacklatch/v23.py +159 -0
- stacklatch/v24.py +199 -0
- stacklatch/v24_contract.py +188 -0
- stacklatch/v24_graph.py +338 -0
- stacklatch/v25.py +156 -0
- stacklatch/v31.py +380 -0
- stacklatch/v31_checkout.py +146 -0
- stacklatch/v31_graph.py +600 -0
- stacklatch/v31_graph_legacy.py +506 -0
- stacklatch-0.1.0a1.dist-info/METADATA +70 -0
- stacklatch-0.1.0a1.dist-info/RECORD +21 -0
- stacklatch-0.1.0a1.dist-info/WHEEL +5 -0
- stacklatch-0.1.0a1.dist-info/entry_points.txt +2 -0
- stacklatch-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- stacklatch-0.1.0a1.dist-info/licenses/LICENSE_INVENTORY.md +12 -0
- stacklatch-0.1.0a1.dist-info/top_level.txt +1 -0
stacklatch/v31_graph.py
ADDED
|
@@ -0,0 +1,600 @@
|
|
|
1
|
+
"""V3.1 static package graph and conservative dictionary return evidence.
|
|
2
|
+
|
|
3
|
+
This parser never imports repository modules on the host. Unsupported syntax
|
|
4
|
+
fails closed for a selected closure; unrelated modules remain unselected.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
from dataclasses import replace
|
|
10
|
+
import hashlib
|
|
11
|
+
import re
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .core import Analysis, InspectionError, _copy_checked
|
|
15
|
+
from .v31_checkout import ClosureAnalysis
|
|
16
|
+
from .v31_graph_legacy import index_graph as legacy_index_graph
|
|
17
|
+
from .v22 import (DENIED, IDENT, STDLIB, _constant_value, _literal,
|
|
18
|
+
_metadata_v22, _read_ast, _return_shape, _score)
|
|
19
|
+
|
|
20
|
+
MAX_MODULES = 32
|
|
21
|
+
MAX_GRAPH_DEPTH = 6
|
|
22
|
+
MAX_FUNCTIONS = 128
|
|
23
|
+
MAX_FIELDS = 24
|
|
24
|
+
PRIMITIVE_CONSTRUCTORS = frozenset({"len", "str", "int", "float", "bool", "isinstance"})
|
|
25
|
+
SAFE_BUILTIN_CALLS = frozenset({"len", "str", "int", "float", "bool", "abs",
|
|
26
|
+
"min", "max", "sum", "round", "sorted",
|
|
27
|
+
"range", "enumerate", "zip", "any", "all",
|
|
28
|
+
"isinstance"})
|
|
29
|
+
NUMERIC_NAMESPACE = re.compile(r"[0-9]{1,2}_[A-Za-z_][A-Za-z0-9_]*\Z")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _value_type(node, assignments, shadowed=frozenset(), helpers=None):
|
|
33
|
+
"""Infer only expressions whose JSON primitive type follows from syntax."""
|
|
34
|
+
if isinstance(node, ast.Constant):
|
|
35
|
+
return {str: "string", int: "integer", float: "number", bool: "boolean",
|
|
36
|
+
type(None): "null"}.get(type(node.value))
|
|
37
|
+
if isinstance(node, ast.Name):
|
|
38
|
+
return assignments.get(node.id)
|
|
39
|
+
if (helpers and isinstance(node, ast.Subscript) and
|
|
40
|
+
isinstance(node.slice, ast.Constant) and type(node.slice.value) is str and
|
|
41
|
+
isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name)):
|
|
42
|
+
call = node.value
|
|
43
|
+
name = call.func.id
|
|
44
|
+
if (name in helpers and name not in shadowed and not call.keywords and
|
|
45
|
+
len(call.args) <= 8 and all(isinstance(arg, (ast.Name, ast.Constant))
|
|
46
|
+
for arg in call.args)):
|
|
47
|
+
helper = helpers[name]
|
|
48
|
+
if (not helper.decorator_list and not helper.args.vararg and
|
|
49
|
+
not helper.args.kwarg and len(call.args) ==
|
|
50
|
+
len(helper.args.posonlyargs + helper.args.args) and
|
|
51
|
+
not helper.args.kwonlyargs):
|
|
52
|
+
fields, _ = _dict_fields(helper, helpers=None)
|
|
53
|
+
return fields.get(node.slice.value, {}).get("type")
|
|
54
|
+
if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
|
|
55
|
+
and node.func.id in {"len", "str", "int", "float", "bool"}
|
|
56
|
+
and node.func.id not in shadowed
|
|
57
|
+
and len(node.args) == 1 and not node.keywords):
|
|
58
|
+
return {"len": "integer", "str": "string", "int": "integer",
|
|
59
|
+
"float": "number", "bool": "boolean"}[node.func.id]
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _dict_fields(fn, helpers=None):
|
|
64
|
+
"""All return branches must prove identical literal keys and field types."""
|
|
65
|
+
assignments = {}
|
|
66
|
+
# Python makes assignments anywhere in a function local throughout that
|
|
67
|
+
# function. A call spelled len(...) is not evidence of an integer when a
|
|
68
|
+
# parameter, import or binding shadows the builtin, even in a later branch.
|
|
69
|
+
shadowed = {a.arg for a in (fn.args.posonlyargs + fn.args.args + fn.args.kwonlyargs)}
|
|
70
|
+
for node in ast.walk(fn):
|
|
71
|
+
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
|
|
72
|
+
shadowed.add(node.id)
|
|
73
|
+
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
74
|
+
shadowed.update(a.asname or a.name.split('.')[0] for a in node.names)
|
|
75
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn:
|
|
76
|
+
shadowed.add(node.name)
|
|
77
|
+
elif isinstance(node, ast.ExceptHandler) and node.name:
|
|
78
|
+
shadowed.add(node.name)
|
|
79
|
+
if not fn.body or not _guaranteed_return(fn.body):
|
|
80
|
+
return {}, "Return path can fall through without a dictionary"
|
|
81
|
+
if any(isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda))
|
|
82
|
+
for statement in fn.body for node in ast.walk(statement)):
|
|
83
|
+
return {}, "Nested callable prevents return-flow proof"
|
|
84
|
+
for node in fn.body:
|
|
85
|
+
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
|
|
86
|
+
name = node.targets[0].id
|
|
87
|
+
inferred = _value_type(node.value, assignments, shadowed, helpers)
|
|
88
|
+
assignments[name] = inferred if name not in assignments or assignments[name] == inferred else None
|
|
89
|
+
for statement in fn.body:
|
|
90
|
+
if isinstance(statement, (ast.Global, ast.Nonlocal)):
|
|
91
|
+
return {}, "Nonlocal or global return field dependency"
|
|
92
|
+
if isinstance(statement, ast.Assign) and len(statement.targets) == 1 and isinstance(statement.targets[0], ast.Name):
|
|
93
|
+
descendants = ast.walk(statement.value)
|
|
94
|
+
else:
|
|
95
|
+
descendants = ast.walk(statement)
|
|
96
|
+
if any(isinstance(node, (ast.Global, ast.Nonlocal)) or
|
|
97
|
+
isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del))
|
|
98
|
+
and node.id in assignments for node in descendants):
|
|
99
|
+
return {}, "A return field local is mutated in nested control flow"
|
|
100
|
+
returns = [node for node in ast.walk(fn) if isinstance(node, ast.Return)]
|
|
101
|
+
if not returns or len(returns) > 16:
|
|
102
|
+
return {}, "No bounded returns"
|
|
103
|
+
result = None
|
|
104
|
+
for node in returns:
|
|
105
|
+
value = node.value
|
|
106
|
+
if not isinstance(value, ast.Dict) or len(value.keys) > MAX_FIELDS:
|
|
107
|
+
return {}, "Return is not a bounded literal dictionary"
|
|
108
|
+
fields = {}
|
|
109
|
+
for key, expression in zip(value.keys, value.values, strict=True):
|
|
110
|
+
if not isinstance(key, ast.Constant) or not isinstance(key.value, str) or not IDENT.fullmatch(key.value):
|
|
111
|
+
return {}, "Dynamic or invalid dictionary key"
|
|
112
|
+
kind = _value_type(expression, assignments, shadowed, helpers)
|
|
113
|
+
if kind is None or key.value in fields:
|
|
114
|
+
return {}, "Unproven or duplicate dictionary field type"
|
|
115
|
+
fields[key.value] = {"type": kind, "line": expression.lineno}
|
|
116
|
+
if result is not None and {k: v["type"] for k, v in fields.items()} != {k: v["type"] for k, v in result.items()}:
|
|
117
|
+
return {}, "Conflicting dictionary return branches"
|
|
118
|
+
result = fields
|
|
119
|
+
return result or {}, "All literal return branches agree"
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _guaranteed_return(statements):
|
|
123
|
+
"""Recognize only bounded, unconditional terminal return control flow."""
|
|
124
|
+
for statement in statements:
|
|
125
|
+
if isinstance(statement, ast.Return):
|
|
126
|
+
return True
|
|
127
|
+
if isinstance(statement, ast.If) and statement.orelse:
|
|
128
|
+
if _guaranteed_return(statement.body) and _guaranteed_return(statement.orelse):
|
|
129
|
+
return True
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _string_parameter_evidence(fn, parameter):
|
|
134
|
+
"""A behavioral hint, never a static proof or an original annotation."""
|
|
135
|
+
methods = {}
|
|
136
|
+
for node in ast.walk(fn):
|
|
137
|
+
if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
|
|
138
|
+
and isinstance(node.func.value, ast.Name)
|
|
139
|
+
and node.func.value.id == parameter and not node.keywords
|
|
140
|
+
and node.func.attr in {"strip", "lower", "replace", "split"}):
|
|
141
|
+
if (node.func.attr in {"strip", "lower"} and not node.args or
|
|
142
|
+
node.func.attr == "replace" and len(node.args) == 2 and
|
|
143
|
+
all(isinstance(a, ast.Constant) and isinstance(a.value, str) for a in node.args) or
|
|
144
|
+
node.func.attr == "split" and (not node.args or len(node.args) == 1 and
|
|
145
|
+
isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str))):
|
|
146
|
+
methods[node.func.attr] = node.lineno
|
|
147
|
+
for first, second in (("strip", "lower"), ("replace", "split")):
|
|
148
|
+
if first in methods and second in methods:
|
|
149
|
+
return [methods[first], methods[second]]
|
|
150
|
+
return []
|
|
151
|
+
AUDITED_MODULES = STDLIB | {"unicodedata", "string", "collections", "dataclasses", "hashlib", "json", "base64", "html"}
|
|
152
|
+
AUDITED_FROM = {"collections": {"Counter"}, "dataclasses": {"dataclass"},
|
|
153
|
+
"typing": {"Final", "Dict", "List", "Tuple", "Optional", "Any", "Sequence", "Literal", "Callable"},
|
|
154
|
+
"pathlib": {"Path"}, "urllib.parse": {"quote", "unquote"},
|
|
155
|
+
"math": {"pi"}, "functools": {"partial"}}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _annotation(node):
|
|
159
|
+
if node is None or isinstance(node, ast.Name):
|
|
160
|
+
return True
|
|
161
|
+
# re.Match is a standard-library type object with no source-controlled
|
|
162
|
+
# attribute lookup. All other attribute expressions remain unsupported.
|
|
163
|
+
if (isinstance(node, ast.Attribute) and node.attr == "Match"
|
|
164
|
+
and isinstance(node.value, ast.Name) and node.value.id == "re"):
|
|
165
|
+
return True
|
|
166
|
+
if isinstance(node, ast.Constant) and node.value is None:
|
|
167
|
+
return True
|
|
168
|
+
if isinstance(node, ast.Subscript):
|
|
169
|
+
return _annotation(node.value) and _annotation(node.slice)
|
|
170
|
+
if isinstance(node, ast.Tuple):
|
|
171
|
+
return all(_annotation(x) for x in node.elts)
|
|
172
|
+
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
|
173
|
+
return _annotation(node.left) and _annotation(node.right)
|
|
174
|
+
return False
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _safe_constant(node, names):
|
|
178
|
+
if _constant_value(node, names):
|
|
179
|
+
return True
|
|
180
|
+
if (isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name)
|
|
181
|
+
and node.value.id == "Literal" and
|
|
182
|
+
isinstance(node.slice, (ast.Tuple, ast.Constant))):
|
|
183
|
+
values = node.slice.elts if isinstance(node.slice, ast.Tuple) else [node.slice]
|
|
184
|
+
return 1 <= len(values) <= MAX_FIELDS and all(
|
|
185
|
+
isinstance(item, ast.Constant) and isinstance(item.value, str)
|
|
186
|
+
and len(item.value) <= 64 for item in values)
|
|
187
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "frozenset" and len(node.args) == 1 and not node.keywords and _literal(node.args[0]):
|
|
188
|
+
return True
|
|
189
|
+
return (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
|
|
190
|
+
and isinstance(node.func.value, ast.Name) and node.func.value.id == "re"
|
|
191
|
+
and node.func.attr == "compile" and len(node.args) in (1, 2) and not node.keywords
|
|
192
|
+
and _literal(node.args[0]) and (len(node.args) == 1 or
|
|
193
|
+
(isinstance(node.args[1], ast.Attribute)
|
|
194
|
+
and isinstance(node.args[1].value, ast.Name) and node.args[1].value.id == "re"
|
|
195
|
+
and node.args[1].attr in {"UNICODE", "IGNORECASE", "MULTILINE", "DOTALL"})))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _modules(analysis: Analysis) -> dict[str, str]:
|
|
199
|
+
result = {}
|
|
200
|
+
for path in analysis.files:
|
|
201
|
+
if not path.endswith(".py") or Path(path).name == "setup.py" or any(
|
|
202
|
+
p in {"tests", "test", ".github"} for p in Path(path).parts):
|
|
203
|
+
continue
|
|
204
|
+
parts = list(Path(path).parts)
|
|
205
|
+
if parts[0] == "src":
|
|
206
|
+
parts.pop(0)
|
|
207
|
+
# Implicit namespace packages are valid Python and have no initializer
|
|
208
|
+
# to execute. Any present initializer is audited before staging.
|
|
209
|
+
if parts[-1] == "__init__.py":
|
|
210
|
+
parts.pop()
|
|
211
|
+
else:
|
|
212
|
+
parts[-1] = parts[-1][:-3]
|
|
213
|
+
# importlib accepts numeric-leading namespace directories even
|
|
214
|
+
# though Python's import statement cannot spell them. This bounded
|
|
215
|
+
# form is staged at the same verified path; source imports still
|
|
216
|
+
# undergo the strict _scan audit.
|
|
217
|
+
if parts and all(IDENT.fullmatch(p) or i < len(parts) - 1 and
|
|
218
|
+
NUMERIC_NAMESPACE.fullmatch(p) for i, p in enumerate(parts)):
|
|
219
|
+
name = ".".join(parts)
|
|
220
|
+
if name in result:
|
|
221
|
+
raise InspectionError("Ambiguous source module path")
|
|
222
|
+
result[name] = path
|
|
223
|
+
return result
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _project_import(node: ast.ImportFrom, module: str, initializer: bool,
|
|
227
|
+
modules: dict[str, str]) -> tuple[str, dict[str, tuple[str, str]]]:
|
|
228
|
+
if node.level:
|
|
229
|
+
package = module if initializer else module.rpartition(".")[0]
|
|
230
|
+
parts = package.split(".") if package else []
|
|
231
|
+
if node.level > len(parts):
|
|
232
|
+
raise InspectionError("Relative import escapes package")
|
|
233
|
+
base = parts[:len(parts) - node.level + 1]
|
|
234
|
+
requested = ".".join([*base, *(node.module.split(".") if node.module else [])])
|
|
235
|
+
else:
|
|
236
|
+
requested = node.module or ""
|
|
237
|
+
if requested not in modules:
|
|
238
|
+
raise InspectionError("Unresolved in-repository import")
|
|
239
|
+
aliases = {}
|
|
240
|
+
for name in node.names:
|
|
241
|
+
if name.name == "*" or not IDENT.fullmatch(name.name):
|
|
242
|
+
raise InspectionError("Wildcard or invalid import name")
|
|
243
|
+
target = f"{requested}.{name.name}"
|
|
244
|
+
if target in modules: # from . import helper_module
|
|
245
|
+
aliases[name.asname or name.name] = (target, "")
|
|
246
|
+
else:
|
|
247
|
+
aliases[name.asname or name.name] = (requested, name.name)
|
|
248
|
+
return requested, aliases
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _scan(root: Path, analysis: Analysis, module: str, modules: dict[str, str]):
|
|
252
|
+
path = modules[module]
|
|
253
|
+
tree = _read_ast(root, analysis, path)
|
|
254
|
+
imports = {}
|
|
255
|
+
edges = set()
|
|
256
|
+
constants = set()
|
|
257
|
+
definitions = {n.name for n in tree.body if isinstance(n, (ast.FunctionDef, ast.ClassDef))}
|
|
258
|
+
if definitions & PRIMITIVE_CONSTRUCTORS:
|
|
259
|
+
raise InspectionError("Import-time definition shadows a primitive")
|
|
260
|
+
if any(isinstance(n, ast.Attribute) and n.attr == "Match"
|
|
261
|
+
and isinstance(n.value, ast.Name) and n.value.id == "re"
|
|
262
|
+
for statement in tree.body if isinstance(statement, ast.FunctionDef)
|
|
263
|
+
for argument in ([a.annotation for a in statement.args.posonlyargs +
|
|
264
|
+
statement.args.args + statement.args.kwonlyargs] + [statement.returns])
|
|
265
|
+
if argument is not None for n in ast.walk(argument)):
|
|
266
|
+
if (not any(isinstance(n, ast.Import) and len(n.names) == 1
|
|
267
|
+
and n.names[0].name == "re" and not n.names[0].asname
|
|
268
|
+
for n in tree.body) or "re" in definitions or any(
|
|
269
|
+
isinstance(n, (ast.Assign, ast.AnnAssign)) and
|
|
270
|
+
any(isinstance(t, ast.Name) and t.id == "re" for t in
|
|
271
|
+
(n.targets if isinstance(n, ast.Assign) else [n.target]))
|
|
272
|
+
for n in tree.body)):
|
|
273
|
+
raise InspectionError("Unproven re.Match annotation binding")
|
|
274
|
+
for node in tree.body:
|
|
275
|
+
if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
|
|
276
|
+
continue
|
|
277
|
+
if isinstance(node, ast.Import):
|
|
278
|
+
for alias in node.names:
|
|
279
|
+
if alias.name.split('.')[0] in PRIMITIVE_CONSTRUCTORS | definitions:
|
|
280
|
+
raise InspectionError("Import shadows a primitive or callable")
|
|
281
|
+
if alias.name in AUDITED_MODULES and not alias.asname:
|
|
282
|
+
continue
|
|
283
|
+
if alias.name not in modules or alias.asname:
|
|
284
|
+
raise InspectionError("Unaudited or aliased module import")
|
|
285
|
+
edges.add(alias.name)
|
|
286
|
+
elif isinstance(node, ast.ImportFrom):
|
|
287
|
+
if node.level == 0 and node.module == "__future__" and [a.name for a in node.names] == ["annotations"]:
|
|
288
|
+
continue
|
|
289
|
+
if node.level == 0 and node.module in AUDITED_FROM and all(
|
|
290
|
+
a.name in AUDITED_FROM[node.module] and not a.asname for a in node.names):
|
|
291
|
+
if any(a.name in PRIMITIVE_CONSTRUCTORS | definitions for a in node.names):
|
|
292
|
+
raise InspectionError("Import shadows a primitive or callable")
|
|
293
|
+
continue
|
|
294
|
+
target, aliases = _project_import(node, module, path.endswith("/__init__.py"), modules)
|
|
295
|
+
if imports.keys() & aliases.keys() or aliases.keys() & (PRIMITIVE_CONSTRUCTORS | definitions | constants):
|
|
296
|
+
raise InspectionError("Ambiguous imported alias")
|
|
297
|
+
imports.update(aliases)
|
|
298
|
+
edges.add(target)
|
|
299
|
+
edges.update(m for m, name in aliases.values() if not name)
|
|
300
|
+
elif isinstance(node, (ast.Assign, ast.AnnAssign)):
|
|
301
|
+
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
302
|
+
if len(targets) != 1 or not isinstance(targets[0], ast.Name) or node.value is None or not _safe_constant(node.value, constants):
|
|
303
|
+
raise InspectionError("Dynamic import-time constant")
|
|
304
|
+
if targets[0].id in PRIMITIVE_CONSTRUCTORS | definitions or targets[0].id in constants:
|
|
305
|
+
raise InspectionError("Import-time binding shadows a primitive or callable")
|
|
306
|
+
constants.add(targets[0].id)
|
|
307
|
+
elif isinstance(node, ast.FunctionDef):
|
|
308
|
+
if node.decorator_list:
|
|
309
|
+
raise InspectionError("Unsupported function definition")
|
|
310
|
+
if any(not _literal(default) for default in
|
|
311
|
+
node.args.defaults + [d for d in node.args.kw_defaults if d is not None]):
|
|
312
|
+
raise InspectionError("Evaluated function default")
|
|
313
|
+
if not any(isinstance(n, ast.ImportFrom) and n.module == "__future__" for n in tree.body):
|
|
314
|
+
anns = [a.annotation for a in node.args.posonlyargs + node.args.args + node.args.kwonlyargs] + [node.returns]
|
|
315
|
+
if any(not _annotation(a) for a in anns):
|
|
316
|
+
raise InspectionError("Evaluated annotation")
|
|
317
|
+
elif isinstance(node, ast.ClassDef):
|
|
318
|
+
if (len(node.bases) == 1 and isinstance(node.bases[0], ast.Name)
|
|
319
|
+
and node.bases[0].id in {"ValueError", "TypeError"}
|
|
320
|
+
and not node.keywords and not node.decorator_list
|
|
321
|
+
and all(isinstance(item, ast.Pass) or isinstance(item, ast.Expr)
|
|
322
|
+
and isinstance(item.value, ast.Constant)
|
|
323
|
+
and isinstance(item.value.value, str) for item in node.body)):
|
|
324
|
+
continue
|
|
325
|
+
decorator = node.decorator_list[0] if len(node.decorator_list) == 1 else None
|
|
326
|
+
safe_decorator = (isinstance(decorator, ast.Name) and decorator.id == "dataclass") or (
|
|
327
|
+
isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name)
|
|
328
|
+
and decorator.func.id == "dataclass" and not decorator.args
|
|
329
|
+
and all(k.arg in {"frozen", "slots"} and isinstance(k.value, ast.Constant)
|
|
330
|
+
and type(k.value.value) is bool for k in decorator.keywords))
|
|
331
|
+
if (not safe_decorator or node.bases or node.keywords
|
|
332
|
+
or not any(isinstance(n, ast.ImportFrom) and n.module == "dataclasses"
|
|
333
|
+
and [a.name for a in n.names] == ["dataclass"] for n in tree.body)):
|
|
334
|
+
raise InspectionError("Unsupported class initialization")
|
|
335
|
+
if any(not (isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name)
|
|
336
|
+
and _annotation(n.annotation) and (n.value is None or _literal(n.value)))
|
|
337
|
+
and not (isinstance(n, ast.FunctionDef) and not n.decorator_list
|
|
338
|
+
and not n.args.vararg and not n.args.kwarg
|
|
339
|
+
and all(_literal(d) for d in n.args.defaults))
|
|
340
|
+
and not (isinstance(n, ast.Expr) and isinstance(n.value, ast.Constant)
|
|
341
|
+
and isinstance(n.value.value, str)) for n in node.body):
|
|
342
|
+
raise InspectionError("Unsupported dataclass body")
|
|
343
|
+
elif (isinstance(node, ast.If) and isinstance(node.test, ast.Compare)
|
|
344
|
+
and isinstance(node.test.left, ast.Name) and node.test.left.id == "__name__"
|
|
345
|
+
and len(node.test.ops) == 1 and isinstance(node.test.ops[0], ast.Eq)
|
|
346
|
+
and len(node.test.comparators) == 1 and isinstance(node.test.comparators[0], ast.Constant)
|
|
347
|
+
and node.test.comparators[0].value == "__main__" and not node.orelse):
|
|
348
|
+
continue
|
|
349
|
+
else:
|
|
350
|
+
raise InspectionError("Unsupported import-time statement")
|
|
351
|
+
return tree, imports, edges
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _closure(root, analysis, module, function, modules, checked, active, functions):
|
|
355
|
+
key = (module, function)
|
|
356
|
+
if key in active:
|
|
357
|
+
raise InspectionError("Cyclic helper closure")
|
|
358
|
+
if key in functions:
|
|
359
|
+
return
|
|
360
|
+
if len(functions) >= MAX_FUNCTIONS:
|
|
361
|
+
raise InspectionError("Helper closure exceeds limit")
|
|
362
|
+
tree, imports, _ = checked[module]
|
|
363
|
+
defs = {n.name: n for n in tree.body if isinstance(n, ast.FunctionDef)}
|
|
364
|
+
if function not in defs:
|
|
365
|
+
raise InspectionError("Unresolved helper function")
|
|
366
|
+
if defs[function].args.vararg or defs[function].args.kwarg:
|
|
367
|
+
raise InspectionError("Selected variadic function unsupported")
|
|
368
|
+
active.add(key)
|
|
369
|
+
# A repository may declare dependencies for unrelated modules. For such
|
|
370
|
+
# repositories demand positive proof of every bare callable used by a
|
|
371
|
+
# selected function before admitting its closure. No package is installed.
|
|
372
|
+
strict = bool(analysis.dependencies)
|
|
373
|
+
local_bindings = {a.arg for a in (defs[function].args.posonlyargs +
|
|
374
|
+
defs[function].args.args + defs[function].args.kwonlyargs)}
|
|
375
|
+
local_bindings.update(n.id for n in ast.walk(defs[function])
|
|
376
|
+
if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store))
|
|
377
|
+
constants = {n.targets[0].id if isinstance(n, ast.Assign) else n.target.id
|
|
378
|
+
for n in tree.body if isinstance(n, (ast.Assign, ast.AnnAssign))
|
|
379
|
+
and (n.targets if isinstance(n, ast.Assign) else [n.target])
|
|
380
|
+
and isinstance((n.targets if isinstance(n, ast.Assign) else [n.target])[0], ast.Name)}
|
|
381
|
+
annotation_nodes = {id(child) for annotation in
|
|
382
|
+
[a.annotation for a in (defs[function].args.posonlyargs +
|
|
383
|
+
defs[function].args.args + defs[function].args.kwonlyargs)] +
|
|
384
|
+
[defs[function].returns]
|
|
385
|
+
if annotation is not None for child in ast.walk(annotation)}
|
|
386
|
+
for node in ast.walk(defs[function]):
|
|
387
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
388
|
+
raise InspectionError("Dynamic function import")
|
|
389
|
+
if isinstance(node, ast.Name) and node.id in DENIED:
|
|
390
|
+
raise InspectionError("Unsafe function name")
|
|
391
|
+
if (strict and id(node) not in annotation_nodes and isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
|
392
|
+
and node.id not in local_bindings | constants | defs.keys() | imports.keys() | SAFE_BUILTIN_CALLS
|
|
393
|
+
and node.id not in {"True", "False", "None"}):
|
|
394
|
+
# Imported standard-library module names are still audited by
|
|
395
|
+
# _scan; bare unknown globals can hide an unstaged dependency.
|
|
396
|
+
imported = {a.name.split('.')[0] for statement in tree.body if isinstance(statement, ast.Import)
|
|
397
|
+
for a in statement.names}
|
|
398
|
+
from_names = {a.name for statement in tree.body if isinstance(statement, ast.ImportFrom)
|
|
399
|
+
and statement.level == 0 and statement.module in AUDITED_FROM
|
|
400
|
+
for a in statement.names}
|
|
401
|
+
if node.id not in imported | from_names:
|
|
402
|
+
raise InspectionError("Unproven global in selected function")
|
|
403
|
+
if (isinstance(node, ast.Attribute) and node.attr.startswith("__")
|
|
404
|
+
and not (node.attr == "__name__"
|
|
405
|
+
and isinstance(node.value, ast.Call)
|
|
406
|
+
and isinstance(node.value.func, ast.Name)
|
|
407
|
+
and node.value.func.id == "type"
|
|
408
|
+
and len(node.value.args) == 1
|
|
409
|
+
and isinstance(node.value.args[0], ast.Name)
|
|
410
|
+
and not node.value.keywords)):
|
|
411
|
+
raise InspectionError("Unsafe function attribute")
|
|
412
|
+
if isinstance(node, ast.Call):
|
|
413
|
+
if isinstance(node.func, ast.Name):
|
|
414
|
+
name = node.func.id
|
|
415
|
+
if name in DENIED:
|
|
416
|
+
raise InspectionError("Unsafe function call")
|
|
417
|
+
if name == "isinstance":
|
|
418
|
+
# Only a check of a JSON primitive input against a
|
|
419
|
+
# non-shadowed builtin primitive is admissible. Arbitrary
|
|
420
|
+
# class objects could invoke an attacker metaclass.
|
|
421
|
+
if (len(node.args) != 2 or node.keywords
|
|
422
|
+
or not isinstance(node.args[0], ast.Name)
|
|
423
|
+
or node.args[0].id not in {p.arg for p in
|
|
424
|
+
(defs[function].args.posonlyargs + defs[function].args.args +
|
|
425
|
+
defs[function].args.kwonlyargs)}
|
|
426
|
+
or not isinstance(node.args[1], ast.Name)
|
|
427
|
+
or node.args[1].id not in {"str", "int", "float", "bool"}
|
|
428
|
+
or name in local_bindings or node.args[1].id in local_bindings):
|
|
429
|
+
raise InspectionError("Unproven isinstance primitive check")
|
|
430
|
+
continue
|
|
431
|
+
if name in defs:
|
|
432
|
+
_closure(root, analysis, module, name, modules, checked, active, functions)
|
|
433
|
+
elif name in imports:
|
|
434
|
+
other, called = imports[name]
|
|
435
|
+
if not called:
|
|
436
|
+
raise InspectionError("Module used as callable")
|
|
437
|
+
_closure(root, analysis, other, called, modules, checked, active, functions)
|
|
438
|
+
elif strict and name not in SAFE_BUILTIN_CALLS:
|
|
439
|
+
raise InspectionError("Unproven callable in selected function")
|
|
440
|
+
elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
|
|
441
|
+
alias = node.func.value.id
|
|
442
|
+
if alias in imports and not imports[alias][1]:
|
|
443
|
+
other = imports[alias][0]
|
|
444
|
+
_closure(root, analysis, other, node.func.attr, modules, checked, active, functions)
|
|
445
|
+
active.remove(key)
|
|
446
|
+
functions.add(key)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def index_graph(root: Path, analysis: Analysis):
|
|
450
|
+
if not isinstance(analysis, ClosureAnalysis):
|
|
451
|
+
return legacy_index_graph(root, analysis)
|
|
452
|
+
if analysis.language != "python" or analysis.license != "MIT-like (heuristic)":
|
|
453
|
+
raise InspectionError("Only MIT-like Python is supported")
|
|
454
|
+
modules = _modules(analysis)
|
|
455
|
+
# The repository is independently bounded by MAX_FILES and MAX_BYTES.
|
|
456
|
+
# A selected dependency closure, rather than unrelated files, owns the
|
|
457
|
+
# smaller import-graph limit.
|
|
458
|
+
checked = {}
|
|
459
|
+
rejected = {}
|
|
460
|
+
|
|
461
|
+
def visit(module, active, depth):
|
|
462
|
+
if module in checked:
|
|
463
|
+
return
|
|
464
|
+
if module in active or depth > MAX_GRAPH_DEPTH:
|
|
465
|
+
raise InspectionError("Cyclic or deep package imports")
|
|
466
|
+
tree, imports, edges = _scan(root, analysis, module, modules)
|
|
467
|
+
for edge in sorted(edges):
|
|
468
|
+
# A package initializer may re-export the child currently being
|
|
469
|
+
# audited. This is the only allowed back edge; all other cycles
|
|
470
|
+
# fail closed.
|
|
471
|
+
if edge in active and modules[module].endswith("/__init__.py") and edge.startswith(module + "."):
|
|
472
|
+
continue
|
|
473
|
+
visit(edge, active | {module}, depth + 1)
|
|
474
|
+
# Python imports parent initializers before submodules.
|
|
475
|
+
parts = module.split(".")
|
|
476
|
+
for n in range(1, len(parts)):
|
|
477
|
+
parent = ".".join(parts[:n])
|
|
478
|
+
if parent in modules and parent not in active:
|
|
479
|
+
visit(parent, active | {module}, depth + 1)
|
|
480
|
+
checked[module] = (tree, imports, edges)
|
|
481
|
+
|
|
482
|
+
for name in sorted(modules):
|
|
483
|
+
try:
|
|
484
|
+
visit(name, set(), 1)
|
|
485
|
+
except InspectionError as exc:
|
|
486
|
+
rejected[name] = str(exc)
|
|
487
|
+
indexed = []
|
|
488
|
+
for module in sorted(checked):
|
|
489
|
+
tree, _, _ = checked[module]
|
|
490
|
+
for fn in (n for n in tree.body if isinstance(n, ast.FunctionDef) and not n.name.startswith("_")):
|
|
491
|
+
params = fn.args.posonlyargs + fn.args.args
|
|
492
|
+
# Optional source controls do not need a binding. Keep a bounded
|
|
493
|
+
# signature while allowing ordinary APIs with five parameters.
|
|
494
|
+
if len(params) + len(fn.args.kwonlyargs) > 8:
|
|
495
|
+
continue
|
|
496
|
+
shape, keys, evidence = _return_shape(fn)
|
|
497
|
+
helpers = {n.name: n for n in tree.body if isinstance(n, ast.FunctionDef) and n is not fn}
|
|
498
|
+
fields, field_evidence = _dict_fields(fn, helpers) if shape == "dict" else ({}, "Not a dictionary return")
|
|
499
|
+
item = {"module": module, "path": modules[module], "name": fn.name,
|
|
500
|
+
"line": fn.lineno, "file_sha256": analysis.file_sha256[modules[module]],
|
|
501
|
+
"parameters": [{"name": p.arg, "annotation": ast.unparse(p.annotation) if p.annotation else "unknown",
|
|
502
|
+
"inferred_type": "str" if not p.annotation and _string_parameter_evidence(fn, p.arg) else None,
|
|
503
|
+
"type_evidence": _string_parameter_evidence(fn, p.arg) if not p.annotation else [],
|
|
504
|
+
"required": i < len(params) - len(fn.args.defaults),
|
|
505
|
+
"kind": "positional_only" if i < len(fn.args.posonlyargs) else "positional_or_keyword"}
|
|
506
|
+
for i, p in enumerate(params)] +
|
|
507
|
+
[{"name": p.arg, "annotation": ast.unparse(p.annotation) if p.annotation else "unknown",
|
|
508
|
+
"required": fn.args.kw_defaults[i] is None, "kind": "keyword_only"}
|
|
509
|
+
for i, p in enumerate(fn.args.kwonlyargs)],
|
|
510
|
+
"return_shape": shape, "return_keys": keys, "shape_evidence": evidence,
|
|
511
|
+
"return_fields": fields, "field_evidence": field_evidence,
|
|
512
|
+
"docstring": ast.get_docstring(fn) or "", "supported": True,
|
|
513
|
+
"declared_dependencies_outside_closure": sorted(analysis.dependencies)}
|
|
514
|
+
try:
|
|
515
|
+
# Metadata in an unrelated example directory cannot affect
|
|
516
|
+
# an imported selected module. Audit only the root and the
|
|
517
|
+
# module's own directory (including its package ancestors).
|
|
518
|
+
parent = Path(modules[module]).parent
|
|
519
|
+
relevant = [path for path in analysis.manifests
|
|
520
|
+
if Path(path).parent == Path('.') or
|
|
521
|
+
Path(path).parent == parent or
|
|
522
|
+
Path(path).parent in parent.parents]
|
|
523
|
+
_metadata_v22(root, replace(analysis, manifests=relevant))
|
|
524
|
+
closure = set()
|
|
525
|
+
_closure(root, analysis, module, fn.name, modules, checked, set(), closure)
|
|
526
|
+
staged = {m for m, _ in closure} | {module}
|
|
527
|
+
pending = list(staged)
|
|
528
|
+
while pending:
|
|
529
|
+
current = pending.pop()
|
|
530
|
+
parents = [".".join(current.split(".")[:n])
|
|
531
|
+
for n in range(1, len(current.split(".")))]
|
|
532
|
+
for dependency in checked[current][2] | set(parents):
|
|
533
|
+
if dependency in modules and dependency not in staged:
|
|
534
|
+
if dependency not in checked:
|
|
535
|
+
raise InspectionError("Unresolved package initializer")
|
|
536
|
+
staged.add(dependency)
|
|
537
|
+
pending.append(dependency)
|
|
538
|
+
if len(staged) > MAX_MODULES:
|
|
539
|
+
raise InspectionError("Selected module graph exceeds limit")
|
|
540
|
+
item["closure"] = sorted(staged)
|
|
541
|
+
except InspectionError as exc:
|
|
542
|
+
item["supported"], item["reason"] = False, str(exc)
|
|
543
|
+
indexed.append(item)
|
|
544
|
+
return indexed, rejected
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def propose_graph(left: Analysis, right: Analysis, left_root: Path, right_root: Path, goal: str):
|
|
548
|
+
if not isinstance(goal, str) or not goal.strip() or any(
|
|
549
|
+
a.provenance != "fetched_git_commit" for a in (left, right)):
|
|
550
|
+
raise InspectionError("Pinned fetched commits and goal required")
|
|
551
|
+
indexes = [index_graph(root, analysis) for root, analysis in
|
|
552
|
+
zip((left_root, right_root), (left, right), strict=True)]
|
|
553
|
+
candidates = []
|
|
554
|
+
for source in indexes[0][0]:
|
|
555
|
+
if (not source["supported"] or not source["parameters"]
|
|
556
|
+
or not source["parameters"][0]["required"]
|
|
557
|
+
or any(p["required"] for p in source["parameters"][1:])):
|
|
558
|
+
continue
|
|
559
|
+
for target in indexes[1][0]:
|
|
560
|
+
if not target["supported"]:
|
|
561
|
+
continue
|
|
562
|
+
required = [p for p in target["parameters"] if p["required"]]
|
|
563
|
+
pattern = ("direct" if len(required) == 1 else "fields" if len(required) == 2
|
|
564
|
+
and source["return_shape"] == "dict" and source["return_keys"] else None)
|
|
565
|
+
if pattern is None:
|
|
566
|
+
continue
|
|
567
|
+
score, evidence = _score(source, target, pattern)
|
|
568
|
+
candidates.append({"pattern": pattern, "score": score, "score_kind": "ranking_evidence_not_probability",
|
|
569
|
+
"contributions": evidence, "source": {k: source[k] for k in ("module", "path", "name", "closure")},
|
|
570
|
+
"target": {k: target[k] for k in ("module", "path", "name", "closure")},
|
|
571
|
+
"target_required": [p["name"] for p in required], "available_fields": source["return_keys"],
|
|
572
|
+
"mapping_suggestion": ({required[0]["name"]: "$output"} if pattern == "direct" else
|
|
573
|
+
{p["name"]: p["name"] for p in required
|
|
574
|
+
if p["name"] in source["return_keys"]}),
|
|
575
|
+
"requires_confirmation": True, "verified": False})
|
|
576
|
+
candidates.sort(key=lambda c: (-c["score"], c["source"]["path"], c["source"]["name"],
|
|
577
|
+
c["target"]["path"], c["target"]["name"]))
|
|
578
|
+
for i, candidate in enumerate(candidates):
|
|
579
|
+
candidate["id"] = i
|
|
580
|
+
return {"schema_version": 24, "goal": goal, "proposed": True, "verified": False,
|
|
581
|
+
"sources": [{"url": a.source, "commit": a.commit, "content_sha256": a.content_sha256,
|
|
582
|
+
"file_sha256": a.file_sha256, "kind": a.provenance} for a in (left, right)],
|
|
583
|
+
"indexes": [x[0] for x in indexes], "rejected_modules": [x[1] for x in indexes],
|
|
584
|
+
"candidates": candidates, "risk_flags": ["AST cannot establish semantic correctness"]}
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def stage_graph(root: Path, analysis: Analysis, selected: dict, target: Path):
|
|
588
|
+
modules = _modules(analysis)
|
|
589
|
+
for name in selected["closure"]:
|
|
590
|
+
path = modules[name]
|
|
591
|
+
parts = list(Path(path).parts)
|
|
592
|
+
if parts[0] == "src":
|
|
593
|
+
parts.pop(0)
|
|
594
|
+
_copy_checked(root, path, target.joinpath(*parts), analysis.file_sha256[path])
|
|
595
|
+
for n in range(1, len(parts)):
|
|
596
|
+
parent = Path("src") if path.startswith("src/") else Path()
|
|
597
|
+
init = (parent / Path(*parts[:n]) / "__init__.py").as_posix()
|
|
598
|
+
if init in analysis.file_sha256:
|
|
599
|
+
_copy_checked(root, init, target / Path(*parts[:n]) / "__init__.py", analysis.file_sha256[init])
|
|
600
|
+
return selected["module"]
|