ballpython 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ballpython/__init__.py +9 -0
- ballpython/__main__.py +8 -0
- ballpython/cli.py +7 -0
- ballpython-2.0.0.dist-info/METADATA +322 -0
- ballpython-2.0.0.dist-info/RECORD +24 -0
- ballpython-2.0.0.dist-info/WHEEL +5 -0
- ballpython-2.0.0.dist-info/entry_points.txt +3 -0
- ballpython-2.0.0.dist-info/top_level.txt +2 -0
- pycleaner/__init__.py +53 -0
- pycleaner/__main__.py +8 -0
- pycleaner/cli.py +1548 -0
- pycleaner/complexity_analyzer.py +473 -0
- pycleaner/config.py +254 -0
- pycleaner/dead_code_detector.py +515 -0
- pycleaner/dependency_auditor.py +331 -0
- pycleaner/import_resolver.py +832 -0
- pycleaner/linter_formatter.py +590 -0
- pycleaner/pipeline.py +349 -0
- pycleaner/security_scanner.py +563 -0
- pycleaner/syntax_healer.py +577 -0
- pycleaner/taint_engine.py +720 -0
- pycleaner/test_generator.py +444 -0
- pycleaner/type_checker.py +989 -0
- pycleaner/typeshed_resolver.py +395 -0
|
@@ -0,0 +1,832 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Static undefined symbol detector and missing import resolver.
|
|
3
|
+
|
|
4
|
+
Analyzes AST scopes to find undefined loaded symbols and matches them
|
|
5
|
+
against Python standard library modules, common typing/dataclass constructs,
|
|
6
|
+
well-known third-party packages, and aliases.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ast
|
|
12
|
+
import builtins
|
|
13
|
+
import sys
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import ClassVar
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(slots=True)
|
|
19
|
+
class MissingImportDiagnostic:
|
|
20
|
+
"""Diagnostic detail for a missing import."""
|
|
21
|
+
|
|
22
|
+
symbol: str
|
|
23
|
+
import_statement: str
|
|
24
|
+
value: str = "missing-import"
|
|
25
|
+
module: str = ""
|
|
26
|
+
status: str = "resolved"
|
|
27
|
+
|
|
28
|
+
def to_dict(self) -> dict[str, str]:
|
|
29
|
+
return {
|
|
30
|
+
"value": self.value,
|
|
31
|
+
"symbol": self.symbol,
|
|
32
|
+
"import_statement": self.import_statement,
|
|
33
|
+
"module": self.module,
|
|
34
|
+
"status": self.status,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(slots=True)
|
|
39
|
+
class ImportResolutionResult:
|
|
40
|
+
"""Outcome of attempting to resolve missing imports in source code."""
|
|
41
|
+
|
|
42
|
+
code: str
|
|
43
|
+
resolved_imports: list[str] = field(default_factory=list)
|
|
44
|
+
unresolved_symbols: list[str] = field(default_factory=list)
|
|
45
|
+
diagnostics: list[MissingImportDiagnostic] = field(default_factory=list)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(slots=True)
|
|
49
|
+
class _ClassificationContext:
|
|
50
|
+
type_checking_symbols: set[str]
|
|
51
|
+
existing_lines: set[str]
|
|
52
|
+
regular_imports: list[str]
|
|
53
|
+
tc_imports: list[str]
|
|
54
|
+
unresolved: list[str]
|
|
55
|
+
diagnostics: list[MissingImportDiagnostic]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class UndefinedSymbolFinder(ast.NodeVisitor):
|
|
59
|
+
"""AST visitor that detects loaded names that are not defined in any accessible scope."""
|
|
60
|
+
|
|
61
|
+
def __init__(self) -> None:
|
|
62
|
+
self.builtin_names: set[str] = set(dir(builtins))
|
|
63
|
+
self.scopes: list[set[str]] = [set()]
|
|
64
|
+
# Parallel to `scopes`: what kind of scope each entry is. Needed to
|
|
65
|
+
# resolve two things correctly per real Python 3 semantics:
|
|
66
|
+
# 1. List/set/dict comprehensions and generator expressions get
|
|
67
|
+
# their own scope -- their loop variables do NOT leak into the
|
|
68
|
+
# enclosing function/module scope (unlike Python 2).
|
|
69
|
+
# 2. Assignment expressions (walrus `:=`) inside a comprehension
|
|
70
|
+
# bind to the nearest enclosing scope that is a function or
|
|
71
|
+
# module scope, skipping over both comprehension scopes and
|
|
72
|
+
# class scopes (PEP 572).
|
|
73
|
+
self.scope_kinds: list[str] = ["module"]
|
|
74
|
+
self.undefined_names: set[str] = set()
|
|
75
|
+
self.annotation_undefined_names: set[str] = set()
|
|
76
|
+
self.runtime_undefined_names: set[str] = set()
|
|
77
|
+
self._in_annotation: bool = False
|
|
78
|
+
|
|
79
|
+
def _current_scope(self) -> set[str]:
|
|
80
|
+
return self.scopes[-1]
|
|
81
|
+
|
|
82
|
+
def _is_defined(self, name: str) -> bool:
|
|
83
|
+
if name in self.builtin_names:
|
|
84
|
+
return True
|
|
85
|
+
for scope in reversed(self.scopes):
|
|
86
|
+
if name in scope:
|
|
87
|
+
return True
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
def visit_Import(self, node: ast.Import) -> None:
|
|
91
|
+
for alias in node.names:
|
|
92
|
+
name = alias.asname or alias.name.split(".")[0]
|
|
93
|
+
self._current_scope().add(name)
|
|
94
|
+
self.generic_visit(node)
|
|
95
|
+
|
|
96
|
+
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
97
|
+
for alias in node.names:
|
|
98
|
+
name = alias.asname or alias.name
|
|
99
|
+
self._current_scope().add(name)
|
|
100
|
+
self.generic_visit(node)
|
|
101
|
+
|
|
102
|
+
def visit_If(self, node: ast.If) -> None:
|
|
103
|
+
is_type_checking = (
|
|
104
|
+
isinstance(node.test, ast.Name) and node.test.id == "TYPE_CHECKING"
|
|
105
|
+
) or (
|
|
106
|
+
isinstance(node.test, ast.Attribute) and node.test.attr == "TYPE_CHECKING"
|
|
107
|
+
)
|
|
108
|
+
if is_type_checking:
|
|
109
|
+
self._register_type_checking_imports(node.body)
|
|
110
|
+
self.generic_visit(node)
|
|
111
|
+
|
|
112
|
+
def _register_type_checking_imports(self, body: list[ast.stmt]) -> None:
|
|
113
|
+
for stmt in body:
|
|
114
|
+
if isinstance(stmt, ast.Import):
|
|
115
|
+
for alias in stmt.names:
|
|
116
|
+
self._current_scope().add(alias.asname or alias.name.split(".")[0])
|
|
117
|
+
elif isinstance(stmt, ast.ImportFrom):
|
|
118
|
+
for alias in stmt.names:
|
|
119
|
+
self._current_scope().add(alias.asname or alias.name)
|
|
120
|
+
|
|
121
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
122
|
+
self._current_scope().add(node.name)
|
|
123
|
+
# Class decorator and base expressions are evaluated in outer scope
|
|
124
|
+
for decorator in node.decorator_list:
|
|
125
|
+
self.visit(decorator)
|
|
126
|
+
for base in node.bases:
|
|
127
|
+
self.visit(base)
|
|
128
|
+
for keyword in node.keywords:
|
|
129
|
+
self.visit(keyword)
|
|
130
|
+
|
|
131
|
+
# Enter class body scope
|
|
132
|
+
self.scopes.append(set())
|
|
133
|
+
self.scope_kinds.append("class")
|
|
134
|
+
for statement in node.body:
|
|
135
|
+
self.visit(statement)
|
|
136
|
+
self.scopes.pop()
|
|
137
|
+
self.scope_kinds.pop()
|
|
138
|
+
|
|
139
|
+
def _visit_arg_annotation(self, arg: ast.arg | None) -> None:
|
|
140
|
+
if arg and arg.annotation:
|
|
141
|
+
self._in_annotation = True
|
|
142
|
+
try:
|
|
143
|
+
self.visit(arg.annotation)
|
|
144
|
+
finally:
|
|
145
|
+
self._in_annotation = False
|
|
146
|
+
|
|
147
|
+
def _register_function_args(self, args: ast.arguments) -> None:
|
|
148
|
+
all_args = args.posonlyargs + args.args + args.kwonlyargs
|
|
149
|
+
for arg in all_args:
|
|
150
|
+
self._current_scope().add(arg.arg)
|
|
151
|
+
self._visit_arg_annotation(arg)
|
|
152
|
+
if args.vararg:
|
|
153
|
+
self._current_scope().add(args.vararg.arg)
|
|
154
|
+
self._visit_arg_annotation(args.vararg)
|
|
155
|
+
if args.kwarg:
|
|
156
|
+
self._current_scope().add(args.kwarg.arg)
|
|
157
|
+
self._visit_arg_annotation(args.kwarg)
|
|
158
|
+
|
|
159
|
+
def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
|
|
160
|
+
self._current_scope().add(node.name)
|
|
161
|
+
for decorator in node.decorator_list:
|
|
162
|
+
self.visit(decorator)
|
|
163
|
+
if node.returns:
|
|
164
|
+
self._in_annotation = True
|
|
165
|
+
try:
|
|
166
|
+
self.visit(node.returns)
|
|
167
|
+
finally:
|
|
168
|
+
self._in_annotation = False
|
|
169
|
+
|
|
170
|
+
self.scopes.append(set())
|
|
171
|
+
self.scope_kinds.append("function")
|
|
172
|
+
|
|
173
|
+
self._register_function_args(node.args)
|
|
174
|
+
|
|
175
|
+
for statement in node.body:
|
|
176
|
+
self.visit(statement)
|
|
177
|
+
|
|
178
|
+
self.scopes.pop()
|
|
179
|
+
self.scope_kinds.pop()
|
|
180
|
+
|
|
181
|
+
visit_AsyncFunctionDef = visit_FunctionDef
|
|
182
|
+
|
|
183
|
+
def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
|
|
184
|
+
if node.value:
|
|
185
|
+
self.visit(node.value)
|
|
186
|
+
if isinstance(node.target, ast.Name):
|
|
187
|
+
self._current_scope().add(node.target.id)
|
|
188
|
+
else:
|
|
189
|
+
self.visit(node.target)
|
|
190
|
+
self._in_annotation = True
|
|
191
|
+
try:
|
|
192
|
+
self.visit(node.annotation)
|
|
193
|
+
finally:
|
|
194
|
+
self._in_annotation = False
|
|
195
|
+
|
|
196
|
+
def visit_Lambda(self, node: ast.Lambda) -> None:
|
|
197
|
+
self.scopes.append(set())
|
|
198
|
+
self.scope_kinds.append("function")
|
|
199
|
+
all_args = node.args.posonlyargs + node.args.args + node.args.kwonlyargs
|
|
200
|
+
for arg in all_args:
|
|
201
|
+
self._current_scope().add(arg.arg)
|
|
202
|
+
if node.args.vararg:
|
|
203
|
+
self._current_scope().add(node.args.vararg.arg)
|
|
204
|
+
if node.args.kwarg:
|
|
205
|
+
self._current_scope().add(node.args.kwarg.arg)
|
|
206
|
+
|
|
207
|
+
self.visit(node.body)
|
|
208
|
+
self.scopes.pop()
|
|
209
|
+
self.scope_kinds.pop()
|
|
210
|
+
|
|
211
|
+
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
|
|
212
|
+
# PEP 572: an assignment expression's target binds in the nearest
|
|
213
|
+
# enclosing scope that is a function or module scope, explicitly
|
|
214
|
+
# skipping over comprehension scopes (so `[y := x for x in data]`
|
|
215
|
+
# binds `y` in the scope containing the comprehension, not inside
|
|
216
|
+
# it) and class scopes (a walrus inside a class body targets the
|
|
217
|
+
# nearest enclosing function/module scope, not the class namespace).
|
|
218
|
+
if isinstance(node.target, ast.Name):
|
|
219
|
+
target_scope = self.scopes[0] # module scope is always a safe fallback
|
|
220
|
+
for scope, kind in zip(reversed(self.scopes), reversed(self.scope_kinds)):
|
|
221
|
+
if kind not in ("comprehension", "class"):
|
|
222
|
+
target_scope = scope
|
|
223
|
+
break
|
|
224
|
+
target_scope.add(node.target.id)
|
|
225
|
+
self.visit(node.value)
|
|
226
|
+
|
|
227
|
+
def visit_Name(self, node: ast.Name) -> None:
|
|
228
|
+
if isinstance(node.ctx, (ast.Store, ast.Del)):
|
|
229
|
+
self._current_scope().add(node.id)
|
|
230
|
+
elif isinstance(node.ctx, ast.Load) and not self._is_defined(node.id):
|
|
231
|
+
self.undefined_names.add(node.id)
|
|
232
|
+
if self._in_annotation:
|
|
233
|
+
self.annotation_undefined_names.add(node.id)
|
|
234
|
+
else:
|
|
235
|
+
self.runtime_undefined_names.add(node.id)
|
|
236
|
+
|
|
237
|
+
def visit_Global(self, node: ast.Global) -> None:
|
|
238
|
+
for name in node.names:
|
|
239
|
+
self.scopes[0].add(name)
|
|
240
|
+
|
|
241
|
+
def visit_Nonlocal(self, node: ast.Nonlocal) -> None:
|
|
242
|
+
# Nonlocal is assumed defined in some outer scope
|
|
243
|
+
for name in node.names:
|
|
244
|
+
self._current_scope().add(name)
|
|
245
|
+
|
|
246
|
+
def visit_For(self, node: ast.For | ast.AsyncFor) -> None:
|
|
247
|
+
# Loop target is stored in current scope
|
|
248
|
+
self._extract_store_names(node.target, self._current_scope())
|
|
249
|
+
self.visit(node.iter)
|
|
250
|
+
for item in node.body:
|
|
251
|
+
self.visit(item)
|
|
252
|
+
for item in node.orelse:
|
|
253
|
+
self.visit(item)
|
|
254
|
+
|
|
255
|
+
visit_AsyncFor = visit_For
|
|
256
|
+
|
|
257
|
+
def visit_With(self, node: ast.With | ast.AsyncWith) -> None:
|
|
258
|
+
for item in node.items:
|
|
259
|
+
self.visit(item.context_expr)
|
|
260
|
+
if item.optional_vars:
|
|
261
|
+
self._extract_store_names(item.optional_vars, self._current_scope())
|
|
262
|
+
for statement in node.body:
|
|
263
|
+
self.visit(statement)
|
|
264
|
+
|
|
265
|
+
visit_AsyncWith = visit_With
|
|
266
|
+
|
|
267
|
+
def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None:
|
|
268
|
+
if node.type:
|
|
269
|
+
self.visit(node.type)
|
|
270
|
+
if node.name:
|
|
271
|
+
self._current_scope().add(node.name)
|
|
272
|
+
for statement in node.body:
|
|
273
|
+
self.visit(statement)
|
|
274
|
+
|
|
275
|
+
def visit_ListComp(self, node: ast.ListComp) -> None:
|
|
276
|
+
self._visit_comprehension(node.generators, [node.elt])
|
|
277
|
+
|
|
278
|
+
def visit_SetComp(self, node: ast.SetComp) -> None:
|
|
279
|
+
self._visit_comprehension(node.generators, [node.elt])
|
|
280
|
+
|
|
281
|
+
def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
|
|
282
|
+
self._visit_comprehension(node.generators, [node.elt])
|
|
283
|
+
|
|
284
|
+
def visit_DictComp(self, node: ast.DictComp) -> None:
|
|
285
|
+
self._visit_comprehension(node.generators, [node.key, node.value])
|
|
286
|
+
|
|
287
|
+
def _visit_comprehension(
|
|
288
|
+
self,
|
|
289
|
+
generators: list[ast.comprehension],
|
|
290
|
+
value_exprs: list[ast.AST],
|
|
291
|
+
) -> None:
|
|
292
|
+
"""Model real Python 3 comprehension scoping.
|
|
293
|
+
|
|
294
|
+
A comprehension is a genuine nested scope: its loop variables do not
|
|
295
|
+
leak into the enclosing function/module scope (this was a Python 2
|
|
296
|
+
behavior removed in Python 3). The single exception is the iterable
|
|
297
|
+
of the *first* `for` clause, which is evaluated in the enclosing
|
|
298
|
+
scope before the comprehension's own scope is entered -- this is
|
|
299
|
+
why `[x for x in undefined_name]` reports `undefined_name` as
|
|
300
|
+
undefined at the call site, not inside the comprehension.
|
|
301
|
+
"""
|
|
302
|
+
if not generators:
|
|
303
|
+
return
|
|
304
|
+
|
|
305
|
+
# First iterable: evaluated in the *current* (enclosing) scope.
|
|
306
|
+
self.visit(generators[0].iter)
|
|
307
|
+
|
|
308
|
+
self.scopes.append(set())
|
|
309
|
+
self.scope_kinds.append("comprehension")
|
|
310
|
+
try:
|
|
311
|
+
self._extract_store_names(generators[0].target, self._current_scope())
|
|
312
|
+
for if_clause in generators[0].ifs:
|
|
313
|
+
self.visit(if_clause)
|
|
314
|
+
|
|
315
|
+
for gen in generators[1:]:
|
|
316
|
+
# Subsequent iterables execute inside the comprehension's
|
|
317
|
+
# own scope (they can reference earlier loop variables).
|
|
318
|
+
self.visit(gen.iter)
|
|
319
|
+
self._extract_store_names(gen.target, self._current_scope())
|
|
320
|
+
for if_clause in gen.ifs:
|
|
321
|
+
self.visit(if_clause)
|
|
322
|
+
|
|
323
|
+
for expr in value_exprs:
|
|
324
|
+
self.visit(expr)
|
|
325
|
+
finally:
|
|
326
|
+
self.scopes.pop()
|
|
327
|
+
self.scope_kinds.pop()
|
|
328
|
+
|
|
329
|
+
def _extract_store_names(self, node: ast.AST, scope: set[str]) -> None:
|
|
330
|
+
if isinstance(node, ast.Name):
|
|
331
|
+
scope.add(node.id)
|
|
332
|
+
elif isinstance(node, (ast.Tuple, ast.List)):
|
|
333
|
+
for elt in node.elts:
|
|
334
|
+
self._extract_store_names(elt, scope)
|
|
335
|
+
elif isinstance(node, ast.Starred):
|
|
336
|
+
self._extract_store_names(node.value, scope)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
class ImportResolver:
|
|
340
|
+
"""Resolves missing imports by inspecting AST and static module databases."""
|
|
341
|
+
|
|
342
|
+
# Static mapping for standard library members and typing constructs
|
|
343
|
+
_STATIC_SYMBOLS: ClassVar[dict[str, str]] = {
|
|
344
|
+
# pathlib
|
|
345
|
+
"Path": "from pathlib import Path",
|
|
346
|
+
"PurePath": "from pathlib import PurePath",
|
|
347
|
+
"PurePosixPath": "from pathlib import PurePosixPath",
|
|
348
|
+
"PureWindowsPath": "from pathlib import PureWindowsPath",
|
|
349
|
+
# collections
|
|
350
|
+
"defaultdict": "from collections import defaultdict",
|
|
351
|
+
"deque": "from collections import deque",
|
|
352
|
+
"Counter": "from collections import Counter",
|
|
353
|
+
"OrderedDict": "from collections import OrderedDict",
|
|
354
|
+
"namedtuple": "from collections import namedtuple",
|
|
355
|
+
# datetime
|
|
356
|
+
"datetime": "from datetime import datetime",
|
|
357
|
+
"timedelta": "from datetime import timedelta",
|
|
358
|
+
"timezone": "from datetime import timezone",
|
|
359
|
+
"date": "from datetime import date",
|
|
360
|
+
# typing
|
|
361
|
+
"Any": "from typing import Any",
|
|
362
|
+
"Callable": "from typing import Callable",
|
|
363
|
+
"ClassVar": "from typing import ClassVar",
|
|
364
|
+
"Dict": "from typing import Dict",
|
|
365
|
+
"Final": "from typing import Final",
|
|
366
|
+
"Generator": "from typing import Generator",
|
|
367
|
+
"Generic": "from typing import Generic",
|
|
368
|
+
"Iterable": "from typing import Iterable",
|
|
369
|
+
"Iterator": "from typing import Iterator",
|
|
370
|
+
"List": "from typing import List",
|
|
371
|
+
"Literal": "from typing import Literal",
|
|
372
|
+
"Mapping": "from typing import Mapping",
|
|
373
|
+
"Optional": "from typing import Optional",
|
|
374
|
+
"Protocol": "from typing import Protocol",
|
|
375
|
+
"Sequence": "from typing import Sequence",
|
|
376
|
+
"Set": "from typing import Set",
|
|
377
|
+
"Tuple": "from typing import Tuple",
|
|
378
|
+
"Type": "from typing import Type",
|
|
379
|
+
"TypeVar": "from typing import TypeVar",
|
|
380
|
+
"TypedDict": "from typing import TypedDict",
|
|
381
|
+
"Union": "from typing import Union",
|
|
382
|
+
"cast": "from typing import cast",
|
|
383
|
+
"overload": "from typing import overload",
|
|
384
|
+
"TYPE_CHECKING": "from typing import TYPE_CHECKING",
|
|
385
|
+
"Annotated": "from typing import Annotated",
|
|
386
|
+
"ParamSpec": "from typing import ParamSpec",
|
|
387
|
+
"Concatenate": "from typing import Concatenate",
|
|
388
|
+
# typing_extensions / 3.11+
|
|
389
|
+
"Self": "from typing import Self",
|
|
390
|
+
"assert_never": "from typing import assert_never",
|
|
391
|
+
# dataclasses
|
|
392
|
+
"dataclass": "from dataclasses import dataclass",
|
|
393
|
+
"field": "from dataclasses import field",
|
|
394
|
+
"asdict": "from dataclasses import asdict",
|
|
395
|
+
"astuple": "from dataclasses import astuple",
|
|
396
|
+
# enum
|
|
397
|
+
"Enum": "from enum import Enum",
|
|
398
|
+
"IntEnum": "from enum import IntEnum",
|
|
399
|
+
"StrEnum": "from enum import StrEnum",
|
|
400
|
+
"Flag": "from enum import Flag",
|
|
401
|
+
"IntFlag": "from enum import IntFlag",
|
|
402
|
+
"auto": "from enum import auto",
|
|
403
|
+
# functools
|
|
404
|
+
"lru_cache": "from functools import lru_cache",
|
|
405
|
+
"cache": "from functools import cache",
|
|
406
|
+
"partial": "from functools import partial",
|
|
407
|
+
"reduce": "from functools import reduce",
|
|
408
|
+
"wraps": "from functools import wraps",
|
|
409
|
+
"total_ordering": "from functools import total_ordering",
|
|
410
|
+
# itertools
|
|
411
|
+
"chain": "from itertools import chain",
|
|
412
|
+
"cycle": "from itertools import cycle",
|
|
413
|
+
"islice": "from itertools import islice",
|
|
414
|
+
"repeat": "from itertools import repeat",
|
|
415
|
+
"accumulate": "from itertools import accumulate",
|
|
416
|
+
"groupby": "from itertools import groupby",
|
|
417
|
+
"product": "from itertools import product",
|
|
418
|
+
"permutations": "from itertools import permutations",
|
|
419
|
+
"combinations": "from itertools import combinations",
|
|
420
|
+
"combinations_with_replacement": "from itertools import combinations_with_replacement",
|
|
421
|
+
# contextlib
|
|
422
|
+
"contextmanager": "from contextlib import contextmanager",
|
|
423
|
+
"asynccontextmanager": "from contextlib import asynccontextmanager",
|
|
424
|
+
"suppress": "from contextlib import suppress",
|
|
425
|
+
"nullcontext": "from contextlib import nullcontext",
|
|
426
|
+
"closing": "from contextlib import closing",
|
|
427
|
+
# abc
|
|
428
|
+
"ABC": "from abc import ABC",
|
|
429
|
+
"abstractmethod": "from abc import abstractmethod",
|
|
430
|
+
"abstractproperty": "from abc import abstractproperty",
|
|
431
|
+
# operator
|
|
432
|
+
"itemgetter": "from operator import itemgetter",
|
|
433
|
+
"attrgetter": "from operator import attrgetter",
|
|
434
|
+
# copy
|
|
435
|
+
"deepcopy": "from copy import deepcopy",
|
|
436
|
+
# pprint
|
|
437
|
+
"pprint": "from pprint import pprint",
|
|
438
|
+
"pformat": "from pprint import pformat",
|
|
439
|
+
# urllib.parse
|
|
440
|
+
"urlparse": "from urllib.parse import urlparse",
|
|
441
|
+
"urlunparse": "from urllib.parse import urlunparse",
|
|
442
|
+
"quote": "from urllib.parse import quote",
|
|
443
|
+
"unquote": "from urllib.parse import unquote",
|
|
444
|
+
# pydantic
|
|
445
|
+
"BaseModel": "from pydantic import BaseModel",
|
|
446
|
+
"Field": "from pydantic import Field",
|
|
447
|
+
"validator": "from pydantic import validator",
|
|
448
|
+
"field_validator": "from pydantic import field_validator",
|
|
449
|
+
# fastapi
|
|
450
|
+
"FastAPI": "from fastapi import FastAPI",
|
|
451
|
+
"APIRouter": "from fastapi import APIRouter",
|
|
452
|
+
"Depends": "from fastapi import Depends",
|
|
453
|
+
"HTTPException": "from fastapi import HTTPException",
|
|
454
|
+
# Popular aliases & data science
|
|
455
|
+
"np": "import numpy as np",
|
|
456
|
+
"pd": "import pandas as pd",
|
|
457
|
+
"DataFrame": "from pandas import DataFrame",
|
|
458
|
+
"Series": "from pandas import Series",
|
|
459
|
+
"plt": "import matplotlib.pyplot as plt",
|
|
460
|
+
"sns": "import seaborn as sns",
|
|
461
|
+
"tf": "import tensorflow as tf",
|
|
462
|
+
"nn": "from torch import nn",
|
|
463
|
+
# concurrent.futures
|
|
464
|
+
"ThreadPoolExecutor": "from concurrent.futures import ThreadPoolExecutor",
|
|
465
|
+
"ProcessPoolExecutor": "from concurrent.futures import ProcessPoolExecutor",
|
|
466
|
+
"as_completed": "from concurrent.futures import as_completed",
|
|
467
|
+
"Future": "from concurrent.futures import Future",
|
|
468
|
+
# threading
|
|
469
|
+
"Thread": "from threading import Thread",
|
|
470
|
+
"Lock": "from threading import Lock",
|
|
471
|
+
"RLock": "from threading import RLock",
|
|
472
|
+
"Event": "from threading import Event",
|
|
473
|
+
"Semaphore": "from threading import Semaphore",
|
|
474
|
+
"BoundedSemaphore": "from threading import BoundedSemaphore",
|
|
475
|
+
"Condition": "from threading import Condition",
|
|
476
|
+
"Barrier": "from threading import Barrier",
|
|
477
|
+
"local": "from threading import local",
|
|
478
|
+
# queue
|
|
479
|
+
"Queue": "from queue import Queue",
|
|
480
|
+
"LifoQueue": "from queue import LifoQueue",
|
|
481
|
+
"PriorityQueue": "from queue import PriorityQueue",
|
|
482
|
+
"Empty": "from queue import Empty",
|
|
483
|
+
"Full": "from queue import Full",
|
|
484
|
+
# re
|
|
485
|
+
"Pattern": "from re import Pattern",
|
|
486
|
+
"Match": "from re import Match",
|
|
487
|
+
# http.server
|
|
488
|
+
"HTTPServer": "from http.server import HTTPServer",
|
|
489
|
+
"BaseHTTPRequestHandler": "from http.server import BaseHTTPRequestHandler",
|
|
490
|
+
"SimpleHTTPRequestHandler": "from http.server import SimpleHTTPRequestHandler",
|
|
491
|
+
# http
|
|
492
|
+
"HTTPStatus": "from http import HTTPStatus",
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
_COMMON_STDLIB_EXPORTS: ClassVar[dict[str, str]] = {
|
|
496
|
+
"sqrt": "from math import sqrt",
|
|
497
|
+
"ceil": "from math import ceil",
|
|
498
|
+
"floor": "from math import floor",
|
|
499
|
+
"sin": "from math import sin",
|
|
500
|
+
"cos": "from math import cos",
|
|
501
|
+
"tan": "from math import tan",
|
|
502
|
+
"log": "from math import log",
|
|
503
|
+
"exp": "from math import exp",
|
|
504
|
+
"pi": "from math import pi",
|
|
505
|
+
"choice": "from random import choice",
|
|
506
|
+
"randint": "from random import randint",
|
|
507
|
+
"shuffle": "from random import shuffle",
|
|
508
|
+
"sample": "from random import sample",
|
|
509
|
+
"dumps": "from json import dumps",
|
|
510
|
+
"loads": "from json import loads",
|
|
511
|
+
"sleep": "from time import sleep",
|
|
512
|
+
"perf_counter": "from time import perf_counter",
|
|
513
|
+
"uuid4": "from uuid import uuid4",
|
|
514
|
+
"sha256": "from hashlib import sha256",
|
|
515
|
+
"md5": "from hashlib import md5",
|
|
516
|
+
"NamedTemporaryFile": "from tempfile import NamedTemporaryFile",
|
|
517
|
+
"TemporaryDirectory": "from tempfile import TemporaryDirectory",
|
|
518
|
+
"copytree": "from shutil import copytree",
|
|
519
|
+
"rmtree": "from shutil import rmtree",
|
|
520
|
+
"format_exc": "from traceback import format_exc",
|
|
521
|
+
"print_exc": "from traceback import print_exc",
|
|
522
|
+
"Popen": "from subprocess import Popen",
|
|
523
|
+
"PIPE": "from subprocess import PIPE",
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
def __init__(
|
|
527
|
+
self,
|
|
528
|
+
custom_import_map: dict[str, str] | None = None,
|
|
529
|
+
auto_add_future_annotations: bool = False,
|
|
530
|
+
) -> None:
|
|
531
|
+
"""
|
|
532
|
+
Args:
|
|
533
|
+
custom_import_map: Extra/override symbol -> import-statement
|
|
534
|
+
mappings, applied before the built-in static registry.
|
|
535
|
+
auto_add_future_annotations: When True, always add
|
|
536
|
+
`from __future__ import annotations` even if no
|
|
537
|
+
TYPE_CHECKING-only symbols require it (PyCleanerConfig's
|
|
538
|
+
auto_add_future_annotations). It is always added regardless
|
|
539
|
+
of this flag when a TYPE_CHECKING-only import is injected,
|
|
540
|
+
since deferred annotation evaluation is required for that
|
|
541
|
+
pattern to work at all.
|
|
542
|
+
"""
|
|
543
|
+
self.stdlib_names: set[str] = set(sys.stdlib_module_names)
|
|
544
|
+
self.custom_import_map: dict[str, str] = (
|
|
545
|
+
dict(custom_import_map) if custom_import_map else {}
|
|
546
|
+
)
|
|
547
|
+
self.auto_add_future_annotations = auto_add_future_annotations
|
|
548
|
+
|
|
549
|
+
def find_undefined(self, source: str, filename: str = "<unknown>") -> list[str]:
|
|
550
|
+
"""Parse source into AST and return list of undefined symbol names."""
|
|
551
|
+
try:
|
|
552
|
+
tree = ast.parse(source, filename=filename)
|
|
553
|
+
except SyntaxError:
|
|
554
|
+
return []
|
|
555
|
+
|
|
556
|
+
finder = UndefinedSymbolFinder()
|
|
557
|
+
finder.visit(tree)
|
|
558
|
+
return sorted(finder.undefined_names)
|
|
559
|
+
|
|
560
|
+
def _classify_undefined_symbol(
|
|
561
|
+
self,
|
|
562
|
+
symbol: str,
|
|
563
|
+
ctx: _ClassificationContext,
|
|
564
|
+
) -> None:
|
|
565
|
+
import_stmt, mod_name = self._get_import_info(symbol)
|
|
566
|
+
if not import_stmt:
|
|
567
|
+
ctx.unresolved.append(symbol)
|
|
568
|
+
ctx.diagnostics.append(
|
|
569
|
+
MissingImportDiagnostic(
|
|
570
|
+
symbol=symbol,
|
|
571
|
+
import_statement="",
|
|
572
|
+
value="missing-import",
|
|
573
|
+
module="",
|
|
574
|
+
status="unresolved",
|
|
575
|
+
)
|
|
576
|
+
)
|
|
577
|
+
return
|
|
578
|
+
|
|
579
|
+
ctx.diagnostics.append(
|
|
580
|
+
MissingImportDiagnostic(
|
|
581
|
+
symbol=symbol,
|
|
582
|
+
import_statement=import_stmt,
|
|
583
|
+
value="missing-import",
|
|
584
|
+
module=mod_name,
|
|
585
|
+
status="resolved",
|
|
586
|
+
)
|
|
587
|
+
)
|
|
588
|
+
is_tc_only = (
|
|
589
|
+
symbol in ctx.type_checking_symbols
|
|
590
|
+
and mod_name != "typing"
|
|
591
|
+
and not import_stmt.startswith("from __future__")
|
|
592
|
+
)
|
|
593
|
+
target = ctx.tc_imports if is_tc_only else ctx.regular_imports
|
|
594
|
+
if import_stmt not in ctx.existing_lines and import_stmt not in target:
|
|
595
|
+
target.append(import_stmt)
|
|
596
|
+
|
|
597
|
+
def _resolve_symbols(
|
|
598
|
+
self,
|
|
599
|
+
undefined: list[str],
|
|
600
|
+
finder: UndefinedSymbolFinder,
|
|
601
|
+
source: str,
|
|
602
|
+
) -> _ClassificationContext:
|
|
603
|
+
ctx = _ClassificationContext(
|
|
604
|
+
type_checking_symbols=finder.annotation_undefined_names
|
|
605
|
+
- finder.runtime_undefined_names,
|
|
606
|
+
existing_lines={line.strip() for line in source.splitlines()},
|
|
607
|
+
regular_imports=[],
|
|
608
|
+
tc_imports=[],
|
|
609
|
+
unresolved=[],
|
|
610
|
+
diagnostics=[],
|
|
611
|
+
)
|
|
612
|
+
for symbol in undefined:
|
|
613
|
+
self._classify_undefined_symbol(symbol, ctx)
|
|
614
|
+
|
|
615
|
+
if ctx.tc_imports:
|
|
616
|
+
tc_stmt = "from typing import TYPE_CHECKING"
|
|
617
|
+
if (
|
|
618
|
+
"TYPE_CHECKING" not in finder.builtin_names
|
|
619
|
+
and tc_stmt not in ctx.existing_lines
|
|
620
|
+
and tc_stmt not in ctx.regular_imports
|
|
621
|
+
):
|
|
622
|
+
ctx.regular_imports.append(tc_stmt)
|
|
623
|
+
return ctx
|
|
624
|
+
|
|
625
|
+
def resolve(
|
|
626
|
+
self, source: str, filename: str = "<unknown>"
|
|
627
|
+
) -> ImportResolutionResult:
|
|
628
|
+
"""Find undefined symbols in source and inject corresponding imports."""
|
|
629
|
+
try:
|
|
630
|
+
tree = ast.parse(source, filename=filename)
|
|
631
|
+
except SyntaxError:
|
|
632
|
+
return ImportResolutionResult(code=source)
|
|
633
|
+
|
|
634
|
+
finder = UndefinedSymbolFinder()
|
|
635
|
+
finder.visit(tree)
|
|
636
|
+
undefined = sorted(finder.undefined_names)
|
|
637
|
+
if not undefined:
|
|
638
|
+
return ImportResolutionResult(code=source)
|
|
639
|
+
|
|
640
|
+
ctx = self._resolve_symbols(undefined, finder, source)
|
|
641
|
+
all_new = ctx.regular_imports + ctx.tc_imports
|
|
642
|
+
if not all_new:
|
|
643
|
+
return ImportResolutionResult(
|
|
644
|
+
code=source,
|
|
645
|
+
unresolved_symbols=ctx.unresolved,
|
|
646
|
+
diagnostics=ctx.diagnostics,
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
updated_code = self._inject_imports(
|
|
650
|
+
source,
|
|
651
|
+
regular_imports=ctx.regular_imports,
|
|
652
|
+
type_checking_imports=ctx.tc_imports,
|
|
653
|
+
add_future_annotations=self.auto_add_future_annotations
|
|
654
|
+
or bool(ctx.tc_imports),
|
|
655
|
+
parsed_tree=tree,
|
|
656
|
+
)
|
|
657
|
+
return ImportResolutionResult(
|
|
658
|
+
code=updated_code,
|
|
659
|
+
resolved_imports=all_new,
|
|
660
|
+
unresolved_symbols=ctx.unresolved,
|
|
661
|
+
diagnostics=ctx.diagnostics,
|
|
662
|
+
)
|
|
663
|
+
|
|
664
|
+
_KNOWN_THIRD_PARTY_ROOTS: ClassVar[frozenset[str]] = frozenset(
|
|
665
|
+
{
|
|
666
|
+
"requests",
|
|
667
|
+
"httpx",
|
|
668
|
+
"pytest",
|
|
669
|
+
"yaml",
|
|
670
|
+
"torch",
|
|
671
|
+
"scipy",
|
|
672
|
+
"click",
|
|
673
|
+
"typer",
|
|
674
|
+
"rich",
|
|
675
|
+
"pydantic",
|
|
676
|
+
"fastapi",
|
|
677
|
+
"uvicorn",
|
|
678
|
+
"jinja2",
|
|
679
|
+
"PIL",
|
|
680
|
+
"cv2",
|
|
681
|
+
"dotenv",
|
|
682
|
+
"sklearn",
|
|
683
|
+
}
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
def _get_custom_import_info(self, symbol: str) -> tuple[str, str] | None:
|
|
687
|
+
if not (self.custom_import_map and symbol in self.custom_import_map):
|
|
688
|
+
return None
|
|
689
|
+
stmt = self.custom_import_map[symbol]
|
|
690
|
+
if stmt.startswith("from "):
|
|
691
|
+
mod = stmt.split("import")[0].replace("from", "").strip()
|
|
692
|
+
else:
|
|
693
|
+
mod = stmt.replace("import", "").strip().split()[0]
|
|
694
|
+
return stmt, mod
|
|
695
|
+
|
|
696
|
+
def _get_import_info(self, symbol: str) -> tuple[str | None, str]:
|
|
697
|
+
"""Determine the import statement and origin module for a given undefined symbol."""
|
|
698
|
+
custom = self._get_custom_import_info(symbol)
|
|
699
|
+
if custom is not None:
|
|
700
|
+
return custom
|
|
701
|
+
|
|
702
|
+
catalog_stmt = self._STATIC_SYMBOLS.get(
|
|
703
|
+
symbol
|
|
704
|
+
) or self._COMMON_STDLIB_EXPORTS.get(symbol)
|
|
705
|
+
if catalog_stmt:
|
|
706
|
+
mod = catalog_stmt.split("import")[0].replace("from", "").strip()
|
|
707
|
+
return catalog_stmt, mod
|
|
708
|
+
|
|
709
|
+
if symbol in self.stdlib_names or symbol in self._KNOWN_THIRD_PARTY_ROOTS:
|
|
710
|
+
return f"import {symbol}", symbol
|
|
711
|
+
|
|
712
|
+
return None, ""
|
|
713
|
+
|
|
714
|
+
def _get_import_statement(self, symbol: str) -> str | None:
|
|
715
|
+
"""Determine the import statement for a given undefined symbol."""
|
|
716
|
+
stmt, _ = self._get_import_info(symbol)
|
|
717
|
+
return stmt
|
|
718
|
+
|
|
719
|
+
def _skip_pragmas_and_comments(self, lines: list[str]) -> int:
|
|
720
|
+
idx = 0
|
|
721
|
+
n = len(lines)
|
|
722
|
+
if idx < n and lines[idx].startswith("#!"):
|
|
723
|
+
idx += 1
|
|
724
|
+
if idx < n and ("coding:" in lines[idx] or "coding=" in lines[idx]):
|
|
725
|
+
idx += 1
|
|
726
|
+
while idx < n and (
|
|
727
|
+
not lines[idx].strip() or lines[idx].strip().startswith("#")
|
|
728
|
+
):
|
|
729
|
+
if "from __future__ import" in lines[idx]:
|
|
730
|
+
break
|
|
731
|
+
idx += 1
|
|
732
|
+
return idx
|
|
733
|
+
|
|
734
|
+
def _find_docstring_end_line(
|
|
735
|
+
self, parsed_tree: ast.Module | None, source: str
|
|
736
|
+
) -> int | None:
|
|
737
|
+
tree = parsed_tree
|
|
738
|
+
if tree is None:
|
|
739
|
+
try:
|
|
740
|
+
tree = ast.parse(source)
|
|
741
|
+
except SyntaxError:
|
|
742
|
+
tree = None
|
|
743
|
+
if tree and tree.body:
|
|
744
|
+
first = tree.body[0]
|
|
745
|
+
if (
|
|
746
|
+
isinstance(first, ast.Expr)
|
|
747
|
+
and isinstance(first.value, ast.Constant)
|
|
748
|
+
and isinstance(first.value.value, str)
|
|
749
|
+
):
|
|
750
|
+
return first.value.end_lineno
|
|
751
|
+
return None
|
|
752
|
+
|
|
753
|
+
def _find_header_insert_idx(
|
|
754
|
+
self, lines: list[str], parsed_tree: ast.Module | None, source: str
|
|
755
|
+
) -> int:
|
|
756
|
+
insert_idx = self._skip_pragmas_and_comments(lines)
|
|
757
|
+
doc_end = self._find_docstring_end_line(parsed_tree, source)
|
|
758
|
+
if doc_end is not None:
|
|
759
|
+
insert_idx = max(insert_idx, doc_end)
|
|
760
|
+
return insert_idx
|
|
761
|
+
|
|
762
|
+
def _inject_future_and_regular(
|
|
763
|
+
self,
|
|
764
|
+
lines: list[str],
|
|
765
|
+
insert_idx: int,
|
|
766
|
+
regular_imports: list[str],
|
|
767
|
+
add_future: bool,
|
|
768
|
+
source: str,
|
|
769
|
+
) -> int:
|
|
770
|
+
if add_future and "from __future__ import annotations" not in source:
|
|
771
|
+
lines.insert(insert_idx, "from __future__ import annotations\n\n")
|
|
772
|
+
insert_idx += 1
|
|
773
|
+
|
|
774
|
+
while insert_idx < len(lines):
|
|
775
|
+
stripped = lines[insert_idx].strip()
|
|
776
|
+
if stripped.startswith("from __future__ import") or not stripped:
|
|
777
|
+
insert_idx += 1
|
|
778
|
+
else:
|
|
779
|
+
break
|
|
780
|
+
|
|
781
|
+
if regular_imports:
|
|
782
|
+
import_block = "".join(f"{stmt}\n" for stmt in regular_imports)
|
|
783
|
+
if insert_idx < len(lines) and lines[insert_idx].strip():
|
|
784
|
+
import_block += "\n"
|
|
785
|
+
lines.insert(insert_idx, import_block)
|
|
786
|
+
insert_idx += 1
|
|
787
|
+
return insert_idx
|
|
788
|
+
|
|
789
|
+
def _inject_type_checking(
|
|
790
|
+
self,
|
|
791
|
+
lines: list[str],
|
|
792
|
+
insert_idx: int,
|
|
793
|
+
tc_imports: list[str],
|
|
794
|
+
) -> None:
|
|
795
|
+
current_text = "".join(lines)
|
|
796
|
+
if "if TYPE_CHECKING:" in current_text:
|
|
797
|
+
tc_idx = next(
|
|
798
|
+
(
|
|
799
|
+
i
|
|
800
|
+
for i, line in enumerate(lines)
|
|
801
|
+
if line.strip().startswith("if TYPE_CHECKING:")
|
|
802
|
+
),
|
|
803
|
+
-1,
|
|
804
|
+
)
|
|
805
|
+
if tc_idx != -1:
|
|
806
|
+
tc_block = "".join(f" {stmt}\n" for stmt in tc_imports)
|
|
807
|
+
lines.insert(tc_idx + 1, tc_block)
|
|
808
|
+
return
|
|
809
|
+
tc_block = (
|
|
810
|
+
"if TYPE_CHECKING:\n"
|
|
811
|
+
+ "".join(f" {stmt}\n" for stmt in tc_imports)
|
|
812
|
+
+ "\n"
|
|
813
|
+
)
|
|
814
|
+
lines.insert(insert_idx, tc_block)
|
|
815
|
+
|
|
816
|
+
def _inject_imports(
|
|
817
|
+
self,
|
|
818
|
+
source: str,
|
|
819
|
+
regular_imports: list[str],
|
|
820
|
+
type_checking_imports: list[str] | None = None,
|
|
821
|
+
add_future_annotations: bool = False,
|
|
822
|
+
parsed_tree: ast.Module | None = None,
|
|
823
|
+
) -> str:
|
|
824
|
+
"""Insert import statements after shebangs, encoding pragmas, and module docstring."""
|
|
825
|
+
lines = source.splitlines(keepends=True)
|
|
826
|
+
insert_idx = self._find_header_insert_idx(lines, parsed_tree, source)
|
|
827
|
+
insert_idx = self._inject_future_and_regular(
|
|
828
|
+
lines, insert_idx, regular_imports, add_future_annotations, source
|
|
829
|
+
)
|
|
830
|
+
if type_checking_imports:
|
|
831
|
+
self._inject_type_checking(lines, insert_idx, type_checking_imports)
|
|
832
|
+
return "".join(lines)
|