skylos 1.1.12__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of skylos might be problematic. Click here for more details.
- skylos/analyzer.py +93 -133
- skylos/cli.py +20 -2
- skylos/constants.py +34 -27
- skylos/framework_aware.py +158 -0
- skylos/test_aware.py +66 -0
- skylos/visitor.py +28 -3
- {skylos-1.1.12.dist-info → skylos-1.2.0.dist-info}/METADATA +1 -1
- skylos-1.2.0.dist-info/RECORD +32 -0
- test/test_analyzer.py +223 -195
- test/test_changes_analyzer.py +149 -0
- test/test_constants.py +348 -0
- test/test_framework_aware.py +372 -0
- test/test_test_aware.py +328 -0
- test/test_visitor.py +0 -10
- skylos-1.1.12.dist-info/RECORD +0 -26
- {skylos-1.1.12.dist-info → skylos-1.2.0.dist-info}/WHEEL +0 -0
- {skylos-1.1.12.dist-info → skylos-1.2.0.dist-info}/entry_points.txt +0 -0
- {skylos-1.1.12.dist-info → skylos-1.2.0.dist-info}/top_level.txt +0 -0
skylos/analyzer.py
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
import ast
|
|
2
|
+
import ast
|
|
3
|
+
import sys
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
3
6
|
from pathlib import Path
|
|
4
7
|
from collections import defaultdict
|
|
5
8
|
from skylos.visitor import Visitor
|
|
6
|
-
from skylos.constants import (
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
from skylos.constants import ( DEFAULT_EXCLUDE_FOLDERS, PENALTIES, AUTO_CALLED )
|
|
10
|
+
from skylos.test_aware import TestAwareVisitor
|
|
11
|
+
import os
|
|
12
|
+
import traceback
|
|
13
|
+
from skylos.framework_aware import FrameworkAwareVisitor, detect_framework_usage
|
|
11
14
|
|
|
12
15
|
logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s - %(message)s')
|
|
13
16
|
logger=logging.getLogger('Skylos')
|
|
@@ -131,6 +134,12 @@ class Skylos:
|
|
|
131
134
|
matches = simple_name_lookup.get(simple, [])
|
|
132
135
|
for d in matches:
|
|
133
136
|
d.references += 1
|
|
137
|
+
|
|
138
|
+
for module_name in self.dynamic:
|
|
139
|
+
for def_name, def_obj in self.defs.items():
|
|
140
|
+
if def_obj.name.startswith(f"{module_name}."):
|
|
141
|
+
if def_obj.type in ("function", "method") and not def_obj.simple_name.startswith("_"):
|
|
142
|
+
def_obj.references += 1
|
|
134
143
|
|
|
135
144
|
def _get_base_classes(self, class_name):
|
|
136
145
|
if class_name not in self.defs:
|
|
@@ -143,128 +152,62 @@ class Skylos:
|
|
|
143
152
|
|
|
144
153
|
return []
|
|
145
154
|
|
|
146
|
-
def
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if
|
|
164
|
-
|
|
165
|
-
"test_" in file_str or
|
|
166
|
-
"/test/" in file_str or
|
|
167
|
-
"/tests/" in file_str or
|
|
168
|
-
"\\test\\" in file_str or
|
|
169
|
-
"\\tests\\" in file_str):
|
|
170
|
-
return True
|
|
171
|
-
|
|
172
|
-
return False
|
|
155
|
+
def _apply_penalties(self, def_obj, visitor, framework):
|
|
156
|
+
c = 100
|
|
157
|
+
|
|
158
|
+
if def_obj.simple_name.startswith("_") and not def_obj.simple_name.startswith("__"):
|
|
159
|
+
c -= PENALTIES["private_name"]
|
|
160
|
+
if def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__"):
|
|
161
|
+
c -= PENALTIES["dunder_or_magic"]
|
|
162
|
+
if def_obj.type == "variable" and def_obj.simple_name == "_":
|
|
163
|
+
c -= PENALTIES["underscored_var"]
|
|
164
|
+
if def_obj.in_init and def_obj.type in ("function", "class"):
|
|
165
|
+
c -= PENALTIES["in_init_file"]
|
|
166
|
+
if def_obj.name.split(".")[0] in self.dynamic:
|
|
167
|
+
c -= PENALTIES["dynamic_module"]
|
|
168
|
+
if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
|
|
169
|
+
c -= PENALTIES["test_related"]
|
|
170
|
+
|
|
171
|
+
framework_confidence = detect_framework_usage(def_obj, visitor=framework)
|
|
172
|
+
if framework_confidence is not None:
|
|
173
|
+
c = min(c, framework_confidence)
|
|
173
174
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
175
|
+
if (def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__")):
|
|
176
|
+
c = 0
|
|
177
|
+
|
|
178
|
+
if def_obj.type == "parameter":
|
|
179
|
+
if def_obj.simple_name in ("self", "cls"):
|
|
180
|
+
c = 0
|
|
181
|
+
elif "." in def_obj.name:
|
|
182
|
+
method_name = def_obj.name.split(".")[-2]
|
|
183
|
+
if method_name.startswith("__") and method_name.endswith("__"):
|
|
184
|
+
c = 0
|
|
185
|
+
|
|
186
|
+
if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
|
|
187
|
+
c = 0
|
|
188
|
+
|
|
189
|
+
if (def_obj.type == "import" and def_obj.name.startswith("__future__.") and
|
|
190
|
+
def_obj.simple_name in ("annotations", "absolute_import", "division",
|
|
191
|
+
"print_function", "unicode_literals", "generator_stop")):
|
|
192
|
+
c = 0
|
|
189
193
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
class_simple_name = class_name.split(".")[-1]
|
|
193
|
-
|
|
194
|
-
if (class_simple_name.startswith("Test") or
|
|
195
|
-
class_simple_name.endswith("Test") or
|
|
196
|
-
class_simple_name.endswith("TestCase")):
|
|
197
|
-
return True
|
|
198
|
-
|
|
199
|
-
if (definition.type == "method" and
|
|
200
|
-
(TEST_METHOD_PATTERN.match(definition.simple_name) or
|
|
201
|
-
definition.simple_name in TEST_LIFECYCLE_METHODS)):
|
|
202
|
-
return True
|
|
203
|
-
|
|
204
|
-
# NOT for imports, variables, parameters
|
|
205
|
-
if definition.type in ("function", "method", "class"):
|
|
206
|
-
if self._is_test_file(definition.filename):
|
|
207
|
-
return True
|
|
208
|
-
|
|
209
|
-
if self._has_test_imports(definition.filename):
|
|
210
|
-
return True
|
|
211
|
-
|
|
212
|
-
## check decorators -- test related
|
|
213
|
-
if self._has_test_decorators(definition.filename):
|
|
214
|
-
return True
|
|
215
|
-
|
|
216
|
-
return False
|
|
217
|
-
|
|
194
|
+
def_obj.confidence = max(c, 0)
|
|
195
|
+
|
|
218
196
|
def _apply_heuristics(self):
|
|
219
|
-
class_methods=defaultdict(list)
|
|
197
|
+
class_methods = defaultdict(list)
|
|
220
198
|
for d in self.defs.values():
|
|
221
|
-
if d.type in("method","function") and"." in d.name:
|
|
222
|
-
cls=d.name.rsplit(".",1)[0]
|
|
223
|
-
if cls in self.defs and self.defs[cls].type=="class":
|
|
199
|
+
if d.type in ("method", "function") and "." in d.name:
|
|
200
|
+
cls = d.name.rsplit(".", 1)[0]
|
|
201
|
+
if cls in self.defs and self.defs[cls].type == "class":
|
|
224
202
|
class_methods[cls].append(d)
|
|
225
|
-
|
|
226
|
-
for cls,methods in class_methods.items():
|
|
227
|
-
if self.defs[cls].references>0:
|
|
203
|
+
|
|
204
|
+
for cls, methods in class_methods.items():
|
|
205
|
+
if self.defs[cls].references > 0:
|
|
228
206
|
for m in methods:
|
|
229
|
-
if m.simple_name in AUTO_CALLED:
|
|
230
|
-
|
|
231
|
-
for d in self.defs.values():
|
|
232
|
-
if d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__")):
|
|
233
|
-
d.confidence = 0
|
|
234
|
-
|
|
235
|
-
if d.type == "parameter" and d.simple_name in ("self", "cls"):
|
|
236
|
-
d.confidence = 0
|
|
237
|
-
|
|
238
|
-
if d.type != "parameter" and (d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__"))):
|
|
239
|
-
d.confidence = 0
|
|
240
|
-
|
|
241
|
-
if (d.type == "import" and d.name.startswith("__future__.") and
|
|
242
|
-
d.simple_name in ("annotations", "absolute_import", "division",
|
|
243
|
-
"print_function", "unicode_literals", "generator_stop")):
|
|
244
|
-
d.confidence = 0
|
|
245
|
-
|
|
246
|
-
if (d.simple_name.startswith("_") and
|
|
247
|
-
not d.simple_name.startswith("__") and
|
|
248
|
-
d.simple_name != "_"):
|
|
249
|
-
d.confidence = 0
|
|
250
|
-
|
|
251
|
-
if not d.simple_name.startswith("_") and d.type in ("function", "method", "class"):
|
|
252
|
-
d.confidence = min(d.confidence, 90)
|
|
253
|
-
|
|
254
|
-
if d.in_init and d.type in ("function", "class"):
|
|
255
|
-
d.confidence = min(d.confidence, 85)
|
|
256
|
-
|
|
257
|
-
if d.name.split(".")[0] in self.dynamic:
|
|
258
|
-
d.confidence = min(d.confidence, 60)
|
|
259
|
-
|
|
260
|
-
if d.type == "variable" and d.simple_name == "_":
|
|
261
|
-
d.confidence = 0
|
|
262
|
-
|
|
263
|
-
if self._is_test_related(d):
|
|
264
|
-
d.confidence = 0
|
|
207
|
+
if m.simple_name in AUTO_CALLED: # __init__, __enter__, __exit__
|
|
208
|
+
m.references += 1
|
|
265
209
|
|
|
266
210
|
def analyze(self, path, thr=60, exclude_folders=None):
|
|
267
|
-
|
|
268
211
|
files, root = self._get_python_files(path, exclude_folders)
|
|
269
212
|
|
|
270
213
|
if not files:
|
|
@@ -289,25 +232,28 @@ class Skylos:
|
|
|
289
232
|
|
|
290
233
|
for file in files:
|
|
291
234
|
mod = modmap[file]
|
|
292
|
-
defs, refs, dyn, exports = proc_file(file, mod)
|
|
293
|
-
|
|
294
|
-
|
|
235
|
+
defs, refs, dyn, exports, test_flags, framework_flags = proc_file(file, mod)
|
|
236
|
+
|
|
237
|
+
# apply penalties while we still have the file-specific flags
|
|
238
|
+
for d in defs:
|
|
239
|
+
self._apply_penalties(d, test_flags, framework_flags)
|
|
295
240
|
self.defs[d.name] = d
|
|
241
|
+
|
|
296
242
|
self.refs.extend(refs)
|
|
297
243
|
self.dynamic.update(dyn)
|
|
298
244
|
self.exports[mod].update(exports)
|
|
299
|
-
|
|
245
|
+
|
|
300
246
|
self._mark_refs()
|
|
301
|
-
self._apply_heuristics()
|
|
247
|
+
self._apply_heuristics()
|
|
302
248
|
self._mark_exports()
|
|
303
|
-
|
|
249
|
+
|
|
304
250
|
thr = max(0, thr)
|
|
305
251
|
|
|
306
252
|
unused = []
|
|
307
253
|
for d in self.defs.values():
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
254
|
+
if d.references == 0 and not d.is_exported and d.confidence > 0 and d.confidence >= thr:
|
|
255
|
+
unused.append(d.to_dict())
|
|
256
|
+
|
|
311
257
|
result = {
|
|
312
258
|
"unused_functions": [],
|
|
313
259
|
"unused_imports": [],
|
|
@@ -341,13 +287,27 @@ def proc_file(file_or_args, mod=None):
|
|
|
341
287
|
file = file_or_args
|
|
342
288
|
|
|
343
289
|
try:
|
|
344
|
-
|
|
345
|
-
|
|
290
|
+
source = Path(file).read_text(encoding="utf-8")
|
|
291
|
+
tree = ast.parse(source)
|
|
292
|
+
|
|
293
|
+
tv = TestAwareVisitor(filename=file)
|
|
294
|
+
tv.visit(tree)
|
|
295
|
+
|
|
296
|
+
fv = FrameworkAwareVisitor(filename=file)
|
|
297
|
+
fv.visit(tree)
|
|
298
|
+
|
|
299
|
+
v = Visitor(mod, file)
|
|
346
300
|
v.visit(tree)
|
|
347
|
-
|
|
301
|
+
|
|
302
|
+
return v.defs, v.refs, v.dyn, v.exports, tv, fv
|
|
348
303
|
except Exception as e:
|
|
349
304
|
logger.error(f"{file}: {e}")
|
|
350
|
-
|
|
305
|
+
if os.getenv("SKYLOS_DEBUG"):
|
|
306
|
+
logger.error(traceback.format_exc())
|
|
307
|
+
dummy_visitor = TestAwareVisitor(filename=file)
|
|
308
|
+
dummy_framework_visitor = FrameworkAwareVisitor(filename=file)
|
|
309
|
+
|
|
310
|
+
return [], [], set(), set(), dummy_visitor, dummy_framework_visitor
|
|
351
311
|
|
|
352
312
|
def analyze(path,conf=60, exclude_folders=None):
|
|
353
313
|
return Skylos().analyze(path,conf, exclude_folders)
|
skylos/cli.py
CHANGED
|
@@ -218,6 +218,13 @@ def main() -> None:
|
|
|
218
218
|
action="store_true",
|
|
219
219
|
help="Enable verbose output"
|
|
220
220
|
)
|
|
221
|
+
parser.add_argument(
|
|
222
|
+
"--confidence",
|
|
223
|
+
"-c",
|
|
224
|
+
type=int,
|
|
225
|
+
default=60,
|
|
226
|
+
help="Confidence threshold (0-100). Lower values include more items. Default: 60"
|
|
227
|
+
)
|
|
221
228
|
parser.add_argument(
|
|
222
229
|
"--interactive", "-i",
|
|
223
230
|
action="store_true",
|
|
@@ -293,7 +300,7 @@ def main() -> None:
|
|
|
293
300
|
logger.info(f"{Colors.GREEN}📁 No folders excluded{Colors.RESET}")
|
|
294
301
|
|
|
295
302
|
try:
|
|
296
|
-
result_json = skylos.analyze(args.path, exclude_folders=list(final_exclude_folders))
|
|
303
|
+
result_json = skylos.analyze(args.path, conf=args.confidence, exclude_folders=list(final_exclude_folders))
|
|
297
304
|
result = json.loads(result_json)
|
|
298
305
|
|
|
299
306
|
except Exception as e:
|
|
@@ -308,6 +315,7 @@ def main() -> None:
|
|
|
308
315
|
unused_imports = result.get("unused_imports", [])
|
|
309
316
|
unused_parameters = result.get("unused_parameters", [])
|
|
310
317
|
unused_variables = result.get("unused_variables", [])
|
|
318
|
+
unused_classes = result.get("unused_classes", [])
|
|
311
319
|
|
|
312
320
|
logger.info(f"{Colors.CYAN}{Colors.BOLD}🔍 Python Static Analysis Results{Colors.RESET}")
|
|
313
321
|
logger.info(f"{Colors.CYAN}{'=' * 35}{Colors.RESET}")
|
|
@@ -317,6 +325,7 @@ def main() -> None:
|
|
|
317
325
|
logger.info(f" • Unused imports: {Colors.YELLOW}{len(unused_imports)}{Colors.RESET}")
|
|
318
326
|
logger.info(f" • Unused parameters: {Colors.YELLOW}{len(unused_parameters)}{Colors.RESET}")
|
|
319
327
|
logger.info(f" • Unused variables: {Colors.YELLOW}{len(unused_variables)}{Colors.RESET}")
|
|
328
|
+
logger.info(f" • Unused classes: {Colors.YELLOW}{len(unused_classes)}{Colors.RESET}")
|
|
320
329
|
|
|
321
330
|
if args.interactive and (unused_functions or unused_imports):
|
|
322
331
|
logger.info(f"\n{Colors.BOLD}Interactive Mode:{Colors.RESET}")
|
|
@@ -402,10 +411,19 @@ def main() -> None:
|
|
|
402
411
|
for i, item in enumerate(unused_variables, 1):
|
|
403
412
|
logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
|
|
404
413
|
logger.info(f" {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
|
|
414
|
+
|
|
415
|
+
if unused_classes:
|
|
416
|
+
logger.info(f"\n{Colors.YELLOW}{Colors.BOLD}📚 Unused Classes{Colors.RESET}")
|
|
417
|
+
logger.info(f"{Colors.YELLOW}{'=' * 18}{Colors.RESET}")
|
|
418
|
+
for i, item in enumerate(unused_classes, 1):
|
|
419
|
+
logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
|
|
420
|
+
logger.info(f" {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
|
|
421
|
+
|
|
405
422
|
else:
|
|
406
423
|
logger.info(f"\n{Colors.GREEN}✓ All variables are being used!{Colors.RESET}")
|
|
407
424
|
|
|
408
|
-
dead_code_count = len(unused_functions) + len(unused_imports)
|
|
425
|
+
dead_code_count = len(unused_functions) + len(unused_imports) + len(unused_variables) + len(unused_classes) + len(unused_parameters)
|
|
426
|
+
|
|
409
427
|
print_badge(dead_code_count, logger)
|
|
410
428
|
|
|
411
429
|
if unused_functions or unused_imports:
|
skylos/constants.py
CHANGED
|
@@ -1,35 +1,42 @@
|
|
|
1
1
|
import re
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
TEST_IMPORT_PATTERNS = {
|
|
12
|
-
"unittest", "unittest.mock", "mock", "pytest", "nose", "nose2",
|
|
13
|
-
"responses", "requests_mock", "freezegun", "factory_boy",
|
|
14
|
-
"hypothesis", "sure", "expects", "testfixtures", "faker"
|
|
4
|
+
PENALTIES = {
|
|
5
|
+
"private_name": 80,
|
|
6
|
+
"dunder_or_magic": 100,
|
|
7
|
+
"underscored_var": 100,
|
|
8
|
+
"in_init_file": 15,
|
|
9
|
+
"dynamic_module": 40,
|
|
10
|
+
"test_related": 100,
|
|
11
|
+
"framework_magic": 40,
|
|
15
12
|
}
|
|
16
13
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
TEST_FILE_RE = re.compile(r"(?:^|[/\\])tests?[/\\]|_test\.py$", re.I)
|
|
15
|
+
TEST_IMPORT_RE = re.compile(r"^(pytest|unittest|nose|mock|responses)(\.|$)")
|
|
16
|
+
TEST_DECOR_RE = re.compile(r"""^(
|
|
17
|
+
pytest\.(fixture|mark) |
|
|
18
|
+
patch(\.|$) |
|
|
19
|
+
responses\.activate |
|
|
20
|
+
freeze_time
|
|
21
|
+
)$""", re.X)
|
|
22
|
+
|
|
23
|
+
AUTO_CALLED = {"__init__", "__enter__", "__exit__"}
|
|
24
|
+
TEST_METHOD_PATTERN = re.compile(r"^test_\w+$")
|
|
25
|
+
|
|
26
|
+
UNITTEST_LIFECYCLE_METHODS = {
|
|
27
|
+
'setUp', 'tearDown', 'setUpClass', 'tearDownClass',
|
|
28
|
+
'setUpModule', 'tearDownModule'
|
|
20
29
|
}
|
|
21
30
|
|
|
31
|
+
FRAMEWORK_FILE_RE = re.compile(r"(?:views|handlers|endpoints|routes|api)\.py$", re.I)
|
|
32
|
+
|
|
22
33
|
DEFAULT_EXCLUDE_FOLDERS = {
|
|
23
|
-
"__pycache__",
|
|
24
|
-
".
|
|
25
|
-
".pytest_cache",
|
|
26
|
-
".mypy_cache",
|
|
27
|
-
".tox",
|
|
28
|
-
"htmlcov",
|
|
29
|
-
".coverage",
|
|
30
|
-
"build",
|
|
31
|
-
"dist",
|
|
32
|
-
"*.egg-info",
|
|
33
|
-
"venv",
|
|
34
|
-
".venv"
|
|
34
|
+
"__pycache__", ".git", ".pytest_cache", ".mypy_cache", ".tox",
|
|
35
|
+
"htmlcov", ".coverage", "build", "dist", "*.egg-info", "venv", ".venv"
|
|
35
36
|
}
|
|
37
|
+
|
|
38
|
+
def is_test_path(p: Path | str) -> bool:
|
|
39
|
+
return bool(TEST_FILE_RE.search(str(p)))
|
|
40
|
+
|
|
41
|
+
def is_framework_path(p: Path | str) -> bool:
|
|
42
|
+
return bool(FRAMEWORK_FILE_RE.search(str(p)))
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import fnmatch
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
# decorator patterns for each of the more popular frameworks
|
|
6
|
+
FRAMEWORK_DECORATORS = [
|
|
7
|
+
# flask
|
|
8
|
+
"@app.route", "@app.get", "@app.post", "@app.put", "@app.delete",
|
|
9
|
+
"@app.patch", "@app.before_request", "@app.after_request",
|
|
10
|
+
"@app.errorhandler", "@app.teardown_*",
|
|
11
|
+
|
|
12
|
+
# fastapi
|
|
13
|
+
"@router.get", "@router.post", "@router.put", "@router.delete",
|
|
14
|
+
"@router.patch", "@router.head", "@router.options",
|
|
15
|
+
"@router.trace", "@router.websocket",
|
|
16
|
+
|
|
17
|
+
# django
|
|
18
|
+
"@*_required", "@login_required", "@permission_required",
|
|
19
|
+
"@user_passes_test", "@staff_member_required",
|
|
20
|
+
|
|
21
|
+
# pydantic
|
|
22
|
+
"@validator", "@field_validator", "@model_validator", "@root_validator",
|
|
23
|
+
|
|
24
|
+
# celery patterns
|
|
25
|
+
"@task", "@shared_task", "@periodic_task", "@celery_task",
|
|
26
|
+
|
|
27
|
+
# generic ones
|
|
28
|
+
"@route", "@get", "@post", "@put", "@delete", "@patch",
|
|
29
|
+
"@middleware", "@depends", "@inject"
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
FRAMEWORK_FUNCTIONS = [
|
|
33
|
+
# django view methods
|
|
34
|
+
"get", "post", "put", "patch", "delete", "head", "options", "trace",
|
|
35
|
+
"*_queryset", "get_queryset", "get_object", "get_context_data",
|
|
36
|
+
"*_form", "form_valid", "form_invalid", "get_form_*",
|
|
37
|
+
|
|
38
|
+
# django model methods
|
|
39
|
+
"save", "delete", "clean", "full_clean", "*_delete", "*_save",
|
|
40
|
+
|
|
41
|
+
# generic API endpoints
|
|
42
|
+
"create_*", "update_*", "delete_*", "list_*", "retrieve_*",
|
|
43
|
+
"handle_*", "process_*", "*_handler", "*_view"
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
FRAMEWORK_IMPORTS = {
|
|
47
|
+
'flask', 'fastapi', 'django', 'pydantic', 'celery', 'starlette',
|
|
48
|
+
'sanic', 'tornado', 'pyramid', 'bottle', 'cherrypy', 'web2py',
|
|
49
|
+
'falcon', 'hug', 'responder', 'quart', 'hypercorn', 'uvicorn'
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class FrameworkAwareVisitor:
|
|
53
|
+
|
|
54
|
+
def __init__(self, filename=None):
|
|
55
|
+
self.is_framework_file = False
|
|
56
|
+
self.framework_decorated_lines = set()
|
|
57
|
+
self.detected_frameworks = set()
|
|
58
|
+
|
|
59
|
+
if filename:
|
|
60
|
+
self._check_framework_imports_in_file(filename)
|
|
61
|
+
|
|
62
|
+
def visit(self, node):
|
|
63
|
+
method = 'visit_' + node.__class__.__name__
|
|
64
|
+
visitor = getattr(self, method, self.generic_visit)
|
|
65
|
+
return visitor(node)
|
|
66
|
+
|
|
67
|
+
def generic_visit(self, node):
|
|
68
|
+
for field, value in ast.iter_fields(node):
|
|
69
|
+
if isinstance(value, list):
|
|
70
|
+
for item in value:
|
|
71
|
+
if isinstance(item, ast.AST):
|
|
72
|
+
self.visit(item)
|
|
73
|
+
elif isinstance(value, ast.AST):
|
|
74
|
+
self.visit(value)
|
|
75
|
+
|
|
76
|
+
def visit_Import(self, node):
|
|
77
|
+
for alias in node.names:
|
|
78
|
+
if any(fw in alias.name.lower() for fw in FRAMEWORK_IMPORTS):
|
|
79
|
+
self.is_framework_file = True
|
|
80
|
+
self.detected_frameworks.add(alias.name.split('.')[0])
|
|
81
|
+
self.generic_visit(node)
|
|
82
|
+
|
|
83
|
+
def visit_ImportFrom(self, node):
|
|
84
|
+
if node.module:
|
|
85
|
+
module_name = node.module.split('.')[0].lower()
|
|
86
|
+
if module_name in FRAMEWORK_IMPORTS:
|
|
87
|
+
self.is_framework_file = True
|
|
88
|
+
self.detected_frameworks.add(module_name)
|
|
89
|
+
self.generic_visit(node)
|
|
90
|
+
|
|
91
|
+
def visit_FunctionDef(self, node):
|
|
92
|
+
for deco in node.decorator_list:
|
|
93
|
+
decorator_str = self._normalize_decorator(deco)
|
|
94
|
+
if self._matches_framework_pattern(decorator_str, FRAMEWORK_DECORATORS):
|
|
95
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
96
|
+
self.is_framework_file = True
|
|
97
|
+
|
|
98
|
+
if self._matches_framework_pattern(node.name, FRAMEWORK_FUNCTIONS):
|
|
99
|
+
if self.is_framework_file:
|
|
100
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
101
|
+
|
|
102
|
+
self.generic_visit(node)
|
|
103
|
+
|
|
104
|
+
def visit_AsyncFunctionDef(self, node):
|
|
105
|
+
self.visit_FunctionDef(node)
|
|
106
|
+
|
|
107
|
+
def visit_ClassDef(self, node):
|
|
108
|
+
for base in node.bases:
|
|
109
|
+
if isinstance(base, ast.Name):
|
|
110
|
+
if any(pattern in base.id.lower() for pattern in ['view', 'viewset', 'api', 'handler']):
|
|
111
|
+
if self.is_framework_file:
|
|
112
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
113
|
+
|
|
114
|
+
self.generic_visit(node)
|
|
115
|
+
|
|
116
|
+
def _normalize_decorator(self, decorator_node):
|
|
117
|
+
if isinstance(decorator_node, ast.Name):
|
|
118
|
+
return f"@{decorator_node.id}"
|
|
119
|
+
elif isinstance(decorator_node, ast.Attribute):
|
|
120
|
+
if isinstance(decorator_node.value, ast.Name):
|
|
121
|
+
return f"@{decorator_node.value.id}.{decorator_node.attr}"
|
|
122
|
+
else:
|
|
123
|
+
return f"@{decorator_node.attr}"
|
|
124
|
+
elif isinstance(decorator_node, ast.Call):
|
|
125
|
+
return self._normalize_decorator(decorator_node.func)
|
|
126
|
+
return "@unknown"
|
|
127
|
+
|
|
128
|
+
def _matches_framework_pattern(self, text, patterns):
|
|
129
|
+
text_clean = text.lstrip('@')
|
|
130
|
+
return any(fnmatch.fnmatch(text_clean, pattern.lstrip('@')) for pattern in patterns)
|
|
131
|
+
|
|
132
|
+
def _check_framework_imports_in_file(self, filename):
|
|
133
|
+
try:
|
|
134
|
+
content = Path(filename).read_text(encoding='utf-8')
|
|
135
|
+
for framework in FRAMEWORK_IMPORTS:
|
|
136
|
+
if (f'import {framework}' in content or
|
|
137
|
+
f'from {framework}' in content):
|
|
138
|
+
self.is_framework_file = True
|
|
139
|
+
self.detected_frameworks.add(framework)
|
|
140
|
+
break
|
|
141
|
+
except:
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
def detect_framework_usage(definition, decorator_nodes=None, visitor=None):
|
|
145
|
+
if not visitor:
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
# very low confidence - likely framework magic
|
|
149
|
+
if definition.line in visitor.framework_decorated_lines:
|
|
150
|
+
return 20
|
|
151
|
+
|
|
152
|
+
# framework file but no direct markers
|
|
153
|
+
if visitor.is_framework_file:
|
|
154
|
+
if (not definition.simple_name.startswith('_') and
|
|
155
|
+
definition.type in ('function', 'method')):
|
|
156
|
+
return 40
|
|
157
|
+
|
|
158
|
+
return None
|
skylos/test_aware.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
from skylos.constants import TEST_IMPORT_RE, TEST_DECOR_RE, TEST_FILE_RE
|
|
3
|
+
|
|
4
|
+
class TestAwareVisitor:
|
|
5
|
+
|
|
6
|
+
def __init__(self, filename=None):
|
|
7
|
+
self.is_test_file = False
|
|
8
|
+
self.test_decorated_lines = set()
|
|
9
|
+
|
|
10
|
+
# mark as test file based on the file path/name NOT imports
|
|
11
|
+
if filename and TEST_FILE_RE.search(str(filename)):
|
|
12
|
+
self.is_test_file = True
|
|
13
|
+
|
|
14
|
+
def visit(self, node):
|
|
15
|
+
method = 'visit_' + node.__class__.__name__
|
|
16
|
+
visitor = getattr(self, method, self.generic_visit)
|
|
17
|
+
return visitor(node)
|
|
18
|
+
|
|
19
|
+
def generic_visit(self, node):
|
|
20
|
+
for field, value in ast.iter_fields(node):
|
|
21
|
+
if isinstance(value, list):
|
|
22
|
+
for item in value:
|
|
23
|
+
if isinstance(item, ast.AST):
|
|
24
|
+
self.visit(item)
|
|
25
|
+
elif isinstance(value, ast.AST):
|
|
26
|
+
self.visit(value)
|
|
27
|
+
|
|
28
|
+
def visit_Import(self, node):
|
|
29
|
+
|
|
30
|
+
if self.is_test_file:
|
|
31
|
+
for alias in node.names:
|
|
32
|
+
if TEST_IMPORT_RE.match(alias.name):
|
|
33
|
+
pass
|
|
34
|
+
self.generic_visit(node)
|
|
35
|
+
|
|
36
|
+
def visit_ImportFrom(self, node):
|
|
37
|
+
if self.is_test_file:
|
|
38
|
+
if node.module and TEST_IMPORT_RE.match(node.module):
|
|
39
|
+
pass
|
|
40
|
+
self.generic_visit(node)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def visit_FunctionDef(self, node):
|
|
44
|
+
if (node.name.startswith('test_') or
|
|
45
|
+
node.name.endswith('_test') or
|
|
46
|
+
any(node.name.startswith(prefix) for prefix in ['setup', 'teardown']) or
|
|
47
|
+
node.name in ['setUp', 'tearDown', 'setUpClass', 'tearDownClass',
|
|
48
|
+
'setUpModule', 'tearDownModule']):
|
|
49
|
+
self.test_decorated_lines.add(node.lineno)
|
|
50
|
+
|
|
51
|
+
for deco in node.decorator_list:
|
|
52
|
+
name = self._decorator_name(deco)
|
|
53
|
+
if name and (TEST_DECOR_RE.match(name) or 'pytest' in name or 'fixture' in name):
|
|
54
|
+
self.test_decorated_lines.add(node.lineno)
|
|
55
|
+
self.generic_visit(node)
|
|
56
|
+
|
|
57
|
+
def visit_AsyncFunctionDef(self, node):
|
|
58
|
+
self.visit_FunctionDef(node)
|
|
59
|
+
|
|
60
|
+
def _decorator_name(self, deco):
|
|
61
|
+
if isinstance(deco, ast.Name):
|
|
62
|
+
return deco.id
|
|
63
|
+
if isinstance(deco, ast.Attribute):
|
|
64
|
+
parent = self._decorator_name(deco.value)
|
|
65
|
+
return f"{parent}.{deco.attr}" if parent else deco.attr
|
|
66
|
+
return ""
|