skylos 1.1.11__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of skylos might be problematic. Click here for more details.
- skylos/analyzer.py +96 -73
- skylos/cli.py +20 -2
- skylos/constants.py +42 -0
- skylos/framework_aware.py +158 -0
- skylos/test_aware.py +66 -0
- skylos/visitor.py +28 -3
- {skylos-1.1.11.dist-info → skylos-1.2.0.dist-info}/METADATA +1 -1
- skylos-1.2.0.dist-info/RECORD +32 -0
- test/test_analyzer.py +223 -195
- test/test_changes_analyzer.py +149 -0
- test/test_constants.py +348 -0
- test/test_framework_aware.py +372 -0
- test/test_test_aware.py +328 -0
- test/test_visitor.py +0 -10
- skylos-1.1.11.dist-info/RECORD +0 -25
- {skylos-1.1.11.dist-info → skylos-1.2.0.dist-info}/WHEEL +0 -0
- {skylos-1.1.11.dist-info → skylos-1.2.0.dist-info}/entry_points.txt +0 -0
- {skylos-1.1.11.dist-info → skylos-1.2.0.dist-info}/top_level.txt +0 -0
skylos/analyzer.py
CHANGED
|
@@ -1,31 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
import ast
|
|
2
|
+
import ast
|
|
3
|
+
import sys
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
3
6
|
from pathlib import Path
|
|
4
7
|
from collections import defaultdict
|
|
5
8
|
from skylos.visitor import Visitor
|
|
9
|
+
from skylos.constants import ( DEFAULT_EXCLUDE_FOLDERS, PENALTIES, AUTO_CALLED )
|
|
10
|
+
from skylos.test_aware import TestAwareVisitor
|
|
11
|
+
import os
|
|
12
|
+
import traceback
|
|
13
|
+
from skylos.framework_aware import FrameworkAwareVisitor, detect_framework_usage
|
|
6
14
|
|
|
7
15
|
logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s - %(message)s')
|
|
8
16
|
logger=logging.getLogger('Skylos')
|
|
9
17
|
|
|
10
|
-
AUTO_CALLED={"__init__","__enter__","__exit__"}
|
|
11
|
-
TEST_METHOD_PATTERN = re.compile(r"^test_\w+$")
|
|
12
|
-
MAGIC_METHODS={f"__{n}__"for n in["init","new","call","getattr","getattribute","enter","exit","str","repr","hash","eq","ne","lt","gt","le","ge","iter","next","contains","len","getitem","setitem","delitem","iadd","isub","imul","itruediv","ifloordiv","imod","ipow","ilshift","irshift","iand","ixor","ior","round","format","dir","abs","complex","int","float","bool","bytes","reduce","await","aiter","anext","add","sub","mul","truediv","floordiv","mod","divmod","pow","lshift","rshift","and","or","xor","radd","rsub","rmul","rtruediv","rfloordiv","rmod","rdivmod","rpow","rlshift","rrshift","rand","ror","rxor"]}
|
|
13
|
-
|
|
14
|
-
DEFAULT_EXCLUDE_FOLDERS = {
|
|
15
|
-
"__pycache__",
|
|
16
|
-
".git",
|
|
17
|
-
".pytest_cache",
|
|
18
|
-
".mypy_cache",
|
|
19
|
-
".tox",
|
|
20
|
-
"htmlcov",
|
|
21
|
-
".coverage",
|
|
22
|
-
"build",
|
|
23
|
-
"dist",
|
|
24
|
-
"*.egg-info",
|
|
25
|
-
"venv",
|
|
26
|
-
".venv"
|
|
27
|
-
}
|
|
28
|
-
|
|
29
18
|
def parse_exclude_folders(user_exclude_folders, use_defaults=True, include_folders=None):
|
|
30
19
|
exclude_set = set()
|
|
31
20
|
|
|
@@ -145,6 +134,12 @@ class Skylos:
|
|
|
145
134
|
matches = simple_name_lookup.get(simple, [])
|
|
146
135
|
for d in matches:
|
|
147
136
|
d.references += 1
|
|
137
|
+
|
|
138
|
+
for module_name in self.dynamic:
|
|
139
|
+
for def_name, def_obj in self.defs.items():
|
|
140
|
+
if def_obj.name.startswith(f"{module_name}."):
|
|
141
|
+
if def_obj.type in ("function", "method") and not def_obj.simple_name.startswith("_"):
|
|
142
|
+
def_obj.references += 1
|
|
148
143
|
|
|
149
144
|
def _get_base_classes(self, class_name):
|
|
150
145
|
if class_name not in self.defs:
|
|
@@ -156,52 +151,63 @@ class Skylos:
|
|
|
156
151
|
return class_def.base_classes
|
|
157
152
|
|
|
158
153
|
return []
|
|
159
|
-
|
|
154
|
+
|
|
155
|
+
def _apply_penalties(self, def_obj, visitor, framework):
|
|
156
|
+
c = 100
|
|
157
|
+
|
|
158
|
+
if def_obj.simple_name.startswith("_") and not def_obj.simple_name.startswith("__"):
|
|
159
|
+
c -= PENALTIES["private_name"]
|
|
160
|
+
if def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__"):
|
|
161
|
+
c -= PENALTIES["dunder_or_magic"]
|
|
162
|
+
if def_obj.type == "variable" and def_obj.simple_name == "_":
|
|
163
|
+
c -= PENALTIES["underscored_var"]
|
|
164
|
+
if def_obj.in_init and def_obj.type in ("function", "class"):
|
|
165
|
+
c -= PENALTIES["in_init_file"]
|
|
166
|
+
if def_obj.name.split(".")[0] in self.dynamic:
|
|
167
|
+
c -= PENALTIES["dynamic_module"]
|
|
168
|
+
if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
|
|
169
|
+
c -= PENALTIES["test_related"]
|
|
170
|
+
|
|
171
|
+
framework_confidence = detect_framework_usage(def_obj, visitor=framework)
|
|
172
|
+
if framework_confidence is not None:
|
|
173
|
+
c = min(c, framework_confidence)
|
|
174
|
+
|
|
175
|
+
if (def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__")):
|
|
176
|
+
c = 0
|
|
177
|
+
|
|
178
|
+
if def_obj.type == "parameter":
|
|
179
|
+
if def_obj.simple_name in ("self", "cls"):
|
|
180
|
+
c = 0
|
|
181
|
+
elif "." in def_obj.name:
|
|
182
|
+
method_name = def_obj.name.split(".")[-2]
|
|
183
|
+
if method_name.startswith("__") and method_name.endswith("__"):
|
|
184
|
+
c = 0
|
|
185
|
+
|
|
186
|
+
if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
|
|
187
|
+
c = 0
|
|
188
|
+
|
|
189
|
+
if (def_obj.type == "import" and def_obj.name.startswith("__future__.") and
|
|
190
|
+
def_obj.simple_name in ("annotations", "absolute_import", "division",
|
|
191
|
+
"print_function", "unicode_literals", "generator_stop")):
|
|
192
|
+
c = 0
|
|
193
|
+
|
|
194
|
+
def_obj.confidence = max(c, 0)
|
|
195
|
+
|
|
160
196
|
def _apply_heuristics(self):
|
|
161
|
-
class_methods=defaultdict(list)
|
|
197
|
+
class_methods = defaultdict(list)
|
|
162
198
|
for d in self.defs.values():
|
|
163
|
-
if d.type in("method","function") and"." in d.name:
|
|
164
|
-
cls=d.name.rsplit(".",1)[0]
|
|
165
|
-
if cls in self.defs and self.defs[cls].type=="class":
|
|
199
|
+
if d.type in ("method", "function") and "." in d.name:
|
|
200
|
+
cls = d.name.rsplit(".", 1)[0]
|
|
201
|
+
if cls in self.defs and self.defs[cls].type == "class":
|
|
166
202
|
class_methods[cls].append(d)
|
|
167
|
-
|
|
168
|
-
for cls,methods in class_methods.items():
|
|
169
|
-
if self.defs[cls].references>0:
|
|
203
|
+
|
|
204
|
+
for cls, methods in class_methods.items():
|
|
205
|
+
if self.defs[cls].references > 0:
|
|
170
206
|
for m in methods:
|
|
171
|
-
if m.simple_name in AUTO_CALLED:
|
|
172
|
-
|
|
173
|
-
for d in self.defs.values():
|
|
174
|
-
if d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__")):
|
|
175
|
-
d.confidence = 0
|
|
176
|
-
|
|
177
|
-
if d.type == "parameter" and d.simple_name in ("self", "cls"):
|
|
178
|
-
d.confidence = 0
|
|
179
|
-
|
|
180
|
-
if d.type != "parameter" and (d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__"))):
|
|
181
|
-
d.confidence = 0
|
|
182
|
-
|
|
183
|
-
if not d.simple_name.startswith("_") and d.type in ("function", "method", "class"):
|
|
184
|
-
d.confidence = min(d.confidence, 90)
|
|
185
|
-
|
|
186
|
-
if d.in_init and d.type in ("function", "class"):
|
|
187
|
-
d.confidence = min(d.confidence, 85)
|
|
188
|
-
|
|
189
|
-
if d.name.split(".")[0] in self.dynamic:
|
|
190
|
-
d.confidence = min(d.confidence, 60)
|
|
191
|
-
|
|
192
|
-
if d.type == "variable" and d.simple_name == "_":
|
|
193
|
-
d.confidence = 0
|
|
194
|
-
|
|
195
|
-
if d.type == "method" and TEST_METHOD_PATTERN.match(d.simple_name):
|
|
196
|
-
class_name = d.name.rsplit(".", 1)[0]
|
|
197
|
-
class_simple_name = class_name.split(".")[-1]
|
|
198
|
-
if (class_simple_name.startswith("Test") or
|
|
199
|
-
class_simple_name.endswith("Test") or
|
|
200
|
-
class_simple_name.endswith("TestCase")):
|
|
201
|
-
d.confidence = 0
|
|
207
|
+
if m.simple_name in AUTO_CALLED: # __init__, __enter__, __exit__
|
|
208
|
+
m.references += 1
|
|
202
209
|
|
|
203
210
|
def analyze(self, path, thr=60, exclude_folders=None):
|
|
204
|
-
|
|
205
211
|
files, root = self._get_python_files(path, exclude_folders)
|
|
206
212
|
|
|
207
213
|
if not files:
|
|
@@ -226,25 +232,28 @@ class Skylos:
|
|
|
226
232
|
|
|
227
233
|
for file in files:
|
|
228
234
|
mod = modmap[file]
|
|
229
|
-
defs, refs, dyn, exports = proc_file(file, mod)
|
|
230
|
-
|
|
231
|
-
|
|
235
|
+
defs, refs, dyn, exports, test_flags, framework_flags = proc_file(file, mod)
|
|
236
|
+
|
|
237
|
+
# apply penalties while we still have the file-specific flags
|
|
238
|
+
for d in defs:
|
|
239
|
+
self._apply_penalties(d, test_flags, framework_flags)
|
|
232
240
|
self.defs[d.name] = d
|
|
241
|
+
|
|
233
242
|
self.refs.extend(refs)
|
|
234
243
|
self.dynamic.update(dyn)
|
|
235
244
|
self.exports[mod].update(exports)
|
|
236
|
-
|
|
245
|
+
|
|
237
246
|
self._mark_refs()
|
|
238
|
-
self._apply_heuristics()
|
|
247
|
+
self._apply_heuristics()
|
|
239
248
|
self._mark_exports()
|
|
240
|
-
|
|
249
|
+
|
|
241
250
|
thr = max(0, thr)
|
|
242
251
|
|
|
243
252
|
unused = []
|
|
244
253
|
for d in self.defs.values():
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
254
|
+
if d.references == 0 and not d.is_exported and d.confidence > 0 and d.confidence >= thr:
|
|
255
|
+
unused.append(d.to_dict())
|
|
256
|
+
|
|
248
257
|
result = {
|
|
249
258
|
"unused_functions": [],
|
|
250
259
|
"unused_imports": [],
|
|
@@ -278,13 +287,27 @@ def proc_file(file_or_args, mod=None):
|
|
|
278
287
|
file = file_or_args
|
|
279
288
|
|
|
280
289
|
try:
|
|
281
|
-
|
|
282
|
-
|
|
290
|
+
source = Path(file).read_text(encoding="utf-8")
|
|
291
|
+
tree = ast.parse(source)
|
|
292
|
+
|
|
293
|
+
tv = TestAwareVisitor(filename=file)
|
|
294
|
+
tv.visit(tree)
|
|
295
|
+
|
|
296
|
+
fv = FrameworkAwareVisitor(filename=file)
|
|
297
|
+
fv.visit(tree)
|
|
298
|
+
|
|
299
|
+
v = Visitor(mod, file)
|
|
283
300
|
v.visit(tree)
|
|
284
|
-
|
|
301
|
+
|
|
302
|
+
return v.defs, v.refs, v.dyn, v.exports, tv, fv
|
|
285
303
|
except Exception as e:
|
|
286
304
|
logger.error(f"{file}: {e}")
|
|
287
|
-
|
|
305
|
+
if os.getenv("SKYLOS_DEBUG"):
|
|
306
|
+
logger.error(traceback.format_exc())
|
|
307
|
+
dummy_visitor = TestAwareVisitor(filename=file)
|
|
308
|
+
dummy_framework_visitor = FrameworkAwareVisitor(filename=file)
|
|
309
|
+
|
|
310
|
+
return [], [], set(), set(), dummy_visitor, dummy_framework_visitor
|
|
288
311
|
|
|
289
312
|
def analyze(path,conf=60, exclude_folders=None):
|
|
290
313
|
return Skylos().analyze(path,conf, exclude_folders)
|
skylos/cli.py
CHANGED
|
@@ -218,6 +218,13 @@ def main() -> None:
|
|
|
218
218
|
action="store_true",
|
|
219
219
|
help="Enable verbose output"
|
|
220
220
|
)
|
|
221
|
+
parser.add_argument(
|
|
222
|
+
"--confidence",
|
|
223
|
+
"-c",
|
|
224
|
+
type=int,
|
|
225
|
+
default=60,
|
|
226
|
+
help="Confidence threshold (0-100). Lower values include more items. Default: 60"
|
|
227
|
+
)
|
|
221
228
|
parser.add_argument(
|
|
222
229
|
"--interactive", "-i",
|
|
223
230
|
action="store_true",
|
|
@@ -293,7 +300,7 @@ def main() -> None:
|
|
|
293
300
|
logger.info(f"{Colors.GREEN}📁 No folders excluded{Colors.RESET}")
|
|
294
301
|
|
|
295
302
|
try:
|
|
296
|
-
result_json = skylos.analyze(args.path, exclude_folders=list(final_exclude_folders))
|
|
303
|
+
result_json = skylos.analyze(args.path, conf=args.confidence, exclude_folders=list(final_exclude_folders))
|
|
297
304
|
result = json.loads(result_json)
|
|
298
305
|
|
|
299
306
|
except Exception as e:
|
|
@@ -308,6 +315,7 @@ def main() -> None:
|
|
|
308
315
|
unused_imports = result.get("unused_imports", [])
|
|
309
316
|
unused_parameters = result.get("unused_parameters", [])
|
|
310
317
|
unused_variables = result.get("unused_variables", [])
|
|
318
|
+
unused_classes = result.get("unused_classes", [])
|
|
311
319
|
|
|
312
320
|
logger.info(f"{Colors.CYAN}{Colors.BOLD}🔍 Python Static Analysis Results{Colors.RESET}")
|
|
313
321
|
logger.info(f"{Colors.CYAN}{'=' * 35}{Colors.RESET}")
|
|
@@ -317,6 +325,7 @@ def main() -> None:
|
|
|
317
325
|
logger.info(f" • Unused imports: {Colors.YELLOW}{len(unused_imports)}{Colors.RESET}")
|
|
318
326
|
logger.info(f" • Unused parameters: {Colors.YELLOW}{len(unused_parameters)}{Colors.RESET}")
|
|
319
327
|
logger.info(f" • Unused variables: {Colors.YELLOW}{len(unused_variables)}{Colors.RESET}")
|
|
328
|
+
logger.info(f" • Unused classes: {Colors.YELLOW}{len(unused_classes)}{Colors.RESET}")
|
|
320
329
|
|
|
321
330
|
if args.interactive and (unused_functions or unused_imports):
|
|
322
331
|
logger.info(f"\n{Colors.BOLD}Interactive Mode:{Colors.RESET}")
|
|
@@ -402,10 +411,19 @@ def main() -> None:
|
|
|
402
411
|
for i, item in enumerate(unused_variables, 1):
|
|
403
412
|
logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
|
|
404
413
|
logger.info(f" {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
|
|
414
|
+
|
|
415
|
+
if unused_classes:
|
|
416
|
+
logger.info(f"\n{Colors.YELLOW}{Colors.BOLD}📚 Unused Classes{Colors.RESET}")
|
|
417
|
+
logger.info(f"{Colors.YELLOW}{'=' * 18}{Colors.RESET}")
|
|
418
|
+
for i, item in enumerate(unused_classes, 1):
|
|
419
|
+
logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
|
|
420
|
+
logger.info(f" {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
|
|
421
|
+
|
|
405
422
|
else:
|
|
406
423
|
logger.info(f"\n{Colors.GREEN}✓ All variables are being used!{Colors.RESET}")
|
|
407
424
|
|
|
408
|
-
dead_code_count = len(unused_functions) + len(unused_imports)
|
|
425
|
+
dead_code_count = len(unused_functions) + len(unused_imports) + len(unused_variables) + len(unused_classes) + len(unused_parameters)
|
|
426
|
+
|
|
409
427
|
print_badge(dead_code_count, logger)
|
|
410
428
|
|
|
411
429
|
if unused_functions or unused_imports:
|
skylos/constants.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
PENALTIES = {
|
|
5
|
+
"private_name": 80,
|
|
6
|
+
"dunder_or_magic": 100,
|
|
7
|
+
"underscored_var": 100,
|
|
8
|
+
"in_init_file": 15,
|
|
9
|
+
"dynamic_module": 40,
|
|
10
|
+
"test_related": 100,
|
|
11
|
+
"framework_magic": 40,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
TEST_FILE_RE = re.compile(r"(?:^|[/\\])tests?[/\\]|_test\.py$", re.I)
|
|
15
|
+
TEST_IMPORT_RE = re.compile(r"^(pytest|unittest|nose|mock|responses)(\.|$)")
|
|
16
|
+
TEST_DECOR_RE = re.compile(r"""^(
|
|
17
|
+
pytest\.(fixture|mark) |
|
|
18
|
+
patch(\.|$) |
|
|
19
|
+
responses\.activate |
|
|
20
|
+
freeze_time
|
|
21
|
+
)$""", re.X)
|
|
22
|
+
|
|
23
|
+
AUTO_CALLED = {"__init__", "__enter__", "__exit__"}
|
|
24
|
+
TEST_METHOD_PATTERN = re.compile(r"^test_\w+$")
|
|
25
|
+
|
|
26
|
+
UNITTEST_LIFECYCLE_METHODS = {
|
|
27
|
+
'setUp', 'tearDown', 'setUpClass', 'tearDownClass',
|
|
28
|
+
'setUpModule', 'tearDownModule'
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
FRAMEWORK_FILE_RE = re.compile(r"(?:views|handlers|endpoints|routes|api)\.py$", re.I)
|
|
32
|
+
|
|
33
|
+
DEFAULT_EXCLUDE_FOLDERS = {
|
|
34
|
+
"__pycache__", ".git", ".pytest_cache", ".mypy_cache", ".tox",
|
|
35
|
+
"htmlcov", ".coverage", "build", "dist", "*.egg-info", "venv", ".venv"
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
def is_test_path(p: Path | str) -> bool:
|
|
39
|
+
return bool(TEST_FILE_RE.search(str(p)))
|
|
40
|
+
|
|
41
|
+
def is_framework_path(p: Path | str) -> bool:
|
|
42
|
+
return bool(FRAMEWORK_FILE_RE.search(str(p)))
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import fnmatch
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
# decorator patterns for each of the more popular frameworks
|
|
6
|
+
FRAMEWORK_DECORATORS = [
|
|
7
|
+
# flask
|
|
8
|
+
"@app.route", "@app.get", "@app.post", "@app.put", "@app.delete",
|
|
9
|
+
"@app.patch", "@app.before_request", "@app.after_request",
|
|
10
|
+
"@app.errorhandler", "@app.teardown_*",
|
|
11
|
+
|
|
12
|
+
# fastapi
|
|
13
|
+
"@router.get", "@router.post", "@router.put", "@router.delete",
|
|
14
|
+
"@router.patch", "@router.head", "@router.options",
|
|
15
|
+
"@router.trace", "@router.websocket",
|
|
16
|
+
|
|
17
|
+
# django
|
|
18
|
+
"@*_required", "@login_required", "@permission_required",
|
|
19
|
+
"@user_passes_test", "@staff_member_required",
|
|
20
|
+
|
|
21
|
+
# pydantic
|
|
22
|
+
"@validator", "@field_validator", "@model_validator", "@root_validator",
|
|
23
|
+
|
|
24
|
+
# celery patterns
|
|
25
|
+
"@task", "@shared_task", "@periodic_task", "@celery_task",
|
|
26
|
+
|
|
27
|
+
# generic ones
|
|
28
|
+
"@route", "@get", "@post", "@put", "@delete", "@patch",
|
|
29
|
+
"@middleware", "@depends", "@inject"
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
FRAMEWORK_FUNCTIONS = [
|
|
33
|
+
# django view methods
|
|
34
|
+
"get", "post", "put", "patch", "delete", "head", "options", "trace",
|
|
35
|
+
"*_queryset", "get_queryset", "get_object", "get_context_data",
|
|
36
|
+
"*_form", "form_valid", "form_invalid", "get_form_*",
|
|
37
|
+
|
|
38
|
+
# django model methods
|
|
39
|
+
"save", "delete", "clean", "full_clean", "*_delete", "*_save",
|
|
40
|
+
|
|
41
|
+
# generic API endpoints
|
|
42
|
+
"create_*", "update_*", "delete_*", "list_*", "retrieve_*",
|
|
43
|
+
"handle_*", "process_*", "*_handler", "*_view"
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
FRAMEWORK_IMPORTS = {
|
|
47
|
+
'flask', 'fastapi', 'django', 'pydantic', 'celery', 'starlette',
|
|
48
|
+
'sanic', 'tornado', 'pyramid', 'bottle', 'cherrypy', 'web2py',
|
|
49
|
+
'falcon', 'hug', 'responder', 'quart', 'hypercorn', 'uvicorn'
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class FrameworkAwareVisitor:
|
|
53
|
+
|
|
54
|
+
def __init__(self, filename=None):
|
|
55
|
+
self.is_framework_file = False
|
|
56
|
+
self.framework_decorated_lines = set()
|
|
57
|
+
self.detected_frameworks = set()
|
|
58
|
+
|
|
59
|
+
if filename:
|
|
60
|
+
self._check_framework_imports_in_file(filename)
|
|
61
|
+
|
|
62
|
+
def visit(self, node):
|
|
63
|
+
method = 'visit_' + node.__class__.__name__
|
|
64
|
+
visitor = getattr(self, method, self.generic_visit)
|
|
65
|
+
return visitor(node)
|
|
66
|
+
|
|
67
|
+
def generic_visit(self, node):
|
|
68
|
+
for field, value in ast.iter_fields(node):
|
|
69
|
+
if isinstance(value, list):
|
|
70
|
+
for item in value:
|
|
71
|
+
if isinstance(item, ast.AST):
|
|
72
|
+
self.visit(item)
|
|
73
|
+
elif isinstance(value, ast.AST):
|
|
74
|
+
self.visit(value)
|
|
75
|
+
|
|
76
|
+
def visit_Import(self, node):
|
|
77
|
+
for alias in node.names:
|
|
78
|
+
if any(fw in alias.name.lower() for fw in FRAMEWORK_IMPORTS):
|
|
79
|
+
self.is_framework_file = True
|
|
80
|
+
self.detected_frameworks.add(alias.name.split('.')[0])
|
|
81
|
+
self.generic_visit(node)
|
|
82
|
+
|
|
83
|
+
def visit_ImportFrom(self, node):
|
|
84
|
+
if node.module:
|
|
85
|
+
module_name = node.module.split('.')[0].lower()
|
|
86
|
+
if module_name in FRAMEWORK_IMPORTS:
|
|
87
|
+
self.is_framework_file = True
|
|
88
|
+
self.detected_frameworks.add(module_name)
|
|
89
|
+
self.generic_visit(node)
|
|
90
|
+
|
|
91
|
+
def visit_FunctionDef(self, node):
|
|
92
|
+
for deco in node.decorator_list:
|
|
93
|
+
decorator_str = self._normalize_decorator(deco)
|
|
94
|
+
if self._matches_framework_pattern(decorator_str, FRAMEWORK_DECORATORS):
|
|
95
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
96
|
+
self.is_framework_file = True
|
|
97
|
+
|
|
98
|
+
if self._matches_framework_pattern(node.name, FRAMEWORK_FUNCTIONS):
|
|
99
|
+
if self.is_framework_file:
|
|
100
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
101
|
+
|
|
102
|
+
self.generic_visit(node)
|
|
103
|
+
|
|
104
|
+
def visit_AsyncFunctionDef(self, node):
|
|
105
|
+
self.visit_FunctionDef(node)
|
|
106
|
+
|
|
107
|
+
def visit_ClassDef(self, node):
|
|
108
|
+
for base in node.bases:
|
|
109
|
+
if isinstance(base, ast.Name):
|
|
110
|
+
if any(pattern in base.id.lower() for pattern in ['view', 'viewset', 'api', 'handler']):
|
|
111
|
+
if self.is_framework_file:
|
|
112
|
+
self.framework_decorated_lines.add(node.lineno)
|
|
113
|
+
|
|
114
|
+
self.generic_visit(node)
|
|
115
|
+
|
|
116
|
+
def _normalize_decorator(self, decorator_node):
|
|
117
|
+
if isinstance(decorator_node, ast.Name):
|
|
118
|
+
return f"@{decorator_node.id}"
|
|
119
|
+
elif isinstance(decorator_node, ast.Attribute):
|
|
120
|
+
if isinstance(decorator_node.value, ast.Name):
|
|
121
|
+
return f"@{decorator_node.value.id}.{decorator_node.attr}"
|
|
122
|
+
else:
|
|
123
|
+
return f"@{decorator_node.attr}"
|
|
124
|
+
elif isinstance(decorator_node, ast.Call):
|
|
125
|
+
return self._normalize_decorator(decorator_node.func)
|
|
126
|
+
return "@unknown"
|
|
127
|
+
|
|
128
|
+
def _matches_framework_pattern(self, text, patterns):
|
|
129
|
+
text_clean = text.lstrip('@')
|
|
130
|
+
return any(fnmatch.fnmatch(text_clean, pattern.lstrip('@')) for pattern in patterns)
|
|
131
|
+
|
|
132
|
+
def _check_framework_imports_in_file(self, filename):
|
|
133
|
+
try:
|
|
134
|
+
content = Path(filename).read_text(encoding='utf-8')
|
|
135
|
+
for framework in FRAMEWORK_IMPORTS:
|
|
136
|
+
if (f'import {framework}' in content or
|
|
137
|
+
f'from {framework}' in content):
|
|
138
|
+
self.is_framework_file = True
|
|
139
|
+
self.detected_frameworks.add(framework)
|
|
140
|
+
break
|
|
141
|
+
except:
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
def detect_framework_usage(definition, decorator_nodes=None, visitor=None):
|
|
145
|
+
if not visitor:
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
# very low confidence - likely framework magic
|
|
149
|
+
if definition.line in visitor.framework_decorated_lines:
|
|
150
|
+
return 20
|
|
151
|
+
|
|
152
|
+
# framework file but no direct markers
|
|
153
|
+
if visitor.is_framework_file:
|
|
154
|
+
if (not definition.simple_name.startswith('_') and
|
|
155
|
+
definition.type in ('function', 'method')):
|
|
156
|
+
return 40
|
|
157
|
+
|
|
158
|
+
return None
|
skylos/test_aware.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
from skylos.constants import TEST_IMPORT_RE, TEST_DECOR_RE, TEST_FILE_RE
|
|
3
|
+
|
|
4
|
+
class TestAwareVisitor:
|
|
5
|
+
|
|
6
|
+
def __init__(self, filename=None):
|
|
7
|
+
self.is_test_file = False
|
|
8
|
+
self.test_decorated_lines = set()
|
|
9
|
+
|
|
10
|
+
# mark as test file based on the file path/name NOT imports
|
|
11
|
+
if filename and TEST_FILE_RE.search(str(filename)):
|
|
12
|
+
self.is_test_file = True
|
|
13
|
+
|
|
14
|
+
def visit(self, node):
|
|
15
|
+
method = 'visit_' + node.__class__.__name__
|
|
16
|
+
visitor = getattr(self, method, self.generic_visit)
|
|
17
|
+
return visitor(node)
|
|
18
|
+
|
|
19
|
+
def generic_visit(self, node):
|
|
20
|
+
for field, value in ast.iter_fields(node):
|
|
21
|
+
if isinstance(value, list):
|
|
22
|
+
for item in value:
|
|
23
|
+
if isinstance(item, ast.AST):
|
|
24
|
+
self.visit(item)
|
|
25
|
+
elif isinstance(value, ast.AST):
|
|
26
|
+
self.visit(value)
|
|
27
|
+
|
|
28
|
+
def visit_Import(self, node):
|
|
29
|
+
|
|
30
|
+
if self.is_test_file:
|
|
31
|
+
for alias in node.names:
|
|
32
|
+
if TEST_IMPORT_RE.match(alias.name):
|
|
33
|
+
pass
|
|
34
|
+
self.generic_visit(node)
|
|
35
|
+
|
|
36
|
+
def visit_ImportFrom(self, node):
|
|
37
|
+
if self.is_test_file:
|
|
38
|
+
if node.module and TEST_IMPORT_RE.match(node.module):
|
|
39
|
+
pass
|
|
40
|
+
self.generic_visit(node)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def visit_FunctionDef(self, node):
|
|
44
|
+
if (node.name.startswith('test_') or
|
|
45
|
+
node.name.endswith('_test') or
|
|
46
|
+
any(node.name.startswith(prefix) for prefix in ['setup', 'teardown']) or
|
|
47
|
+
node.name in ['setUp', 'tearDown', 'setUpClass', 'tearDownClass',
|
|
48
|
+
'setUpModule', 'tearDownModule']):
|
|
49
|
+
self.test_decorated_lines.add(node.lineno)
|
|
50
|
+
|
|
51
|
+
for deco in node.decorator_list:
|
|
52
|
+
name = self._decorator_name(deco)
|
|
53
|
+
if name and (TEST_DECOR_RE.match(name) or 'pytest' in name or 'fixture' in name):
|
|
54
|
+
self.test_decorated_lines.add(node.lineno)
|
|
55
|
+
self.generic_visit(node)
|
|
56
|
+
|
|
57
|
+
def visit_AsyncFunctionDef(self, node):
|
|
58
|
+
self.visit_FunctionDef(node)
|
|
59
|
+
|
|
60
|
+
def _decorator_name(self, deco):
|
|
61
|
+
if isinstance(deco, ast.Name):
|
|
62
|
+
return deco.id
|
|
63
|
+
if isinstance(deco, ast.Attribute):
|
|
64
|
+
parent = self._decorator_name(deco.value)
|
|
65
|
+
return f"{parent}.{deco.attr}" if parent else deco.attr
|
|
66
|
+
return ""
|
skylos/visitor.py
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import ast
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
import os
|
|
5
|
+
import re, logging, traceback
|
|
6
|
+
DBG = bool(int(os.getenv("SKYLOS_DEBUG", "0")))
|
|
7
|
+
log = logging.getLogger("Skylos")
|
|
8
|
+
|
|
4
9
|
|
|
5
10
|
PYTHON_BUILTINS={"print","len","str","int","float","list","dict","set","tuple","range","open","super","object","type","enumerate","zip","map","filter","sorted","reversed","sum","min","max","all","any","next","iter","repr","chr","ord","bytes","bytearray","memoryview","format","round","abs","pow","divmod","complex","hash","id","bool","callable","getattr","setattr","delattr","hasattr","isinstance","issubclass","globals","locals","vars","dir","property","classmethod","staticmethod"}
|
|
6
11
|
DYNAMIC_PATTERNS={"getattr","globals","eval","exec"}
|
|
@@ -74,11 +79,23 @@ class Visitor(ast.NodeVisitor):
|
|
|
74
79
|
self.visit(node)
|
|
75
80
|
|
|
76
81
|
def visit_string_annotation(self, annotation_str):
|
|
82
|
+
if not isinstance(annotation_str, str):
|
|
83
|
+
return
|
|
84
|
+
|
|
85
|
+
if DBG:
|
|
86
|
+
log.debug(f"[visitor] parsing string annotation {annotation_str!r} "
|
|
87
|
+
f"in {self.filename}:{getattr(self, 'line', '?')}")
|
|
88
|
+
|
|
77
89
|
try:
|
|
78
|
-
parsed = ast.parse(annotation_str, mode=
|
|
90
|
+
parsed = ast.parse(annotation_str, mode="eval")
|
|
79
91
|
self.visit(parsed.body)
|
|
80
|
-
except:
|
|
81
|
-
|
|
92
|
+
except Exception:
|
|
93
|
+
if DBG:
|
|
94
|
+
log.debug("[visitor] inner-annotation parse failed:\n" +
|
|
95
|
+
traceback.format_exc())
|
|
96
|
+
# keep going but dont swallow symbol names:
|
|
97
|
+
for tok in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", annotation_str):
|
|
98
|
+
self.add_ref(tok)
|
|
82
99
|
|
|
83
100
|
def visit_Import(self,node):
|
|
84
101
|
for a in node.names:
|
|
@@ -250,6 +267,11 @@ class Visitor(ast.NodeVisitor):
|
|
|
250
267
|
if module_name != "self":
|
|
251
268
|
qualified_name = f"{self.qual(module_name)}.{attr_name}"
|
|
252
269
|
self.add_ref(qualified_name)
|
|
270
|
+
|
|
271
|
+
elif isinstance(node.args[0], ast.Name):
|
|
272
|
+
target_name = node.args[0].id
|
|
273
|
+
if target_name != "self":
|
|
274
|
+
self.dyn.add(self.mod.split(".")[0] if self.mod else "")
|
|
253
275
|
|
|
254
276
|
elif isinstance(node.func, ast.Name) and node.func.id == "globals":
|
|
255
277
|
parent = getattr(node, 'parent', None)
|
|
@@ -259,6 +281,9 @@ class Visitor(ast.NodeVisitor):
|
|
|
259
281
|
func_name = parent.slice.value
|
|
260
282
|
self.add_ref(func_name)
|
|
261
283
|
self.add_ref(f"{self.mod}.{func_name}")
|
|
284
|
+
|
|
285
|
+
elif isinstance(node.func, ast.Name) and node.func.id in ("eval", "exec"):
|
|
286
|
+
self.dyn.add(self.mod.split(".")[0] if self.mod else "")
|
|
262
287
|
|
|
263
288
|
def visit_Name(self,node):
|
|
264
289
|
if isinstance(node.ctx,ast.Load):
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
skylos/__init__.py,sha256=ZZWhq0TZ3G-yDi9inbYvMn8OBes-pqo1aYB5EivuxFI,152
|
|
2
|
+
skylos/analyzer.py,sha256=MVO16T58bPpcISQKiHWtcQaeHG6r2awH6IuxHUmG56E,13858
|
|
3
|
+
skylos/cli.py,sha256=6udZY4vLU6PFzVMkaiCLCRcLXgquyHdmf4rIOAPosWc,18266
|
|
4
|
+
skylos/constants.py,sha256=F1kMjuTxfw2hJjd0SeOQcgex5WhHMUhXCzOlVmwuACs,1230
|
|
5
|
+
skylos/framework_aware.py,sha256=p7BGoFnzkpaLJoE3M5qgyIeZvXx17tOkdyXgeqGKmqU,5804
|
|
6
|
+
skylos/test_aware.py,sha256=cduaWMcFsuzIEQWAMFPC58xGk8NUU3SbUS0ChRPedv8,2372
|
|
7
|
+
skylos/visitor.py,sha256=MnWyzs0b2JOD2Nj1Iu7ZIZHCr7fRC92PY1bsT8bdgXg,12796
|
|
8
|
+
test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
test/compare_tools.py,sha256=0g9PDeJlbst-7hOaQzrL4MiJFQKpqM8q8VeBGzpPczg,22738
|
|
10
|
+
test/conftest.py,sha256=57sTF6vLL5U0CVKwGQFJcRs6n7t1dEHIriQoSluNmAI,6418
|
|
11
|
+
test/diagnostics.py,sha256=ExuFOCVpc9BDwNYapU96vj9RXLqxji32Sv6wVF4nJYU,13802
|
|
12
|
+
test/test_analyzer.py,sha256=c8gbZuUls3N8xGGDXdPzsaoma6AFR2fIDqp9cNJlg7c,22104
|
|
13
|
+
test/test_changes_analyzer.py,sha256=l1hspCFz-sF8gqOilJvntUDuGckwhYsVtzvSRB13ZWw,5085
|
|
14
|
+
test/test_cli.py,sha256=rtdKzanDRJT_F92jKkCQFdhvlfwVJxfXKO8Hrbn-mIg,13180
|
|
15
|
+
test/test_constants.py,sha256=pMuDy0UpC81zENMDCeK6Bqmm3BR_HHZQSlMG-9TgOm0,12602
|
|
16
|
+
test/test_framework_aware.py,sha256=tJ7bnhiGeSdsAvrWaGJO5ovTrw9n9BBk6o1HCw15yDA,11693
|
|
17
|
+
test/test_integration.py,sha256=bNKGUe-w0xEZEdnoQNHbssvKMGs9u9fmFQTOz1lX9_k,12398
|
|
18
|
+
test/test_skylos.py,sha256=kz77STrS4k3Eez5RDYwGxOg2WH3e7zNZPUYEaTLbGTs,15608
|
|
19
|
+
test/test_test_aware.py,sha256=VmbR_MQY0m941CAxxux8OxJHIr7l8crfWRouSeBMhIo,9390
|
|
20
|
+
test/test_visitor.py,sha256=xAbGv-XaozKm_0WJJhr0hMb6mLaJcbPz57G9-SWkxFU,22764
|
|
21
|
+
test/sample_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
+
test/sample_repo/app.py,sha256=M5XgoAn-LPz50mKAj_ZacRKf-Pg7I4HbjWP7Z9jE4a0,226
|
|
23
|
+
test/sample_repo/sample_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
test/sample_repo/sample_repo/commands.py,sha256=b6gQ9YDabt2yyfqGbOpLo0osF7wya8O4Lm7m8gtCr3g,2575
|
|
25
|
+
test/sample_repo/sample_repo/models.py,sha256=xXIg3pToEZwKuUCmKX2vTlCF_VeFA0yZlvlBVPIy5Qw,3320
|
|
26
|
+
test/sample_repo/sample_repo/routes.py,sha256=8yITrt55BwS01G7nWdESdx8LuxmReqop1zrGUKPeLi8,2475
|
|
27
|
+
test/sample_repo/sample_repo/utils.py,sha256=S56hEYh8wkzwsD260MvQcmUFOkw2EjFU27nMLFE6G2k,1103
|
|
28
|
+
skylos-1.2.0.dist-info/METADATA,sha256=FqeObdsFPYrqrKzpBngriBQbKZXnOJUnqDKJQSn-N4k,224
|
|
29
|
+
skylos-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
30
|
+
skylos-1.2.0.dist-info/entry_points.txt,sha256=zzRpN2ByznlQoLeuLolS_TFNYSQxUGBL1EXQsAd6bIA,43
|
|
31
|
+
skylos-1.2.0.dist-info/top_level.txt,sha256=f8GA_7KwfaEopPMP8-EXDQXaqd4IbsOQPakZy01LkdQ,12
|
|
32
|
+
skylos-1.2.0.dist-info/RECORD,,
|