skylos 1.1.12__py3-none-any.whl → 1.2.1__py3-none-any.whl

skylos/analyzer.py

@@ -1,13 +1,20 @@
 #!/usr/bin/env python3
-import ast,sys,json,logging,re
+import ast
+import sys
+import json
+import logging
 from pathlib import Path
 from collections import defaultdict
 from skylos.visitor import Visitor
-from skylos.constants import (
-    AUTO_CALLED, TEST_METHOD_PATTERN, MAGIC_METHODS,
-    TEST_LIFECYCLE_METHODS, TEST_IMPORT_PATTERNS, TEST_DECORATORS,
-    DEFAULT_EXCLUDE_FOLDERS
-)
+from skylos.constants import ( DEFAULT_EXCLUDE_FOLDERS, PENALTIES, AUTO_CALLED )
+from skylos.test_aware import TestAwareVisitor
+import os
+import traceback
+from skylos.framework_aware import FrameworkAwareVisitor, detect_framework_usage
+import io
+import tokenize
+import re
+import warnings
 
 logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s - %(message)s')
 logger=logging.getLogger('Skylos')
@@ -27,12 +34,27 @@ def parse_exclude_folders(user_exclude_folders, use_defaults=True, include_folde
     
     return exclude_set
 
+IGNORE_PATTERNS = (
+    r"#\s*pragma:\s*no\s+skylos", ## our own pragma
+    r"#\s*pragma:\s*no\s+cover",
+    r"#\s*noqa(?:\b|:)", # flake8 style
+)
+
+def _collect_ignored_lines(source: str) -> set[int]:
+    ignores = set()
+    for tok in tokenize.generate_tokens(io.StringIO(source).readline):
+        if tok.type == tokenize.COMMENT:
+            if any(re.search(pat, tok.string, flags=re.I) for pat in IGNORE_PATTERNS):
+                ignores.add(tok.start[0])
+    return ignores
+
 class Skylos:
     def __init__(self):
         self.defs={}
         self.refs=[]
         self.dynamic=set()
         self.exports=defaultdict(set)
+        self.ignored_lines:set[int]=set()
 
     def _module(self,root,f):
         p=list(f.relative_to(root).parts)
@@ -131,6 +153,12 @@ class Skylos:
             matches = simple_name_lookup.get(simple, [])
             for d in matches:
                 d.references += 1
+ 
+        for module_name in self.dynamic:
+            for def_name, def_obj in self.defs.items():
+                if def_obj.name.startswith(f"{module_name}."):
+                    if def_obj.type in ("function", "method") and not def_obj.simple_name.startswith("_"):
+                        def_obj.references += 1
     
     def _get_base_classes(self, class_name):
         if class_name not in self.defs:
@@ -143,128 +171,62 @@ class Skylos:
         
         return []
     
-    def _has_test_imports(self, file_path):
-        try:
-            with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-                
-            for test_import in TEST_IMPORT_PATTERNS:
-                if f"import {test_import}" in content or f"from {test_import}" in content:
-                    return True
-                    
-            return False
-        except:
-            return False
-    
-    def _is_test_file(self, file_path):
-        """check if file locs indicates its a test file"""
-        file_str = str(file_path).lower()
-        
-        if (file_str.endswith("test.py") or 
-            file_str.endswith("_test.py") or 
-            "test_" in file_str or
-            "/test/" in file_str or
-            "/tests/" in file_str or
-            "\\test\\" in file_str or
-            "\\tests\\" in file_str):
-            return True
-            
-        return False
+    def _apply_penalties(self, def_obj, visitor, framework):
+        c = 100
+        
+        if def_obj.simple_name.startswith("_") and not def_obj.simple_name.startswith("__"):
+            c -= PENALTIES["private_name"] 
+        if def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__"):
+            c -= PENALTIES["dunder_or_magic"]
+        if def_obj.type == "variable" and def_obj.simple_name == "_":
+            c -= PENALTIES["underscored_var"]
+        if def_obj.in_init and def_obj.type in ("function", "class"):
+            c -= PENALTIES["in_init_file"]
+        if def_obj.name.split(".")[0] in self.dynamic:
+            c -= PENALTIES["dynamic_module"]
+        if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
+            c -= PENALTIES["test_related"]
+        
+        framework_confidence = detect_framework_usage(def_obj, visitor=framework)
+        if framework_confidence is not None:
+            c = min(c, framework_confidence)
 
-    def _has_test_decorators(self, file_path):
-        """Check if file uses test-related decorators"""
-        try:
-            with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-                
-            for decorator in TEST_DECORATORS:
-                if f"@{decorator}" in content:
-                    return True
-                    
-            return False
-        except:
-            return False
-        
-    def _is_test_related(self, definition):
+        if (def_obj.simple_name.startswith("__") and def_obj.simple_name.endswith("__")):
+            c = 0
+
+        if def_obj.type == "parameter":
+            if def_obj.simple_name in ("self", "cls"):
+                c = 0
+            elif "." in def_obj.name:
+                method_name = def_obj.name.split(".")[-2]
+                if method_name.startswith("__") and method_name.endswith("__"):
+                    c = 0
+        
+        if visitor.is_test_file or def_obj.line in visitor.test_decorated_lines:
+            c = 0
+        
+        if (def_obj.type == "import" and def_obj.name.startswith("__future__.") and
+            def_obj.simple_name in ("annotations", "absolute_import", "division", 
+                                "print_function", "unicode_literals", "generator_stop")):
+            c = 0
         
-        if "." in definition.name:
-            class_name = definition.name.rsplit(".", 1)[0]
-            class_simple_name = class_name.split(".")[-1]
-            
-            if (class_simple_name.startswith("Test") or 
-                class_simple_name.endswith("Test") or 
-                class_simple_name.endswith("TestCase")):
-                return True
-        
-        if (definition.type == "method" and 
-            (TEST_METHOD_PATTERN.match(definition.simple_name) or
-             definition.simple_name in TEST_LIFECYCLE_METHODS)):
-            return True
-        
-        # NOT for imports, variables, parameters
-        if definition.type in ("function", "method", "class"):
-            if self._is_test_file(definition.filename):
-                return True
-                
-            if self._has_test_imports(definition.filename):
-                return True
-                
-            ## check decorators -- test related
-            if self._has_test_decorators(definition.filename):
-                return True
-            
-        return False
-                    
+        def_obj.confidence = max(c, 0)
+
     def _apply_heuristics(self):
-        class_methods=defaultdict(list)
+        class_methods = defaultdict(list)
         for d in self.defs.values():
-            if d.type in("method","function") and"." in d.name:
-                cls=d.name.rsplit(".",1)[0]
-                if cls in self.defs and self.defs[cls].type=="class":
+            if d.type in ("method", "function") and "." in d.name:
+                cls = d.name.rsplit(".", 1)[0]
+                if cls in self.defs and self.defs[cls].type == "class":
                     class_methods[cls].append(d)
-
-        for cls,methods in class_methods.items():
-            if self.defs[cls].references>0:
+        
+        for cls, methods in class_methods.items():
+            if self.defs[cls].references > 0:
                 for m in methods:
-                    if m.simple_name in AUTO_CALLED:m.references+=1
-                    
-        for d in self.defs.values():
-            if d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__")):
-                d.confidence = 0
-            
-            if d.type == "parameter" and d.simple_name in ("self", "cls"):
-                d.confidence = 0
-
-            if d.type != "parameter" and (d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__"))):
-                d.confidence = 0
-
-            if (d.type == "import" and d.name.startswith("__future__.") and
-                d.simple_name in ("annotations", "absolute_import", "division", 
-                                "print_function", "unicode_literals", "generator_stop")):
-                d.confidence = 0
-
-            if (d.simple_name.startswith("_") and 
-                not d.simple_name.startswith("__") and  
-                d.simple_name != "_"):
-                d.confidence = 0
-
-            if not d.simple_name.startswith("_") and d.type in ("function", "method", "class"):
-                d.confidence = min(d.confidence, 90)
-            
-            if d.in_init and d.type in ("function", "class"):
-                d.confidence = min(d.confidence, 85)
-            
-            if d.name.split(".")[0] in self.dynamic:
-                d.confidence = min(d.confidence, 60)
-            
-            if d.type == "variable" and d.simple_name == "_":
-                d.confidence = 0
-            
-            if self._is_test_related(d):
-                d.confidence = 0
+                    if m.simple_name in AUTO_CALLED:  # __init__, __enter__, __exit__
+                        m.references += 1
 
     def analyze(self, path, thr=60, exclude_folders=None):
-        
         files, root = self._get_python_files(path, exclude_folders)
         
         if not files:
@@ -289,25 +251,43 @@ class Skylos:
         
         for file in files:
             mod = modmap[file]
-            defs, refs, dyn, exports = proc_file(file, mod)
-            
-            for d in defs: 
+
+            result = proc_file(file, mod)
+
+            if len(result) == 7: ##new
+                defs, refs, dyn, exports, test_flags, framework_flags, ignored = result
+                self.ignored_lines.update(ignored)
+            else: ##legacy
+                warnings.warn(
+                    "proc_file() now returns 7 values (added ignored_lines). "
+                    "The 6-value form is deprecated and will disappear.",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+                defs, refs, dyn, exports, test_flags, framework_flags = result
+
+            # apply penalties while we still have the file-specific flags
+            for d in defs:
+                self._apply_penalties(d, test_flags, framework_flags) 
                 self.defs[d.name] = d
+
             self.refs.extend(refs)
             self.dynamic.update(dyn)
             self.exports[mod].update(exports)
-        
+
         self._mark_refs()
-        self._apply_heuristics()
+        self._apply_heuristics() 
         self._mark_exports()
-                
+        
         thr = max(0, thr)
 
         unused = []
         for d in self.defs.values():
-            if d.references == 0 and not d.is_exported and d.confidence >= thr:
-                unused.append(d.to_dict())
-        
+                if (d.references == 0 and not d.is_exported
+                    and d.confidence >= thr
+                    and d.line not in self.ignored_lines):
+                        unused.append(d.to_dict())
+
         result = {
             "unused_functions": [], 
             "unused_imports": [], 
@@ -341,13 +321,28 @@ def proc_file(file_or_args, mod=None):
         file = file_or_args 
 
     try:
-        tree = ast.parse(Path(file).read_text(encoding="utf-8"))
-        v = Visitor(mod, file)
+        source = Path(file).read_text(encoding="utf-8")
+        ignored = _collect_ignored_lines(source)   
+        tree   = ast.parse(source)
+
+        tv = TestAwareVisitor(filename=file)
+        tv.visit(tree)
+
+        fv = FrameworkAwareVisitor(filename=file)
+        fv.visit(tree)
+
+        v  = Visitor(mod, file)
         v.visit(tree)
-        return v.defs, v.refs, v.dyn, v.exports
+
+        return v.defs, v.refs, v.dyn, v.exports, tv, fv, ignored
     except Exception as e:
         logger.error(f"{file}: {e}")
-        return [], [], set(), set()
+        if os.getenv("SKYLOS_DEBUG"):
+            logger.error(traceback.format_exc())
+        dummy_visitor = TestAwareVisitor(filename=file)
+        dummy_framework_visitor = FrameworkAwareVisitor(filename=file)
+
+        return [], [], set(), set(), dummy_visitor, dummy_framework_visitor, set()
 
 def analyze(path,conf=60, exclude_folders=None):
     return Skylos().analyze(path,conf, exclude_folders)

skylos/cli.py

@@ -218,6 +218,13 @@ def main() -> None:
         action="store_true",
         help="Enable verbose output"
     )
+    parser.add_argument(
+        "--confidence",
+        "-c",
+        type=int,
+        default=60,
+        help="Confidence threshold (0-100). Lower values include more items. Default: 60"
+    )
     parser.add_argument(
         "--interactive", "-i",
         action="store_true",
@@ -293,7 +300,7 @@ def main() -> None:
             logger.info(f"{Colors.GREEN}📁 No folders excluded{Colors.RESET}")
 
     try:
-        result_json = skylos.analyze(args.path, exclude_folders=list(final_exclude_folders))
+        result_json = skylos.analyze(args.path, conf=args.confidence, exclude_folders=list(final_exclude_folders))
         result = json.loads(result_json)
 
     except Exception as e:
@@ -308,6 +315,7 @@ def main() -> None:
     unused_imports = result.get("unused_imports", [])
     unused_parameters = result.get("unused_parameters", [])
     unused_variables = result.get("unused_variables", [])
+    unused_classes = result.get("unused_classes", [])
     
     logger.info(f"{Colors.CYAN}{Colors.BOLD}🔍 Python Static Analysis Results{Colors.RESET}")
     logger.info(f"{Colors.CYAN}{'=' * 35}{Colors.RESET}")
@@ -317,6 +325,7 @@ def main() -> None:
     logger.info(f"  • Unused imports: {Colors.YELLOW}{len(unused_imports)}{Colors.RESET}")
     logger.info(f"  • Unused parameters: {Colors.YELLOW}{len(unused_parameters)}{Colors.RESET}")
     logger.info(f"  • Unused variables: {Colors.YELLOW}{len(unused_variables)}{Colors.RESET}")
+    logger.info(f"  • Unused classes: {Colors.YELLOW}{len(unused_classes)}{Colors.RESET}")
 
     if args.interactive and (unused_functions or unused_imports):
         logger.info(f"\n{Colors.BOLD}Interactive Mode:{Colors.RESET}")
@@ -402,10 +411,19 @@ def main() -> None:
             for i, item in enumerate(unused_variables, 1):
                 logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
                 logger.info(f"    {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
+                
+        if unused_classes:
+            logger.info(f"\n{Colors.YELLOW}{Colors.BOLD}📚 Unused Classes{Colors.RESET}")
+            logger.info(f"{Colors.YELLOW}{'=' * 18}{Colors.RESET}")
+            for i, item in enumerate(unused_classes, 1):
+                logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
+                logger.info(f"    {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
+
         else:
             logger.info(f"\n{Colors.GREEN}✓ All variables are being used!{Colors.RESET}")
 
-        dead_code_count = len(unused_functions) + len(unused_imports)
+        dead_code_count = len(unused_functions) + len(unused_imports) + len(unused_variables) + len(unused_classes) + len(unused_parameters)
+
         print_badge(dead_code_count, logger)
 
         if unused_functions or unused_imports:

skylos/constants.py

@@ -1,35 +1,42 @@
 import re
+from pathlib import Path
 
-AUTO_CALLED={"__init__","__enter__","__exit__"}
-TEST_METHOD_PATTERN = re.compile(r"^test_\w+$")
-MAGIC_METHODS={f"__{n}__"for n in["init","new","call","getattr","getattribute","enter","exit","str","repr","hash","eq","ne","lt","gt","le","ge","iter","next","contains","len","getitem","setitem","delitem","iadd","isub","imul","itruediv","ifloordiv","imod","ipow","ilshift","irshift","iand","ixor","ior","round","format","dir","abs","complex","int","float","bool","bytes","reduce","await","aiter","anext","add","sub","mul","truediv","floordiv","mod","divmod","pow","lshift","rshift","and","or","xor","radd","rsub","rmul","rtruediv","rfloordiv","rmod","rdivmod","rpow","rlshift","rrshift","rand","ror","rxor"]}
-TEST_LIFECYCLE_METHODS = {
-    "setUp", "tearDown", "setUpClass", "tearDownClass", 
-    "setUpModule", "tearDownModule", "setup_method", "teardown_method",
-    "setup_class", "teardown_class", "setup_function", "teardown_function"
-}
-TEST_IMPORT_PATTERNS = {
-    "unittest", "unittest.mock", "mock", "pytest", "nose", "nose2",
-    "responses", "requests_mock", "freezegun", "factory_boy", 
-    "hypothesis", "sure", "expects", "testfixtures", "faker"
+PENALTIES = {
+    "private_name":       80,   
+    "dunder_or_magic":    100, 
+    "underscored_var":    100, 
+    "in_init_file":       15, 
+    "dynamic_module":     40,
+    "test_related":       100,
+     "framework_magic":    40,
 }
 
-TEST_DECORATORS = {
-    "patch", "mock", "pytest.fixture", "pytest.mark", "given",
-    "responses.activate", "freeze_time", "patch.object", "patch.dict"
+TEST_FILE_RE   = re.compile(r"(?:^|[/\\])tests?[/\\]|_test\.py$", re.I)
+TEST_IMPORT_RE = re.compile(r"^(pytest|unittest|nose|mock|responses)(\.|$)")
+TEST_DECOR_RE  = re.compile(r"""^(
+    pytest\.(fixture|mark) |
+    patch(\.|$) |
+    responses\.activate |
+    freeze_time
+)$""", re.X)
+
+AUTO_CALLED = {"__init__", "__enter__", "__exit__"}
+TEST_METHOD_PATTERN = re.compile(r"^test_\w+$")
+
+UNITTEST_LIFECYCLE_METHODS = {
+    'setUp', 'tearDown', 'setUpClass', 'tearDownClass', 
+    'setUpModule', 'tearDownModule'
 }
 
+FRAMEWORK_FILE_RE = re.compile(r"(?:views|handlers|endpoints|routes|api)\.py$", re.I)
+
 DEFAULT_EXCLUDE_FOLDERS = {
-    "__pycache__",
-    ".git", 
-    ".pytest_cache",
-    ".mypy_cache",
-    ".tox",
-    "htmlcov",
-    ".coverage",
-    "build",
-    "dist",
-    "*.egg-info",
-    "venv",
-    ".venv"
+    "__pycache__", ".git", ".pytest_cache", ".mypy_cache", ".tox",
+    "htmlcov", ".coverage", "build", "dist", "*.egg-info", "venv", ".venv"
 }
+
+def is_test_path(p: Path | str) -> bool:
+    return bool(TEST_FILE_RE.search(str(p)))
+
+def is_framework_path(p: Path | str) -> bool:
+    return bool(FRAMEWORK_FILE_RE.search(str(p)))

skylos/framework_aware.py

@@ -0,0 +1,158 @@
+import ast
+import fnmatch
+from pathlib import Path
+
+# decorator patterns for each of the more popular frameworks
+FRAMEWORK_DECORATORS = [
+    # flask
+    "@app.route", "@app.get", "@app.post", "@app.put", "@app.delete", 
+    "@app.patch", "@app.before_request", "@app.after_request",
+    "@app.errorhandler", "@app.teardown_*",
+    
+    # fastapi  
+    "@router.get", "@router.post", "@router.put", "@router.delete",
+    "@router.patch", "@router.head", "@router.options",
+    "@router.trace", "@router.websocket",
+    
+    # django
+    "@*_required", "@login_required", "@permission_required", 
+    "@user_passes_test", "@staff_member_required",
+    
+    # pydantic 
+    "@validator", "@field_validator", "@model_validator", "@root_validator",
+    
+    # celery patterns
+    "@task", "@shared_task", "@periodic_task", "@celery_task",
+    
+    # generic ones
+    "@route", "@get", "@post", "@put", "@delete", "@patch",
+    "@middleware", "@depends", "@inject"
+]
+
+FRAMEWORK_FUNCTIONS = [
+    # django view methods
+    "get", "post", "put", "patch", "delete", "head", "options", "trace",
+    "*_queryset", "get_queryset", "get_object", "get_context_data",
+    "*_form", "form_valid", "form_invalid", "get_form_*",
+    
+    # django model methods
+    "save", "delete", "clean", "full_clean", "*_delete", "*_save",
+    
+    # generic API endpoints
+    "create_*", "update_*", "delete_*", "list_*", "retrieve_*",
+    "handle_*", "process_*", "*_handler", "*_view"
+]
+
+FRAMEWORK_IMPORTS = {
+    'flask', 'fastapi', 'django', 'pydantic', 'celery', 'starlette', 
+    'sanic', 'tornado', 'pyramid', 'bottle', 'cherrypy', 'web2py',
+    'falcon', 'hug', 'responder', 'quart', 'hypercorn', 'uvicorn'
+}
+
+class FrameworkAwareVisitor:
+    
+    def __init__(self, filename=None):
+        self.is_framework_file = False
+        self.framework_decorated_lines = set()
+        self.detected_frameworks = set()
+        
+        if filename:
+            self._check_framework_imports_in_file(filename)
+
+    def visit(self, node):
+        method = 'visit_' + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(node)
+
+    def generic_visit(self, node):
+        for field, value in ast.iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, ast.AST):
+                        self.visit(item)
+            elif isinstance(value, ast.AST):
+                self.visit(value)
+
+    def visit_Import(self, node):
+        for alias in node.names:
+            if any(fw in alias.name.lower() for fw in FRAMEWORK_IMPORTS):
+                self.is_framework_file = True
+                self.detected_frameworks.add(alias.name.split('.')[0])
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node):
+        if node.module:
+            module_name = node.module.split('.')[0].lower()
+            if module_name in FRAMEWORK_IMPORTS:
+                self.is_framework_file = True
+                self.detected_frameworks.add(module_name)
+        self.generic_visit(node)
+
+    def visit_FunctionDef(self, node):
+        for deco in node.decorator_list:
+            decorator_str = self._normalize_decorator(deco)
+            if self._matches_framework_pattern(decorator_str, FRAMEWORK_DECORATORS):
+                self.framework_decorated_lines.add(node.lineno)
+                self.is_framework_file = True
+        
+        if self._matches_framework_pattern(node.name, FRAMEWORK_FUNCTIONS):
+            if self.is_framework_file:  
+                self.framework_decorated_lines.add(node.lineno)
+        
+        self.generic_visit(node)
+
+    def visit_AsyncFunctionDef(self, node):
+        self.visit_FunctionDef(node)
+
+    def visit_ClassDef(self, node):
+        for base in node.bases:
+            if isinstance(base, ast.Name):
+                if any(pattern in base.id.lower() for pattern in ['view', 'viewset', 'api', 'handler']):
+                    if self.is_framework_file:
+                        self.framework_decorated_lines.add(node.lineno)
+        
+        self.generic_visit(node)
+
+    def _normalize_decorator(self, decorator_node):
+        if isinstance(decorator_node, ast.Name):
+            return f"@{decorator_node.id}"
+        elif isinstance(decorator_node, ast.Attribute):
+            if isinstance(decorator_node.value, ast.Name):
+                return f"@{decorator_node.value.id}.{decorator_node.attr}"
+            else:
+                return f"@{decorator_node.attr}"
+        elif isinstance(decorator_node, ast.Call):
+            return self._normalize_decorator(decorator_node.func)
+        return "@unknown"
+
+    def _matches_framework_pattern(self, text, patterns):
+        text_clean = text.lstrip('@')
+        return any(fnmatch.fnmatch(text_clean, pattern.lstrip('@')) for pattern in patterns)
+
+    def _check_framework_imports_in_file(self, filename):
+        try:
+            content = Path(filename).read_text(encoding='utf-8')
+            for framework in FRAMEWORK_IMPORTS:
+                if (f'import {framework}' in content or 
+                    f'from {framework}' in content):
+                    self.is_framework_file = True
+                    self.detected_frameworks.add(framework)
+                    break
+        except:
+            pass
+
+def detect_framework_usage(definition, decorator_nodes=None, visitor=None):
+    if not visitor:
+        return None
+    
+    # very low confidence - likely framework magic
+    if definition.line in visitor.framework_decorated_lines:
+        return 20  
+    
+    # framework file but no direct markers
+    if visitor.is_framework_file:
+        if (not definition.simple_name.startswith('_') and 
+            definition.type in ('function', 'method')):
+            return 40 
+    
+    return None