gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/type_coverage.py

@@ -0,0 +1,277 @@
+"""Type hint coverage analysis step.
+
+Analyzes Python source files to compute type annotation coverage percentage,
+identifying functions/methods/classes that lack type hints.
+"""
+
+import ast
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Tuple
+
+from .base import StepResult
+from ..context import BundleContext
+from ..filters import should_exclude_from_analysis
+
+
+@dataclass
+class TypeCoverageStats:
+    """Type coverage statistics."""
+
+    total_functions: int = 0
+    typed_functions: int = 0
+    total_classes: int = 0
+    typed_classes: int = 0
+    total_attributes: int = 0
+    typed_attributes: int = 0
+    missing_items: List[Tuple[str, int, str]] = field(
+        default_factory=list
+    )  # (file, line, name)
+
+    @property
+    def function_coverage(self) -> float:
+        """Return function type coverage percentage."""
+        if self.total_functions == 0:
+            return 100.0
+        return (self.typed_functions / self.total_functions) * 100
+
+    @property
+    def class_coverage(self) -> float:
+        """Return class type coverage percentage."""
+        if self.total_classes == 0:
+            return 100.0
+        return (self.typed_classes / self.total_classes) * 100
+
+    @property
+    def attribute_coverage(self) -> float:
+        """Return attribute type coverage percentage."""
+        if self.total_attributes == 0:
+            return 100.0
+        return (self.typed_attributes / self.total_attributes) * 100
+
+    @property
+    def overall_coverage(self) -> float:
+        """Return overall type coverage percentage."""
+        total = self.total_functions + self.total_classes + self.total_attributes
+        typed = self.typed_functions + self.typed_classes + self.typed_attributes
+        if total == 0:
+            return 100.0
+        return (typed / total) * 100
+
+
+class TypeCoverageAnalyzer(ast.NodeVisitor):
+    """AST visitor for analyzing type coverage."""
+
+    def __init__(self, filepath: str) -> None:
+        self.filepath = filepath
+        self.stats = TypeCoverageStats()
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+        """Visit function definition."""
+        self._analyze_function(node)
+        self.generic_visit(node)
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
+        """Visit async function definition."""
+        self._analyze_function(node)
+        self.generic_visit(node)
+
+    def _analyze_function(self, node):
+        """Analyze function/method for type hints."""
+        # Skip special methods and private methods
+        if node.name.startswith("_") and not node.name.startswith("__"):
+            return
+
+        self.stats.total_functions += 1
+
+        # Check if function has type hints
+        has_return_type = node.returns is not None
+        has_param_types = all(
+            arg.annotation is not None
+            for arg in node.args.args
+            if arg.arg not in ("self", "cls")
+        )
+
+        if has_return_type and (
+            has_param_types
+            or len([a for a in node.args.args if a.arg not in ("self", "cls")]) == 0
+        ):
+            self.stats.typed_functions += 1
+        else:
+            self.stats.missing_items.append(
+                (self.filepath, node.lineno, f"function '{node.name}'")
+            )
+
+    def visit_ClassDef(self, node: ast.ClassDef) -> None:
+        """Visit class definition."""
+        # Skip private classes
+        if node.name.startswith("_"):
+            self.generic_visit(node)
+            return
+
+        self.stats.total_classes += 1
+
+        # Check if class has any type-hinted attributes
+        has_annotations = any(isinstance(stmt, ast.AnnAssign) for stmt in node.body)
+
+        # Check __init__ for parameter types
+        init_method = None
+        for stmt in node.body:
+            if isinstance(stmt, ast.FunctionDef) and stmt.name == "__init__":
+                init_method = stmt
+                break
+
+        if init_method:
+            has_init_types = any(
+                arg.annotation is not None
+                for arg in init_method.args.args
+                if arg.arg not in ("self", "cls")
+            )
+            if has_annotations or has_init_types:
+                self.stats.typed_classes += 1
+            else:
+                self.stats.missing_items.append(
+                    (self.filepath, node.lineno, f"class '{node.name}'")
+                )
+        else:
+            # Class without __init__ - check for annotated attributes
+            if has_annotations:
+                self.stats.typed_classes += 1
+            else:
+                self.stats.missing_items.append(
+                    (self.filepath, node.lineno, f"class '{node.name}'")
+                )
+
+        self.generic_visit(node)
+
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
+        """Visit annotated assignment (class/module level)."""
+        # Count as typed attribute
+        self.stats.total_attributes += 1
+        self.stats.typed_attributes += 1
+        self.generic_visit(node)
+
+
+@dataclass
+class TypeCoverageStep:
+    """Step that analyzes type hint coverage."""
+
+    name: str = "type-coverage"
+    outfile: str = "logs/80_type_coverage.txt"
+
+    def run(self, context: BundleContext) -> StepResult:
+        """Analyze type coverage in Python files."""
+        import time
+
+        start = time.time()
+
+        # Get all Python files in the project
+        python_files = self._find_python_files(context.root)
+
+        if not python_files:
+            elapsed = time.time() - start
+            note = "No Python files found"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        overall_stats = TypeCoverageStats()
+
+        for filepath in python_files:
+            try:
+                with open(filepath, "r", encoding="utf-8") as f:
+                    source = f.read()
+
+                tree = ast.parse(source, filename=str(filepath))
+                analyzer = TypeCoverageAnalyzer(str(filepath))
+                analyzer.visit(tree)
+
+                # Merge stats
+                overall_stats.total_functions += analyzer.stats.total_functions
+                overall_stats.typed_functions += analyzer.stats.typed_functions
+                overall_stats.total_classes += analyzer.stats.total_classes
+                overall_stats.typed_classes += analyzer.stats.typed_classes
+                overall_stats.total_attributes += analyzer.stats.total_attributes
+                overall_stats.typed_attributes += analyzer.stats.typed_attributes
+                overall_stats.missing_items.extend(analyzer.stats.missing_items)
+
+            except SyntaxError:
+                # Skip files with syntax errors
+                continue
+            except Exception:
+                # Skip files that can't be parsed
+                continue
+
+        elapsed = time.time() - start
+
+        # Write report
+        log_path = context.workdir / self.outfile
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(log_path, "w") as f:
+            f.write("=" * 80 + "\n")
+            f.write("TYPE HINT COVERAGE ANALYSIS\n")
+            f.write("=" * 80 + "\n\n")
+
+            f.write("Summary:\n")
+            f.write("-" * 80 + "\n")
+            f.write(f"Overall Coverage:    {overall_stats.overall_coverage:6.2f}%\n")
+            f.write(
+                f"Function Coverage:   {overall_stats.function_coverage:6.2f}% "
+                f"({overall_stats.typed_functions}/{overall_stats.total_functions})\n"
+            )
+            f.write(
+                f"Class Coverage:      {overall_stats.class_coverage:6.2f}% "
+                f"({overall_stats.typed_classes}/{overall_stats.total_classes})\n"
+            )
+            f.write(
+                f"Attribute Coverage:  {overall_stats.attribute_coverage:6.2f}% "
+                f"({overall_stats.typed_attributes}/{overall_stats.total_attributes})\n"
+            )
+            f.write("\n")
+
+            if overall_stats.missing_items:
+                f.write("Missing Type Hints:\n")
+                f.write("-" * 80 + "\n")
+
+                # Sort by file, then line number
+                sorted_missing = sorted(
+                    overall_stats.missing_items, key=lambda x: (x[0], x[1])
+                )
+
+                current_file = None
+                for filepath, lineno, name in sorted_missing:
+                    # Show relative path
+                    rel_path = os.path.relpath(filepath, context.root)
+                    if rel_path != current_file:
+                        f.write(f"\n{rel_path}:\n")
+                        current_file = rel_path
+                    f.write(f"  Line {lineno:4d}: {name}\n")
+
+                f.write("\n")
+
+            f.write("=" * 80 + "\n")
+            f.write(f"Analysis complete - {len(python_files)} files analyzed\n")
+            f.write("=" * 80 + "\n")
+
+        # Determine status
+        coverage = overall_stats.overall_coverage
+        if coverage >= 80:
+            status = "OK"
+        elif coverage >= 50:
+            status = "WARN"
+        else:
+            status = "FAIL"
+
+        note = f"{coverage:.1f}% type coverage"
+        return StepResult(self.name, status, int(elapsed), note)
+
+    def _find_python_files(self, root: Path) -> List[Path]:
+        """Find all Python source files, excluding dependencies/caches/build dirs."""
+        python_files = []
+
+        for path in root.rglob("*.py"):
+            # Use comprehensive exclusion filter for PROJECT files only
+            if should_exclude_from_analysis(path):
+                continue
+            python_files.append(path)
+
+        return python_files

pybundle/steps/unused_deps.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class UnusedDependenciesStep:
+    name: str = "unused dependencies"
+    outfile: str = "meta/31_unused_packages.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        python = ctx.tools.python
+        if not python:
+            out.write_text("python not found; skipping\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing python")
+
+        try:
+            # Get all installed packages
+            pip_freeze_result = subprocess.run(  # nosec B603
+                [python, "-m", "pip", "freeze"],
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=30,
+            )
+
+            if pip_freeze_result.returncode != 0:
+                out.write_text(
+                    f"pip freeze failed: {pip_freeze_result.stderr}\n", encoding="utf-8"
+                )
+                return StepResult(self.name, "FAIL", 0, "pip freeze failed")
+
+            # Parse installed packages (normalize names)
+            installed_packages: list[str] = []
+            for line in pip_freeze_result.stdout.splitlines():
+                if line and not line.startswith("-") and "==" in line:
+                    pkg_name = line.split("==")[0].strip().lower().replace("_", "-")
+                    installed_packages.append(pkg_name)
+
+            # Get imported modules from source code
+            imported_modules = self._get_imported_modules(ctx.root)
+
+            # Find unused packages (installed but not imported)
+            # Note: This is a heuristic - some packages are used indirectly
+            unused = sorted(set(installed_packages) - imported_modules)
+            
+            # Categorize by confidence level
+            likely_unused, maybe_unused = self._categorize_by_confidence(
+                unused, ctx.root
+            )
+
+            # Write results
+            with out.open("w", encoding="utf-8") as f:
+                f.write("=" * 70 + "\n")
+                f.write("UNUSED DEPENDENCIES ANALYSIS\n")
+                f.write("=" * 70 + "\n\n")
+                f.write(f"Total installed packages: {len(installed_packages)}\n")
+                f.write(f"Total imported modules: {len(imported_modules)}\n")
+                f.write(f"Potentially unused packages: {len(unused)}\n\n")
+
+                if likely_unused:
+                    f.write("LIKELY UNUSED (high confidence - no obvious indirect usage):\n")
+                    f.write("These packages are good candidates for removal.\n\n")
+                    for pkg in likely_unused:
+                        f.write(f"  - {pkg}\n")
+                    f.write("\n")
+                
+                if maybe_unused:
+                    f.write("MAYBE UNUSED (lower confidence - might be plugins/indirect deps):\n")
+                    f.write("These may be used indirectly or as plugins. Verify before removing.\n\n")
+                    for pkg in maybe_unused:
+                        f.write(f"  - {pkg}\n")
+                    f.write("\n")
+                
+                if not unused:
+                    f.write("No obviously unused packages detected.\n")
+
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "OK", elapsed, "")
+
+        except subprocess.TimeoutExpired:
+            out.write_text("Analysis timed out\n", encoding="utf-8")
+            return StepResult(
+                self.name, "FAIL", int((time.time() - start) * 1000), "timeout"
+            )
+        except Exception as e:
+            out.write_text(f"Error: {e}\n", encoding="utf-8")
+            return StepResult(
+                self.name, "FAIL", int((time.time() - start) * 1000), str(e)
+            )
+
+    def _get_imported_modules(self, root: Path) -> set[str]:
+        """Extract top-level module names from import statements."""
+        imported = set()
+
+        for py_file in root.rglob("*.py"):
+            # Skip venv and common excluded directories
+            parts = set(py_file.parts)
+            if any(
+                x in parts
+                for x in [
+                    ".venv",
+                    "venv",
+                    "__pycache__",
+                    "node_modules",
+                    "dist",
+                    "build",
+                    "artifacts",
+                    ".git",
+                    ".tox",
+                ]
+            ):
+                continue
+
+            try:
+                content = py_file.read_text(encoding="utf-8", errors="ignore")
+                for line in content.splitlines():
+                    line = line.strip()
+                    # Match: import foo, from foo import bar
+                    if line.startswith("import "):
+                        module = line[7:].split()[0].split(".")[0].split(",")[0].strip()
+                        imported.add(module.lower().replace("_", "-"))
+                    elif line.startswith("from "):
+                        line_parts = line.split()
+                        if len(line_parts) >= 2:
+                            module = line_parts[1].split(".")[0].strip()
+                            imported.add(module.lower().replace("_", "-"))
+            except Exception:
+                continue
+
+        return imported
+    
+    def _categorize_by_confidence(
+        self, unused: list[str], root: Path
+    ) -> tuple[list[str], list[str]]:
+        """Categorize unused packages by confidence level.
+        
+        Returns:
+            (likely_unused, maybe_unused) tuple of package lists
+        """
+        # Packages that are commonly indirect deps or plugins (lower confidence)
+        indirect_patterns = {
+            # Build/dev tools often used indirectly
+            "setuptools", "wheel", "pip", "build",
+            # Testing plugins
+            "pytest-", "coverage", "pluggy",
+            # Type checking stubs
+            "types-", "-stubs",
+            # Common indirect deps
+            "certifi", "charset-normalizer", "idna", "urllib3",
+            "six", "packaging", "pyparsing",
+            # Web framework plugins often auto-discovered
+            "uvicorn", "gunicorn", "hypercorn",
+            # Database drivers (might be dynamically loaded)
+            "psycopg2", "pymysql", "cx-oracle",
+            # Celery/async workers
+            "celery", "redis", "kombu", "amqp",
+        }
+        
+        likely_unused = []
+        maybe_unused = []
+        
+        for pkg in unused:
+            # Check if package matches indirect dependency patterns
+            is_likely_indirect = any(
+                pattern in pkg for pattern in indirect_patterns
+            )
+            
+            # Check if mentioned in config files (pyproject.toml, setup.py, etc.)
+            mentioned_in_config = self._is_mentioned_in_config(pkg, root)
+            
+            if is_likely_indirect or mentioned_in_config:
+                maybe_unused.append(pkg)
+            else:
+                likely_unused.append(pkg)
+        
+        return likely_unused, maybe_unused
+    
+    def _is_mentioned_in_config(self, pkg: str, root: Path) -> bool:
+        """Check if package is mentioned in config files."""
+        config_files = [
+            "pyproject.toml",
+            "setup.py",
+            "setup.cfg",
+            "requirements.txt",
+            "requirements-dev.txt",
+        ]
+        
+        for config_file in config_files:
+            config_path = root / config_file
+            if config_path.exists():
+                try:
+                    content = config_path.read_text(encoding="utf-8", errors="ignore")
+                    # Normalize package name for comparison
+                    if pkg in content or pkg.replace("-", "_") in content:
+                        return True
+                except Exception:
+                    continue
+        
+        return False

pybundle/steps/vulture.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+def _detect_framework_decorators(root: Path) -> set[str]:
+    """Detect common framework decorators that should be whitelisted."""
+    decorators = set()
+    
+    # Check for FastAPI
+    for p in root.rglob("*.py"):
+        try:
+            content = p.read_text(encoding="utf-8", errors="ignore")
+            if "from fastapi" in content or "import fastapi" in content:
+                decorators.update([
+                    "@app.get",
+                    "@app.post", 
+                    "@app.put",
+                    "@app.delete",
+                    "@app.patch",
+                    "@router.get",
+                    "@router.post",
+                    "@router.put",
+                    "@router.delete",
+                    "@router.patch",
+                    "@app.on_event",
+                    "@app.middleware",
+                ])
+            if "from flask" in content or "import flask" in content:
+                decorators.update([
+                    "@app.route",
+                    "@app.before_request",
+                    "@app.after_request",
+                    "@app.errorhandler",
+                ])
+            if "from django" in content or "import django" in content:
+                decorators.update([
+                    "@login_required",
+                    "@permission_required",
+                    "@staff_member_required",
+                ])
+        except Exception:
+            continue
+    
+    return decorators
+
+
+@dataclass
+class VultureStep:
+    name: str = "vulture"
+    target: str = "."
+    outfile: str = "logs/50_vulture.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        vulture = which("vulture")
+        if not vulture:
+            out.write_text(
+                "vulture not found; skipping (pip install vulture)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing vulture")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping vulture\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        target_path = ctx.root / self.target
+        
+        # Create FastAPI/framework decorator whitelist if applicable
+        whitelist_file = self._create_framework_whitelist(ctx)
+        
+        # Build command with ALL OPTIONS FIRST, then paths
+        # This is critical for argparse compatibility
+        cmd = [
+            vulture,
+            "--exclude",
+            "*venv*,*.venv*,.pybundle-venv,venv,env,.env,__pycache__,artifacts,build,dist,.git,.tox,node_modules",
+            "--min-confidence",
+            "60",
+            "--sort-by-size",
+        ]
+        
+        # Add target path
+        cmd.append(str(target_path))
+        
+        # Add whitelist as a regular path argument (not a flag)
+        # Vulture treats whitelist files as regular Python files to analyze
+        if whitelist_file and whitelist_file.exists():
+            cmd.append(str(whitelist_file))
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=120,
+            )
+            out.write_text(result.stdout, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+
+            # Vulture exit codes:
+            # 0 = no dead code found
+            # 1 = usage/configuration error
+            # 3 = dead code found (this is success!)
+            if result.returncode in (0, 3):
+                return StepResult(self.name, "OK", elapsed, "")
+            else:
+                return StepResult(
+                    self.name, "FAIL", elapsed, f"exit {result.returncode}"
+                )
+        except subprocess.TimeoutExpired:
+            out.write_text("vulture timed out after 120s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 120000, "timeout")
+        except Exception as e:
+            out.write_text(f"vulture error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))
+    
+    def _create_framework_whitelist(self, ctx: BundleContext) -> Path | None:
+        """Create a temporary whitelist file for framework decorators."""
+        decorators = _detect_framework_decorators(ctx.root)
+        
+        if not decorators:
+            return None
+        
+        # Put whitelist in meta/whitelists/ not logs/ to avoid path parsing issues
+        whitelist_path = ctx.workdir / "meta" / "whitelists" / "vulture_framework.py"
+        whitelist_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        with whitelist_path.open("w", encoding="utf-8") as f:
+            f.write("# Auto-generated whitelist for framework decorators\n")
+            f.write("# These patterns are commonly used but appear unused to vulture\n\n")
+            for decorator in sorted(decorators):
+                # Create dummy functions to whitelist the decorator pattern
+                func_name = decorator.replace("@", "").replace(".", "_")
+                f.write(f"def {func_name}(): pass\n")
+        
+        return whitelist_path