gwc-pybundle 1.4.5__py3-none-any.whl

pybundle/steps/slow_tests.py

@@ -0,0 +1,170 @@
+"""
+Slow test identification - Milestone 4 (v1.4.1)
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class SlowTestsStep:
+    """
+    Identify slow tests by parsing pytest duration output.
+    
+    Outputs:
+    - logs/71_slow_tests.txt: Ranked list of slowest tests
+    """
+    
+    name: str = "slow_tests"
+    
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        
+        if not ctx.tools.pytest:
+            return StepResult(self.name, "SKIP", 0, "pytest not found")
+        
+        tests_dir = ctx.root / "tests"
+        if not tests_dir.is_dir():
+            return StepResult(self.name, "SKIP", 0, "no tests/ directory")
+        
+        threshold = ctx.options.slow_test_threshold
+        ctx.emit(f"  Identifying tests slower than {threshold}s...")
+        
+        output_file = ctx.workdir / "logs" / "71_slow_tests.txt"
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        try:
+            # Run pytest with duration reporting
+            result = subprocess.run(
+                [
+                    str(ctx.tools.pytest),
+                    "-v",
+                    "--durations=0",  # Show all durations
+                    "--tb=no"  # No traceback to keep output clean
+                ],
+                cwd=ctx.root,
+                capture_output=True,
+                text=True,
+                timeout=180  # 3 minute timeout
+            )
+            
+            # Parse durations from output
+            slow_tests = self._parse_durations(result.stdout, threshold)
+            
+            # Generate report
+            with output_file.open("w") as f:
+                f.write("=" * 70 + "\n")
+                f.write(f"SLOW TEST IDENTIFICATION (threshold: {threshold}s)\n")
+                f.write("=" * 70 + "\n\n")
+                
+                if slow_tests:
+                    f.write(f"Found {len(slow_tests)} test(s) exceeding {threshold}s:\n\n")
+                    
+                    # Sort by duration (descending)
+                    slow_tests.sort(key=lambda x: x[1], reverse=True)
+                    
+                    f.write(f"{'Duration (s)':<15} {'Test'}\n")
+                    f.write("-" * 70 + "\n")
+                    
+                    for test_name, duration in slow_tests:
+                        f.write(f"{duration:>13.2f}  {test_name}\n")
+                    
+                    f.write("\n" + "=" * 70 + "\n")
+                    f.write("STATISTICS:\n")
+                    f.write("-" * 70 + "\n")
+                    total_time = sum(d for _, d in slow_tests)
+                    avg_time = total_time / len(slow_tests)
+                    f.write(f"Total slow test time: {total_time:.2f}s\n")
+                    f.write(f"Average slow test time: {avg_time:.2f}s\n")
+                    f.write(f"Slowest test: {slow_tests[0][1]:.2f}s ({slow_tests[0][0]})\n")
+                    
+                    f.write("\n" + "=" * 70 + "\n")
+                    f.write("RECOMMENDATIONS:\n")
+                    f.write("- Profile slow tests to identify bottlenecks\n")
+                    f.write("- Consider using pytest fixtures to reduce setup time\n")
+                    f.write("- Mock external dependencies (DB, API calls, file I/O)\n")
+                    f.write("- Use pytest-xdist for parallel test execution\n")
+                else:
+                    f.write(f"✅ No tests exceed {threshold}s threshold!\n\n")
+                    
+                    # Still show fastest tests for context
+                    all_tests = self._parse_all_durations(result.stdout)
+                    if all_tests:
+                        all_tests.sort(key=lambda x: x[1], reverse=True)
+                        f.write("Top 10 longest tests (all under threshold):\n\n")
+                        f.write(f"{'Duration (s)':<15} {'Test'}\n")
+                        f.write("-" * 70 + "\n")
+                        for test_name, duration in all_tests[:10]:
+                            f.write(f"{duration:>13.2f}  {test_name}\n")
+                
+                # Append raw duration output for reference
+                f.write("\n" + "=" * 70 + "\n")
+                f.write("RAW PYTEST DURATION OUTPUT:\n")
+                f.write("-" * 70 + "\n")
+                # Find and include the duration section
+                in_duration_section = False
+                for line in result.stdout.splitlines():
+                    if "slowest durations" in line.lower() or "=== " in line:
+                        in_duration_section = True
+                    if in_duration_section:
+                        f.write(line + "\n")
+            
+            elapsed = int((time.time() - start) * 1000)
+            
+            if slow_tests:
+                return StepResult(self.name, "OK", elapsed, f"{len(slow_tests)} slow tests")
+            else:
+                return StepResult(self.name, "OK", elapsed)
+                
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+    
+    def _parse_durations(self, output: str, threshold: float) -> list[tuple[str, float]]:
+        """Parse pytest --durations output for tests exceeding threshold"""
+        slow_tests = []
+        
+        # Look for duration lines like: "0.52s call     test_file.py::test_name"
+        for line in output.splitlines():
+            if "s call" in line or "s setup" in line or "s teardown" in line:
+                parts = line.split()
+                if len(parts) >= 3:
+                    try:
+                        duration_str = parts[0].rstrip('s')
+                        duration = float(duration_str)
+                        
+                        if duration >= threshold:
+                            # Extract test name (usually the last part)
+                            test_name = parts[-1]
+                            slow_tests.append((test_name, duration))
+                    except (ValueError, IndexError):
+                        continue
+        
+        return slow_tests
+    
+    def _parse_all_durations(self, output: str) -> list[tuple[str, float]]:
+        """Parse all test durations"""
+        all_tests = []
+        
+        for line in output.splitlines():
+            if "s call" in line:
+                parts = line.split()
+                if len(parts) >= 3:
+                    try:
+                        duration_str = parts[0].rstrip('s')
+                        duration = float(duration_str)
+                        test_name = parts[-1]
+                        all_tests.append((test_name, duration))
+                    except (ValueError, IndexError):
+                        continue
+        
+        return all_tests

pybundle/steps/test_flakiness.py

@@ -0,0 +1,172 @@
+"""
+Test flakiness detection - Milestone 4 (v1.4.1)
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class TestFlakinessStep:
+    """
+    Run tests multiple times to detect non-deterministic failures (flaky tests).
+    
+    Outputs:
+    - logs/70_test_flakiness.txt: Report of flaky tests with pass/fail patterns
+    """
+    
+    name: str = "test_flakiness"
+    
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        
+        if not ctx.tools.pytest:
+            return StepResult(self.name, "SKIP", 0, "pytest not found")
+        
+        tests_dir = ctx.root / "tests"
+        if not tests_dir.is_dir():
+            return StepResult(self.name, "SKIP", 0, "no tests/ directory")
+        
+        runs = ctx.options.test_flakiness_runs
+        ctx.emit(f"  Running tests {runs}x to detect flakiness...")
+        
+        output_file = ctx.workdir / "logs" / "70_test_flakiness.txt"
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        try:
+            # Run tests multiple times and collect results
+            results = []
+            test_outcomes = {}  # test_name -> [pass/fail/error]
+            
+            for i in range(runs):
+                ctx.emit(f"    Run {i+1}/{runs}...")
+                result = subprocess.run(
+                    [str(ctx.tools.pytest), "-v", "--tb=no"],
+                    cwd=ctx.root,
+                    capture_output=True,
+                    text=True,
+                    timeout=180  # 3 minute timeout per run
+                )
+                results.append(result)
+                
+                # Parse test results
+                self._parse_test_outcomes(result.stdout, test_outcomes, i)
+            
+            # Analyze for flakiness
+            flaky_tests = self._identify_flaky_tests(test_outcomes)
+            
+            # Generate report
+            with output_file.open("w") as f:
+                f.write("=" * 70 + "\n")
+                f.write(f"TEST FLAKINESS DETECTION ({runs} runs)\n")
+                f.write("=" * 70 + "\n\n")
+                
+                if not test_outcomes:
+                    f.write("No test results collected.\n\n")
+                    for i, result in enumerate(results):
+                        f.write(f"Run {i+1} output:\n")
+                        f.write(result.stdout[:500])
+                        f.write("\n\n")
+                else:
+                    total_tests = len(test_outcomes)
+                    f.write(f"Total tests analyzed: {total_tests}\n")
+                    f.write(f"Flaky tests detected: {len(flaky_tests)}\n\n")
+                    
+                    if flaky_tests:
+                        f.write("=" * 70 + "\n")
+                        f.write("FLAKY TESTS (non-deterministic results):\n")
+                        f.write("=" * 70 + "\n\n")
+                        
+                        for test_name, outcomes in flaky_tests.items():
+                            pattern = " -> ".join(outcomes)
+                            f.write(f"⚠️  {test_name}\n")
+                            f.write(f"    Pattern: {pattern}\n\n")
+                    else:
+                        f.write("✅ No flaky tests detected - all tests deterministic!\n\n")
+                    
+                    # Summary of all tests
+                    f.write("=" * 70 + "\n")
+                    f.write("ALL TESTS SUMMARY:\n")
+                    f.write("=" * 70 + "\n\n")
+                    
+                    stable_pass = []
+                    stable_fail = []
+                    flaky = []
+                    
+                    for test_name, outcomes in test_outcomes.items():
+                        unique_outcomes = set(outcomes)
+                        if len(unique_outcomes) == 1:
+                            if "PASSED" in unique_outcomes:
+                                stable_pass.append(test_name)
+                            else:
+                                stable_fail.append(test_name)
+                        else:
+                            flaky.append(test_name)
+                    
+                    f.write(f"Stable passing: {len(stable_pass)}\n")
+                    f.write(f"Stable failing: {len(stable_fail)}\n")
+                    f.write(f"Flaky: {len(flaky)}\n\n")
+                    
+                    if stable_fail:
+                        f.write("Consistently failing tests:\n")
+                        for test in stable_fail[:20]:  # Limit to 20
+                            f.write(f"  - {test}\n")
+                        if len(stable_fail) > 20:
+                            f.write(f"  ... and {len(stable_fail) - 20} more\n")
+                        f.write("\n")
+                    
+                    f.write("=" * 70 + "\n")
+                    f.write("RECOMMENDATIONS:\n")
+                    f.write("- Fix flaky tests by removing non-deterministic behavior\n")
+                    f.write("- Common causes: timing issues, random data, external dependencies\n")
+                    f.write("- Use pytest-randomly to test with different orderings\n")
+            
+            elapsed = int((time.time() - start) * 1000)
+            
+            if flaky_tests:
+                return StepResult(self.name, "OK", elapsed, f"{len(flaky_tests)} flaky tests")
+            else:
+                return StepResult(self.name, "OK", elapsed)
+                
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+    
+    def _parse_test_outcomes(self, output: str, test_outcomes: dict, run_num: int) -> None:
+        """Parse pytest -v output to extract test results"""
+        for line in output.splitlines():
+            # Look for pytest verbose output: "test_file.py::test_name PASSED"
+            if "::" in line and any(status in line for status in ["PASSED", "FAILED", "ERROR", "SKIPPED"]):
+                parts = line.split()
+                if len(parts) >= 2:
+                    test_name = parts[0]
+                    # Find status
+                    status = None
+                    for s in ["PASSED", "FAILED", "ERROR", "SKIPPED"]:
+                        if s in line:
+                            status = s
+                            break
+                    
+                    if status:
+                        if test_name not in test_outcomes:
+                            test_outcomes[test_name] = []
+                        test_outcomes[test_name].append(status)
+    
+    def _identify_flaky_tests(self, test_outcomes: dict) -> dict:
+        """Identify tests with inconsistent results across runs"""
+        flaky = {}
+        for test_name, outcomes in test_outcomes.items():
+            unique_outcomes = set(outcomes)
+            # Flaky if not all the same outcome
+            if len(unique_outcomes) > 1:
+                flaky[test_name] = outcomes
+        return flaky

pybundle/steps/tree.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from pybundle.context import BundleContext
+from pybundle.policy import AIContextPolicy, PathFilter
+
+
+@dataclass
+class TreeStep:
+    name: str = "tree (filtered)"
+    max_depth: int = 4
+    excludes: list[str] | None = None
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        policy = self.policy or AIContextPolicy()
+
+        # allow overrides
+        exclude_dirs = set(self.excludes) if self.excludes else set(policy.exclude_dirs)
+        filt = PathFilter(
+            exclude_dirs=exclude_dirs,
+            exclude_patterns=set(policy.exclude_patterns),
+            exclude_file_exts=set(policy.exclude_file_exts),
+        )
+
+        out = ctx.metadir / "10_tree.txt"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        root = ctx.root
+        lines: list[str] = []
+
+        for dirpath, dirnames, filenames in os.walk(root):
+            dp = Path(dirpath)
+            rel_dp = dp.relative_to(root)
+            depth = 0 if rel_dp == Path(".") else len(rel_dp.parts)
+
+            if depth > self.max_depth:
+                dirnames[:] = []
+                continue
+
+            # prune dirs (name + venv-structure)
+            kept = []
+            for d in dirnames:
+                if filt.should_prune_dir(dp, d):
+                    continue
+                kept.append(d)
+            dirnames[:] = kept
+
+            for fn in filenames:
+                p = dp / fn
+                if not filt.should_include_file(root, p):
+                    continue
+                lines.append(str(p.relative_to(root)))
+
+        lines.sort()
+        out.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
+        dur = int(time.time() - start)
+        return StepResult(self.name, "PASS", dur, "python-walk")
+
+
+@dataclass
+class LargestFilesStep:
+    name: str = "largest files"
+    limit: int = 80
+    excludes: list[str] | None = None
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        policy = self.policy or AIContextPolicy()
+
+        exclude_dirs = set(self.excludes) if self.excludes else set(policy.exclude_dirs)
+        filt = PathFilter(
+            exclude_dirs=exclude_dirs,
+            exclude_patterns=set(policy.exclude_patterns),
+            exclude_file_exts=set(policy.exclude_file_exts),
+        )
+
+        out = ctx.metadir / "11_largest_files.txt"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        files: list[tuple[int, str]] = []
+        root = ctx.root
+
+        for dirpath, dirnames, filenames in os.walk(root):
+            dp = Path(dirpath)
+
+            kept = []
+            for d in dirnames:
+                if filt.should_prune_dir(dp, d):
+                    continue
+                kept.append(d)
+            dirnames[:] = kept
+
+            for fn in filenames:
+                p = dp / fn
+                if not filt.should_include_file(root, p):
+                    continue
+                try:
+                    size = p.stat().st_size
+                except OSError:
+                    continue
+                files.append((size, str(p.relative_to(root))))
+
+        files.sort(key=lambda x: x[0], reverse=True)
+        lines = [f"{size}\t{path}" for size, path in files[: self.limit]]
+        out.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        return StepResult(self.name, "PASS", dur, f"count={len(files)}")

pybundle/steps/unused_deps.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+@dataclass
+class UnusedDependenciesStep:
+    name: str = "unused dependencies"
+    outfile: str = "meta/31_unused_packages.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        python = ctx.tools.python
+        if not python:
+            out.write_text("python not found; skipping\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing python")
+
+        try:
+            # Get all installed packages
+            pip_freeze_result = subprocess.run(  # nosec B603
+                [python, "-m", "pip", "freeze"],
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=30,
+            )
+            
+            if pip_freeze_result.returncode != 0:
+                out.write_text(f"pip freeze failed: {pip_freeze_result.stderr}\n", encoding="utf-8")
+                return StepResult(self.name, "FAIL", 0, "pip freeze failed")
+            
+            # Parse installed packages (normalize names)
+            installed_packages = set()
+            for line in pip_freeze_result.stdout.splitlines():
+                if line and not line.startswith("-") and "==" in line:
+                    pkg_name = line.split("==")[0].strip().lower().replace("_", "-")
+                    installed_packages.add(pkg_name)
+            
+            # Get imported modules from source code
+            imported_modules = self._get_imported_modules(ctx.root)
+            
+            # Find unused packages (installed but not imported)
+            # Note: This is a heuristic - some packages are used indirectly
+            unused = sorted(installed_packages - imported_modules)
+            
+            # Write results
+            with out.open("w", encoding="utf-8") as f:
+                f.write("=" * 70 + "\n")
+                f.write("UNUSED DEPENDENCIES ANALYSIS\n")
+                f.write("=" * 70 + "\n\n")
+                f.write(f"Total installed packages: {len(installed_packages)}\n")
+                f.write(f"Total imported modules: {len(imported_modules)}\n")
+                f.write(f"Potentially unused packages: {len(unused)}\n\n")
+                
+                if unused:
+                    f.write("Packages installed but not directly imported:\n")
+                    f.write("(Note: Some may be indirect dependencies or plugins)\n\n")
+                    for pkg in unused:
+                        f.write(f"  - {pkg}\n")
+                else:
+                    f.write("No obviously unused packages detected.\n")
+            
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "OK", elapsed, None)
+            
+        except subprocess.TimeoutExpired:
+            out.write_text("Analysis timed out\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", int((time.time() - start) * 1000), "timeout")
+        except Exception as e:
+            out.write_text(f"Error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", int((time.time() - start) * 1000), str(e))
+
+    def _get_imported_modules(self, root: Path) -> set[str]:
+        """Extract top-level module names from import statements."""
+        imported = set()
+        
+        for py_file in root.rglob("*.py"):
+            # Skip venv and common excluded directories
+            parts = set(py_file.parts)
+            if any(x in parts for x in [".venv", "venv", "__pycache__", "node_modules", 
+                                         "dist", "build", "artifacts", ".git", ".tox"]):
+                continue
+            
+            try:
+                content = py_file.read_text(encoding="utf-8", errors="ignore")
+                for line in content.splitlines():
+                    line = line.strip()
+                    # Match: import foo, from foo import bar
+                    if line.startswith("import "):
+                        module = line[7:].split()[0].split(".")[0].split(",")[0].strip()
+                        imported.add(module.lower().replace("_", "-"))
+                    elif line.startswith("from "):
+                        parts = line.split()
+                        if len(parts) >= 2:
+                            module = parts[1].split(".")[0].strip()
+                            imported.add(module.lower().replace("_", "-"))
+            except Exception:
+                continue
+        
+        return imported

pybundle/steps/vulture.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class VultureStep:
+    name: str = "vulture"
+    target: str = "."
+    outfile: str = "logs/50_vulture.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        vulture = which("vulture")
+        if not vulture:
+            out.write_text(
+                "vulture not found; skipping (pip install vulture)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing vulture")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text("no .py files detected; skipping vulture\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        target_path = ctx.root / self.target
+        cmd = [
+            vulture,
+            str(target_path),
+            "--exclude", "*venv*,*.venv*,.pybundle-venv,venv,env,.env,__pycache__",
+            "--min-confidence", "60",  # Configurable confidence threshold
+            "--sort-by-size",
+        ]
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=120,
+            )
+            out.write_text(result.stdout, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+            
+            # Vulture returns 0 if no dead code found, 1 if dead code found
+            # Both are "success" for our purposes
+            if result.returncode in (0, 1):
+                return StepResult(self.name, "OK", elapsed, None)
+            else:
+                return StepResult(self.name, "FAIL", elapsed, f"exit {result.returncode}")
+        except subprocess.TimeoutExpired:
+            out.write_text("vulture timed out after 120s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 120000, "timeout")
+        except Exception as e:
+            out.write_text(f"vulture error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))