gwc-pybundle 1.4.5__py3-none-any.whl

pybundle/steps/cprofile_step.py

@@ -0,0 +1,155 @@
+"""
+CPU profiling with cProfile - Milestone 3 (v1.4.0)
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class CProfileStep:
+    """
+    Run cProfile on the project entry point or test suite to identify CPU bottlenecks.
+    
+    Outputs:
+    - logs/60_cprofile.txt: Top 50 slowest functions
+    - meta/60_cprofile.stats: Binary stats file for further analysis
+    """
+    
+    name: str = "cprofile"
+    
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        
+        if ctx.options.no_profile:
+            return StepResult(self.name, "SKIP", 0, "profiling disabled")
+        
+        # Determine default profiling target
+        entry_point = ctx.options.profile_entry_point
+        if not entry_point:
+            # Default: profile pytest if tests/ exists
+            tests_dir = ctx.root / "tests"
+            if not tests_dir.is_dir():
+                return StepResult(self.name, "SKIP", 0, "no tests/ and no entry point")
+        
+        # Determine what to profile
+        if entry_point:
+            target_path = Path(entry_point)
+            if not target_path.is_absolute():
+                target_path = ctx.root / entry_point
+            
+            if not target_path.exists():
+                return StepResult(self.name, "SKIP", 0, f"entry point not found: {entry_point}")
+            
+            if target_path.is_file():
+                # Profile a specific script
+                cmd = [
+                    str(ctx.tools.python),
+                    "-m", "cProfile",
+                    "-o", str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                    str(target_path)
+                ]
+                desc = f"Profiling {target_path.name}"
+            else:
+                # Assume it's a directory, profile pytest
+                cmd = [
+                    str(ctx.tools.python),
+                    "-m", "cProfile",
+                    "-o", str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                    "-m", "pytest",
+                    str(target_path),
+                    "-q"
+                ]
+                desc = f"Profiling pytest in {target_path.name}/"
+        else:
+            # Default: profile pytest
+            cmd = [
+                str(ctx.tools.python),
+                "-m", "cProfile",
+                "-o", str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                "-m", "pytest",
+                "-q"
+            ]
+            desc = "Profiling pytest"
+        
+        ctx.emit(f"  {desc}")
+        
+        try:
+            # Run profiling
+            result = subprocess.run(
+                cmd,
+                cwd=ctx.root,
+                capture_output=True,
+                text=True,
+                timeout=300  # 5 minute timeout for profiling
+            )
+            
+            # Generate human-readable report
+            stats_file = ctx.workdir / "meta" / "60_cprofile.stats"
+            if stats_file.exists():
+                self._generate_report(stats_file, ctx.workdir)
+                elapsed = int((time.time() - start) * 1000)
+                return StepResult(self.name, "OK", elapsed)
+            else:
+                # Still write output for debugging
+                output_file = ctx.workdir / "logs" / "60_cprofile.txt"
+                output_file.parent.mkdir(parents=True, exist_ok=True)
+                
+                with output_file.open("w") as f:
+                    f.write("=" * 70 + "\n")
+                    f.write("CPU PROFILING FAILED\n")
+                    f.write("=" * 70 + "\n\n")
+                    f.write("STDOUT:\n")
+                    f.write(result.stdout)
+                    f.write("\n\nSTDERR:\n")
+                    f.write(result.stderr)
+                
+                elapsed = int((time.time() - start) * 1000)
+                return StepResult(self.name, "FAIL", elapsed, "stats file not created")
+                
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+    
+    def _generate_report(self, stats_file: Path, workdir: Path) -> None:
+        """Generate top 50 slowest functions report"""
+        import pstats
+        
+        output_file = workdir / "logs" / "60_cprofile.txt"
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        with output_file.open("w") as f:
+            f.write("=" * 70 + "\n")
+            f.write("TOP 50 SLOWEST FUNCTIONS (CPU PROFILING)\n")
+            f.write("=" * 70 + "\n\n")
+            
+            # Load stats
+            stats = pstats.Stats(str(stats_file), stream=f)
+            
+            # Remove directory paths for cleaner output
+            stats.strip_dirs()
+            
+            # Sort by cumulative time and print top 50
+            f.write("Sorted by cumulative time:\n")
+            f.write("-" * 70 + "\n")
+            stats.sort_stats('cumulative')
+            stats.print_stats(50)
+            
+            f.write("\n" + "=" * 70 + "\n")
+            f.write("Sorted by total time (time spent in function itself):\n")
+            f.write("-" * 70 + "\n")
+            stats.sort_stats('time')
+            stats.print_stats(50)
+            
+            f.write("\n" + "=" * 70 + "\n")
+            f.write(f"Full binary stats saved to: meta/60_cprofile.stats\n")
+            f.write("Analyze with: python -m pstats meta/60_cprofile.stats\n")

pybundle/steps/dependency_sizes.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class DependencySizesStep:
+    name: str = "dependency sizes"
+    outfile: str = "meta/33_dependency_sizes.txt"
+    top_n: int = 50  # Show top N largest packages
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        python = ctx.tools.python
+        if not python:
+            out.write_text("python not found; skipping\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing python")
+
+        try:
+            # Get list of installed packages
+            list_result = subprocess.run(  # nosec B603
+                [python, "-m", "pip", "list", "--format=json"],
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=30,
+            )
+            
+            if list_result.returncode != 0:
+                out.write_text(f"pip list failed: {list_result.stderr}\n", encoding="utf-8")
+                return StepResult(self.name, "FAIL", 0, "pip list failed")
+            
+            import json
+            packages = json.loads(list_result.stdout)
+            
+            # Get size for each package
+            package_sizes = []
+            for pkg in packages:
+                pkg_name = pkg["name"]
+                try:
+                    show_result = subprocess.run(  # nosec B603
+                        [python, "-m", "pip", "show", pkg_name],
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        text=True,
+                        timeout=5,
+                    )
+                    
+                    if show_result.returncode == 0:
+                        # Parse Location from pip show output
+                        location = None
+                        for line in show_result.stdout.splitlines():
+                            if line.startswith("Location:"):
+                                location = line.split(":", 1)[1].strip()
+                                break
+                        
+                        if location:
+                            # Calculate directory size
+                            from pathlib import Path
+                            pkg_path = Path(location) / pkg_name.replace("-", "_")
+                            if not pkg_path.exists():
+                                pkg_path = Path(location) / pkg_name
+                            
+                            if pkg_path.exists() and pkg_path.is_dir():
+                                size = sum(f.stat().st_size for f in pkg_path.rglob("*") if f.is_file())
+                                package_sizes.append((pkg_name, pkg["version"], size))
+                except Exception:
+                    # Skip packages that fail
+                    continue
+            
+            # Sort by size (descending)
+            package_sizes.sort(key=lambda x: x[2], reverse=True)
+            
+            # Write results
+            with out.open("w", encoding="utf-8") as f:
+                f.write("=" * 70 + "\n")
+                f.write(f"TOP {min(self.top_n, len(package_sizes))} LARGEST DEPENDENCIES\n")
+                f.write("=" * 70 + "\n\n")
+                f.write(f"Total packages analyzed: {len(packages)}\n")
+                f.write(f"Packages with size data: {len(package_sizes)}\n\n")
+                
+                if package_sizes:
+                    # Calculate total size
+                    total_size = sum(size for _, _, size in package_sizes)
+                    f.write(f"Total size: {self._format_size(total_size)}\n\n")
+                    
+                    f.write(f"{'Package':<40} {'Version':<15} {'Size':>15}\n")
+                    f.write("-" * 70 + "\n")
+                    
+                    for pkg_name, version, size in package_sizes[:self.top_n]:
+                        f.write(f"{pkg_name:<40} {version:<15} {self._format_size(size):>15}\n")
+                else:
+                    f.write("No package size data available.\n")
+            
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "OK", elapsed, None)
+            
+        except subprocess.TimeoutExpired:
+            out.write_text("Analysis timed out\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", int((time.time() - start) * 1000), "timeout")
+        except Exception as e:
+            out.write_text(f"Error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", int((time.time() - start) * 1000), str(e))
+
+    def _format_size(self, size_bytes: int) -> str:
+        """Format size in human-readable format."""
+        for unit in ["B", "KB", "MB", "GB"]:
+            if size_bytes < 1024.0:
+                return f"{size_bytes:.1f} {unit}"
+            size_bytes /= 1024.0
+        return f"{size_bytes:.1f} TB"

pybundle/steps/duplication.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class DuplicationStep:
+    name: str = "duplication"
+    target: str = "."
+    outfile: str = "logs/53_duplication.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        pylint = which("pylint")
+        if not pylint:
+            out.write_text(
+                "pylint not found; skipping (pip install pylint)\n", 
+                encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing pylint")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping duplication check\n", 
+                encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        target_path = ctx.root / self.target
+        cmd = [
+            pylint,
+            str(target_path),
+            "--disable=all",  # Disable all other checks
+            "--enable=duplicate-code",  # Only check for duplication
+            "--min-similarity-lines=6",  # Minimum 6 lines to be considered duplicate
+        ]
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=180,  # Duplication detection can be slower
+            )
+            out.write_text(result.stdout, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+            
+            # pylint returns various exit codes:
+            # 0 = no issues
+            # 1, 2, 4, 8, 16 = various issue types (we still want the output)
+            # We consider all of these as success
+            if result.returncode in (0, 1, 2, 4, 8, 16, 24, 32):
+                return StepResult(self.name, "OK", elapsed, None)
+            else:
+                return StepResult(
+                    self.name, 
+                    "FAIL", 
+                    elapsed, 
+                    f"exit {result.returncode}"
+                )
+        except subprocess.TimeoutExpired:
+            out.write_text("duplication check timed out after 180s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 180000, "timeout")
+        except Exception as e:
+            out.write_text(f"duplication check error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))

pybundle/steps/error_refs.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+DEFAULT_EXCLUDE_PREFIXES = (
+    ".git/",
+    ".venv/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    ".pytest_cache/",
+    "__pycache__/",
+    "node_modules/",
+    "dist/",
+    "build/",
+    "artifacts/",
+)
+
+# Patterns based on your bash sed rules:
+# 1) tool-style: path:line(:col)...
+_RE_COLON_LINE = re.compile(r"^([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\d+(?::\d+)?\b.*$")
+
+# 2) pytest traceback: File "path", line N
+_RE_PYTEST_FILE = re.compile(r'^\s*File "([^"]+)", line \d+\b.*$')
+
+# 3) mypy: (optional "mypy:") ./path:line: (error|note|warning):
+_RE_MYPY_LINE = re.compile(
+    r"^(?:mypy:\s*)?(?:\./)?([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\d+:\s*(?:error|note|warning):.*$"
+)
+
+# 4) mypy rare: path: (error|note|warning): ...
+_RE_MYPY_NOLINE = re.compile(
+    r"^(?:mypy:\s*)?(?:\./)?([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\s*(?:error|note|warning):.*$"
+)
+
+
+def _normalize_to_repo_rel(root: Path, p: str) -> str | None:
+    p = p.strip()
+    if not p:
+        return None
+
+    # remove leading ./ for consistency
+    if p.startswith("./"):
+        p = p[2:]
+
+    # absolute path -> must be under repo root
+    if p.startswith("/"):
+        try:
+            rp = Path(p).resolve()
+            rr = rp.relative_to(root.resolve())
+            return str(rr).replace("\\", "/")
+        except Exception:
+            return None
+
+    # relative path
+    return p.replace("\\", "/")
+
+
+def _is_allowed_repo_file(root: Path, rel: str) -> bool:
+    rel = rel.lstrip("./")
+    if not rel or rel.endswith("/"):
+        return False
+
+    # exclude common junk
+    for pref in DEFAULT_EXCLUDE_PREFIXES:
+        if rel.startswith(pref):
+            return False
+    if "/__pycache__/" in f"/{rel}/":
+        return False
+
+    # must exist and be a file inside repo
+    fp = (root / rel).resolve()
+    try:
+        fp.relative_to(root.resolve())
+    except Exception:
+        return False
+
+    return fp.is_file()
+
+
+def _extract_paths_from_text(text: str) -> list[str]:
+    out: list[str] = []
+    for line in text.splitlines():
+        m = _RE_COLON_LINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_PYTEST_FILE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_MYPY_LINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_MYPY_NOLINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+    return out
+
+
+@dataclass
+class ErrorReferencedFilesStep:
+    name: str = "collect error-referenced files"
+    max_files: int = 250
+    # Paths are relative to the bundle workdir
+    log_files: list[str] | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        # Default set aligned to our step numbers
+        log_files = self.log_files or [
+            "logs/31_ruff_check.txt",
+            "logs/32_ruff_format_check.txt",
+            "logs/33_mypy.txt",
+            "logs/34_pytest_q.txt",
+        ]
+
+        out_list = ctx.workdir / "error_files_from_logs.txt"
+        out_count = ctx.workdir / "error_refs_count.txt"
+        report = ctx.metadir / "60_error_refs_report.txt"
+
+        dest_root = ctx.srcdir / "_error_refs"
+        dest_root.mkdir(parents=True, exist_ok=True)
+
+        # Collect candidate paths
+        candidates: set[str] = set()
+        scanned = 0
+        missing_logs = 0
+
+        for lf in log_files:
+            lp = ctx.workdir / lf
+            if not lp.is_file():
+                missing_logs += 1
+                continue
+            scanned += 1
+            try:
+                txt = lp.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+
+            for raw in _extract_paths_from_text(txt):
+                norm = _normalize_to_repo_rel(ctx.root, raw)
+                if norm:
+                    candidates.add(norm)
+
+        # Normalize / filter to real repo files
+        allowed = sorted([p for p in candidates if _is_allowed_repo_file(ctx.root, p)])
+
+        # Write list file (even if empty)
+        out_list.write_text(
+            "\n".join(allowed) + ("\n" if allowed else ""), encoding="utf-8"
+        )
+
+        # Copy up to max_files
+        copied = 0
+        for rel in allowed:
+            if copied >= self.max_files:
+                break
+            src = ctx.root / rel
+            dst = dest_root / rel
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                # preserve mode/timestamps
+                dst.write_bytes(src.read_bytes())
+                copied += 1
+            except Exception:
+                continue
+
+        out_count.write_text(f"{copied}\n", encoding="utf-8")
+
+        report.write_text(
+            "\n".join(
+                [
+                    f"scanned_logs={scanned}",
+                    f"missing_logs={missing_logs}",
+                    f"candidates_total={len(candidates)}",
+                    f"allowed_repo_files={len(allowed)}",
+                    f"copied={copied}",
+                    f"max_files={self.max_files}",
+                    "dest=src/_error_refs",
+                ]
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+
+        dur = int(time.time() - start)
+        note = f"allowed={len(allowed)} copied={copied}"
+        if copied >= self.max_files:
+            note += " (HIT MAX)"
+        return StepResult(self.name, "PASS", dur, note)