gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/copy_pack.py

@@ -0,0 +1,293 @@
+from __future__ import annotations
+
+import os
+import shutil
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from pybundle.context import BundleContext
+from pybundle.policy import AIContextPolicy, PathFilter
+
+
+def _is_venv_root(p: Path) -> bool:
+    if not p.is_dir():
+        return False
+
+    # Strong marker: standard venv metadata
+    if (p / "pyvenv.cfg").is_file():
+        return True
+
+    # Typical venv executables (Linux/macOS)
+    if (p / "bin").is_dir():
+        # venv/virtualenv always has python here
+        if (p / "bin" / "python").exists() or (p / "bin" / "python3").exists():
+            # activation script is common but not guaranteed; still strong signal
+            if (p / "bin" / "activate").is_file():
+                return True
+            # also accept presence of site-packages under lib
+            if any((p / "lib").glob("python*/site-packages")):
+                return True
+
+    # Windows venv layout
+    if (p / "Scripts").is_dir():
+        if (p / "Scripts" / "python.exe").is_file() or (
+            p / "Scripts" / "python"
+        ).exists():
+            if (p / "Scripts" / "activate").is_file():
+                return True
+            if (p / "Lib" / "site-packages").is_dir():
+                return True
+
+    # Some virtualenvs keep a .Python marker (macOS, older tooling)
+    if (p / ".Python").exists():
+        return True
+
+    return False
+
+
+def _is_under_venv(root: Path, rel_path: Path) -> bool:
+    # walk ancestors: a/b/c.py -> check a, a/b, a/b/c
+    cur = root
+    for part in rel_path.parts:
+        cur = cur / part
+        if _is_venv_root(cur):
+            return True
+    return False
+
+
+def _safe_copy_file(src: Path, dst: Path) -> None:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    # preserve mode + timestamps where possible
+    shutil.copy2(src, dst)
+
+
+def _copy_tree_filtered(
+    root: Path,
+    src_dir: Path,
+    dst_dir: Path,
+    filt: "PathFilter",
+) -> tuple[int, int, int]:
+    """
+    Copy directory tree while pruning excluded directories and skipping excluded files.
+
+    Returns: (files_copied, dirs_pruned, files_excluded)
+    """
+    seen_files = 0
+    files_copied = 0
+    pruned_dirs = 0
+
+    for dirpath, dirnames, filenames in os.walk(src_dir):
+        dp = Path(dirpath)
+        rel_dir = dp.relative_to(src_dir)
+
+        # prune dirs in-place so os.walk doesn't descend into them
+        kept: list[str] = []
+        for d in dirnames:
+            if filt.should_prune_dir(dp, d):
+                pruned_dirs += 1
+                continue
+            kept.append(d)
+        dirnames[:] = kept
+
+        for fn in filenames:
+            seen_files += 1
+            sp = dp / fn
+            rel_file = rel_dir / fn
+
+            # single source of truth: PathFilter handles excluded dirs, patterns, and extensions
+            if not filt.should_include_file(root, sp):
+                continue
+
+            tp = dst_dir / rel_file
+            try:
+                _safe_copy_file(sp, tp)
+            except OSError:
+                continue
+
+            files_copied += 1
+
+    files_excluded = max(0, seen_files - files_copied)
+    return files_copied, pruned_dirs, files_excluded
+
+
+def _guess_package_dirs(root: Path, filt: "PathFilter") -> list[Path]:
+    out: list[Path] = []
+    for p in sorted(root.iterdir()):
+        if not p.is_dir():
+            continue
+        if p.name.startswith("."):
+            continue
+        if filt.should_prune_dir(root, p.name):
+            continue
+        if (p / "__init__.py").is_file():
+            out.append(p)
+    return out
+
+
+@dataclass
+class CuratedCopyStep:
+    name: str = "copy curated source pack"
+    include_files: list[str] | None = None
+    include_dirs: list[str] | None = None
+    include_globs: list[str] | None = None
+    exclude_dirs: set[str] | None = None
+    max_files: int = 20000
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        dst_root = ctx.srcdir  # bundle/src
+        dst_root.mkdir(parents=True, exist_ok=True)
+
+        policy = self.policy or AIContextPolicy()
+
+        exclude = (
+            set(self.exclude_dirs) if self.exclude_dirs else set(policy.exclude_dirs)
+        )
+        exclude_patterns = set(policy.exclude_patterns)
+        filt = PathFilter(
+            exclude_dirs=exclude,
+            exclude_patterns=exclude_patterns,
+            exclude_file_exts=set(policy.exclude_file_exts),
+        )
+        include_files = self.include_files or list(policy.include_files)
+        include_dirs = self.include_dirs or list(policy.include_dirs)
+        include_globs = self.include_globs or list(policy.include_globs)
+
+        copied = 0
+        pruned = 0
+        excluded_total = 0
+
+        # 1) Include well-known top-level files if present
+        for rel_file in include_files:
+            if copied >= self.max_files:
+                break
+
+            sp = ctx.root / rel_file
+            if not sp.is_file():
+                continue
+            if not filt.should_include_file(ctx.root, sp):
+                continue
+
+            try:
+                _safe_copy_file(sp, dst_root / rel_file)
+                copied += 1
+                if copied >= self.max_files:
+                    break
+            except OSError:
+                pass
+
+        # 2) Include common top-level dirs (src/tests/tools)
+        for rel_dir in include_dirs:
+            sp = ctx.root / rel_dir
+            if not sp.is_dir():
+                continue
+
+            # policy prune (exact + patterns + venv detection inside PathFilter)
+            if filt.should_prune_dir(ctx.root, rel_dir):
+                pruned += 1
+                continue
+
+            # extra-strong venv detection for oddly-named envs
+            if _is_venv_root(sp):
+                pruned += 1
+                continue
+
+            files_copied, dirs_pruned, files_excluded = _copy_tree_filtered(
+                ctx.root, sp, dst_root / rel_dir, filt
+            )
+            copied += files_copied
+            pruned += dirs_pruned
+            excluded_total += files_excluded
+
+            if copied >= self.max_files:
+                break
+
+        # 3) Include detected package dirs at root (if not already copied)
+        if copied < self.max_files:
+            for pkg_dir in _guess_package_dirs(ctx.root, filt):
+                rel_pkg_name = pkg_dir.name
+                if (dst_root / rel_pkg_name).exists():
+                    continue
+                files_copied, dirs_pruned, files_excluded = _copy_tree_filtered(
+                    ctx.root, pkg_dir, dst_root / rel_pkg_name, filt
+                )
+                copied += files_copied
+                pruned += dirs_pruned
+                excluded_total += files_excluded
+                if copied >= self.max_files:
+                    break
+
+        # 4) Optional globs (best-effort; avoid deep explosion by pruning excluded dirs)
+        # We’ll apply globs but skip anything under excluded dirs.
+        if copied < self.max_files:
+            for g in include_globs:
+                for sp in ctx.root.glob(g):
+                    try:
+                        if not sp.exists():
+                            continue
+
+                        rel_path = sp.relative_to(ctx.root)
+
+                        if _is_under_venv(ctx.root, rel_path):
+                            pruned += 1
+                            continue
+
+                        dst = dst_root / rel_path
+                        if dst.exists():
+                            continue
+
+                        if sp.is_file():
+                            if not filt.should_include_file(ctx.root, sp):
+                                continue
+                            _safe_copy_file(sp, dst)
+                            copied += 1
+
+                        elif sp.is_dir():
+                            # prune dir itself before copying
+                            parent = (
+                                ctx.root
+                                if rel_path.parent == Path(".")
+                                else (ctx.root / rel_path.parent)
+                            )
+                            if filt.should_prune_dir(parent, rel_path.name):
+                                pruned += 1
+                                continue
+                            if _is_venv_root(sp):
+                                pruned += 1
+                                continue
+
+                            files_copied, dirs_pruned, files_excluded = (
+                                _copy_tree_filtered(
+                                    ctx.root, sp, dst_root / rel_path, filt
+                                )
+                            )
+                            copied += files_copied
+                            pruned += dirs_pruned
+                            excluded_total += files_excluded
+
+                        if copied >= self.max_files:
+                            break
+                    except Exception:
+                        continue
+                if copied >= self.max_files:
+                    break
+
+        # write a short manifest for sanity
+        manifest = ctx.workdir / "meta" / "50_copy_manifest.txt"
+        manifest.parent.mkdir(parents=True, exist_ok=True)
+        manifest.write_text(
+            f"copied_files={copied}\n"
+            f"excluded_files={excluded_total}\n"
+            f"pruned_dirs={pruned}\n"
+            f"max_files={self.max_files}\n",
+            encoding="utf-8",
+        )
+
+        dur = int(time.time() - start)
+        note = f"copied={copied} pruned={pruned}"
+        if copied >= self.max_files:
+            note += " (HIT MAX)"
+        return StepResult(self.name, "PASS", dur, note)

pybundle/steps/coverage.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _has_pytest(root: Path) -> bool:
+    """Check if pytest is likely used (tests exist or pytest in deps)."""
+    # Look for common test directories (exclude venv)
+    for test_dir in ["tests", "test"]:
+        test_path = root / test_dir
+        if test_path.is_dir():
+            # Make sure it's not inside a venv
+            if not any(
+                p.name.endswith("venv") or p.name.startswith(".")
+                for p in test_path.parents
+            ):
+                return True
+
+    # Look for test files in the project root and immediate subdirectories
+    # (not deep recursion to avoid finding venv tests)
+    for pattern in ["test_*.py", "*_test.py"]:
+        for p in root.glob(pattern):
+            return True
+        # Check one level deep
+        for subdir in root.iterdir():
+            if (
+                subdir.is_dir()
+                and not subdir.name.startswith(".")
+                and not subdir.name.endswith("venv")
+            ):
+                for p in subdir.glob(pattern):
+                    return True
+
+    return False
+
+
+@dataclass
+class CoverageStep:
+    name: str = "coverage"
+    outfile: str = "logs/35_coverage.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        # Check for pytest first (since we use pytest-cov)
+        pytest_bin = which("pytest")
+        if not pytest_bin:
+            out.write_text(
+                "pytest not found; skipping coverage (pip install pytest pytest-cov)\n",
+                encoding="utf-8",
+            )
+            return StepResult(self.name, "SKIP", 0, "missing pytest")
+
+        # Check if there are tests to run
+        if not _has_pytest(ctx.root):
+            out.write_text("no tests detected; skipping coverage\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "no tests")
+
+        # Run pytest with coverage (including branch coverage for v1.4.1+)
+        cmd = [
+            pytest_bin,
+            "--cov",
+            "--cov-branch",  # Enable branch coverage (v1.4.1+)
+            "--cov-report=term-missing:skip-covered",
+            "--no-cov-on-fail",
+            "-q",
+        ]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+
+        # Combine stdout and stderr
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+
+        # If pytest-cov is not installed, provide helpful message
+        if "pytest: error: unrecognized arguments: --cov" in text:
+            text = (
+                header
+                + "pytest-cov not found; install with: pip install pytest-cov\n\n"
+                + text
+            )
+            out.write_text(ctx.redact_text(text), encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing pytest-cov")
+
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        # Non-zero exit means test failures or coverage threshold not met
+        note = "" if cp.returncode == 0 else f"exit={cp.returncode}"
+        return StepResult(self.name, "PASS", dur, note)

pybundle/steps/cprofile_step.py

@@ -0,0 +1,166 @@
+"""
+CPU profiling with cProfile - Milestone 3 (v1.4.0)
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class CProfileStep:
+    """
+    Run cProfile on the project entry point or test suite to identify CPU bottlenecks.
+
+    Outputs:
+    - logs/60_cprofile.txt: Top 50 slowest functions
+    - meta/60_cprofile.stats: Binary stats file for further analysis
+    """
+
+    name: str = "cprofile"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        if ctx.options.no_profile:
+            return StepResult(self.name, "SKIP", 0, "profiling disabled")
+
+        # Determine default profiling target
+        entry_point = ctx.options.profile_entry_point
+        if not entry_point:
+            # Default: profile pytest if tests/ exists
+            tests_dir = ctx.root / "tests"
+            if not tests_dir.is_dir():
+                return StepResult(self.name, "SKIP", 0, "no tests/ and no entry point")
+
+        # Determine what to profile
+        if entry_point:
+            target_path = Path(entry_point)
+            if not target_path.is_absolute():
+                target_path = ctx.root / entry_point
+
+            if not target_path.exists():
+                return StepResult(
+                    self.name, "SKIP", 0, f"entry point not found: {entry_point}"
+                )
+
+            if target_path.is_file():
+                # Profile a specific script
+                cmd = [
+                    str(ctx.tools.python),
+                    "-m",
+                    "cProfile",
+                    "-o",
+                    str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                    str(target_path),
+                ]
+                desc = f"Profiling {target_path.name}"
+            else:
+                # Assume it's a directory, profile pytest
+                cmd = [
+                    str(ctx.tools.python),
+                    "-m",
+                    "cProfile",
+                    "-o",
+                    str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                    "-m",
+                    "pytest",
+                    str(target_path),
+                    "-q",
+                ]
+                desc = f"Profiling pytest in {target_path.name}/"
+        else:
+            # Default: profile pytest
+            cmd = [
+                str(ctx.tools.python),
+                "-m",
+                "cProfile",
+                "-o",
+                str(ctx.workdir / "meta" / "60_cprofile.stats"),
+                "-m",
+                "pytest",
+                "-q",
+            ]
+            desc = "Profiling pytest"
+
+        ctx.emit(f"  {desc}")
+
+        try:
+            # Run profiling
+            result = subprocess.run(
+                cmd,
+                cwd=ctx.root,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout for profiling
+            )
+
+            # Generate human-readable report
+            stats_file = ctx.workdir / "meta" / "60_cprofile.stats"
+            if stats_file.exists():
+                self._generate_report(stats_file, ctx.workdir)
+                elapsed = int((time.time() - start) * 1000)
+                return StepResult(self.name, "OK", elapsed)
+            else:
+                # Still write output for debugging
+                output_file = ctx.workdir / "logs" / "60_cprofile.txt"
+                output_file.parent.mkdir(parents=True, exist_ok=True)
+
+                with output_file.open("w") as f:
+                    f.write("=" * 70 + "\n")
+                    f.write("CPU PROFILING FAILED\n")
+                    f.write("=" * 70 + "\n\n")
+                    f.write("STDOUT:\n")
+                    f.write(result.stdout)
+                    f.write("\n\nSTDERR:\n")
+                    f.write(result.stderr)
+
+                elapsed = int((time.time() - start) * 1000)
+                return StepResult(self.name, "FAIL", elapsed, "stats file not created")
+
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+
+    def _generate_report(self, stats_file: Path, workdir: Path) -> None:
+        """Generate top 50 slowest functions report"""
+        import pstats
+
+        output_file = workdir / "logs" / "60_cprofile.txt"
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+
+        with output_file.open("w") as f:
+            f.write("=" * 70 + "\n")
+            f.write("TOP 50 SLOWEST FUNCTIONS (CPU PROFILING)\n")
+            f.write("=" * 70 + "\n\n")
+
+            # Load stats
+            stats = pstats.Stats(str(stats_file), stream=f)
+
+            # Remove directory paths for cleaner output
+            stats.strip_dirs()
+
+            # Sort by cumulative time and print top 50
+            f.write("Sorted by cumulative time:\n")
+            f.write("-" * 70 + "\n")
+            stats.sort_stats("cumulative")
+            stats.print_stats(50)
+
+            f.write("\n" + "=" * 70 + "\n")
+            f.write("Sorted by total time (time spent in function itself):\n")
+            f.write("-" * 70 + "\n")
+            stats.sort_stats("time")
+            stats.print_stats(50)
+
+            f.write("\n" + "=" * 70 + "\n")
+            f.write("Full binary stats saved to: meta/60_cprofile.stats\n")
+            f.write("Analyze with: python -m pstats meta/60_cprofile.stats\n")

pybundle/steps/dependency_sizes.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class DependencySizesStep:
+    name: str = "dependency sizes"
+    outfile: str = "meta/33_dependency_sizes.txt"
+    top_n: int = 50  # Show top N largest packages
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        python = ctx.tools.python
+        if not python:
+            out.write_text("python not found; skipping\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing python")
+
+        try:
+            # Get list of installed packages
+            list_result = subprocess.run(  # nosec B603
+                [python, "-m", "pip", "list", "--format=json"],
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=30,
+            )
+
+            if list_result.returncode != 0:
+                out.write_text(
+                    f"pip list failed: {list_result.stderr}\n", encoding="utf-8"
+                )
+                return StepResult(self.name, "FAIL", 0, "pip list failed")
+
+            import json
+
+            packages = json.loads(list_result.stdout)
+
+            # Get size for each package
+            package_sizes = []
+            for pkg in packages:
+                pkg_name = pkg["name"]
+                try:
+                    show_result = subprocess.run(  # nosec B603
+                        [python, "-m", "pip", "show", pkg_name],
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        text=True,
+                        timeout=5,
+                    )
+
+                    if show_result.returncode == 0:
+                        # Parse Location from pip show output
+                        location = None
+                        for line in show_result.stdout.splitlines():
+                            if line.startswith("Location:"):
+                                location = line.split(":", 1)[1].strip()
+                                break
+
+                        if location:
+                            # Calculate directory size
+                            from pathlib import Path
+
+                            pkg_path = Path(location) / pkg_name.replace("-", "_")
+                            if not pkg_path.exists():
+                                pkg_path = Path(location) / pkg_name
+
+                            if pkg_path.exists() and pkg_path.is_dir():
+                                size = sum(
+                                    f.stat().st_size
+                                    for f in pkg_path.rglob("*")
+                                    if f.is_file()
+                                )
+                                package_sizes.append((pkg_name, pkg["version"], size))
+                except Exception:
+                    # Skip packages that fail
+                    continue
+
+            # Sort by size (descending)
+            package_sizes.sort(key=lambda x: x[2], reverse=True)
+
+            # Write results
+            with out.open("w", encoding="utf-8") as f:
+                f.write("=" * 70 + "\n")
+                f.write(
+                    f"TOP {min(self.top_n, len(package_sizes))} LARGEST DEPENDENCIES\n"
+                )
+                f.write("=" * 70 + "\n\n")
+                f.write(f"Total packages analyzed: {len(packages)}\n")
+                f.write(f"Packages with size data: {len(package_sizes)}\n\n")
+
+                if package_sizes:
+                    # Calculate total size
+                    total_size = sum(size for _, _, size in package_sizes)
+                    f.write(f"Total size: {self._format_size(total_size)}\n\n")
+
+                    f.write(f"{'Package':<40} {'Version':<15} {'Size':>15}\n")
+                    f.write("-" * 70 + "\n")
+
+                    for pkg_name, version, size in package_sizes[: self.top_n]:
+                        f.write(
+                            f"{pkg_name:<40} {version:<15} {self._format_size(size):>15}\n"
+                        )
+                else:
+                    f.write("No package size data available.\n")
+
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "OK", elapsed, "")
+
+        except subprocess.TimeoutExpired:
+            out.write_text("Analysis timed out\n", encoding="utf-8")
+            return StepResult(
+                self.name, "FAIL", int((time.time() - start) * 1000), "timeout"
+            )
+        except Exception as e:
+            out.write_text(f"Error: {e}\n", encoding="utf-8")
+            return StepResult(
+                self.name, "FAIL", int((time.time() - start) * 1000), str(e)
+            )
+
+    def _format_size(self, size_bytes: float) -> str:
+        """Format size in human-readable format."""
+        for unit in ["B", "KB", "MB", "GB"]:
+            if size_bytes < 1024.0:
+                return f"{size_bytes:.1f} {unit}"
+            size_bytes /= 1024.0
+        return f"{size_bytes:.1f} TB"