gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/container_image.py

@@ -0,0 +1,294 @@
+"""
+Step: Container Image Analysis
+Analyze Docker images for size and layer optimization.
+"""
+
+import subprocess
+import re
+from pathlib import Path
+from typing import List, Optional
+
+from .base import Step, StepResult
+
+
+class ContainerImageStep(Step):
+    """Analyze Docker images for size and layer information."""
+
+    name = "container image analysis"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Analyze Docker images from docker-compose or specified images."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Try to get image name from options or docker-compose
+        image_name = getattr(ctx.options, "docker_image", None)
+
+        if not image_name:
+            # Try to find docker-compose.yml
+            compose_files = [
+                root / "docker-compose.yml",
+                root / "docker-compose.yaml",
+            ]
+
+            image_name = None
+            for compose_file in compose_files:
+                if compose_file.exists():
+                    image_name = self._extract_image_from_compose(compose_file)
+                    if image_name:
+                        break
+
+        if not image_name:
+            elapsed = int(time.time() - start)
+            return StepResult(
+                self.name, "SKIP", elapsed, "No docker image specified or found"
+            )
+
+        # Check if Docker is available
+        docker_available = self._check_docker()
+        if not docker_available:
+            elapsed = int(time.time() - start)
+            return StepResult(self.name, "SKIP", elapsed, "Docker not available")
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "CONTAINER IMAGE ANALYSIS",
+            "=" * 80,
+            "",
+            f"Target image: {image_name}",
+            "",
+        ]
+
+        # Get image info
+        image_info = self._inspect_image(image_name)
+        if not image_info:
+            lines.extend(
+                [
+                    "⚠ Could not inspect image. Image may not be built or available.",
+                    "",
+                    "Build the image first with:",
+                    f"  docker build -t {image_name} .",
+                    "",
+                ]
+            )
+        else:
+            # Image details
+            lines.extend(
+                [
+                    "=" * 80,
+                    "IMAGE INFORMATION",
+                    "=" * 80,
+                    "",
+                    f"Image ID:       {image_info.get('Id', 'N/A')[:19]}...",
+                    f"Created:        {image_info.get('Created', 'N/A')}",
+                    f"Architecture:   {image_info.get('Architecture', 'N/A')}",
+                    f"OS:             {image_info.get('Os', 'N/A')}",
+                    f"Docker Version: {image_info.get('DockerVersion', 'N/A')}",
+                    "",
+                ]
+            )
+
+            # Size information
+            size = image_info.get("Size", 0)
+            virtual_size = image_info.get("VirtualSize", 0)
+
+            lines.extend(
+                [
+                    "=" * 80,
+                    "SIZE INFORMATION",
+                    "=" * 80,
+                    "",
+                    f"Compressed size:   {self._format_size(size)}",
+                    f"Uncompressed size: {self._format_size(virtual_size)}",
+                    "",
+                ]
+            )
+
+            # Layer analysis
+            layers = self._get_image_history(image_name)
+            if layers:
+                lines.extend(
+                    [
+                        "=" * 80,
+                        "LAYER ANALYSIS (top 20 by size)",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                # Sort by size descending
+                sorted_layers = sorted(
+                    layers, key=lambda x: x.get("Size", 0), reverse=True
+                )
+
+                total_layer_size = 0
+                for i, layer in enumerate(sorted_layers[:20], 1):
+                    size = layer.get("Size", 0)
+                    total_layer_size += size
+                    cmd = layer.get("CreatedBy", "").strip()
+                    
+                    # Truncate long commands
+                    if len(cmd) > 70:
+                        cmd = cmd[:67] + "..."
+
+                    size_str = self._format_size(size)
+                    lines.append(f"{i:2}. {size_str:12} {cmd}")
+
+                if len(sorted_layers) > 20:
+                    lines.append(f"\n... and {len(sorted_layers) - 20} more layers")
+
+                lines.append("")
+
+                # Recommendations for optimization
+                lines.extend(
+                    [
+                        "=" * 80,
+                        "OPTIMIZATION RECOMMENDATIONS",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                # Check for common issues
+                has_large_run = any(
+                    layer.get("Size", 0) > 50 * 1024 * 1024 for layer in layers
+                )
+                layer_count = len(layers)
+
+                if has_large_run:
+                    lines.append("  - Large RUN layers detected (>50MB)")
+                    lines.append("    Consider combining RUN commands and cleaning up in same layer")
+
+                if layer_count > 30:
+                    lines.append(
+                        f"  - Many layers detected ({layer_count})"
+                    )
+                    lines.append("    Consider combining related RUN instructions")
+
+                if total_layer_size > 1024 * 1024 * 1024:
+                    size_gb = total_layer_size / (1024 * 1024 * 1024)
+                    lines.append(f"  - Large total size ({size_gb:.1f}GB)")
+                    lines.append("    Consider using multi-stage builds or base image optimization")
+
+                lines.append("")
+
+        # Recommendations
+        if not image_info:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "NEXT STEPS",
+                    "=" * 80,
+                    "",
+                    f"1. Build the Docker image:",
+                    f"   docker build -t {image_name} .",
+                    "",
+                    f"2. Re-run pybundle to analyze the built image:",
+                    f"   pybundle run analysis --docker-image {image_name}",
+                    "",
+                ]
+            )
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "meta" / "107_container_image.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _check_docker(self) -> bool:
+        """Check if Docker is available and running."""
+        try:
+            result = subprocess.run(
+                ["docker", "version"],
+                capture_output=True,
+                timeout=5,
+            )
+            return result.returncode == 0
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            return False
+
+    def _extract_image_from_compose(self, compose_file: Path) -> Optional[str]:
+        """Extract image name from docker-compose file."""
+        try:
+            content = compose_file.read_text(encoding="utf-8", errors="ignore")
+            # Simple regex to find image references
+            match = re.search(r'image:\s*([^\s\n]+)', content)
+            if match:
+                return match.group(1)
+        except (OSError, UnicodeDecodeError):
+            pass
+
+        return None
+
+    def _inspect_image(self, image_name: str) -> dict:
+        """Use docker inspect to get image information."""
+        try:
+            result = subprocess.run(
+                ["docker", "image", "inspect", image_name],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            if result.returncode == 0:
+                import json
+
+                data = json.loads(result.stdout)
+                if data and len(data) > 0:
+                    return data[0]
+        except (subprocess.TimeoutExpired, FileNotFoundError, Exception):
+            pass
+
+        return {}
+
+    def _get_image_history(self, image_name: str) -> List[dict]:
+        """Get layer history from docker history."""
+        layers = []
+        try:
+            result = subprocess.run(
+                ["docker", "history", "--no-trunc", "--human", image_name],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            if result.returncode == 0:
+                import json
+
+                # Try to get JSON format
+                result = subprocess.run(
+                    ["docker", "history", "--no-trunc", image_name, "--format", "json"],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+
+                if result.returncode == 0:
+                    for line in result.stdout.strip().split("\n"):
+                        if line:
+                            try:
+                                layer = json.loads(line)
+                                layers.append(layer)
+                            except json.JSONDecodeError:
+                                pass
+
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            pass
+
+        return layers
+
+    def _format_size(self, size_bytes: float) -> str:
+        """Format bytes into human readable size."""
+        for unit in ["B", "KB", "MB", "GB", "TB"]:
+            if size_bytes < 1024:
+                return f"{size_bytes:.1f}{unit}"
+            size_bytes /= 1024
+
+        return f"{size_bytes:.1f}PB"

pybundle/steps/context_expand.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import ast
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+def _read_lines(p: Path) -> list[str]:
+    if not p.is_file():
+        return []
+    return [
+        ln.strip()
+        for ln in p.read_text(encoding="utf-8", errors="replace").splitlines()
+        if ln.strip()
+    ]
+
+
+def _is_under(root: Path, p: Path) -> bool:
+    try:
+        p.resolve().relative_to(root.resolve())
+        return True
+    except Exception:
+        return False
+
+
+def _copy_file(src: Path, dst: Path) -> bool:
+    try:
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        dst.write_bytes(src.read_bytes())
+        return True
+    except Exception:
+        return False
+
+
+def _find_repo_candidates(root: Path) -> list[Path]:
+    cands = [root]
+    if (root / "src").is_dir():
+        cands.append(root / "src")
+    return cands
+
+
+def _module_to_path(roots: list[Path], module: str) -> Path | None:
+    """
+    Resolve an absolute module like 'pkg.sub' to a file inside repo roots:
+      - <root>/pkg/sub.py
+      - <root>/pkg/sub/__init__.py
+    """
+    parts = module.split(".")
+    for r in roots:
+        f1 = r.joinpath(*parts).with_suffix(".py")
+        if f1.is_file():
+            return f1
+        f2 = r.joinpath(*parts) / "__init__.py"
+        if f2.is_file():
+            return f2
+    return None
+
+
+def _relative_module_to_path(
+    roots: list[Path], base_file: Path, module: str | None, level: int
+) -> Path | None:
+    """
+    Resolve relative imports like:
+      from . import x      (level=1, module=None)
+      from ..foo import y  (level=2, module="foo")
+    """
+    # Find the package directory for base_file
+    base_dir = base_file.parent
+
+    # Move up `level` package levels
+    rel_dir = base_dir
+    for _ in range(level):
+        rel_dir = rel_dir.parent
+
+    if module:
+        target = rel_dir.joinpath(*module.split("."))
+    else:
+        target = rel_dir
+
+    # Try module as file or package
+    f1 = target.with_suffix(".py")
+    if f1.is_file():
+        return f1
+    f2 = target / "__init__.py"
+    if f2.is_file():
+        return f2
+
+    # If relative resolution fails, try absolute resolution as fallback
+    if module:
+        return _module_to_path(roots, module)
+    return None
+
+
+def _extract_import_modules(py_file: Path) -> set[tuple[str | None, int]]:
+    """
+    Returns a set of (module, level) pairs:
+      - absolute imports: (module, 0)
+      - relative imports: (module, level>=1)
+    module can be None for `from . import x` style.
+    """
+    out: set[tuple[str | None, int]] = set()
+    try:
+        src = py_file.read_text(encoding="utf-8", errors="replace")
+        tree = ast.parse(src)
+    except Exception:
+        return out
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                if alias.name:
+                    out.add((alias.name, 0))
+        elif isinstance(node, ast.ImportFrom):
+            # node.module can be None (from . import x)
+            lvl = int(getattr(node, "level", 0) or 0)
+            mod = node.module
+            if mod:
+                out.add((mod, lvl))
+            else:
+                # still useful; we can resolve to the package itself for context
+                out.add((None, lvl))
+    return out
+
+
+def _add_package_chain(files: set[Path], root: Path, py: Path) -> None:
+    """
+    Add __init__.py from file's dir up to repo root.
+    """
+    cur = py.parent
+    root_res = root.resolve()
+    while True:
+        initp = cur / "__init__.py"
+        if initp.is_file():
+            files.add(initp)
+        if cur.resolve() == root_res:
+            break
+        if not _is_under(root, cur):
+            break
+        cur = cur.parent
+
+
+def _add_conftest_chain(files: set[Path], root: Path, py: Path) -> None:
+    """
+    Add conftest.py from file's dir up to repo root (pytest glue).
+    """
+    cur = py.parent
+    root_res = root.resolve()
+    while True:
+        cp = cur / "conftest.py"
+        if cp.is_file():
+            files.add(cp)
+        if cur.resolve() == root_res:
+            break
+        if not _is_under(root, cur):
+            break
+        cur = cur.parent
+
+
+@dataclass
+class ErrorContextExpandStep:
+    name: str = "expand error context"
+    depth: int = 2
+    max_files: int = 600
+    # reads this list produced by step 8
+    source_list_file: str = "error_files_from_logs.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        roots = _find_repo_candidates(ctx.root)
+
+        list_path = ctx.workdir / self.source_list_file
+        rels = _read_lines(list_path)
+
+        dest_root = ctx.srcdir / "_error_context"
+        dest_root.mkdir(parents=True, exist_ok=True)
+
+        to_copy: set[Path] = set()
+        queue: list[tuple[Path, int]] = []
+
+        # Seed with referenced python files only
+        for rel_str in rels:
+            p = (ctx.root / rel_str).resolve()
+            if p.is_file() and p.suffix == ".py" and _is_under(ctx.root, p):
+                queue.append((p, 0))
+                to_copy.add(p)
+
+        visited: set[Path] = set()
+
+        while queue and len(to_copy) < self.max_files:
+            py_file, d = queue.pop(0)
+            if py_file in visited:
+                continue
+            visited.add(py_file)
+
+            # add pytest + package glue around this file
+            _add_package_chain(to_copy, ctx.root, py_file)
+            _add_conftest_chain(to_copy, ctx.root, py_file)
+
+            if d >= self.depth:
+                continue
+
+            # parse imports and resolve to repo files
+            for mod, level in _extract_import_modules(py_file):
+                target: Path | None
+                if level and level > 0:
+                    target = _relative_module_to_path(roots, py_file, mod, level)
+                else:
+                    if not mod:
+                        continue
+                    target = _module_to_path(roots, mod)
+
+                if target and target.is_file() and _is_under(ctx.root, target):
+                    if target.suffix == ".py":
+                        if target not in to_copy:
+                            to_copy.add(target)
+                        queue.append((target, d + 1))
+
+                if len(to_copy) >= self.max_files:
+                    break
+
+        # Always include top-level config files if present (small but high value)
+        for cfg in [
+            "pyproject.toml",
+            "mypy.ini",
+            "ruff.toml",
+            ".ruff.toml",
+            "pytest.ini",
+            "setup.cfg",
+            "requirements.txt",
+        ]:
+            p = ctx.root / cfg
+            if p.is_file():
+                to_copy.add(p)
+
+        copied = 0
+        for p in sorted(to_copy):
+            if copied >= self.max_files:
+                break
+            # copy under src/_error_context/<repo-relative-path>
+            try:
+                rel_path = p.resolve().relative_to(ctx.root.resolve())
+            except Exception:
+                continue
+            dst = dest_root / rel_path
+            if _copy_file(p, dst):
+                copied += 1
+
+        report = ctx.metadir / "61_error_context_report.txt"
+        report.write_text(
+            "\n".join(
+                [
+                    f"seed_files={len([r for r in rels if r.endswith('.py')])}",
+                    f"depth={self.depth}",
+                    f"max_files={self.max_files}",
+                    f"resolved_total={len(to_copy)}",
+                    f"copied={copied}",
+                    "dest=src/_error_context",
+                ]
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+
+        dur = int(time.time() - start)
+        note = f"resolved={len(to_copy)} copied={copied}"
+        if copied >= self.max_files:
+            note += " (HIT MAX)"
+        return StepResult(self.name, "PASS", dur, note)