gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/dockerignore.py

@@ -0,0 +1,311 @@
+"""
+Step: .dockerignore Effectiveness Analysis
+Validate .dockerignore and analyze build context efficiency.
+"""
+
+import fnmatch
+from pathlib import Path
+from typing import Set, List
+
+from .base import Step, StepResult
+
+
+class DockerigoreStep(Step):
+    """Analyze .dockerignore effectiveness and build context."""
+
+    name = "dockerignore analysis"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Analyze .dockerignore patterns and effectiveness."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Check if Dockerfile exists
+        dockerfiles = list(root.rglob("Dockerfile*"))
+        if not dockerfiles:
+            elapsed = int(time.time() - start)
+            return StepResult(self.name, "SKIP", elapsed, "No Dockerfiles found")
+
+        # Find .dockerignore
+        dockerignore_path = root / ".dockerignore"
+        if not dockerignore_path.exists():
+            elapsed = int(time.time() - start)
+            return StepResult(self.name, "SKIP", elapsed, "No .dockerignore found")
+
+        # Parse .dockerignore patterns
+        ignore_patterns = self._parse_dockerignore(dockerignore_path)
+
+        # Analyze what would be included/excluded
+        included_files, excluded_files, analysis = self._analyze_build_context(
+            root, ignore_patterns
+        )
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            ".DOCKERIGNORE EFFECTIVENESS ANALYSIS",
+            "=" * 80,
+            "",
+            f"Build context root: {root}",
+            f".dockerignore location: {dockerignore_path}",
+            "",
+        ]
+
+        # Stats
+        lines.extend(
+            [
+                "=" * 80,
+                "BUILD CONTEXT STATISTICS",
+                "=" * 80,
+                "",
+                f"Total files in build context: {len(included_files) + len(excluded_files)}",
+                f"Included files: {len(included_files)}",
+                f"Excluded files: {len(excluded_files)}",
+                f"Exclusion rate: {len(excluded_files) / (len(included_files) + len(excluded_files)) * 100:.1f}%",
+                "",
+            ]
+        )
+
+        # Show ignored files by category
+        if excluded_files:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "EXCLUDED FILES BY CATEGORY",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            categories = {
+                "version_control": [],
+                "cache": [],
+                "tests": [],
+                "docs": [],
+                "build_artifacts": [],
+                "env": [],
+                "venv": [],
+                "node": [],
+                "other": [],
+            }
+
+            for file_path in sorted(excluded_files):
+                path_str = str(file_path).lower()
+
+                if any(
+                    x in path_str
+                    for x in [".git", ".hg", ".svn", ".bzr", ".gitignore"]
+                ):
+                    categories["version_control"].append(file_path)
+                elif any(
+                    x in path_str for x in ["__pycache__", ".pytest_cache", ".mypy_cache"]
+                ):
+                    categories["cache"].append(file_path)
+                elif any(x in path_str for x in ["test", "spec"]):
+                    categories["tests"].append(file_path)
+                elif any(x in path_str for x in ["docs", "readme", "doc"]):
+                    categories["docs"].append(file_path)
+                elif any(
+                    x in path_str
+                    for x in ["build", "dist", ".egg", "*.pyc", "*.o", "*.so"]
+                ):
+                    categories["build_artifacts"].append(file_path)
+                elif any(x in path_str for x in [".env", "secrets"]):
+                    categories["env"].append(file_path)
+                elif any(x in path_str for x in ["venv", ".venv", "env/"]):
+                    categories["venv"].append(file_path)
+                elif any(x in path_str for x in ["node_modules", "npm", "yarn"]):
+                    categories["node"].append(file_path)
+                else:
+                    categories["other"].append(file_path)
+
+            for category, files in categories.items():
+                if files:
+                    cat_name = category.replace("_", " ").title()
+                    lines.append(f"\n{cat_name} ({len(files)} files):")
+                    for f in files[:10]:  # Show first 10
+                        lines.append(f"  - {f}")
+                    if len(files) > 10:
+                        lines.append(f"  ... and {len(files) - 10} more")
+
+            lines.append("")
+
+        # Show .dockerignore patterns
+        lines.extend(
+            [
+                "=" * 80,
+                ".DOCKERIGNORE PATTERNS",
+                "=" * 80,
+                "",
+                f"Total patterns: {len(ignore_patterns)}",
+                "",
+            ]
+        )
+
+        for i, pattern in enumerate(ignore_patterns[:50], 1):
+            lines.append(f"{i:3}. {pattern}")
+
+        if len(ignore_patterns) > 50:
+            lines.append(f"    ... and {len(ignore_patterns) - 50} more patterns")
+
+        lines.append("")
+
+        # Analysis of included files that might be unnecessary
+        if included_files:
+            large_files = []
+            for file_path in included_files:
+                try:
+                    size = file_path.stat().st_size
+                    if size > 1024 * 1024:  # > 1MB
+                        large_files.append((file_path, size))
+                except OSError:
+                    pass
+
+            if large_files:
+                large_files.sort(key=lambda x: x[1], reverse=True)
+
+                lines.extend(
+                    [
+                        "=" * 80,
+                        "LARGE INCLUDED FILES (potential optimization targets)",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                for file_path, size in large_files[:20]:
+                    size_mb = size / (1024 * 1024)
+                    lines.append(f"  {str(file_path):50} {size_mb:8.2f} MB")
+
+                if len(large_files) > 20:
+                    lines.append(f"  ... and {len(large_files) - 20} more")
+
+                lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if analysis["has_venv"]:
+            lines.append("  - Virtual environments should be excluded: add 'venv/' or '.venv/'")
+        if analysis["has_tests"]:
+            lines.append("  - Consider excluding test files: add 'test*' or 'tests/'")
+        if analysis["has_docs"]:
+            lines.append("  - Consider excluding documentation: add 'docs/'")
+        if analysis["has_cache"]:
+            lines.append("  - Cache directories should be excluded: add '__pycache__/'")
+        if analysis["has_git"]:
+            lines.append("  - VCS files should be excluded: add '.git/'")
+
+        if not (
+            analysis["has_venv"]
+            or analysis["has_tests"]
+            or analysis["has_cache"]
+            or analysis["has_git"]
+        ):
+            lines.append("  - .dockerignore is well configured")
+
+        # Image size impact
+        if excluded_files:
+            estimated_excluded = 0
+            for file_path in excluded_files:
+                try:
+                    estimated_excluded += file_path.stat().st_size
+                except OSError:
+                    pass
+
+            if estimated_excluded > 1024 * 1024:
+                excluded_mb = estimated_excluded / (1024 * 1024)
+                lines.append(f"  - Estimated space saved: ~{excluded_mb:.1f} MB")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "meta" / "106_dockerignore_analysis.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _parse_dockerignore(self, dockerignore_path: Path) -> List[str]:
+        """Parse .dockerignore file and return patterns."""
+        patterns = []
+        try:
+            content = dockerignore_path.read_text(encoding="utf-8", errors="ignore")
+            for line in content.split("\n"):
+                line = line.strip()
+                # Skip empty lines and comments
+                if line and not line.startswith("#"):
+                    patterns.append(line)
+        except (OSError, UnicodeDecodeError):
+            pass
+
+        return patterns
+
+    def _analyze_build_context(
+        self, root: Path, ignore_patterns: List[str]
+    ) -> tuple[Set[Path], Set[Path], dict]:
+        """Analyze build context and determine included/excluded files."""
+        included = set()
+        excluded = set()
+        analysis = {
+            "has_venv": False,
+            "has_tests": False,
+            "has_docs": False,
+            "has_cache": False,
+            "has_git": False,
+        }
+
+        # Walk directory tree
+        for file_path in root.rglob("*"):
+            if not file_path.is_file():
+                continue
+
+            rel_path = file_path.relative_to(root)
+            rel_str = str(rel_path)
+
+            # Check if matches any ignore pattern
+            is_ignored = False
+            for pattern in ignore_patterns:
+                # Handle patterns with/without trailing slash
+                if pattern.endswith("/"):
+                    # Directory pattern
+                    if fnmatch.fnmatch(rel_str, pattern.rstrip("/") + "*"):
+                        is_ignored = True
+                        break
+                else:
+                    # File or glob pattern
+                    if fnmatch.fnmatch(rel_str, pattern):
+                        is_ignored = True
+                        break
+
+            if is_ignored:
+                excluded.add(rel_path)
+
+                # Track what's being excluded
+                rel_lower = rel_str.lower()
+                if "venv" in rel_lower or ".venv" in rel_lower:
+                    analysis["has_venv"] = True
+                if "test" in rel_lower:
+                    analysis["has_tests"] = True
+                if "docs" in rel_lower or "readme" in rel_lower:
+                    analysis["has_docs"] = True
+                if "__pycache__" in rel_lower or "cache" in rel_lower:
+                    analysis["has_cache"] = True
+                if ".git" in rel_lower:
+                    analysis["has_git"] = True
+            else:
+                included.add(rel_path)
+
+        return included, excluded, analysis

pybundle/steps/duplication.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class DuplicationStep:
+    name: str = "duplication"
+    target: str = "."
+    outfile: str = "logs/53_duplication.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        pylint = which("pylint")
+        if not pylint:
+            out.write_text(
+                "pylint not found; skipping (pip install pylint)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing pylint")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping duplication check\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        # Find the main package directory (directory with __init__.py)
+        # This is typically the directory with the same name as the project
+        # or explicitly named "src", "lib", etc.
+        target_path = ctx.root / self.target
+
+        # If target is ".", try to find the actual package directory
+        if self.target == ".":
+            # Look for a directory with __init__.py at root level
+            for item in ctx.root.iterdir():
+                if item.is_dir() and not item.name.startswith("."):
+                    init_file = item / "__init__.py"
+                    if init_file.exists():
+                        target_path = item
+                        break
+
+        cmd = [
+            pylint,
+            str(target_path),
+            "--disable=all",  # Disable all other checks
+            "--enable=duplicate-code",  # Only check for duplication
+            "--min-similarity-lines=6",  # Minimum 6 lines to be considered duplicate
+        ]
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=180,  # Duplication detection can be slower
+            )
+            out.write_text(result.stdout, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+
+            # pylint returns various exit codes:
+            # 0 = no issues
+            # 1, 2, 4, 8, 16 = various issue types (we still want the output)
+            # We consider all of these as success
+            if result.returncode in (0, 1, 2, 4, 8, 16, 24, 32):
+                return StepResult(self.name, "OK", elapsed, "")
+            else:
+                return StepResult(
+                    self.name, "FAIL", elapsed, f"exit {result.returncode}"
+                )
+        except subprocess.TimeoutExpired:
+            out.write_text("duplication check timed out after 180s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 180000, "timeout")
+        except Exception as e:
+            out.write_text(f"duplication check error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))

pybundle/steps/env_completeness.py

@@ -0,0 +1,269 @@
+"""
+Step: Environment Variable Completeness Check
+Verify that all required environment variables are documented.
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Set, Tuple, Optional
+
+from .base import Step, StepResult
+
+
+class EnvCompletenessStep(Step):
+    """Check environment variable completeness and documentation."""
+
+    name = "env completeness"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Analyze environment variable completeness."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Find all env var usages
+        used_vars = self._find_env_var_usage(root)
+
+        # Find documented vars
+        documented_vars = self._find_documented_vars(root)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "ENVIRONMENT VARIABLE COMPLETENESS REPORT",
+            "=" * 80,
+            "",
+            f"Project root: {root}",
+            "",
+        ]
+
+        # Summary
+        lines.extend(
+            [
+                "SUMMARY",
+                "=" * 80,
+                "",
+                f"Environment variables referenced: {len(used_vars['all_vars'])}",
+                f"Documented variables: {len(documented_vars['all_vars'])}",
+                "",
+            ]
+        )
+
+        # Used but not documented
+        undocumented = used_vars["all_vars"] - documented_vars["all_vars"]
+        if undocumented:
+            lines.append(f"⚠ NOT DOCUMENTED ({len(undocumented)}):")
+            for var in sorted(undocumented):
+                lines.append(f"  - {var}")
+            lines.append("")
+        else:
+            lines.append("✓ All referenced variables are documented")
+            lines.append("")
+
+        # Documented but not used
+        unused = documented_vars["all_vars"] - used_vars["all_vars"]
+        if unused:
+            lines.append(f"ℹ DOCUMENTED BUT NOT USED ({len(unused)}):")
+            for var in sorted(unused):
+                lines.append(f"  - {var}")
+            lines.append("")
+        else:
+            lines.append("✓ All documented variables are referenced")
+            lines.append("")
+
+        # Detailed analysis
+        lines.extend(
+            [
+                "=" * 80,
+                "DETAILED ANALYSIS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        # Env usage by type
+        if used_vars["getenv"] or used_vars["environ"]:
+            lines.append("ENVIRONMENT VARIABLE ACCESS PATTERNS:")
+            lines.append("")
+
+            if used_vars["getenv"]:
+                lines.append(f"  os.getenv() calls: {len(used_vars['getenv'])}")
+                for var in sorted(list(used_vars["getenv"]))[:10]:
+                    lines.append(f"    - {var}")
+                if len(used_vars["getenv"]) > 10:
+                    lines.append(f"    ... and {len(used_vars['getenv']) - 10} more")
+                lines.append("")
+
+            if used_vars["environ"]:
+                lines.append(f"  os.environ[] access: {len(used_vars['environ'])}")
+                for var in sorted(list(used_vars["environ"]))[:10]:
+                    lines.append(f"    - {var}")
+                if len(used_vars["environ"]) > 10:
+                    lines.append(f"    ... and {len(used_vars['environ']) - 10} more")
+                lines.append("")
+
+        # Documentation sources
+        if documented_vars["env_example"] or documented_vars["env_file"]:
+            lines.append("DOCUMENTATION SOURCES:")
+            lines.append("")
+
+            if documented_vars["env_example"]:
+                lines.append(f"  .env.example: {len(documented_vars['env_example'])} variables")
+                for var in sorted(list(documented_vars["env_example"]))[:10]:
+                    lines.append(f"    - {var}")
+                if len(documented_vars["env_example"]) > 10:
+                    lines.append(f"    ... and {len(documented_vars['env_example']) - 10} more")
+                lines.append("")
+
+            if documented_vars["env_file"]:
+                lines.append(f"  .env: {len(documented_vars['env_file'])} variables")
+                for var in sorted(list(documented_vars["env_file"]))[:10]:
+                    lines.append(f"    - {var}")
+                if len(documented_vars["env_file"]) > 10:
+                    lines.append(f"    ... and {len(documented_vars['env_file']) - 10} more")
+                lines.append("")
+
+        # Documentation completeness score
+        total_vars = len(used_vars["all_vars"])
+        documented_count = len(used_vars["all_vars"] & documented_vars["all_vars"])
+
+        if total_vars > 0:
+            completeness = (documented_count / total_vars) * 100
+            lines.append("=" * 80)
+            lines.append("COMPLETENESS SCORE")
+            lines.append("=" * 80)
+            lines.append("")
+            lines.append(
+                f"  {documented_count}/{total_vars} variables documented ({completeness:.1f}%)"
+            )
+            lines.append("")
+
+            if completeness == 100:
+                lines.append("  ✓ EXCELLENT: All variables properly documented")
+            elif completeness >= 80:
+                lines.append("  ✓ GOOD: Most variables documented")
+            elif completeness >= 50:
+                lines.append("  ⚠ FAIR: Several variables missing documentation")
+            else:
+                lines.append("  ✗ POOR: Majority of variables not documented")
+
+            lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if undocumented:
+            lines.append("  - Add missing variables to .env.example:")
+            for var in sorted(list(undocumented))[:5]:
+                lines.append(f"    {var}=<value>")
+            if len(undocumented) > 5:
+                lines.append(f"    ... and {len(undocumented) - 5} more")
+
+        if unused:
+            lines.append("  - Remove or deprecate unused documented variables")
+
+        if not (documented_vars["env_example"] or documented_vars["env_file"]):
+            lines.append("  - Create .env.example to document all environment variables")
+
+        lines.append("  - Use descriptive comments in .env.example")
+        lines.append("  - Keep .env.example in version control, .env in .gitignore")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "meta" / "120_config_completeness.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _find_env_var_usage(self, root: Path) -> Dict[str, Set[str]]:
+        """Find all environment variable usage patterns."""
+        getenv_vars = set()
+        environ_vars = set()
+
+        python_files = list(root.rglob("*.py"))
+
+        for py_file in python_files:
+            # Skip venv and cache
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+
+                # Find os.getenv patterns
+                getenv_pattern = r'os\.getenv\(["\']([A-Z_][A-Z0-9_]*)["\']'
+                for match in re.finditer(getenv_pattern, source):
+                    getenv_vars.add(match.group(1))
+
+                # Find os.environ patterns
+                environ_pattern = r'os\.environ\[["\']([A-Z_][A-Z0-9_]*)["\']'
+                for match in re.finditer(environ_pattern, source):
+                    environ_vars.add(match.group(1))
+
+            except (OSError, UnicodeDecodeError):
+                continue
+
+        all_vars = getenv_vars | environ_vars
+
+        return {
+            "all_vars": all_vars,
+            "getenv": getenv_vars,
+            "environ": environ_vars,
+        }
+
+    def _find_documented_vars(self, root: Path) -> Dict[str, Set[str]]:
+        """Find documented environment variables."""
+        env_example_vars = set()
+        env_file_vars = set()
+
+        # Check .env.example
+        env_example = root / ".env.example"
+        if env_example.exists():
+            try:
+                content = env_example.read_text(encoding="utf-8", errors="ignore")
+                for line in content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        var_name = line.split("=")[0].strip()
+                        if var_name and var_name.isupper():
+                            env_example_vars.add(var_name)
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        # Check .env
+        env_file = root / ".env"
+        if env_file.exists():
+            try:
+                content = env_file.read_text(encoding="utf-8", errors="ignore")
+                for line in content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        var_name = line.split("=")[0].strip()
+                        if var_name and var_name.isupper():
+                            env_file_vars.add(var_name)
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        all_vars = env_example_vars | env_file_vars
+
+        return {
+            "all_vars": all_vars,
+            "env_example": env_example_vars,
+            "env_file": env_file_vars,
+        }