gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/interrogate.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+from ..filters import should_exclude_from_analysis
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class InterrogateStep:
+    name: str = "interrogate"
+    target: str = "."
+    outfile: str = "logs/52_docstring_coverage.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        interrogate = which("interrogate")
+        if not interrogate:
+            out.write_text(
+                "interrogate not found; skipping (pip install interrogate)\n",
+                encoding="utf-8",
+            )
+            return StepResult(self.name, "SKIP", 0, "missing interrogate")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping interrogate\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        target_path = ctx.root / self.target
+        
+        # Build comprehensive exclusion list for interrogate
+        # Exclude ALL venvs, caches, and dependency directories
+        exclude_patterns = [
+            "*venv*",  # Catches .venv, venv, .freeze-venv, .gaslog-venv, etc.
+            "*site-packages*",
+            "__pycache__",
+            ".pytest_cache",
+            ".mypy_cache",
+            ".ruff_cache",
+            ".tox",
+            ".nox",
+            "artifacts",
+            "build",
+            "dist",
+            ".git",
+            "node_modules",
+        ]
+        
+        cmd = [
+            interrogate,
+            str(target_path),
+            "-v",  # Verbose output
+            "--fail-under",
+            "0",  # Don't fail the step based on coverage percentage
+            "--color",
+            "--exclude",
+            ",".join(exclude_patterns),
+        ]
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=120,
+            )
+            out.write_text(result.stdout, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+
+            # interrogate returns 0 even with missing docstrings when --fail-under=0
+            # We consider any execution a success
+            return StepResult(self.name, "OK", elapsed, "")
+        except subprocess.TimeoutExpired:
+            out.write_text("interrogate timed out after 120s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 120000, "timeout")
+        except Exception as e:
+            out.write_text(f"interrogate error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))

pybundle/steps/license_scan.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+@dataclass
+class LicenseScanStep:
+    name: str = "license scan"
+    outfile: str = "meta/32_licenses.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        pip_licenses = which("pip-licenses")
+        if not pip_licenses:
+            out.write_text(
+                "pip-licenses not found; skipping (pip install pip-licenses)\n",
+                encoding="utf-8",
+            )
+            return StepResult(self.name, "SKIP", 0, "missing pip-licenses")
+
+        # Run pip-licenses with detailed output
+        cmd = [
+            pip_licenses,
+            "--format=markdown",  # Markdown table format
+            "--with-urls",  # Include project URLs
+            "--with-description",  # Include package descriptions
+        ]
+
+        try:
+            result = subprocess.run(  # nosec B603 - Using full path from which()
+                cmd,
+                cwd=ctx.root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=60,
+            )
+
+            output = result.stdout
+
+            # Add license compatibility warnings
+            warnings = self._check_license_compatibility(output)
+            if warnings:
+                output += "\n\n" + "=" * 70 + "\n"
+                output += "LICENSE COMPATIBILITY WARNINGS\n"
+                output += "=" * 70 + "\n\n"
+                output += "\n".join(warnings)
+
+            out.write_text(output, encoding="utf-8")
+            elapsed = int((time.time() - start) * 1000)
+
+            return StepResult(self.name, "OK", elapsed, "")
+        except subprocess.TimeoutExpired:
+            out.write_text("pip-licenses timed out after 60s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 60000, "timeout")
+        except Exception as e:
+            out.write_text(f"pip-licenses error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))
+
+    def _check_license_compatibility(self, output: str) -> list[str]:
+        """Check for common license compatibility issues."""
+        warnings = []
+
+        # Simple heuristic: look for GPL + permissive license mixing
+        has_gpl = any(gpl in output for gpl in ["GPL", "AGPL", "LGPL"])
+        has_mit = "MIT" in output
+        has_apache = "Apache" in output
+        has_bsd = "BSD" in output
+
+        if has_gpl and (has_mit or has_apache or has_bsd):
+            warnings.append(
+                "⚠️  Potential GPL compatibility issue detected:\n"
+                "   - GPL/LGPL/AGPL licenses found alongside permissive licenses (MIT/Apache/BSD)\n"
+                "   - Review GPL obligations if redistributing\n"
+                "   - LGPL is generally compatible with permissive licenses\n"
+                "   - Consult legal counsel for production use"
+            )
+
+        # Check for proprietary or unknown licenses
+        if "UNKNOWN" in output:
+            warnings.append(
+                "⚠️  Packages with UNKNOWN licenses detected:\n"
+                "   - Review manually before distribution\n"
+                "   - May indicate missing license metadata"
+            )
+
+        return warnings

pybundle/steps/line_profiler.py

@@ -0,0 +1,117 @@
+"""
+Line-by-line profiling with line_profiler - Milestone 3 (v1.4.0)
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class LineProfilerStep:
+    """
+    Line-by-line profiling using line_profiler (optional, requires manual annotation).
+
+    This step is disabled by default and requires:
+    1. line_profiler installed
+    2. Functions decorated with @profile or listed in config
+
+    Outputs:
+    - logs/63_line_profile.txt: Line-by-line execution times
+    """
+
+    name: str = "line_profiler"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        # Only run if explicitly enabled
+        if ctx.options.no_profile or not ctx.options.enable_line_profiler:
+            return StepResult(self.name, "SKIP", 0, "line profiler not enabled")
+
+        # Check if line_profiler is installed
+        if not ctx.tools.line_profiler:
+            return StepResult(self.name, "SKIP", 0, "line_profiler not installed")
+
+        # Check if entry point exists
+        entry_point = ctx.options.profile_entry_point
+        if not entry_point:
+            return StepResult(self.name, "SKIP", 0, "no entry point specified")
+
+        target_path = Path(entry_point)
+        if not target_path.is_absolute():
+            target_path = ctx.root / entry_point
+
+        if not target_path.exists():
+            return StepResult(
+                self.name, "SKIP", 0, f"entry point not found: {entry_point}"
+            )
+
+        ctx.emit(f"  Running line profiler on {target_path.name}")
+        ctx.emit("  Note: Functions must be decorated with @profile")
+
+        try:
+            # Run line_profiler via kernprof
+            result = subprocess.run(
+                [
+                    str(ctx.tools.line_profiler),
+                    "-l",  # Line-by-line
+                    "-v",  # Verbose output
+                    str(target_path),
+                ],
+                cwd=ctx.root,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout
+            )
+
+            # Write output
+            output_file = ctx.workdir / "logs" / "63_line_profile.txt"
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            with output_file.open("w") as f:
+                f.write("=" * 70 + "\n")
+                f.write("LINE-BY-LINE PROFILING (line_profiler)\n")
+                f.write("=" * 70 + "\n\n")
+
+                if result.returncode == 0:
+                    f.write(result.stdout)
+                    if result.stderr:
+                        f.write("\n\nWarnings/Errors:\n")
+                        f.write(result.stderr)
+                else:
+                    f.write(
+                        "Line profiling failed or no functions decorated with @profile\n\n"
+                    )
+                    f.write("To use line_profiler:\n")
+                    f.write("1. Install: pip install line_profiler\n")
+                    f.write("2. Decorate functions with @profile\n")
+                    f.write(
+                        "3. Specify entry point: --profile-entry-point path/to/script.py\n"
+                    )
+                    f.write("4. Enable: --enable-line-profiler\n\n")
+                    f.write("STDOUT:\n")
+                    f.write(result.stdout)
+                    f.write("\n\nSTDERR:\n")
+                    f.write(result.stderr)
+
+            elapsed = int((time.time() - start) * 1000)
+            if result.returncode == 0:
+                return StepResult(self.name, "OK", elapsed)
+            else:
+                return StepResult(
+                    self.name, "FAIL", elapsed, f"exit {result.returncode}"
+                )
+
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))

pybundle/steps/link_validation.py

@@ -0,0 +1,287 @@
+"""README and documentation link validation step.
+
+Checks all markdown files for broken links (HTTP/HTTPS URLs).
+"""
+
+import re
+import time
+from pathlib import Path
+from typing import Dict, List, Tuple
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+from ..filters import should_exclude_from_analysis
+
+
+@dataclass
+class LinkValidationStep:
+    """Step that validates links in markdown files."""
+
+    name: str = "link-validation"
+    outfile: str = "logs/81_link_validation.txt"
+
+    def run(self, context: BundleContext) -> StepResult:
+        """Validate links in markdown documentation."""
+        start = time.time()
+
+        # Find all markdown files
+        md_files = self._find_markdown_files(context.root)
+
+        if not md_files:
+            elapsed = time.time() - start
+            note = "No markdown files found"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        # Extract all links from markdown files
+        all_links = self._extract_links(md_files, context.root)
+
+        if not all_links:
+            elapsed = time.time() - start
+            note = "No HTTP(S) links found"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        # Check links
+        results = self._check_links(all_links)
+
+        elapsed = time.time() - start
+
+        # Write report
+        log_path = context.workdir / self.outfile
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(log_path, "w") as f:
+            f.write("=" * 80 + "\n")
+            f.write("LINK VALIDATION REPORT\n")
+            f.write("=" * 80 + "\n\n")
+
+            # Summary
+            total_links = len(results)
+            broken_links = sum(1 for status, _ in results.values() if status != "OK")
+            success_rate = (
+                ((total_links - broken_links) / total_links * 100)
+                if total_links > 0
+                else 100
+            )
+
+            f.write("Summary:\n")
+            f.write("-" * 80 + "\n")
+            f.write(f"Total URLs checked:  {total_links}\n")
+            f.write(f"Broken links:        {broken_links}\n")
+            f.write(f"Success rate:        {success_rate:.1f}%\n")
+            f.write("\n")
+
+            # Group results by file
+            links_by_file: Dict[str, List[Tuple[str, str, str]]] = {}
+            for (filepath, url), (status, message) in results.items():
+                if filepath not in links_by_file:
+                    links_by_file[filepath] = []
+                links_by_file[filepath].append((url, status, message))
+
+            # Write results
+            for filepath in sorted(links_by_file.keys()):
+                f.write(f"\n{filepath}:\n")
+                f.write("-" * 80 + "\n")
+
+                for url, status, message in links_by_file[filepath]:
+                    status_icon = "✓" if status == "OK" else "✗"
+                    f.write(f"{status_icon} [{status:6s}] {url}\n")
+                    if message and status != "OK":
+                        f.write(f"           {message}\n")
+
+            f.write("\n" + "=" * 80 + "\n")
+            f.write(f"Validation complete - {len(md_files)} markdown files scanned\n")
+            f.write("=" * 80 + "\n")
+
+        # Determine overall status (WARN not FAIL - broken links shouldn't block shipping)
+        if broken_links == 0:
+            status = "OK"
+            note = f"All {total_links} links valid"
+        else:
+            status = "WARN"  # Changed from FAIL - documentation links are quality issues
+            note = f"{broken_links}/{total_links} broken links"
+
+        return StepResult(self.name, status, int(elapsed), note)
+
+    def _find_markdown_files(self, root: Path) -> List[Path]:
+        """Find all markdown files, excluding dependencies/caches/build dirs."""
+        md_files = []
+
+        for path in root.rglob("*.md"):
+            # Use comprehensive exclusion filter for PROJECT files only
+            if should_exclude_from_analysis(path):
+                continue
+            md_files.append(path)
+
+        return md_files
+
+    def _extract_links(
+        self, md_files: List[Path], root: Path
+    ) -> Dict[Tuple[str, str], None]:
+        """Extract HTTP/HTTPS links from markdown files.
+
+        Returns dict with (filepath, url) as keys for deduplication.
+        """
+        links: Dict[Tuple[str, str], None] = {}
+        # Regex for markdown links and bare URLs
+        link_pattern = re.compile(
+            r"\[([^\]]+)\]\(([^)]+)\)|(?:^|[^(])(https?://[^\s\)<>]+)", re.MULTILINE
+        )
+
+        for filepath in md_files:
+            try:
+                with open(filepath, "r", encoding="utf-8") as f:
+                    content = f.read()
+
+                rel_path = str(filepath.relative_to(root))
+
+                for match in link_pattern.finditer(content):
+                    # Group 2 is markdown link URL, group 3 is bare URL
+                    url = match.group(2) or match.group(3)
+                    if url and (
+                        url.startswith("http://") or url.startswith("https://")
+                    ):
+                        # Clean up URL (remove trailing punctuation)
+                        url = url.rstrip(".,;:!?")
+                        links[(rel_path, url)] = None
+
+            except Exception:
+                # Skip files that can't be read
+                continue
+
+        return links
+
+    def _check_links(
+        self, links: Dict[Tuple[str, str], None]
+    ) -> Dict[Tuple[str, str], Tuple[str, str]]:
+        """Check if links are valid.
+
+        Returns dict with (filepath, url) -> (status, message).
+        """
+        import subprocess
+
+        results = {}
+
+        # Check if we have curl available
+        try:
+            subprocess.run(
+                ["curl", "--version"],
+                capture_output=True,
+                check=True,
+                timeout=5,
+            )
+            has_curl = True
+        except (subprocess.SubprocessError, FileNotFoundError):
+            has_curl = False
+
+        # If no curl, try Python requests
+        has_requests = False
+        if not has_curl:
+            try:
+                import requests  # type: ignore[import-untyped]
+
+                has_requests = True
+            except ImportError:
+                pass
+
+        if not has_curl and not has_requests:
+            # Can't validate links without tools
+            for key in links:
+                results[key] = ("SKIP", "curl or requests not available")
+            return results
+
+        # Limit number of links to check (prevent hanging on large docs)
+        max_links = 50
+        links_to_check = list(links.keys())[:max_links]
+        
+        if len(links) > max_links:
+            # Mark excess links as skipped
+            for key in list(links.keys())[max_links:]:
+                results[key] = ("SKIP", f"Exceeded {max_links} link limit")
+
+        # Check each link with overall timeout protection
+        check_start = time.time()
+        max_check_time = 120  # 2 minutes max for all link checking
+        
+        for filepath, url in links_to_check:
+            # Check if we've exceeded overall time budget
+            if time.time() - check_start > max_check_time:
+                results[(filepath, url)] = ("SKIP", "Overall timeout exceeded")
+                # Mark remaining as skipped
+                remaining_idx = links_to_check.index((filepath, url)) + 1
+                for key in links_to_check[remaining_idx:]:
+                    results[key] = ("SKIP", "Overall timeout exceeded")
+                break
+                
+            status, message = self._check_single_link(url, has_curl, has_requests)
+            results[(filepath, url)] = (status, message)
+
+        return results
+
+    def _check_single_link(
+        self, url: str, has_curl: bool, has_requests: bool
+    ) -> Tuple[str, str]:
+        """Check a single link.
+
+        Returns (status, message) tuple.
+        """
+        import subprocess
+
+        if has_curl:
+            try:
+                # Use curl with HEAD request, follow redirects, shorter timeout
+                result = subprocess.run(
+                    [
+                        "curl",
+                        "-I",  # HEAD request
+                        "-L",  # Follow redirects
+                        "-s",  # Silent
+                        "-o",
+                        "/dev/null",  # Discard output
+                        "-w",
+                        "%{http_code}",  # Write HTTP code
+                        "--connect-timeout",
+                        "3",  # 3 second connection timeout
+                        "--max-time",
+                        "5",  # 5 second total timeout
+                        url,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=6,  # Python timeout slightly higher than curl's
+                )
+
+                http_code = result.stdout.strip()
+                if http_code.startswith("2") or http_code.startswith("3"):
+                    return ("OK", "")
+                elif http_code == "000":
+                    return ("FAIL", "Connection failed")
+                else:
+                    return ("FAIL", f"HTTP {http_code}")
+
+            except subprocess.TimeoutExpired:
+                return ("FAIL", "Timeout")
+            except Exception as e:
+                return ("FAIL", f"Error: {str(e)[:50]}")
+
+        elif has_requests:
+            try:
+                import requests
+
+                response = requests.head(
+                    url,
+                    allow_redirects=True,
+                    timeout=5,  # Reduced from 10 to 5 seconds
+                    headers={"User-Agent": "pybundle-link-checker"},
+                )
+
+                if response.status_code < 400:
+                    return ("OK", "")
+                else:
+                    return ("FAIL", f"HTTP {response.status_code}")
+
+            except requests.exceptions.Timeout:
+                return ("FAIL", "Timeout")
+            except requests.exceptions.RequestException as e:
+                return ("FAIL", f"Error: {str(e)[:50]}")
+
+        return ("SKIP", "No validation method available")