gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/api_docs.py

@@ -0,0 +1,219 @@
+"""API documentation generation step using pdoc.
+
+Generates HTML API documentation and validates docstring parsing.
+"""
+
+import shutil
+import subprocess
+import time
+from pathlib import Path
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class ApiDocsStep:
+    """Step that generates API documentation using pdoc."""
+
+    name: str = "api-docs"
+    outfile: str = "logs/82_api_docs.txt"
+
+    def run(self, context: BundleContext) -> StepResult:
+        """Generate API documentation."""
+        start = time.time()
+
+        # Check if pdoc is available and get version
+        pdoc_path = shutil.which("pdoc")
+        if not pdoc_path:
+            elapsed = time.time() - start
+            note = "pdoc not installed (pip install pdoc)"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        # Get pdoc version for diagnostics
+        pdoc_version = None
+        try:
+            version_result = subprocess.run(
+                ["pdoc", "--version"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            pdoc_version = version_result.stdout.strip() or version_result.stderr.strip()
+        except Exception:
+            pdoc_version = "unknown"
+
+        # Find Python package(s) in project
+        packages = self._find_packages(context.root)
+
+        if not packages:
+            elapsed = time.time() - start
+            note = "No Python packages found"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        # Create output directory in meta
+        output_dir = context.workdir / "meta/82_api_docs"
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Generate docs for each package
+        success_count = 0
+        error_count = 0
+        errors = []
+
+        for package in packages:
+            try:
+                # Use pdoc v14+ syntax (--output-dir instead of --html + --output-dir)
+                # Try modern syntax first, fallback to legacy if it fails
+                result = subprocess.run(
+                    [
+                        "pdoc",
+                        "--output-dir",
+                        str(output_dir),
+                        str(package),
+                    ],
+                    cwd=context.root,
+                    capture_output=True,
+                    text=True,
+                    timeout=60,
+                )
+
+                if result.returncode == 0:
+                    success_count += 1
+                else:
+                    error_count += 1
+                    error_msg = result.stderr or result.stdout
+                    errors.append((package.name, result.returncode, error_msg))
+
+            except subprocess.TimeoutExpired:
+                error_count += 1
+                errors.append((package.name, -1, "Timeout after 60s"))
+            except Exception as e:
+                error_count += 1
+                errors.append((package.name, -1, str(e)))
+
+        elapsed = time.time() - start
+
+        # Write summary report
+        log_path = context.workdir / self.outfile
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(log_path, "w") as f:
+            f.write("=" * 80 + "\n")
+            f.write("API DOCUMENTATION GENERATION\n")
+            f.write("=" * 80 + "\n\n")
+
+            f.write("Environment:\n")
+            f.write("-" * 80 + "\n")
+            f.write(f"pdoc path:       {pdoc_path}\n")
+            f.write(f"pdoc version:    {pdoc_version}\n")
+            f.write("\n")
+
+            f.write("Summary:\n")
+            f.write("-" * 80 + "\n")
+            f.write(f"Packages processed:  {len(packages)}\n")
+            f.write(f"Successful:          {success_count}\n")
+            f.write(f"Failed:              {error_count}\n")
+            f.write(f"Output directory:    {output_dir.relative_to(context.workdir)}\n")
+            f.write("\n")
+
+            if errors:
+                f.write("Errors:\n")
+                f.write("-" * 80 + "\n")
+                for package_name, retcode, error_msg in errors:
+                    f.write(f"\n{package_name}:\n")
+                    f.write(f"  Exit code: {retcode}\n")
+                    # Limit error message length
+                    if len(error_msg) > 500:
+                        error_msg = error_msg[:500] + "\n... (truncated)"
+                    f.write(f"  Error: {error_msg}\n")
+                    
+                    # Add actionable fix suggestions
+                    f.write("\n  💡 How to fix:\n")
+                    if "--html" in error_msg or "unrecognized arguments" in error_msg:
+                        f.write("     - Your pdoc version may not support --html flag\n")
+                        f.write("     - pdoc < 14.0: use 'pdoc --html'\n")
+                        f.write("     - pdoc >= 14.0: use 'pdoc --output-dir'\n")
+                        f.write(f"     - Detected version: {pdoc_version}\n")
+                        f.write("     - Try: pip install --upgrade pdoc\n")
+                    elif "Timeout" in error_msg:
+                        f.write("     - Package is too large or has import errors\n")
+                        f.write("     - Check for circular imports or missing dependencies\n")
+                    else:
+                        f.write("     - Check that package imports work: python -c 'import <package>'\n")
+                        f.write("     - Verify dependencies are installed\n")
+                        f.write("     - Run manually: pdoc --output-dir docs/ <package>\n")
+
+            if success_count > 0:
+                f.write("\nGenerated Documentation:\n")
+                f.write("-" * 80 + "\n")
+                # List generated HTML files
+                html_files = sorted(output_dir.rglob("*.html"))
+                for html_file in html_files[:20]:  # Limit to first 20
+                    rel_path = html_file.relative_to(context.workdir)
+                    f.write(f"  {rel_path}\n")
+                if len(html_files) > 20:
+                    f.write(f"  ... and {len(html_files) - 20} more files\n")
+
+            f.write("\n" + "=" * 80 + "\n")
+            f.write("Documentation generation complete\n")
+            f.write("=" * 80 + "\n")
+
+        # Determine status
+        if error_count == 0:
+            status = "OK"
+            note = f"Generated docs for {success_count} package(s)"
+        elif success_count > 0:
+            status = "WARN"
+            note = f"{success_count} OK, {error_count} failed"
+        else:
+            status = "FAIL"
+            note = f"All {error_count} package(s) failed"
+
+        return StepResult(self.name, status, int(elapsed), note)
+
+    def _find_packages(self, root: Path) -> list[Path]:
+        """Find Python packages (directories with __init__.py).
+
+        Returns top-level packages only, excluding common directories.
+        """
+        packages = []
+        exclude_dirs = {
+            "__pycache__",
+            ".git",
+            ".tox",
+            "venv",
+            "env",
+            ".venv",
+            ".env",
+            "node_modules",
+            "artifacts",
+            "build",
+            "dist",
+            ".pytest_cache",
+            ".mypy_cache",
+            ".ruff_cache",
+            "htmlcov",
+            ".egg-info",
+            "tests",
+            "test",
+            ".pybundle-venv",  # pybundle's venv
+        }
+
+        # Look for directories with __init__.py at root level first
+        for item in root.iterdir():
+            if item.is_dir() and item.name not in exclude_dirs:
+                init_file = item / "__init__.py"
+                if init_file.exists():
+                    packages.append(item)
+
+        # If no packages found at root, look one level deeper
+        if not packages:
+            for item in root.iterdir():
+                if item.is_dir() and item.name not in exclude_dirs:
+                    for subitem in item.iterdir():
+                        if subitem.is_dir() and subitem.name not in exclude_dirs:
+                            init_file = subitem / "__init__.py"
+                            if init_file.exists():
+                                packages.append(subitem)
+
+        return packages

pybundle/steps/asyncio_analysis.py

@@ -0,0 +1,358 @@
+"""
+Step: AsyncIO Task Analysis
+Static analysis of async/await patterns in codebase.
+"""
+
+import re
+import ast
+from pathlib import Path
+from typing import Dict, List, Set, Tuple, Optional
+
+from .base import Step, StepResult
+
+
+class AsyncioAnalysisStep(Step):
+    """Analyze async/await patterns in Python code."""
+
+    name = "asyncio analysis"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Analyze asyncio usage and patterns."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Find all async definitions
+        async_funcs = self._find_async_functions(root)
+
+        # Analyze async patterns
+        patterns = self._analyze_async_patterns(root, async_funcs)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "ASYNCIO ANALYSIS REPORT",
+            "=" * 80,
+            "",
+        ]
+
+        # Summary
+        lines.extend(
+            [
+                "SUMMARY",
+                "=" * 80,
+                "",
+                f"Async functions found: {len(async_funcs['all_async'])}",
+                f"Coroutines: {len(async_funcs['coroutines'])}",
+                f"Async generators: {len(async_funcs['async_generators'])}",
+                f"Async context managers: {len(async_funcs['async_context_managers'])}",
+                "",
+            ]
+        )
+
+        if not async_funcs["all_async"]:
+            lines.extend(
+                [
+                    "⊘ No async code detected in project",
+                    "",
+                    "This project does not appear to use asyncio patterns.",
+                    "If this is incorrect, ensure async functions are in analyzed files.",
+                    "",
+                ]
+            )
+        else:
+            # Detailed breakdown
+            lines.extend(
+                [
+                    "ASYNC FUNCTIONS BY TYPE",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            if async_funcs["coroutines"]:
+                lines.append(f"COROUTINES ({len(async_funcs['coroutines'])}):")
+                for file_path, func_name, line_no in sorted(
+                    async_funcs["coroutines"], key=lambda x: (x[0], x[2])
+                )[:20]:
+                    lines.append(f"  {file_path}:{line_no} - {func_name}")
+                if len(async_funcs["coroutines"]) > 20:
+                    lines.append(f"  ... and {len(async_funcs['coroutines']) - 20} more")
+                lines.append("")
+
+            if async_funcs["async_generators"]:
+                lines.append(f"ASYNC GENERATORS ({len(async_funcs['async_generators'])}):")
+                for file_path, func_name, line_no in sorted(
+                    async_funcs["async_generators"], key=lambda x: (x[0], x[2])
+                )[:20]:
+                    lines.append(f"  {file_path}:{line_no} - {func_name}")
+                if len(async_funcs["async_generators"]) > 20:
+                    lines.append(
+                        f"  ... and {len(async_funcs['async_generators']) - 20} more"
+                    )
+                lines.append("")
+
+            if async_funcs["async_context_managers"]:
+                lines.append(
+                    f"ASYNC CONTEXT MANAGERS ({len(async_funcs['async_context_managers'])}):"
+                )
+                for file_path, func_name, line_no in sorted(
+                    async_funcs["async_context_managers"], key=lambda x: (x[0], x[2])
+                )[:20]:
+                    lines.append(f"  {file_path}:{line_no} - {func_name}")
+                if len(async_funcs["async_context_managers"]) > 20:
+                    lines.append(
+                        f"  ... and {len(async_funcs['async_context_managers']) - 20} more"
+                    )
+                lines.append("")
+
+        # Pattern analysis
+        if async_funcs["all_async"]:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "ASYNC PATTERNS",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            if patterns["missing_awaits"]:
+                lines.append(
+                    f"⚠ POTENTIAL MISSING AWAITS ({len(patterns['missing_awaits'])}):"
+                )
+                for item in patterns["missing_awaits"][:15]:
+                    lines.append(f"  {item['file']}:{item['line']}")
+                    if item.get("context"):
+                        context_line = item["context"].strip()
+                        if len(context_line) > 70:
+                            context_line = context_line[:67] + "..."
+                        lines.append(f"    > {context_line}")
+                if len(patterns["missing_awaits"]) > 15:
+                    lines.append(
+                        f"  ... and {len(patterns['missing_awaits']) - 15} more"
+                    )
+                lines.append("")
+            else:
+                lines.append("✓ No obvious missing await calls detected")
+                lines.append("")
+
+            if patterns["async_context_managers"]:
+                lines.append(
+                    f"✓ ASYNC CONTEXT MANAGERS ({len(patterns['async_context_managers'])}):"
+                )
+                for item in patterns["async_context_managers"][:10]:
+                    lines.append(f"  {item}")
+                if len(patterns["async_context_managers"]) > 10:
+                    lines.append(
+                        f"  ... and {len(patterns['async_context_managers']) - 10} more"
+                    )
+                lines.append("")
+
+            # Python version features
+            lines.extend(
+                [
+                    "PYTHON 3.9+ FEATURES",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            if patterns["taskgroup_usage"]:
+                lines.append(f"✓ TaskGroup usage (Python 3.11+): {len(patterns['taskgroup_usage'])}")
+                for item in patterns["taskgroup_usage"][:5]:
+                    lines.append(f"  {item}")
+                lines.append("")
+            else:
+                lines.append("ℹ No TaskGroup usage (available in Python 3.11+)")
+                lines.append("")
+
+            if patterns["exception_groups"]:
+                lines.append(f"✓ Exception Groups usage: {len(patterns['exception_groups'])}")
+                for item in patterns["exception_groups"][:5]:
+                    lines.append(f"  {item}")
+                lines.append("")
+            else:
+                lines.append("ℹ No exception groups detected")
+                lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if async_funcs["all_async"]:
+            if patterns["missing_awaits"]:
+                lines.append("  - Review potential missing await calls marked above")
+                lines.append("  - Use linters (ruff, pylint) with async checks enabled")
+            else:
+                lines.append("  - ✓ Good: No obvious missing awaits detected")
+
+            lines.append("  - Use asyncio.gather() for concurrent execution")
+            lines.append("  - Prefer TaskGroup (Python 3.11+) for structured concurrency")
+            lines.append("  - Use asyncio.timeout() for deadline management")
+            lines.append("  - Avoid blocking calls (use async equivalents)")
+            lines.append("  - Use pytest-asyncio for testing async code")
+        else:
+            lines.append("  - No async code detected")
+            lines.append("  - If planning to add asyncio, use structured concurrency patterns")
+            lines.append("  - Consider asyncio for I/O-bound operations")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "logs" / "130_async_analysis.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _find_async_functions(self, root: Path) -> Dict[str, List[Tuple[str, str, int]]]:
+        """Find all async function definitions."""
+        coroutines = []
+        async_generators = []
+        async_context_managers = []
+
+        python_files = list(root.rglob("*.py"))
+
+        for py_file in python_files:
+            # Skip venv and cache
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                rel_path = str(py_file.relative_to(root))
+
+                tree = ast.parse(source)
+
+                for node in ast.walk(tree):
+                    if isinstance(node, ast.AsyncFunctionDef):
+                        func_name = node.name
+                        line_no = node.lineno
+
+                        # Determine type by checking for yield/yield from
+                        has_yield = any(
+                            isinstance(n, (ast.Yield, ast.YieldFrom))
+                            for n in ast.walk(node)
+                        )
+
+                        if has_yield:
+                            async_generators.append((rel_path, func_name, line_no))
+                        elif func_name.startswith("__aenter__") or func_name.startswith(
+                            "__aexit__"
+                        ):
+                            async_context_managers.append((rel_path, func_name, line_no))
+                        else:
+                            coroutines.append((rel_path, func_name, line_no))
+
+            except (OSError, UnicodeDecodeError, SyntaxError):
+                continue
+
+        all_async = coroutines + async_generators + async_context_managers
+
+        return {
+            "all_async": all_async,
+            "coroutines": coroutines,
+            "async_generators": async_generators,
+            "async_context_managers": async_context_managers,
+        }
+
+    def _analyze_async_patterns(self, root: Path, async_funcs: Dict) -> Dict:
+        """Analyze async patterns and best practices."""
+        missing_awaits = []
+        async_context_managers = []
+        taskgroup_usage = []
+        exception_groups = []
+
+        python_files = list(root.rglob("*.py"))
+
+        for py_file in python_files:
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                rel_path = str(py_file.relative_to(root))
+
+                tree = ast.parse(source)
+
+                for node in ast.walk(tree):
+                    if isinstance(node, ast.AsyncFunctionDef):
+                        # Check for missing awaits
+                        for call_node in ast.walk(node):
+                            if isinstance(call_node, ast.Call):
+                                # Check if it's a coroutine call without await
+                                if isinstance(call_node.func, ast.Name):
+                                    func_name = call_node.func.id
+                                    # Look for common async functions not being awaited
+                                    if func_name in [
+                                        "sleep",
+                                        "gather",
+                                        "create_task",
+                                    ] or any(
+                                        n[1] == func_name
+                                        for n in async_funcs["all_async"]
+                                    ):
+                                        # Check if parent is not Await
+                                        if not any(
+                                            isinstance(p, ast.Await)
+                                            for p in ast.walk(node)
+                                            if call_node in list(ast.walk(p))
+                                        ):
+                                            # This is a simple heuristic
+                                            context = source.split("\n")[
+                                                call_node.lineno - 1
+                                            ]
+                                            missing_awaits.append(
+                                                {
+                                                    "file": rel_path,
+                                                    "line": call_node.lineno,
+                                                    "context": context,
+                                                }
+                                            )
+
+                        # Check for async with
+                        for stmt in ast.walk(node):
+                            if isinstance(stmt, ast.AsyncWith):
+                                async_context_managers.append(
+                                    f"{rel_path}:{node.lineno}"
+                                )
+
+                # Global patterns
+                for line_num, line in enumerate(source.split("\n"), 1):
+                    if "TaskGroup" in line:
+                        taskgroup_usage.append(f"{rel_path}:{line_num}")
+
+                    if "ExceptionGroup" in line:
+                        exception_groups.append(f"{rel_path}:{line_num}")
+
+            except (OSError, UnicodeDecodeError, SyntaxError):
+                continue
+
+        # Deduplicate
+        async_context_managers = list(set(async_context_managers))
+        taskgroup_usage = list(set(taskgroup_usage))
+        exception_groups = list(set(exception_groups))
+
+        return {
+            "missing_awaits": missing_awaits,
+            "async_context_managers": async_context_managers,
+            "taskgroup_usage": taskgroup_usage,
+            "exception_groups": exception_groups,
+        }

pybundle/steps/bandit.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class BanditStep:
+    name: str = "bandit"
+    target: str = "."
+    outfile: str = "logs/50_bandit.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        bandit = which("bandit")
+        if not bandit:
+            out.write_text(
+                "bandit not found; skipping (pip install bandit)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing bandit")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text("no .py files detected; skipping bandit\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        # Run bandit with recursive mode, excluding common directories
+        cmd = [
+            bandit,
+            "-r",
+            self.target,
+            "--exclude",
+            "**/artifacts/**,.venv,venv,__pycache__,.mypy_cache,.ruff_cache,node_modules,dist,build,.git,.tox,*.egg-info",
+            "--format",
+            "txt",
+        ]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        # Bandit exit codes: 0=no issues, 1=issues found
+        note = "" if cp.returncode == 0 else f"exit={cp.returncode} (security issues)"
+        return StepResult(self.name, "PASS", dur, note)

pybundle/steps/base.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from ..context import BundleContext
+
+
+@dataclass
+class StepResult:
+    name: str
+    status: str  # "PASS" | "FAIL" | "SKIP"
+    seconds: int
+    note: str = ""
+
+
+class Step(Protocol):
+    name: str
+
+    def run(self, ctx: BundleContext) -> StepResult: ...