gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/query_pattern_analysis.py

@@ -0,0 +1,334 @@
+"""
+Step: Query Pattern Analysis
+Analyze database query patterns and detect performance issues.
+"""
+
+import re
+import ast
+from pathlib import Path
+from typing import Dict, List, Set, Tuple, Optional
+
+from .base import Step, StepResult
+
+
+class QueryPatternAnalysisStep(Step):
+    """Analyze database query patterns for performance issues."""
+
+    name = "query pattern analysis"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Analyze query patterns in codebase."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Analyze query patterns
+        patterns = self._analyze_query_patterns(root)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "QUERY PATTERN ANALYSIS REPORT",
+            "=" * 80,
+            "",
+        ]
+
+        # Summary
+        lines.extend(
+            [
+                "SUMMARY",
+                "=" * 80,
+                "",
+                f"ORM framework detected: {patterns['orm_type']}",
+                f"Model definitions found: {patterns['model_count']}",
+                f"Query patterns analyzed: {patterns['query_count']}",
+                "",
+            ]
+        )
+
+        if not patterns["orm_type"]:
+            lines.extend(
+                [
+                    "⊘ No ORM detected",
+                    "",
+                    "This project does not appear to use an ORM.",
+                    "If this is incorrect, ensure ORM imports are in analyzed files.",
+                    "",
+                ]
+            )
+        else:
+            # ORM Details
+            lines.extend(
+                [
+                    f"{patterns['orm_type'].upper()} ANALYSIS",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            lines.append(f"Models/Entities found: {patterns['model_count']}")
+            if patterns["models"]:
+                for model in sorted(patterns["models"])[:15]:
+                    lines.append(f"  - {model}")
+                if len(patterns["models"]) > 15:
+                    lines.append(f"  ... and {len(patterns['models']) - 15} more")
+
+            lines.append("")
+
+            # N+1 Query Patterns
+            if patterns["suspected_n_plus_1"]:
+                lines.extend(
+                    [
+                        "SUSPECTED N+1 QUERY PATTERNS",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                for issue in patterns["suspected_n_plus_1"][:15]:
+                    lines.append(f"File: {issue['file']}")
+                    lines.append(f"Line: {issue['line']}")
+                    lines.append(f"Pattern: {issue['pattern']}")
+                    if issue.get("context"):
+                        context = issue["context"].strip()
+                        if len(context) > 70:
+                            context = context[:67] + "..."
+                        lines.append(f"Context: {context}")
+                    lines.append("")
+
+                if len(patterns["suspected_n_plus_1"]) > 15:
+                    lines.append(
+                        f"... and {len(patterns['suspected_n_plus_1']) - 15} more suspected N+1 patterns"
+                    )
+                    lines.append("")
+
+            else:
+                lines.append("✓ No obvious N+1 query patterns detected")
+                lines.append("")
+
+            # Lazy Loading Patterns
+            if patterns["lazy_loading"]:
+                lines.extend(
+                    [
+                        "LAZY LOADING PATTERNS (Potential Performance Issues)",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                for issue in patterns["lazy_loading"][:10]:
+                    lines.append(f"File: {issue['file']}")
+                    lines.append(f"Line: {issue['line']}")
+                    lines.append(f"Type: {issue['type']}")
+                    lines.append("")
+
+                if len(patterns["lazy_loading"]) > 10:
+                    lines.append(
+                        f"... and {len(patterns['lazy_loading']) - 10} more lazy loading patterns"
+                    )
+                    lines.append("")
+
+            # Relationship Access
+            if patterns["relationship_access"]:
+                lines.extend(
+                    [
+                        "RELATIONSHIP ACCESS PATTERNS",
+                        "=" * 80,
+                        "",
+                    ]
+                )
+
+                lines.append(
+                    f"Foreign key accesses: {patterns['relationship_access'].get('foreign_keys', 0)}"
+                )
+                lines.append(
+                    f"Many-to-many accesses: {patterns['relationship_access'].get('many_to_many', 0)}"
+                )
+                lines.append(
+                    f"Reverse relationship accesses: {patterns['relationship_access'].get('reverse', 0)}"
+                )
+
+                lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if patterns["orm_type"]:
+            if patterns["suspected_n_plus_1"]:
+                if patterns["orm_type"].lower() == "django":
+                    lines.append("  N+1 Query Fixes (Django):")
+                    lines.append("    - Use select_related() for ForeignKey relationships")
+                    lines.append("    - Use prefetch_related() for ManyToMany and reverse ForeignKey")
+                    lines.append("")
+                    lines.append("  Example:")
+                    lines.append(
+                        "    users = User.objects.select_related('profile').prefetch_related('posts')"
+                    )
+
+                elif patterns["orm_type"].lower() == "sqlalchemy":
+                    lines.append("  N+1 Query Fixes (SQLAlchemy):")
+                    lines.append("    - Use joinedload() for eager loading")
+                    lines.append("    - Use contains_eager() with joins")
+                    lines.append("    - Use selectinload() for relationships")
+                    lines.append("")
+                    lines.append("  Example:")
+                    lines.append(
+                        "    query.options(joinedload(User.posts)).all()"
+                    )
+
+                lines.append("")
+
+            if patterns["lazy_loading"]:
+                lines.append("  Lazy Loading Best Practices:")
+                lines.append("    - Load related objects within query, not after retrieval")
+                lines.append("    - Use batch loading for sets of objects")
+                lines.append("    - Consider caching for frequently accessed relationships")
+                lines.append("")
+
+            lines.append("  General Recommendations:")
+            lines.append("    - Use database query profiling (Django Debug Toolbar, etc.)")
+            lines.append("    - Review query execution plans (EXPLAIN)")
+            lines.append("    - Add indexes to frequently filtered columns")
+            lines.append("    - Monitor query count and execution time")
+
+        else:
+            lines.append("  - No ORM detected; static query analysis not applicable")
+            lines.append("  - If using raw SQL, consider adopting an ORM for consistency")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "logs" / "140_query_patterns.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _analyze_query_patterns(self, root: Path) -> Dict:
+        """Analyze query patterns in codebase."""
+        orm_type = None
+        models = set()
+        model_count = 0
+        query_count = 0
+        suspected_n_plus_1 = []
+        lazy_loading = []
+        relationship_access = {"foreign_keys": 0, "many_to_many": 0, "reverse": 0}
+
+        python_files = list(root.rglob("*.py"))
+
+        for py_file in python_files:
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                rel_path = str(py_file.relative_to(root))
+
+                # Detect ORM
+                if "from django.db import models" in source or "from django.db.models" in source:
+                    if not orm_type:
+                        orm_type = "Django"
+                elif (
+                    "from sqlalchemy" in source
+                    and "declarative_base" in source
+                ):
+                    if not orm_type:
+                        orm_type = "SQLAlchemy"
+                elif "from tortoise import fields" in source:
+                    if not orm_type:
+                        orm_type = "Tortoise ORM"
+
+                # Count models
+                for line in source.split("\n"):
+                    if re.search(r"class\s+(\w+)\s*\(.*Model.*\):", line):
+                        match = re.search(r"class\s+(\w+)\s*\(", line)
+                        if match:
+                            model_name = match.group(1)
+                            models.add(model_name)
+                            model_count += 1
+
+                # Detect query patterns
+                for line_num, line in enumerate(source.split("\n"), 1):
+                    # N+1 patterns
+                    if re.search(
+                        r"for\s+\w+\s+in\s+.*\.\s*(all|filter|get)\(\)",
+                        line
+                    ):
+                        query_count += 1
+                        suspected_n_plus_1.append(
+                            {
+                                "file": rel_path,
+                                "line": line_num,
+                                "pattern": "Loop with query",
+                                "context": line,
+                            }
+                        )
+
+                    # Django patterns
+                    if "select_related" not in source and "for obj in" in line:
+                        if ".objects.all()" in source or ".objects.filter" in source:
+                            query_count += 1
+
+                    # Lazy loading patterns (accessing attributes after query)
+                    if (
+                        re.search(r"\.[\w_]+\s*(?:$|#)", line)
+                        and "select_related" not in line
+                        and "prefetch" not in line
+                    ):
+                        # Potential lazy loading
+                        if any(
+                            kw in line
+                            for kw in ["for ", r"\.all(", r"\.filter("]
+                        ):
+                            pass
+                        else:
+                            lazy_loading.append(
+                                {
+                                    "file": rel_path,
+                                    "line": line_num,
+                                    "type": "Potential lazy loading",
+                                }
+                            )
+
+                    # Relationship access patterns
+                    if re.search(r"\.\w+_set\.", line):  # Reverse FK Django
+                        relationship_access["reverse"] += 1
+                    elif re.search(r"\.objects\.through", line):  # Many-to-many
+                        relationship_access["many_to_many"] += 1
+                    elif re.search(r"\..*_id\b", line):  # FK access
+                        relationship_access["foreign_keys"] += 1
+
+            except (OSError, UnicodeDecodeError, SyntaxError):
+                continue
+
+        # Deduplicate and limit
+        suspected_n_plus_1 = list(
+            {(item["file"], item["line"]): item for item in suspected_n_plus_1}.values()
+        )
+        lazy_loading = list(
+            {(item["file"], item["line"]): item for item in lazy_loading}.values()
+        )
+
+        return {
+            "orm_type": orm_type,
+            "models": models,
+            "model_count": model_count,
+            "query_count": query_count,
+            "suspected_n_plus_1": suspected_n_plus_1,
+            "lazy_loading": lazy_loading,
+            "relationship_access": relationship_access,
+        }

pybundle/steps/radon.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class RadonStep:
+    name: str = "radon"
+    target: str = "."
+    outfile: str = "logs/51_radon_complexity.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        radon = which("radon")
+        if not radon:
+            out.write_text(
+                "radon not found; skipping (pip install radon)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing radon")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text("no .py files detected; skipping radon\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        target_path = ctx.root / self.target
+
+        # Build exclude patterns to avoid scanning artifacts, venvs, caches
+        # CRITICAL: Radon scans everything by default, including prior pybundle runs
+        # Use simple patterns without wildcards - radon's --exclude is finicky
+        excludes = [
+            # Artifacts from prior pybundle runs (CRITICAL - prevents duplicate reports)
+            "artifacts",
+            # Virtual environments (all common patterns)
+            ".venv",
+            "venv",
+            "env",
+            ".env",
+            ".freeze-venv",
+            ".pybundle-venv",
+            # Also catch custom venv names with glob patterns
+            "*-venv",
+            "*_venv",
+            ".gaslog-venv",
+            # Caches
+            "__pycache__",
+            ".mypy_cache",
+            ".pytest_cache",
+            ".ruff_cache",
+            ".tox",
+            ".nox",
+            # Build outputs
+            "node_modules",
+            "dist",
+            "build",
+            "target",
+            # Version control
+            ".git",
+        ]
+        
+        # Radon --exclude takes comma-separated patterns
+        exclude_arg = ",".join(excludes)
+
+        # Run cyclomatic complexity check
+        cmd_cc = [
+            radon,
+            "cc",
+            str(target_path),
+            "-s",  # Show complexity score
+            "-a",  # Average complexity
+            "-nc",  # No color
+            "--exclude",
+            exclude_arg,
+        ]
+
+        # Run maintainability index check
+        cmd_mi = [
+            radon,
+            "mi",
+            str(target_path),
+            "-s",  # Show maintainability index
+            "-nc",  # No color
+            "--exclude",
+            exclude_arg,
+        ]
+
+        try:
+            # Collect both metrics in one output file
+            with out.open("w", encoding="utf-8") as f:
+                f.write("=" * 70 + "\n")
+                f.write("CYCLOMATIC COMPLEXITY\n")
+                f.write("=" * 70 + "\n\n")
+
+                result_cc = subprocess.run(  # nosec B603 - Using full path from which()
+                    cmd_cc,
+                    cwd=ctx.root,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    timeout=120,
+                )
+                f.write(result_cc.stdout)
+
+                f.write("\n\n")
+                f.write("=" * 70 + "\n")
+                f.write("MAINTAINABILITY INDEX\n")
+                f.write("=" * 70 + "\n\n")
+
+                result_mi = subprocess.run(  # nosec B603 - Using full path from which()
+                    cmd_mi,
+                    cwd=ctx.root,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    timeout=120,
+                )
+                f.write(result_mi.stdout)
+
+            elapsed = int((time.time() - start) * 1000)
+
+            # Radon returns 0 on success
+            if result_cc.returncode == 0 and result_mi.returncode == 0:
+                return StepResult(self.name, "OK", elapsed, "")
+            else:
+                return StepResult(
+                    self.name,
+                    "FAIL",
+                    elapsed,
+                    f"exit cc:{result_cc.returncode} mi:{result_mi.returncode}",
+                )
+        except subprocess.TimeoutExpired:
+            out.write_text("radon timed out after 120s\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 120000, "timeout")
+        except Exception as e:
+            out.write_text(f"radon error: {e}\n", encoding="utf-8")
+            return StepResult(self.name, "FAIL", 0, str(e))

pybundle/steps/repro_md.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import platform
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+@dataclass
+class ReproMarkdownStep:
+    name: str = "generate REPRO.md"
+    outfile: str = "REPRO.md"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        repro = ctx.workdir / self.outfile
+
+        # ---- tool detection ----
+        tool_names = [
+            "python",
+            "pip",
+            "git",
+            "ruff",
+            "mypy",
+            "pytest",
+            "rg",
+            "zip",
+            "tar",
+        ]
+        detected = {t: which(t) for t in tool_names}
+
+        # Prefer ctx.tools.python if you have it
+        if getattr(ctx, "tools", None) and getattr(ctx.tools, "python", None):
+            detected["python"] = ctx.tools.python
+
+        # ---- file inventory (what actually exists) ----
+        def list_txt(dirpath: Path) -> list[str]:
+            if not dirpath.is_dir():
+                return []
+            return sorted(
+                str(p.relative_to(ctx.workdir)) for p in dirpath.rglob("*.txt")
+            )
+
+        logs_list = list_txt(ctx.logdir)
+        meta_list = list_txt(ctx.metadir)
+
+        # Also include key top-level files if present
+        top_files = []
+        for name in [
+            "RUN_LOG.txt",
+            "SUMMARY.json",
+            "error_files_from_logs.txt",
+            "error_refs_count.txt",
+        ]:
+            p = ctx.workdir / name
+            if p.exists():
+                top_files.append(name)
+
+        # ---- step summary (best-effort, never crash) ----
+        results = getattr(ctx, "results", [])
+        ctx.results = results  # ensure it's set for future steps
+
+        summary_lines = []
+        for r in results:
+            note = f" ({r.note})" if getattr(r, "note", "") else ""
+            summary_lines.append(f"- **{r.name}**: {r.status}{note}")
+
+        # ---- environment ----
+        pyver = sys.version.split()[0]
+        plat = platform.platform()
+        profile = ctx.profile_name
+        utc_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+        # ---- build markdown ----
+        def fmt_tool(t: str) -> str:
+            path = detected.get(t)
+            return f"- `{t}`: ✅ `{path}`" if path else f"- `{t}`: ❌ not found"
+
+        md = []
+        md += ["# Reproduction Guide", ""]
+        md += [
+            "This bundle captures diagnostic outputs and the minimum relevant project context",
+            "to reproduce issues reliably on another system.",
+            "",
+            "## Overview",
+            f"- Profile: `{profile}`",
+            f"- Generated (UTC): `{utc_now}`",
+            f"- Project root: `{ctx.root}`",
+            "",
+            "## Environment Snapshot",
+            f"- OS: `{plat}`",
+            f"- Python: `{pyver}`",
+            "",
+            "## Tools Detected",
+            *[fmt_tool(t) for t in tool_names],
+            "",
+        ]
+
+        if summary_lines:
+            md += ["## Steps Executed", *summary_lines, ""]
+
+        md += [
+            "## How to Reproduce",
+            "",
+            "From the project root:",
+            "",
+            "```bash",
+            f"python -m pybundle run {profile}",
+            "```",
+            "",
+            "Re-run individual tools (if installed):",
+            "",
+            "```bash",
+            "python -m compileall .",
+            "ruff check .",
+            "ruff format --check .",
+            "mypy .",
+            "pytest -q",
+            "```",
+            "",
+            "## Produced Artifacts",
+            "",
+        ]
+
+        if top_files:
+            md += ["### Top-level", *[f"- `{p}`" for p in top_files], ""]
+
+        md += (
+            ["### logs/", *(f"- `{p}`" for p in logs_list)]
+            if logs_list
+            else ["### logs/", "- (none)", ""]
+        )
+        md += (
+            ["", "### meta/", *(f"- `{p}`" for p in meta_list)]
+            if meta_list
+            else ["", "### meta/", "- (none)"]
+        )
+
+        md += [
+            "",
+            "## Context Packs",
+            "",
+            "- `src/_error_refs/` – files directly referenced by tool output",
+            "- `src/_error_context/` – related imports + pytest glue (conftest/__init__) + configs",
+            "",
+            "## Notes",
+            "",
+            "- Non-zero exits from linters/tests are recorded for diagnosis; bundle creation continues.",
+            "- Missing tools typically produce SKIP logs rather than failing the bundle.",
+            "",
+        ]
+
+        repro.write_text("\n".join(md) + "\n", encoding="utf-8")
+
+        dur = int(time.time() - start)
+        return StepResult(self.name, "PASS", dur, "")