gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/rg_scans.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+@dataclass
+class RipgrepScanStep:
+    name: str
+    pattern: str
+    outfile: str
+    target: str = "."  # directory or file
+    extra_args: list[str] | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        rg = which("rg")
+        if not rg:
+            out.write_text(
+                "rg (ripgrep) not found; skipping (install ripgrep)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing rg")
+
+        args = self.extra_args or []
+        # -n line numbers, --no-heading keeps it grep-like, -S smart case can be handy
+        cmd = [rg, "-n", "--no-heading", "-S", *args, self.pattern, self.target]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+        # rg exit codes:
+        # 0 = matches found
+        # 1 = no matches found (not an error!)
+        # 2 = actual error
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        note = ""
+        if cp.returncode == 2:
+            note = "rg error (exit=2) recorded"
+        elif cp.returncode == 1:
+            note = "no matches"
+
+        # Always PASS; we’re collecting info, not enforcing policy (yet).
+        return StepResult(self.name, "PASS", dur, note)
+
+
+def default_rg_steps(target: str = ".") -> list[RipgrepScanStep]:
+    return [
+        RipgrepScanStep(
+            name="rg TODO/FIXME/HACK",
+            pattern=r"TODO|FIXME|HACK",
+            outfile="logs/40_rg_todos.txt",
+            target=target,
+        ),
+        RipgrepScanStep(
+            name="rg print(",
+            pattern=r"^\s*print\(",
+            outfile="logs/41_rg_prints.txt",
+            target=target,
+        ),
+        RipgrepScanStep(
+            name="rg except patterns",
+            pattern=r"except\s+Exception|except\s*:",
+            outfile="logs/42_rg_bare_excepts.txt",
+            target=target,
+        ),
+    ]

pybundle/steps/roadmap.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from .base import StepResult
+from ..context import BundleContext
+from ..policy import AIContextPolicy
+from ..roadmap_scan import build_roadmap  # keep only if you actually use it
+
+
+class RoadmapGraph(Protocol):
+    entrypoints: list[Any]
+    nodes: list[Any]
+    edges: list[Any]
+    stats: dict[str, Any]
+
+
+@dataclass
+class RoadmapStep:
+    name: str = "roadmap (project map)"
+    out_md: str = "meta/70_roadmap.md"
+    out_json: str = "meta/70_roadmap.json"
+    include: list[str] | None = None
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        policy = self.policy or AIContextPolicy()
+
+        # Include dirs: explicit override wins; otherwise policy candidates (with fallback)
+        if self.include:
+            include_dirs = [
+                ctx.root / p for p in self.include if (ctx.root / p).exists()
+            ]
+            if not include_dirs:
+                include_dirs = [ctx.root]
+        else:
+            include_dirs = policy.include_dir_candidates(
+                ctx.root
+            )  # includes fallback to [root]
+
+        exclude_dirs = set(policy.exclude_dirs)
+
+        graph = build_roadmap(
+            root=ctx.root,
+            include_dirs=include_dirs,
+            exclude_dirs=exclude_dirs,
+            max_files=policy.roadmap_max_files,
+        )
+
+        # JSON
+        out_json_path = ctx.workdir / self.out_json
+        out_json_path.parent.mkdir(parents=True, exist_ok=True)
+        out_json_path.write_text(
+            json.dumps(graph.to_dict(), indent=2), encoding="utf-8"
+        )
+
+        # Markdown (policy-driven Mermaid knobs)
+        out_md_path = ctx.workdir / self.out_md
+        out_md_path.parent.mkdir(parents=True, exist_ok=True)
+        out_md_path.write_text(self._render_md(graph, policy), encoding="utf-8")
+
+        langs = sorted({n.lang for n in graph.nodes if getattr(n, "lang", None)})
+        summary = {
+            "languages": langs,
+            "entrypoints": [ep.node for ep in graph.entrypoints[:50]],
+            "stats": graph.stats,
+        }
+        (ctx.workdir / "meta" / "71_roadmap_summary.json").write_text(
+            json.dumps(summary, indent=2), encoding="utf-8"
+        )
+
+        dur = int(time.time() - start)
+        note = f"nodes={len(graph.nodes)} edges={len(graph.edges)} entrypoints={len(graph.entrypoints)}"
+        return StepResult(self.name, "PASS", dur, note)
+
+    def _render_md(self, graph: RoadmapGraph, policy: AIContextPolicy) -> str:
+        depth = policy.roadmap_mermaid_depth
+        max_edges = policy.roadmap_mermaid_max_edges
+
+        lines: list[str] = []
+        lines.append("# Project Roadmap")
+        lines.append("")
+        lines.append("## Entrypoints")
+        if not graph.entrypoints:
+            lines.append("- (none detected)")
+        else:
+            for ep in graph.entrypoints[:50]:
+                lines.append(
+                    f"- `{ep.node}` — {ep.reason} (confidence {ep.confidence}/3)"
+                )
+        lines.append("")
+        lines.append("## High-level map")
+        lines.append("```mermaid")
+        lines.append("flowchart LR")
+        lines.extend(
+            self._render_mermaid_bfs(graph, max_depth=depth, max_edges=max_edges)
+        )
+        lines.append("```")
+        lines.append("")
+        lines.append("## Stats")
+        for k in sorted(graph.stats.keys()):
+            lines.append(f"- **{k}**: {graph.stats[k]}")
+        lines.append("")
+        lines.append("## Notes")
+        lines.append(
+            "- Destinations like `py:...`, `js:...`, `rs:...` are dependency specs (not resolved to paths yet)."
+        )
+        lines.append(
+            "- This is designed to be deterministic and readable, not a perfect compiler-grade call graph."
+        )
+        lines.append("")
+        return "\n".join(lines)
+
+    def _render_mermaid_bfs(
+        self, graph: RoadmapGraph, max_depth: int = 2, max_edges: int = 180
+    ) -> list[str]:
+        from collections import deque
+
+        adj: dict[str, list[str]] = {}
+        for e in graph.edges:
+            adj.setdefault(e.src, []).append(e.dst)
+
+        entry = [ep.node for ep in graph.entrypoints]
+        if not entry:
+            return ['  A["(no entrypoints)"]']
+
+        q = deque([(n, 0) for n in entry])
+        seen_edges: set[tuple[str, str]] = set()
+        shown: list[str] = []
+        seen_nodes: set[str] = set(entry)
+
+        while q and len(shown) < max_edges:
+            node, depth = q.popleft()
+            if depth >= max_depth:
+                continue
+            for dst in adj.get(node, []):
+                key = (node, dst)
+                if key in seen_edges:
+                    continue
+                seen_edges.add(key)
+                shown.append(f'  "{node}" --> "{dst}"')
+                if dst not in seen_nodes:
+                    seen_nodes.add(dst)
+                    q.append((dst, depth + 1))
+                if len(shown) >= max_edges:
+                    break
+
+        return shown or ['  A["(no edges rendered)"]']

pybundle/steps/ruff.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    # Fast-ish heuristic: look for any .py file in top couple levels
+    # (Avoid walking deep trees; ruff itself can handle it.)
+    for p in root.rglob("*.py"):
+        # ignore common junk dirs
+        parts = set(p.parts)
+        if (
+            ".venv" in parts
+            or "__pycache__" in parts
+            or ".mypy_cache" in parts
+            or ".ruff_cache" in parts
+        ):
+            continue
+        if (
+            "node_modules" in parts
+            or "dist" in parts
+            or "build" in parts
+            or "artifacts" in parts
+        ):
+            continue
+        return True
+    return False
+
+
+@dataclass
+class RuffCheckStep:
+    name: str = "ruff check"
+    target: str = "."
+    outfile: str = "logs/31_ruff_check.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        ruff = which("ruff")
+        if not ruff:
+            out.write_text(
+                "ruff not found; skipping (pip install ruff)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing ruff")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping ruff check\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        cmd = [ruff, "check", self.target]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        # ruff nonzero = lint failures; that’s *valuable*, but for bundling we record it.
+        note = "" if cp.returncode == 0 else f"exit={cp.returncode} (lint findings)"
+        return StepResult(self.name, "PASS", dur, note)
+
+
+@dataclass
+class RuffFormatCheckStep:
+    name: str = "ruff format --check"
+    target: str = "."
+    outfile: str = "logs/32_ruff_format_check.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        ruff = which("ruff")
+        if not ruff:
+            out.write_text(
+                "ruff not found; skipping (pip install ruff)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing ruff")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text(
+                "no .py files detected; skipping ruff format check\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        cmd = [ruff, "format", "--check", self.target]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        
+        # Exit code 0 = formatted correctly, non-zero = needs formatting
+        if cp.returncode == 0:
+            return StepResult(self.name, "PASS", dur, "")
+        else:
+            # Format drift should be WARN, not PASS - it's actionable
+            note = f"exit={cp.returncode} (format drift detected)"
+            return StepResult(self.name, "WARN", dur, note)

pybundle/steps/secrets_detection.py

@@ -0,0 +1,235 @@
+"""
+Step: Enhanced Secrets Detection
+Advanced secrets detection using entropy analysis and patterns.
+"""
+
+import re
+import math
+import json
+from pathlib import Path
+from typing import Dict, List, Set, Tuple, Optional
+
+from .base import Step, StepResult
+
+
+class SecretsDetectionStep(Step):
+    """Detect secrets in codebase using entropy and regex patterns."""
+
+    name = "secrets detection"
+
+    # Common secret patterns
+    SECRET_PATTERNS = {
+        "AWS_KEY_ID": r"AKIA[0-9A-Z]{16}",
+        "AWS_SECRET": r"aws_secret_access_key[\"']?\s*[:=]\s*[\"']([^\"'\n]+)[\"']",
+        "GITHUB_TOKEN": r"gh[pousr]_[A-Za-z0-9_]{36,255}",
+        "PRIVATE_KEY": r"-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----",
+        "API_KEY": r"api[_-]?key[\"']?\s*[:=]\s*[\"']([^\"'\n]+)[\"']",
+        "DATABASE_PASSWORD": r"(?:password|passwd)[\"']?\s*[:=]\s*[\"']([^\"'\n]+)[\"']",
+        "JWT_SECRET": r"jwt[_-]?secret[\"']?\s*[:=]\s*[\"']([^\"'\n]+)[\"']",
+        "SLACK_TOKEN": r"xox[baprs]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,32}",
+        "STRIPE_KEY": r"sk_live_[0-9a-zA-Z]{24}",
+    }
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Detect secrets in codebase."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Scan for secrets
+        secrets_found = self._scan_for_secrets(root)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "SECRETS DETECTION REPORT",
+            "=" * 80,
+            "",
+        ]
+
+        if secrets_found["matches"]:
+            lines.append("⚠ POTENTIAL SECRETS DETECTED (pattern-based)")
+            lines.append("")
+
+            for file_path, details in secrets_found["matches"].items():
+                lines.append(f"File: {file_path}")
+
+                for issue in details:
+                    lines.append(f"  Line {issue['line']}: {issue['type']}")
+                    if issue.get("context"):
+                        context_line = issue["context"].strip()
+                        if len(context_line) > 70:
+                            context_line = context_line[:67] + "..."
+                        lines.append(f"    Context: {context_line}")
+                    if issue.get("entropy"):
+                        lines.append(f"    Entropy: {issue['entropy']:.2f}")
+
+                lines.append("")
+
+            lines.append(f"Total files with potential secrets: {len(secrets_found['matches'])}")
+            lines.append(f"Total potential secrets: {secrets_found['total_matches']}")
+        else:
+            lines.append("✓ No pattern-based secrets detected")
+            lines.append("  (API keys, AWS keys, GitHub tokens, private keys, etc.)")
+
+        lines.extend(
+            [
+                "",
+                "=" * 80,
+                "ENTROPY ANALYSIS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        # High entropy strings analysis (LIMITED OUTPUT)
+        high_entropy = secrets_found.get("high_entropy", [])
+        if high_entropy:
+            lines.append(f"Found {len(high_entropy)} high-entropy strings (may include hashes, tokens, UUIDs):")
+            lines.append("")
+            lines.append("NOTE: High-entropy detection produces many false positives.")
+            lines.append("      Focus on pattern-based findings above for actual secrets.")
+            lines.append("")
+
+            # Show only top 10 highest-entropy, not all
+            display_count = min(10, len(high_entropy))
+            for item in high_entropy[:display_count]:
+                lines.append(f"  File: {item['file']}")
+                lines.append(f"  Line: {item['line']}")
+                lines.append(f"  Entropy: {item['entropy']:.3f}")
+                if item.get("context"):
+                    context = item["context"].strip()
+                    if len(context) > 60:
+                        context = context[:57] + "..."
+                    lines.append(f"  Value preview: {context}")
+                lines.append("")
+
+            if len(high_entropy) > display_count:
+                lines.append(f"  ... and {len(high_entropy) - display_count} more (suppressed for readability)")
+                lines.append(f"      Run with --deep-scan to see full entropy analysis")
+                lines.append("")
+
+        else:
+            lines.append("✓ No high-entropy strings detected")
+            lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if secrets_found["matches"] or high_entropy:
+            lines.append("  - Review and rotate any exposed secrets immediately")
+            lines.append("  - Use a secrets manager (AWS Secrets Manager, HashiCorp Vault)")
+            lines.append("  - Configure git hooks to prevent committing secrets")
+            lines.append("  - Use .gitignore to exclude .env and secrets files")
+            lines.append("  - Consider using detect-secrets or similar tools in CI/CD")
+
+        else:
+            lines.append("  - ✓ Good security practice: no obvious secrets detected")
+            lines.append("  - Continue to use secrets management best practices")
+            lines.append("  - Store sensitive data in .env or secrets manager")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "logs" / "121_secrets_advanced.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _scan_for_secrets(self, root: Path) -> Dict:
+        """Scan files for secrets using patterns and entropy analysis."""
+        matches = {}
+        high_entropy = []
+        total_matches = 0
+
+        python_files = list(root.rglob("*.py")) + list(root.rglob("*.json")) + list(
+            root.rglob("*.yaml")
+        ) + list(root.rglob("*.yml"))
+
+        for py_file in python_files:
+            # Skip venv, cache, and dependency directories (PROJECT SCOPE ONLY)
+            if any(
+                part in py_file.parts
+                for part in [
+                    "venv", ".venv", "env", "__pycache__", "site-packages",
+                    ".mypy_cache", ".pytest_cache", ".ruff_cache", ".freeze-venv",
+                    "node_modules", "dist", "build", "target"
+                ]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                rel_path = str(py_file.relative_to(root))
+
+                file_matches = []
+
+                # Check against secret patterns
+                for line_num, line in enumerate(source.split("\n"), 1):
+                    for secret_type, pattern in self.SECRET_PATTERNS.items():
+                        if re.search(pattern, line, re.IGNORECASE):
+                            file_matches.append(
+                                {
+                                    "line": line_num,
+                                    "type": secret_type,
+                                    "context": line,
+                                }
+                            )
+                            total_matches += 1
+
+                # Check entropy of quoted strings
+                string_pattern = r'["\']([A-Za-z0-9_\-\.]{20,})["\']'
+                for line_num, line in enumerate(source.split("\n"), 1):
+                    for match in re.finditer(string_pattern, line):
+                        string_val = match.group(1)
+                        entropy = self._calculate_entropy(string_val)
+
+                        # Flag high entropy strings (likely encrypted or random)
+                        if entropy > 4.0:
+                            high_entropy.append(
+                                {
+                                    "file": rel_path,
+                                    "line": line_num,
+                                    "entropy": entropy,
+                                    "context": string_val,
+                                }
+                            )
+
+                if file_matches:
+                    matches[rel_path] = file_matches
+
+            except (OSError, UnicodeDecodeError):
+                continue
+
+        # Sort high entropy by entropy score
+        high_entropy.sort(key=lambda x: x["entropy"], reverse=True)
+
+        return {
+            "matches": matches,
+            "total_matches": total_matches,
+            "high_entropy": high_entropy,
+        }
+
+    def _calculate_entropy(self, s: str) -> float:
+        """Calculate Shannon entropy of a string."""
+        if not s:
+            return 0.0
+
+        entropy = 0.0
+        for byte in set(s):
+            freq = s.count(byte) / len(s)
+            entropy -= freq * math.log2(freq)
+
+        return entropy