gwc-pybundle 1.4.5__py3-none-any.whl

pybundle/roadmap_scan.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import ast
+import os
+import re
+from pathlib import Path
+from typing import Literal, Optional
+from .steps.copy_pack import _is_venv_root, _is_under_venv
+
+from .roadmap_model import Node, Edge, EntryPoint, RoadmapGraph
+
+PY_EXT = {".py"}
+JS_EXT = {".js", ".jsx", ".mjs", ".cjs"}
+TS_EXT = {".ts", ".tsx"}
+RUST_EXT = {".rs"}
+Lang = Literal["python", "js", "ts", "rust", "html", "css", "config", "unknown"]
+
+IMPORT_RE = re.compile(r'^\s*import\s+.*?\s+from\s+[\'"](.+?)[\'"]\s*;?\s*$', re.M)
+REQUIRE_RE = re.compile(r'require\(\s*[\'"](.+?)[\'"]\s*\)')
+RUST_USE_RE = re.compile(r"^\s*use\s+([a-zA-Z0-9_:]+)", re.M)
+RUST_MOD_RE = re.compile(r"^\s*mod\s+([a-zA-Z0-9_]+)\s*;", re.M)
+
+
+def _rel(root: Path, p: Path) -> str:
+    return str(p.resolve().relative_to(root.resolve())).replace("\\", "/")
+
+
+def guess_lang(p: Path) -> Lang:
+    suf = p.suffix.lower()
+    if suf in PY_EXT:
+        return "python"
+    if suf in TS_EXT:
+        return "ts"
+    if suf in JS_EXT:
+        return "js"
+    if suf in RUST_EXT:
+        return "rust"
+    if suf in {".html", ".jinja", ".j2"}:
+        return "html"
+    if suf in {".css", ".scss", ".sass"}:
+        return "css"
+    if suf in {".toml", ".yaml", ".yml", ".json", ".ini", ".cfg"}:
+        return "config"
+    return "unknown"
+
+
+def scan_python_imports(root: Path, file_path: Path) -> list[str]:
+    # returns import module strings (not resolved paths)
+    try:
+        tree = ast.parse(file_path.read_text(encoding="utf-8", errors="replace"))
+    except Exception:
+        return []
+    mods: list[str] = []
+    for n in ast.walk(tree):
+        if isinstance(n, ast.Import):
+            for a in n.names:
+                mods.append(a.name)
+        elif isinstance(n, ast.ImportFrom):
+            if n.module:
+                mods.append(n.module)
+    return mods
+
+
+def scan_js_imports(text: str) -> list[str]:
+    out = []
+    out += IMPORT_RE.findall(text)
+    out += REQUIRE_RE.findall(text)
+    return out
+
+
+def scan_rust_uses(text: str) -> tuple[list[str], list[str]]:
+    uses = RUST_USE_RE.findall(text)
+    mods = RUST_MOD_RE.findall(text)
+    return uses, mods
+
+
+def detect_entrypoints(root: Path) -> list[EntryPoint]:
+    eps: list[EntryPoint] = []
+
+    # Python CLI entry: __main__.py
+    p = root / "src"
+    if p.exists():
+        for main in p.rglob("__main__.py"):
+            eps.append(
+                EntryPoint(
+                    node=_rel(root, main), reason="python __main__.py", confidence=3
+                )
+            )
+
+    # Rust main.rs (including tauri src-tauri)
+    for mr in root.rglob("main.rs"):
+        if "target/" in str(mr):  # safety
+            continue
+        eps.append(EntryPoint(node=_rel(root, mr), reason="rust main.rs", confidence=3))
+
+    # package.json scripts as entrypoints (synthetic)
+    pkg = root / "package.json"
+    if pkg.is_file():
+        eps.append(EntryPoint(node="package.json", reason="node scripts", confidence=2))
+
+    return eps
+
+
+def detect_entrypoints_from_nodes(nodes: dict[str, Node]) -> list[EntryPoint]:
+    """Derive entrypoints from the scanned node list (deterministic, no FS scope issues)."""
+    eps: list[EntryPoint] = []
+
+    for nid, n in nodes.items():
+        path = n.path
+        if path.endswith("__main__.py"):
+            eps.append(EntryPoint(node=nid, reason="python __main__.py", confidence=3))
+        elif path.endswith("main.rs"):
+            eps.append(EntryPoint(node=nid, reason="rust main.rs", confidence=3))
+        elif path == "package.json":
+            eps.append(
+                EntryPoint(node=nid, reason="node package.json scripts", confidence=2)
+            )
+        elif path == "pyproject.toml":
+            eps.append(
+                EntryPoint(
+                    node=nid,
+                    reason="python pyproject.toml (scripts/entrypoints likely)",
+                    confidence=1,
+                )
+            )
+
+    # Optional hints (useful for library-ish layouts)
+    for hint in ("src/pybundle/cli.py", "src/pybundle/__init__.py"):
+        if hint in nodes:
+            eps.append(
+                EntryPoint(node=hint, reason="likely CLI/module entry", confidence=1)
+            )
+
+    # Deduplicate deterministically
+    uniq = {(e.node, e.reason, e.confidence) for e in eps}
+    eps = [EntryPoint(node=a, reason=b, confidence=c) for (a, b, c) in uniq]
+    return sorted(eps, key=lambda e: (e.node, -e.confidence, e.reason))
+
+
+def _resolve_py_to_node(root: Path, src_rel: str, mod: str) -> Optional[str]:
+    """
+    Resolve a Python import module string to a local file node (relative path),
+    if it exists in the scanned repo. Deterministic, no sys.path tricks.
+    """
+    # Normalize relative imports like ".cli" or "..utils"
+    # We only support relative imports within the src file's package directory.
+    if mod.startswith("."):
+        # count leading dots
+        dots = 0
+        for ch in mod:
+            if ch == ".":
+                dots += 1
+            else:
+                break
+        tail = mod[dots:]  # remaining name after dots
+        src_dir = Path(src_rel).parent  # e.g. pybundle/
+        # go up (dots-1) levels: from . = same package, .. = parent, etc
+        base = src_dir
+        for _ in range(max(dots - 1, 0)):
+            base = base.parent
+        if tail:
+            parts = tail.split(".")
+            cand = base.joinpath(*parts)
+        else:
+            cand = base
+    else:
+        cand = Path(*mod.split("."))
+
+    # candidate file paths relative to root
+    py_file = (root / cand).with_suffix(".py")
+    init_file = root / cand / "__init__.py"
+
+    if py_file.is_file():
+        return _rel(root, py_file)
+    if init_file.is_file():
+        return _rel(root, init_file)
+    return None
+
+
+def build_roadmap(
+    root: Path, include_dirs: list[Path], exclude_dirs: set[str], max_files: int = 20000
+) -> RoadmapGraph:
+    nodes: dict[str, Node] = {}
+    edges: list[Edge] = []
+
+    # Walk selected dirs
+    files: list[Path] = []
+    skipped_big = 0
+
+    for d in include_dirs:
+        if not d.exists():
+            continue
+
+        if _is_venv_root(d):
+            continue
+
+        for dirpath, dirnames, filenames in os.walk(d):
+            dirpath_p = Path(dirpath)
+
+            # 1) prune excluded dirs by name
+            dirnames[:] = [dn for dn in dirnames if dn not in exclude_dirs]
+
+            # 2) prune venv dirs by structure (ANY name)
+            dirnames[:] = [
+                dn
+                for dn in dirnames
+                if dn not in exclude_dirs and dn != ".pybundle-venv"
+            ]
+            dirnames[:] = [dn for dn in dirnames if not _is_venv_root(dirpath_p / dn)]
+
+            for fn in filenames:
+                p = dirpath_p / fn
+
+                # 3️⃣ skip anything under a venv (belt + suspenders)
+                rel_p = Path(_rel(root, p))
+                if _is_under_venv(root, rel_p):
+                    continue
+
+                rel_s = _rel(root, p)
+                if rel_s.startswith(".pybundle-venv/") or "/site-packages/" in rel_s:
+                    continue
+                if _is_under_venv(root, Path(rel_s)):
+                    continue
+
+                try:
+                    if p.stat().st_size > 2_000_000:
+                        skipped_big += 1
+                        continue
+                except OSError:
+                    continue
+
+                files.append(p)
+                if len(files) >= max_files:
+                    break
+
+            if len(files) >= max_files:
+                break
+
+    # Create nodes
+    for f in files:
+        rel = _rel(root, f)
+        nodes[rel] = Node(id=rel, path=rel, lang=guess_lang(f))
+
+    # Scan edges
+    for f in files:
+        rel = _rel(root, f)
+        lang = nodes[rel].lang
+        text = None
+
+        if lang in {"js", "ts", "rust", "html", "config"}:
+            text = f.read_text(encoding="utf-8", errors="replace")
+
+        if lang == "python":
+            for mod in scan_python_imports(root, f):
+                resolved = _resolve_py_to_node(root, rel, mod)
+                if resolved and resolved in nodes:
+                    edges.append(Edge(src=rel, dst=resolved, type="import"))
+                else:
+                    edges.append(Edge(src=rel, dst=f"py:{mod}", type="import"))
+        elif lang in {"js", "ts"} and text is not None:
+            for spec in scan_js_imports(text):
+                edges.append(Edge(src=rel, dst=f"js:{spec}", type="import"))
+        elif lang == "rust" and text is not None:
+            uses, mods = scan_rust_uses(text)
+            for u in uses:
+                edges.append(Edge(src=rel, dst=f"rs:{u}", type="use"))
+            for m in mods:
+                edges.append(Edge(src=rel, dst=f"rsmod:{m}", type="mod"))
+
+        # TODO: add template includes, docker compose, pyproject scripts, etc.
+
+    # Entrypoints
+    eps = detect_entrypoints_from_nodes(nodes)
+
+    # Stats
+    stats: dict[str, int] = {}
+    for n in nodes.values():
+        stats[f"nodes_{n.lang}"] = stats.get(f"nodes_{n.lang}", 0) + 1
+    for e in edges:
+        stats[f"edges_{e.type}"] = stats.get(f"edges_{e.type}", 0) + 1
+    stats["skipped_big_files"] = skipped_big
+
+    # determinism: sort
+    node_list = sorted(nodes.values(), key=lambda x: x.id)
+    edge_list = sorted(edges, key=lambda e: (e.src, e.dst, e.type, e.note))
+    eps_sorted = sorted(eps, key=lambda e: (e.node, -e.confidence, e.reason))
+
+    return RoadmapGraph(
+        version=1,
+        root=str(root),
+        nodes=node_list,
+        edges=edge_list,
+        entrypoints=eps_sorted,
+        stats=stats,
+    )

pybundle/root_detect.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+_MARKERS = [".git", "pyproject.toml", "requirements.txt", "setup.cfg", "setup.py"]
+
+
+def detect_project_root(start: Path) -> Path | None:
+    cur = start.resolve()
+    for p in [cur, *cur.parents]:
+        for m in _MARKERS:
+            if (p / m).exists():
+                return p
+    return None

pybundle/runner.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import time
+from dataclasses import asdict
+
+try:
+    from colorama import Fore, Style, init as colorama_init
+    # Force colors in xterm and other terminals
+    # strip=False keeps ANSI codes, autoreset=True resets after each print
+    colorama_init(autoreset=True, strip=False)
+    COLORS_AVAILABLE = True
+except ImportError:
+    COLORS_AVAILABLE = False
+    # Fallback if colorama not available
+    class Fore:
+        RED = ""
+        YELLOW = ""
+        GREEN = ""
+        CYAN = ""
+        RESET = ""
+    class Style:
+        BRIGHT = ""
+        RESET_ALL = ""
+
+from .context import BundleContext
+from .packaging import archive_output_path, make_archive, resolve_archive_format
+from .manifest import write_manifest
+from .steps.base import StepResult
+
+
+def _format_duration(milliseconds: int) -> str:
+    """Format milliseconds into human-readable duration (hr min sec)."""
+    seconds = milliseconds / 1000.0
+    
+    if seconds < 1:
+        return f"{milliseconds}ms"
+    elif seconds < 60:
+        return f"{seconds:.1f}s"
+    elif seconds < 3600:
+        minutes = int(seconds // 60)
+        secs = int(seconds % 60)
+        return f"{minutes}m {secs}s"
+    else:
+        hours = int(seconds // 3600)
+        minutes = int((seconds % 3600) // 60)
+        secs = int(seconds % 60)
+        return f"{hours}h {minutes}m {secs}s"
+
+
+def _emit_progress(msg: str, color: str = "cyan") -> None:
+    """Emit a progress message with optional color."""
+    color_code = getattr(Fore, color.upper(), Fore.CYAN)
+    print(f"{color_code}{msg}{Style.RESET_ALL}", file=sys.stderr, flush=True)
+
+
+def _emit_step_result(idx: int, total: int, step_name: str, result: StepResult) -> None:
+    """Emit colored step result based on status."""
+    # Determine color and symbol based on status
+    if result.status == "OK":
+        color = Fore.GREEN
+        symbol = "✓"
+    elif result.status == "SKIP":
+        color = Fore.YELLOW
+        symbol = "⊘"
+    elif result.status == "FAIL":
+        color = Fore.RED
+        symbol = "✗"
+    else:
+        color = Fore.CYAN
+        symbol = "•"
+    
+    # Format duration in human-readable format
+    duration = _format_duration(result.seconds)
+    
+    # Build status line with colorful step counter and status-colored step name
+    step_counter = f"{Fore.BLACK}{Style.BRIGHT}[{Fore.MAGENTA}{idx}{Fore.WHITE}/{Fore.CYAN}{total}{Fore.BLACK}{Style.BRIGHT}]{Style.RESET_ALL}"
+    status_msg = f"{step_counter} {color}{Style.BRIGHT}{symbol} {step_name}{Style.RESET_ALL}"
+    
+    # Add note and duration in terminal default color
+    if result.note:
+        status_msg += f" - {result.note}"
+    status_msg += f" ({duration})"
+    
+    print(status_msg, file=sys.stderr, flush=True)
+
+
+def run_profile(ctx: BundleContext, profile) -> int:
+    t0 = time.time()
+    ctx.write_runlog(f"=== pybundle run {profile.name} ===")
+    ctx.write_runlog(f"ROOT: {ctx.root}")
+    ctx.write_runlog(f"WORK: {ctx.workdir}")
+
+    ctx.results.clear()
+    results: list[StepResult] = ctx.results
+    any_fail = False
+
+    total_steps = len(profile.steps)
+    for idx, step in enumerate(profile.steps, 1):
+        # Progress indicator with colorful step count: [magenta#white/cyan#] Running: step
+        step_counter = f"{Fore.BLACK}{Style.BRIGHT}[{Fore.MAGENTA}{idx}{Fore.WHITE}/{Fore.CYAN}{total_steps}{Fore.BLACK}{Style.BRIGHT}]{Style.RESET_ALL}"
+        print(f"{step_counter} {Fore.WHITE}Running: {step.name}...{Style.RESET_ALL}", file=sys.stderr, flush=True)
+        ctx.write_runlog(f"-- START: {step.name}")
+        
+        r = step.run(ctx)
+        results.append(r)
+        ctx.results = results
+        
+        # Colored status output
+        _emit_step_result(idx, total_steps, step.name, r)
+        ctx.write_runlog(
+            f"-- DONE:  {step.name} [{r.status}] ({r.seconds}s) {r.note}".rstrip()
+        )
+        
+        if r.status == "FAIL":
+            any_fail = True
+            if ctx.strict:
+                break
+
+    ctx.summary_json.write_text(
+        json.dumps(
+            {
+                "profile": profile.name,
+                "root": str(ctx.root),
+                "workdir": str(ctx.workdir),
+                "results": [asdict(r) for r in results],
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+
+    ctx.results = results
+
+    # Write the manifest BEFORE archiving so it's included inside the bundle.
+    archive_fmt_used = resolve_archive_format(ctx)
+    archive_path = archive_output_path(ctx, archive_fmt_used)
+
+    write_manifest(
+        ctx=ctx,
+        profile_name=profile.name,
+        archive_path=archive_path,
+        archive_format_used=archive_fmt_used,
+    )
+
+    archive_path, archive_fmt_used = make_archive(ctx)
+    ctx.archive_path = archive_path
+    ctx.duration_ms = int((time.time() - t0) * 1000)
+
+    ctx.write_runlog(f"ARCHIVE: {archive_path}")
+
+    ctx.emit(f"✅ Archive created: {archive_path}")
+    if ctx.keep_workdir:
+        ctx.emit(f"📁 Workdir kept:     {ctx.workdir}")
+
+    if not ctx.keep_workdir:
+        shutil.rmtree(ctx.workdir, ignore_errors=True)
+
+    if any_fail and ctx.strict:
+        return 10
+    return 0

pybundle/steps/__init__.py

@@ -0,0 +1,26 @@
+"""
+pybundle steps - individual analysis and packaging steps.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "bandit",
+    "base",
+    "compileall",
+    "context_expand",
+    "coverage",
+    "copy_pack",
+    "error_refs",
+    "handoff_md",
+    "mypy",
+    "pip_audit",
+    "pylance",
+    "pytest",
+    "repro_md",
+    "rg_scans",
+    "roadmap",
+    "ruff",
+    "shell",
+    "tree",
+]

pybundle/steps/bandit.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+from ..tools import which
+
+
+def _repo_has_py_files(root: Path) -> bool:
+    """Fast check if there are Python files to scan."""
+    for p in root.rglob("*.py"):
+        parts = set(p.parts)
+        if (
+            ".venv" not in parts
+            and "__pycache__" not in parts
+            and "node_modules" not in parts
+            and "dist" not in parts
+            and "build" not in parts
+            and "artifacts" not in parts
+        ):
+            return True
+    return False
+
+
+@dataclass
+class BanditStep:
+    name: str = "bandit"
+    target: str = "."
+    outfile: str = "logs/50_bandit.txt"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.workdir / self.outfile
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        bandit = which("bandit")
+        if not bandit:
+            out.write_text(
+                "bandit not found; skipping (pip install bandit)\n", encoding="utf-8"
+            )
+            return StepResult(self.name, "SKIP", 0, "missing bandit")
+
+        if not _repo_has_py_files(ctx.root):
+            out.write_text("no .py files detected; skipping bandit\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "no python files")
+
+        # Run bandit with recursive mode, excluding common directories
+        cmd = [
+            bandit,
+            "-r",
+            self.target,
+            "--exclude",
+            "**/artifacts/**,.venv,venv,__pycache__,.mypy_cache,.ruff_cache,node_modules,dist,build,.git,.tox,*.egg-info",
+            "--format",
+            "txt",
+        ]
+        header = f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n\n"
+
+        cp = subprocess.run(  # nosec B603
+            cmd, cwd=str(ctx.root), text=True, capture_output=True, check=False
+        )
+        text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+        out.write_text(ctx.redact_text(text), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        # Bandit exit codes: 0=no issues, 1=issues found
+        note = "" if cp.returncode == 0 else f"exit={cp.returncode} (security issues)"
+        return StepResult(self.name, "PASS", dur, note)

pybundle/steps/base.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from ..context import BundleContext
+
+
+@dataclass
+class StepResult:
+    name: str
+    status: str  # "PASS" | "FAIL" | "SKIP"
+    seconds: int
+    note: str = ""
+
+
+class Step(Protocol):
+    name: str
+
+    def run(self, ctx: BundleContext) -> StepResult: ...

pybundle/steps/compileall.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import subprocess  # nosec B404 - Required for tool execution, paths validated
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+def _guess_targets(root: Path) -> list[str]:
+    """
+    Heuristic targets:
+    - If there are top-level Python package dirs (contain __init__.py), compile those.
+    - Otherwise compile '.' (repo root).
+    """
+    targets: list[str] = []
+
+    for p in sorted(root.iterdir()):
+        if not p.is_dir():
+            continue
+        if p.name.startswith("."):
+            continue
+        if (p / "__init__.py").is_file():
+            targets.append(p.name)
+
+    return targets or ["."]
+
+
+@dataclass
+class CompileAllStep:
+    name: str = "compileall"
+    quiet: bool = True
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        out = ctx.logdir / "30_compileall.txt"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        py = ctx.tools.python
+        if not py:
+            out.write_text("python not found; skipping compileall\n", encoding="utf-8")
+            return StepResult(self.name, "SKIP", 0, "missing python")
+
+        targets = _guess_targets(ctx.root)
+        cmd = [py, "-m", "compileall"]
+        if self.quiet:
+            cmd.append("-q")
+        cmd.extend(targets)
+
+        header = (
+            f"## PWD: {ctx.root}\n## CMD: {' '.join(cmd)}\n## TARGETS: {targets}\n\n"
+        )
+
+        try:
+            cp = subprocess.run(  # nosec B603
+                cmd,
+                cwd=str(ctx.root),
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+            text = header + (cp.stdout or "") + ("\n" + cp.stderr if cp.stderr else "")
+            out.write_text(ctx.redact_text(text), encoding="utf-8")
+            dur = int(time.time() - start)
+
+            # compileall uses non-zero for compile failures; we record it but don't fail bundling.
+            note = "" if cp.returncode == 0 else f"exit={cp.returncode} (recorded)"
+            return StepResult(self.name, "PASS", dur, note)
+        except Exception as e:
+            out.write_text(
+                ctx.redact_text(header + f"\nEXCEPTION: {e}\n"), encoding="utf-8"
+            )
+            dur = int(time.time() - start)
+            return StepResult(self.name, "PASS", dur, f"exception recorded: {e}")