gwc-pybundle 0.4.2__py3-none-any.whl

pybundle/policy.py

@@ -0,0 +1,176 @@
+# pybundle/policy.py
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable
+
+
+# Common junk that AI should not ingest by default
+DEFAULT_EXCLUDE_DIRS: set[str] = {
+    ".git", ".hg", ".svn", ".venv", "venv", ".direnv", ".cache",
+    ".mypy_cache", ".ruff_cache", ".pytest_cache", "__pycache__",
+    "node_modules", ".pybundle-venv", "binaries", "dist", "build", 
+    "target", "out", ".next", ".nuxt", ".svelte-kit", "artifacts",
+}
+
+# File extensions that commonly produce noise or massive distraction in AI mode
+DEFAULT_EXCLUDE_FILE_EXTS: set[str] = {
+    # packaging/installers/binaries
+    ".appimage", ".deb", ".rpm", ".exe", ".msi", ".dmg", ".pkg",
+    ".so", ".dll", ".dylib",
+
+    # runtime DBs
+    ".db", ".sqlite", ".sqlite3",
+
+    # archives (often huge)
+    ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z",
+}
+
+# Manifests + config files that are often essential for polyglot projects
+DEFAULT_INCLUDE_FILES: list[str] = [
+    # Python
+    "pyproject.toml",
+    "requirements.txt",
+    "requirements-dev.txt",
+    "poetry.lock",
+    "pdm.lock",
+    "uv.lock",
+    "setup.cfg",
+    "setup.py",
+    "mypy.ini",
+    "ruff.toml",
+    ".ruff.toml",
+    "pytest.ini",
+    "tox.ini",
+    ".python-version",
+
+    # Docs / meta
+    "README.md",
+    "README.rst",
+    "README.txt",
+    "CHANGELOG.md",
+    "LICENSE",
+    "LICENSE.md",
+
+    # Node / frontend
+    "package.json",
+    "package-lock.json",
+    "pnpm-lock.yaml",
+    "yarn.lock",
+    "tsconfig.json",
+    "vite.config.js",
+    "vite.config.ts",
+    "webpack.config.js",
+    "webpack.config.ts",
+
+    # Rust / Tauri
+    "Cargo.toml",
+    "Cargo.lock",
+    "tauri.conf.json",
+    "tauri.conf.json5",
+    "tauri.conf.toml",
+]
+
+DEFAULT_INCLUDE_DIRS: list[str] = [
+    # Python-ish
+    "src", "app", "tests", "tools", "docs", ".github", "templates",
+    "static", "src-tauri", "frontend", "web", "ui",
+]
+
+DEFAULT_INCLUDE_GLOBS: list[str] = [
+    "*.py", "*/**/*.py", "templates/**/*", "static/**/*",
+]
+
+
+@dataclass(frozen=True)
+class AIContextPolicy:
+    # path filters
+    exclude_dirs: set[str] = field(default_factory=lambda: set(DEFAULT_EXCLUDE_DIRS))
+    exclude_file_exts: set[str] = field(default_factory=lambda: set(DEFAULT_EXCLUDE_FILE_EXTS))
+
+    # curated inclusion
+    include_files: list[str] = field(default_factory=lambda: list(DEFAULT_INCLUDE_FILES))
+    include_dirs: list[str] = field(default_factory=lambda: list(DEFAULT_INCLUDE_DIRS))
+    include_globs: list[str] = field(default_factory=lambda: list(DEFAULT_INCLUDE_GLOBS))
+
+    # AI-friendly knobs
+    tree_max_depth: int = 4
+    largest_limit: int = 80
+    roadmap_max_files: int = 20000
+    roadmap_mermaid_depth: int = 2
+    roadmap_mermaid_max_edges: int = 180
+
+    def include_dir_candidates(self, root: Path) -> list[Path]:
+        out: list[Path] = []
+        for d in self.include_dirs:
+            p = root / d
+            if p.exists():
+                out.append(p)
+        return out or [root]
+
+
+@dataclass
+class PathFilter:
+    """
+    Shared filtering logic across steps:
+    - prune excluded dir names
+    - prune venvs by structure (any name)
+    - optionally exclude noisy file types by extension
+    """
+    exclude_dirs: set[str]
+    exclude_file_exts: set[str]
+    detect_venvs: bool = True
+
+    def is_venv_root(self, p: Path) -> bool:
+        if not p.is_dir():
+            return False
+
+        if (p / "pyvenv.cfg").is_file():
+            return True
+
+        if (p / "bin").is_dir():
+            if (p / "bin" / "activate").is_file() and (
+                (p / "bin" / "python").exists() or (p / "bin" / "python3").exists()
+            ):
+                return True
+            if any((p / "lib").glob("python*/site-packages")):
+                return True
+
+        if (p / "Scripts").is_dir():
+            if (p / "Scripts" / "activate").is_file() and (
+                (p / "Scripts" / "python.exe").is_file() or (p / "Scripts" / "python").exists()
+            ):
+                return True
+            if (p / "Lib" / "site-packages").is_dir():
+                return True
+
+        if (p / ".Python").exists():
+            return True
+
+        return False
+
+    def should_prune_dir(self, parent_dir: Path, child_name: str) -> bool:
+        if child_name in self.exclude_dirs:
+            return True
+        if self.detect_venvs and self.is_venv_root(parent_dir / child_name):
+            return True
+        return False
+
+    def should_include_file(self, root: Path, p: Path) -> bool:
+        try:
+            rel = p.relative_to(root)
+        except Exception:
+            return False
+
+        # reject files under excluded dirs by name
+        for part in rel.parts[:-1]:
+            if part in self.exclude_dirs:
+                return False
+
+        # reject excluded extensions
+        ext = p.suffix.lower()
+        if ext in self.exclude_file_exts:
+            return False
+
+        return True

pybundle/profiles.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import dataclasses
+from .context import RunOptions
+from .steps.shell import ShellStep
+from .steps.tree import TreeStep, LargestFilesStep
+from .steps.compileall import CompileAllStep
+from .steps.ruff import RuffCheckStep, RuffFormatCheckStep
+from .steps.mypy import MypyStep
+from .steps.pytest import PytestStep
+from .steps.rg_scans import default_rg_steps
+from .steps.error_refs import ErrorReferencedFilesStep
+from .steps.context_expand import ErrorContextExpandStep
+from .steps.copy_pack import CuratedCopyStep
+from .steps.repro_md import ReproMarkdownStep
+from .steps.handoff_md import HandoffMarkdownStep
+from .steps.roadmap import RoadmapStep
+from .policy import AIContextPolicy
+
+
+@dataclass(frozen=True)
+class Profile:
+    name: str
+    steps: list
+
+def _dedupe_steps(steps: list) -> list:
+    seen = set()
+    out = []
+    for s in steps:
+        key = getattr(s, "out", None) or getattr(s, "out_md", None) or getattr(s, "name", None)
+        # fallback to class name if needed
+        key = key or s.__class__.__name__
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(s)
+    return out
+
+def resolve_defaults(profile: str, opts: RunOptions) -> RunOptions:
+    if profile == "ai":
+        return dataclasses.replace(
+            opts,
+            no_ruff = opts.no_ruff if opts.no_ruff is not None else True,
+            no_mypy = opts.no_mypy if opts.no_mypy is not None else True,
+            no_pytest = opts.no_pytest if opts.no_pytest is not None else True,
+            no_rg = opts.no_rg if opts.no_rg is not None else True,
+            no_error_refs = opts.no_error_refs if opts.no_error_refs is not None else True,
+            no_context = opts.no_context if opts.no_context is not None else True,
+            no_compileall = opts.no_compileall if opts.no_compileall is not None else True,
+        )
+    return opts
+
+def _analysis_steps(options: RunOptions) -> list:
+    policy = AIContextPolicy()
+
+    steps: list = [
+        ShellStep("git status", "meta/00_git_status.txt", ["git", "status"], require_cmd="git"),
+        ShellStep("git diff", "meta/01_git_diff.txt", ["git", "diff"], require_cmd="git"),
+        ShellStep("uname -a", "meta/21_uname.txt", ["uname", "-a"], require_cmd="uname"),
+
+        TreeStep(max_depth=policy.tree_max_depth, policy=policy),
+        LargestFilesStep(limit=policy.largest_limit, policy=policy),
+    ]
+
+    # Lint/type/test (CLI-overridable)
+    if not options.no_ruff:
+        steps += [
+            RuffCheckStep(target=options.ruff_target),
+            RuffFormatCheckStep(target=options.ruff_target),
+        ]
+
+    if not options.no_mypy:
+        steps += [MypyStep(target=options.mypy_target)]
+
+    if not options.no_pytest:
+        steps += [PytestStep(args=options.pytest_args or ["-q"])]
+
+    # Landmine scans
+    if not options.no_rg:
+        steps += list(default_rg_steps(target="."))
+
+    # Error-driven packs
+    if not options.no_error_refs:
+        steps += [ErrorReferencedFilesStep(max_files=options.error_max_files)]
+
+    if not options.no_context:
+        steps += [
+            ErrorContextExpandStep(
+                depth=options.context_depth,
+                max_files=options.context_max_files,
+            )
+        ]
+    
+    if not options.no_compileall:
+        steps.append(CompileAllStep())
+
+    # Curated pack + repro doc
+    steps += [
+        ShellStep("python -V", "meta/20_python_version.txt", ["python", "-V"], require_cmd="python"),
+        ShellStep("pip freeze", "meta/22_pip_freeze.txt", ["python", "-m", "pip", "freeze"], require_cmd="python"),
+
+        CuratedCopyStep(policy=policy),
+        ReproMarkdownStep(),
+        RoadmapStep(policy=policy),
+        HandoffMarkdownStep(),
+    ]
+
+    return _dedupe_steps(steps)
+
+def get_profile(name: str, options: RunOptions) -> Profile:
+    if name == "analysis":
+        return Profile(name="analysis", steps=_analysis_steps(options))
+
+    if name == "debug":
+        # debug inherits analysis but keeps the same options
+        steps = list(_analysis_steps(options))
+        steps.append(
+            ShellStep(
+                "pip check",
+                "logs/25_pip_check.txt",
+                ["python", "-m", "pip", "check"],
+                require_cmd="python",
+            )
+        )
+        return Profile(name="debug", steps=steps)
+
+    if name == "backup":
+        # Scaffold: we'll implement real backup modes next
+        return Profile(
+            name="backup",
+            steps=[
+                ShellStep(
+                    "python -V",
+                    "meta/20_python_version.txt",
+                    ["python", "-V"],
+                    require_cmd="python",
+                ),
+            ],
+        )
+
+    if name == "ai":
+        opts = resolve_defaults("ai", options)
+        return Profile(name="ai", steps=_analysis_steps(opts))
+
+    raise ValueError(f"unknown profile: {name}")

pybundle/roadmap_model.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Literal, Dict, List, Optional
+
+Lang = Literal["python", "js", "ts", "rust", "html", "css", "config", "unknown"]
+EdgeType = Literal["import", "require", "use", "mod", "include", "script", "entrypoint"]
+
+@dataclass(frozen=True)
+class Node:
+    id: str                 # stable id (usually path)
+    path: str               # repo-relative
+    lang: Lang
+
+@dataclass(frozen=True)
+class Edge:
+    src: str                # node id
+    dst: str                # node id (or synthetic id)
+    type: EdgeType
+    note: str = ""          # e.g. "from X import Y", "package.json script: dev"
+
+@dataclass
+class EntryPoint:
+    node: str               # node id
+    reason: str             # why we think it's an entry
+    confidence: int = 2     # 1-3
+
+@dataclass
+class RoadmapGraph:
+    version: int
+    root: str
+    nodes: List[Node]
+    edges: List[Edge]
+    entrypoints: List[EntryPoint]
+    stats: Dict[str, int]   # counts by lang/edge types/etc.
+
+    def to_dict(self) -> dict:
+        return asdict(self)

pybundle/roadmap_scan.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+import ast
+import os
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Optional, Literal
+from .steps.copy_pack import _is_venv_root, _is_under_venv
+
+from .roadmap_model import Node, Edge, EntryPoint, RoadmapGraph
+
+PY_EXT = {".py"}
+JS_EXT = {".js", ".jsx", ".mjs", ".cjs"}
+TS_EXT = {".ts", ".tsx"}
+RUST_EXT = {".rs"}
+Lang = Literal["python", "js", "ts", "rust", "html", "css", "config", "unknown"]
+
+IMPORT_RE = re.compile(r'^\s*import\s+.*?\s+from\s+[\'"](.+?)[\'"]\s*;?\s*$', re.M)
+REQUIRE_RE = re.compile(r'require\(\s*[\'"](.+?)[\'"]\s*\)')
+RUST_USE_RE = re.compile(r'^\s*use\s+([a-zA-Z0-9_:]+)', re.M)
+RUST_MOD_RE = re.compile(r'^\s*mod\s+([a-zA-Z0-9_]+)\s*;', re.M)
+
+def _rel(root: Path, p: Path) -> str:
+    return str(p.resolve().relative_to(root.resolve())).replace("\\", "/")
+
+def guess_lang(p: Path) -> Lang:
+    suf = p.suffix.lower()
+    if suf in PY_EXT: return "python"
+    if suf in TS_EXT: return "ts"
+    if suf in JS_EXT: return "js"
+    if suf in RUST_EXT: return "rust"
+    if suf in {".html", ".jinja", ".j2"}: return "html"
+    if suf in {".css", ".scss", ".sass"}: return "css"
+    if suf in {".toml", ".yaml", ".yml", ".json", ".ini", ".cfg"}: return "config"
+    return "unknown"
+
+def scan_python_imports(root: Path, file_path: Path) -> list[str]:
+    # returns import module strings (not resolved paths)
+    try:
+        tree = ast.parse(file_path.read_text(encoding="utf-8", errors="replace"))
+    except Exception:
+        return []
+    mods: list[str] = []
+    for n in ast.walk(tree):
+        if isinstance(n, ast.Import):
+            for a in n.names:
+                mods.append(a.name)
+        elif isinstance(n, ast.ImportFrom):
+            if n.module:
+                mods.append(n.module)
+    return mods
+
+def scan_js_imports(text: str) -> list[str]:
+    out = []
+    out += IMPORT_RE.findall(text)
+    out += REQUIRE_RE.findall(text)
+    return out
+
+def scan_rust_uses(text: str) -> tuple[list[str], list[str]]:
+    uses = RUST_USE_RE.findall(text)
+    mods = RUST_MOD_RE.findall(text)
+    return uses, mods
+
+def detect_entrypoints(root: Path) -> list[EntryPoint]:
+    eps: list[EntryPoint] = []
+
+    # Python CLI entry: __main__.py
+    p = root / "src"
+    if p.exists():
+        for main in p.rglob("__main__.py"):
+            eps.append(EntryPoint(node=_rel(root, main), reason="python __main__.py", confidence=3))
+
+    # Rust main.rs (including tauri src-tauri)
+    for mr in root.rglob("main.rs"):
+        if "target/" in str(mr):  # safety
+            continue
+        eps.append(EntryPoint(node=_rel(root, mr), reason="rust main.rs", confidence=3))
+
+    # package.json scripts as entrypoints (synthetic)
+    pkg = root / "package.json"
+    if pkg.is_file():
+        eps.append(EntryPoint(node="package.json", reason="node scripts", confidence=2))
+
+    return eps
+
+def detect_entrypoints_from_nodes(nodes: dict[str, Node]) -> list[EntryPoint]:
+    """Derive entrypoints from the scanned node list (deterministic, no FS scope issues)."""
+    eps: list[EntryPoint] = []
+
+    for nid, n in nodes.items():
+        path = n.path
+        if path.endswith("__main__.py"):
+            eps.append(EntryPoint(node=nid, reason="python __main__.py", confidence=3))
+        elif path.endswith("main.rs"):
+            eps.append(EntryPoint(node=nid, reason="rust main.rs", confidence=3))
+        elif path == "package.json":
+            eps.append(EntryPoint(node=nid, reason="node package.json scripts", confidence=2))
+        elif path == "pyproject.toml":
+            eps.append(EntryPoint(node=nid, reason="python pyproject.toml (scripts/entrypoints likely)", confidence=1))
+
+    # Optional hints (useful for library-ish layouts)
+    for hint in ("src/pybundle/cli.py", "src/pybundle/__init__.py"):
+        if hint in nodes:
+            eps.append(EntryPoint(node=hint, reason="likely CLI/module entry", confidence=1))
+
+    # Deduplicate deterministically
+    uniq = {(e.node, e.reason, e.confidence) for e in eps}
+    eps = [EntryPoint(node=a, reason=b, confidence=c) for (a, b, c) in uniq]
+    return sorted(eps, key=lambda e: (e.node, -e.confidence, e.reason))
+
+def _resolve_py_to_node(root: Path, src_rel: str, mod: str) -> Optional[str]:
+    """
+    Resolve a Python import module string to a local file node (relative path),
+    if it exists in the scanned repo. Deterministic, no sys.path tricks.
+    """
+    # Normalize relative imports like ".cli" or "..utils"
+    # We only support relative imports within the src file's package directory.
+    if mod.startswith("."):
+        # count leading dots
+        dots = 0
+        for ch in mod:
+            if ch == ".":
+                dots += 1
+            else:
+                break
+        tail = mod[dots:]  # remaining name after dots
+        src_dir = Path(src_rel).parent  # e.g. pybundle/
+        # go up (dots-1) levels: from . = same package, .. = parent, etc
+        base = src_dir
+        for _ in range(max(dots - 1, 0)):
+            base = base.parent
+        if tail:
+            parts = tail.split(".")
+            cand = base.joinpath(*parts)
+        else:
+            cand = base
+    else:
+        cand = Path(*mod.split("."))
+
+    # candidate file paths relative to root
+    py_file = (root / cand).with_suffix(".py")
+    init_file = root / cand / "__init__.py"
+
+    if py_file.is_file():
+        return _rel(root, py_file)
+    if init_file.is_file():
+        return _rel(root, init_file)
+    return None
+
+def build_roadmap(root: Path, include_dirs: list[Path], exclude_dirs: set[str], max_files: int = 20000) -> RoadmapGraph:
+    nodes: dict[str, Node] = {}
+    edges: list[Edge] = []
+
+    # Walk selected dirs
+    files: list[Path] = []
+    root_res = root.resolve()
+
+    skipped_big = 0
+
+    for d in include_dirs:
+        if not d.exists():
+            continue
+
+        if _is_venv_root(d):
+            continue
+
+        for dirpath, dirnames, filenames in os.walk(d):
+            dirpath_p = Path(dirpath)
+
+            # 1) prune excluded dirs by name
+            dirnames[:] = [dn for dn in dirnames if dn not in exclude_dirs]
+
+            # 2) prune venv dirs by structure (ANY name)
+            dirnames[:] = [dn for dn in dirnames if dn not in exclude_dirs and dn != ".pybundle-venv"]
+            dirnames[:] = [dn for dn in dirnames if not _is_venv_root(dirpath_p / dn)]
+
+            for fn in filenames:
+                p = dirpath_p / fn
+
+                # 3️⃣ skip anything under a venv (belt + suspenders)
+                rel = Path(_rel(root, p))
+                if _is_under_venv(root, rel):
+                    continue
+
+                rel_s = _rel(root, p)
+                if rel_s.startswith(".pybundle-venv/") or "/site-packages/" in rel_s:
+                    continue
+                if _is_under_venv(root, Path(rel_s)):
+                    continue
+
+                try:
+                    if p.stat().st_size > 2_000_000:
+                        skipped_big += 1
+                        continue
+                except OSError:
+                    continue
+
+                files.append(p)
+                if len(files) >= max_files:
+                    break
+
+            if len(files) >= max_files:
+                break
+
+    # Create nodes
+    for f in files:
+        rel = _rel(root, f)
+        nodes[rel] = Node(id=rel, path=rel, lang=guess_lang(f))
+
+    # Scan edges
+    for f in files:
+        rel = _rel(root, f)
+        lang = nodes[rel].lang
+        text = None
+
+        if lang in {"js", "ts", "rust", "html", "config"}:
+            text = f.read_text(encoding="utf-8", errors="replace")
+
+        if lang == "python":
+            for mod in scan_python_imports(root, f):
+                resolved = _resolve_py_to_node(root, rel, mod)
+                if resolved and resolved in nodes:
+                    edges.append(Edge(src=rel, dst=resolved, type="import"))
+                else:
+                    edges.append(Edge(src=rel, dst=f"py:{mod}", type="import"))
+        elif lang in {"js", "ts"} and text is not None:
+            for spec in scan_js_imports(text):
+                edges.append(Edge(src=rel, dst=f"js:{spec}", type="import"))
+        elif lang == "rust" and text is not None:
+            uses, mods = scan_rust_uses(text)
+            for u in uses:
+                edges.append(Edge(src=rel, dst=f"rs:{u}", type="use"))
+            for m in mods:
+                edges.append(Edge(src=rel, dst=f"rsmod:{m}", type="mod"))
+
+        # TODO: add template includes, docker compose, pyproject scripts, etc.
+
+    # Entrypoints
+    eps = detect_entrypoints_from_nodes(nodes)
+
+    # Stats
+    stats: dict[str, int] = {}
+    for n in nodes.values():
+        stats[f"nodes_{n.lang}"] = stats.get(f"nodes_{n.lang}", 0) + 1
+    for e in edges:
+        stats[f"edges_{e.type}"] = stats.get(f"edges_{e.type}", 0) + 1
+    stats["skipped_big_files"] = skipped_big
+
+    # determinism: sort
+    node_list = sorted(nodes.values(), key=lambda x: x.id)
+    edge_list = sorted(edges, key=lambda e: (e.src, e.dst, e.type, e.note))
+    eps_sorted = sorted(eps, key=lambda e: (e.node, -e.confidence, e.reason))
+
+    return RoadmapGraph(
+        version=1,
+        root=str(root),
+        nodes=node_list,
+        edges=edge_list,
+        entrypoints=eps_sorted,
+        stats=stats,
+    )

pybundle/root_detect.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+_MARKERS = [".git", "pyproject.toml", "requirements.txt", "setup.cfg", "setup.py"]
+
+
+def detect_project_root(start: Path) -> Path | None:
+    cur = start.resolve()
+    for p in [cur, *cur.parents]:
+        for m in _MARKERS:
+            if (p / m).exists():
+                return p
+    return None