git-reaper 0.1.0__py3-none-any.whl

git_reaper/__init__.py

@@ -0,0 +1,8 @@
+"""git-reaper: reap structured knowledge from repositories."""
+
+try:
+    from git_reaper._version import __version__
+except ImportError:  # no VCS metadata and no build hook output; should not happen in installs
+    __version__ = "0.0.0"
+
+__all__ = ["__version__"]

git_reaper/__main__.py

@@ -0,0 +1,6 @@
+"""Allow `python -m git_reaper` to invoke the CLI."""
+
+from git_reaper.cli import app
+
+if __name__ == "__main__":
+    app()

git_reaper/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

git_reaper/art.py

@@ -0,0 +1,60 @@
+"""The skull gallery: banners, tombstones, and spinner frames.
+
+Everything here is decoration. Every caller must honor --plain / NO_COLOR
+by simply not calling into this module (see theme.theme_enabled).
+"""
+
+from __future__ import annotations
+
+import random
+
+HERO_SKULL = r"""
+                ______
+             .-"      "-.
+            /            \
+           |,  .-.  .-.  ,|
+           | )(_o/  \o_)( |
+           |/     /\     \|
+           (_     ^^     _)
+            \__|IIIIII|__/
+             | \IIIIII/ |
+             \          /
+              `--------`
+        g i t - r e a p e r
+"""
+
+# Chosen automatically for skinny terminals.
+NARROW_SKULL = r"""
+      .-.
+     (o.o)
+      |=|
+   git-reaper
+"""
+
+MINI_SKULL = ".-.\n|x|\n'-'"
+
+SCYTHE_FRAMES = ["/", "-", "\\", "|"]
+
+TOMBSTONE_DIVIDER = "  _______\n |  RIP  |\n_|_______|_"
+
+
+def banner(version: str, width: int = 80) -> str:
+    """The CLI banner, sized to the terminal."""
+    skull = HERO_SKULL if width >= 40 else NARROW_SKULL
+    return f"{skull.rstrip()}\n        v{version}\n"
+
+
+def tombstone(lines: list[str]) -> str:
+    """Render lines of text inside ASCII tombstone art."""
+    inner = max(len(line) for line in lines) if lines else 0
+    inner = max(inner, 11)
+    top = "         " + "_" * inner
+    body = [f"        /{' ' * inner}\\"]
+    body.extend(f"       | {line.center(inner - 2)} |" for line in ["R I P", "", *lines])
+    body.append("    ___|" + "_" * inner + "|___")
+    return "\n".join([top, *body])
+
+
+def boo() -> str:
+    """A random piece from the gallery, for the hidden `reaper boo`."""
+    return random.choice([HERO_SKULL, NARROW_SKULL, MINI_SKULL, TOMBSTONE_DIVIDER])

git_reaper/cache.py

@@ -0,0 +1,131 @@
+"""The catacombs: the clone cache.
+
+Remote clones land in a content-addressed cache under
+``~/.cache/git-reaper/catacombs/<host>/<owner>/<repo>``, shallow by default,
+reused across runs, and cleared by ``banish``. Local ``file://`` sources are
+buried flat as ``localhost/<name>-<digest>`` to stay inside Windows path
+limits.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import time
+from pathlib import Path
+from urllib.parse import urlparse
+
+from git_reaper import fsutil
+from git_reaper.models import BanishResult, CacheEntry
+
+_SCP_RE = re.compile(r"^(?:\w+@)?(?P<host>[\w.-]+):(?P<path>.+)$")
+
+
+def catacombs_root() -> Path:
+    """Cache root, overridable via GIT_REAPER_CACHE for tests and CI."""
+    override = os.environ.get("GIT_REAPER_CACHE")
+    if override:
+        return Path(override)
+    xdg = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(xdg) / "git-reaper" / "catacombs"
+
+
+def _sanitize(part: str) -> str:
+    part = part.strip("/").removesuffix(".git")
+    return re.sub(r"[^\w.-]", "_", part) or "_"
+
+
+def grave_path(url: str) -> Path:
+    """Map a remote URL to its plot in the catacombs."""
+    parsed = urlparse(url)
+    if parsed.scheme == "file":
+        # Tolerate Windows spellings (file://C:\repos\x): backslashes never
+        # delimit for urlparse, so the drive letter lands in netloc.
+        path = parsed.path.replace("\\", "/")
+        if re.match(r"^[A-Za-z]:", parsed.netloc):
+            path = parsed.netloc.replace("\\", "/") + path
+        path = path.strip("/")
+        if not path:
+            raise ValueError(f"URL has no repository path: {url!r}")
+        # Local paths can be arbitrarily deep; mirroring them under the
+        # catacombs would breach Windows' 260-char path limit. Bury them
+        # flat: basename plus a short digest of the full path.
+        digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:12]
+        name = _sanitize(path.rsplit("/", 1)[-1])
+        return catacombs_root() / "localhost" / f"{name}-{digest}"
+    if parsed.scheme:
+        host, path = parsed.netloc or "localhost", parsed.path
+    else:
+        scp = _SCP_RE.match(url)
+        if not scp:
+            raise ValueError(f"cannot read this incantation as a repo URL: {url!r}")
+        host, path = scp.group("host"), scp.group("path")
+    segments = [_sanitize(seg) for seg in path.strip("/").split("/") if seg]
+    if not segments:
+        raise ValueError(f"URL has no repository path: {url!r}")
+    return catacombs_root() / _sanitize(host) / Path(*segments)
+
+
+def _dir_size(path: Path) -> int:
+    return sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
+
+
+def list_graves() -> list[CacheEntry]:
+    """Every interred repo, oldest first."""
+    root = catacombs_root()
+    entries: list[CacheEntry] = []
+    if not root.is_dir():
+        return entries
+    for git_dir in sorted(root.rglob(".git")):
+        repo = git_dir.parent
+        marker = repo / ".git-reaper-url"
+        url = marker.read_text(encoding="utf-8").strip() if marker.is_file() else ""
+        entries.append(
+            CacheEntry(
+                path=str(repo),
+                url=url,
+                size_bytes=_dir_size(repo),
+                last_used=repo.stat().st_mtime,
+            )
+        )
+    entries.sort(key=lambda e: e.last_used)
+    return entries
+
+
+def mark_grave(repo_path: Path, url: str) -> None:
+    """Record the source URL and refresh the last-used stamp."""
+    (repo_path / ".git-reaper-url").write_text(url + "\n", encoding="utf-8")
+    os.utime(repo_path)
+
+
+def banish(older_than_seconds: float | None = None) -> BanishResult:
+    """Clear the catacombs. With older_than, a partial exorcism."""
+    result = BanishResult()
+    cutoff = time.time() - older_than_seconds if older_than_seconds is not None else None
+    for entry in list_graves():
+        if cutoff is not None and entry.last_used > cutoff:
+            result.kept.append(entry)
+            continue
+        try:
+            fsutil.force_rmtree(entry.path)
+        except OSError:
+            # A grave we cannot dig up (locked file?) is kept, not "removed".
+            result.kept.append(entry)
+            continue
+        result.removed.append(entry)
+        result.reclaimed_bytes += entry.size_bytes
+    return result
+
+
+_AGE_RE = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$", re.IGNORECASE)
+_AGE_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
+
+
+def parse_age(text: str) -> float:
+    """Parse '7d', '12h', '90m' into seconds. Raises ValueError."""
+    match = _AGE_RE.match(text)
+    if not match:
+        raise ValueError(f"unreadable age: {text!r} (try '7d', '12h', '30m')")
+    value, unit = match.groups()
+    return int(value) * _AGE_UNITS[unit.lower()]

git_reaper/cli.py

@@ -0,0 +1,340 @@
+"""The CLI face of the reaper: a thin Typer adapter over git_reaper.core.
+
+Rules of the house:
+- Artifacts go to --out or stdout. Narration goes to stderr, always.
+- Every themed message still carries the plain cause and a next step.
+- Exit codes: 0 rest in peace, 1 the ritual failed, 2 bad incantation.
+"""
+
+from __future__ import annotations
+
+import json
+import shlex
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+from rich.table import Table
+
+from git_reaper import __version__, art, cache, fsutil, schemas
+from git_reaper.core import harvest as harvest_core
+from git_reaper.core import pulse as pulse_core
+from git_reaper.core import tree as tree_core
+from git_reaper.core.source import resolve_source
+from git_reaper.formatters import jsonfmt, markdown
+from git_reaper.gitio import GitError
+from git_reaper.theme import make_console, theme_enabled
+
+app = typer.Typer(
+    name="reaper",
+    help="A spooky utility for data mining git repositories.",
+    no_args_is_help=True,
+    context_settings={"help_option_names": ["-h", "--help"]},
+)
+
+
+@dataclass
+class State:
+    plain: bool = False
+    verbosity: int = 0  # -1 whisper, 0 default, 1 moan, 2 shriek
+    console: Console = None  # type: ignore[assignment]
+
+
+state = State()
+
+
+def _say(style: str, message: str, level: int = 0) -> None:
+    """Narrate to stderr if the current verbosity allows it."""
+    if state.verbosity >= level:
+        state.console.print(f"[{style}]\\[{style}][/{style}] {escape(message)}")
+
+
+def _die(message: str, hint: str | None = None) -> typer.Exit:
+    _say("blood", f"the ritual failed: {message}")
+    if hint:
+        _say("ash", f"next step: {hint}")
+    return typer.Exit(code=1)
+
+
+def _invocation() -> str:
+    return "reaper " + " ".join(shlex.quote(a) for a in sys.argv[1:])
+
+
+def _emit(text: str, out: Path | None) -> None:
+    """Write an artifact to --out or stdout. Chatter never comes here."""
+    if out:
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(text, encoding="utf-8")
+    else:
+        sys.stdout.write(text)
+
+
+def _print_schema(command: str) -> None:
+    schema = schemas.schema_for(schemas.COMMAND_MODELS[command])
+    sys.stdout.write(json.dumps(schema, indent=2) + "\n")
+
+
+def _validate_format(fmt: str) -> None:
+    if fmt not in ("md", "json"):
+        raise _die(f"unknown format {fmt!r}", "use --format md or --format json")
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        sys.stdout.write(f"git-reaper {__version__}\n")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    plain: bool = typer.Option(
+        False, "--plain", "--no-theme", help="Clean ASCII output; no color, no art."
+    ),
+    whisper: bool = typer.Option(False, "-q", "--whisper", help="Only errors."),
+    verbose: int = typer.Option(
+        0, "-v", "--moan", count=True, help="More narration; -vv (--shriek) for debug."
+    ),
+    version: bool = typer.Option(
+        False, "--version", callback=_version_callback, is_eager=True, help="Print version."
+    ),
+) -> None:
+    state.plain = plain
+    state.verbosity = -1 if whisper else verbose
+    state.console = make_console(plain=plain, quiet=False)
+
+
+def _banner() -> None:
+    if theme_enabled(state.plain) and state.verbosity >= 0:
+        state.console.print(f"[eldritch]{art.MINI_SKULL}[/eldritch]", highlight=False)
+
+
+# --------------------------------------------------------------------------
+# harvest (reap)
+# --------------------------------------------------------------------------
+
+
+def _harvest_impl(
+    source: str,
+    pattern: list[str],
+    exclude: list[str],
+    out: Path | None,
+    ref: str | None,
+    depth: int,
+    max_file_size: str | None,
+    max_total_size: str | None,
+    include_binary: bool,
+) -> None:
+    """Gather files matching a pattern and concatenate them into one artifact."""
+    _banner()
+    try:
+        file_cap = fsutil.parse_size(max_file_size) if max_file_size else None
+        total_cap = fsutil.parse_size(max_total_size) if max_total_size else None
+    except ValueError as exc:
+        raise _die(str(exc)) from exc
+
+    try:
+        resolved = resolve_source(source, ref=ref, depth=depth)
+    except (FileNotFoundError, ValueError, GitError) as exc:
+        raise _die(str(exc), "check the path or URL; `reaper pulse` checks your setup") from exc
+    if resolved.cached:
+        _say("necro", f"catacombs hit: {resolved.repo.source} already interred, reusing")
+
+    patterns = tuple(pattern) if pattern else harvest_core.DEFAULT_PATTERNS
+    try:
+        result = harvest_core.harvest(
+            resolved.repo,
+            patterns=patterns,
+            excludes=exclude,
+            max_file_size=file_cap,
+            max_total_size=total_cap,
+            include_binary=include_binary,
+            invoked=_invocation(),
+        )
+    except harvest_core.CapExceeded as exc:
+        raise _die(str(exc)) from exc
+
+    _say(
+        "necro",
+        f"gathered {len(result.files)} souls ({', '.join(patterns)}) ... "
+        f"{result.total_lines:,} lines, {fsutil.human_size(result.total_bytes)}",
+    )
+    for skipped in result.skipped:
+        _say("ember", f"skipped {skipped.path}: {skipped.skip_reason}", level=0)
+
+    if out:
+        with out.open("w", encoding="utf-8") as fh:
+            markdown.write_harvest(result, fh)
+        _say(
+            "bone",
+            f"wrote {out}  ({len(result.files)} files, ~{result.token_estimate:,} tokens)",
+        )
+    else:
+        markdown.write_harvest(result, sys.stdout)
+    _say("ash", "the reaping is complete.")
+
+
+@app.command("harvest")
+def harvest_cmd(
+    source: str | None = typer.Argument(None, help="Local path or repo URL."),
+    pattern: list[str] = typer.Option(
+        [], "--pattern", "--glob", "-p", help="Glob(s) to gather (default: *.md)."
+    ),
+    exclude: list[str] = typer.Option([], "--exclude", "-x", help="Glob(s) to skip."),
+    out: Path | None = typer.Option(None, "--out", "-o", help="Output file (default stdout)."),
+    ref: str | None = typer.Option(None, "--ref", help="Branch, tag, or sha (remote sources)."),
+    depth: int = typer.Option(1, "--depth", help="Clone depth for remote sources."),
+    max_file_size: str | None = typer.Option(
+        None, "--max-file-size", help="Skip files larger than this (e.g. 1MB)."
+    ),
+    max_total_size: str | None = typer.Option(
+        None, "--max-total-size", help="Abort past this total (e.g. 100MB)."
+    ),
+    include_binary: bool = typer.Option(False, "--include-binary", help="Do not skip binaries."),
+    schema: bool = typer.Option(False, "--schema", help="Print the JSON schema and exit."),
+) -> None:
+    """Gather files matching a pattern into one flat artifact."""
+    if schema:
+        _print_schema("harvest")
+        return
+    if source is None:
+        raise _die("no source given", "pass a local path or a repo URL")
+    _harvest_impl(
+        source,
+        pattern,
+        exclude,
+        out,
+        ref,
+        depth,
+        max_file_size,
+        max_total_size,
+        include_binary,
+    )
+
+
+# --------------------------------------------------------------------------
+# tree (map)
+# --------------------------------------------------------------------------
+
+
+@app.command("tree")
+def tree_cmd(
+    source: str = typer.Argument(".", help="Local path or repo URL."),
+    depth: int | None = typer.Option(None, "--depth", "-d", help="Max depth."),
+    dirs_only: bool = typer.Option(False, "--dirs-only", help="Directories only."),
+    sizes: bool = typer.Option(False, "--sizes", help="Show file sizes."),
+    lines: bool = typer.Option(False, "--lines", help="Show line counts."),
+    exclude: list[str] = typer.Option([], "--exclude", "-x", help="Glob(s) to skip."),
+    fmt: str = typer.Option("md", "--format", "-f", help="md or json."),
+    out: Path | None = typer.Option(None, "--out", "-o", help="Output file (default stdout)."),
+    ref: str | None = typer.Option(None, "--ref", help="Branch, tag, or sha (remote sources)."),
+    schema: bool = typer.Option(False, "--schema", help="Print the JSON schema and exit."),
+) -> None:
+    """Emit a hierarchical file listing."""
+    if schema:
+        _print_schema("tree")
+        return
+    _validate_format(fmt)
+    _banner()
+    try:
+        resolved = resolve_source(source, ref=ref)
+    except (FileNotFoundError, ValueError, GitError) as exc:
+        raise _die(str(exc), "check the path or URL; `reaper pulse` checks your setup") from exc
+
+    result = tree_core.tree(
+        resolved.repo,
+        max_depth=depth,
+        dirs_only=dirs_only,
+        with_sizes=sizes,
+        with_lines=lines,
+        excludes=exclude,
+        invoked=_invocation(),
+    )
+    _say("necro", f"mapped {result.dir_count} crypts, {result.file_count} souls")
+    if fmt == "json":
+        _emit(jsonfmt.render(result), out)
+    else:
+        _emit(markdown.render_tree(result, with_sizes=sizes, with_lines=lines), out)
+
+
+# --------------------------------------------------------------------------
+# pulse (doctor)
+# --------------------------------------------------------------------------
+
+
+@app.command("pulse")
+def pulse_cmd(
+    fmt: str = typer.Option("md", "--format", "-f", help="md or json."),
+    schema: bool = typer.Option(False, "--schema", help="Print the JSON schema and exit."),
+) -> None:
+    """Signs-of-life check: git, extras, cache health."""
+    if schema:
+        _print_schema("pulse")
+        return
+    _validate_format(fmt)
+    result = pulse_core.pulse()
+    if fmt == "json":
+        _emit(jsonfmt.render(result), None)
+    else:
+        table = Table(title="signs of life", title_style="eldritch", border_style="grave")
+        table.add_column("check", style="bone")
+        table.add_column("", justify="center")
+        table.add_column("detail", style="ash")
+        for check in result.checks:
+            mark = "[necro]ok[/necro]" if check.ok else "[blood]DEAD[/blood]"
+            # escape: details like "[git] extra" are text, not Rich markup
+            table.add_row(check.name, mark, escape(check.detail))
+        state.console.print(table)
+    if not result.ok:
+        _say("blood", "the patient is unwell; fix the DEAD rows above")
+        raise typer.Exit(code=1)
+    _say("necro", "there is a pulse. faint, but there.")
+
+
+# --------------------------------------------------------------------------
+# banish (purge)
+# --------------------------------------------------------------------------
+
+
+@app.command("banish")
+def banish_cmd(
+    older_than: str | None = typer.Option(
+        None, "--older-than", help="Only clear graves older than this (e.g. 7d, 12h)."
+    ),
+    schema: bool = typer.Option(False, "--schema", help="Print the JSON schema and exit."),
+) -> None:
+    """Clear the catacombs (the clone cache)."""
+    if schema:
+        _print_schema("banish")
+        return
+    _banner()
+    try:
+        age = cache.parse_age(older_than) if older_than else None
+    except ValueError as exc:
+        raise _die(str(exc)) from exc
+    result = cache.banish(older_than_seconds=age)
+    for entry in result.removed:
+        _say("ember", f"banished {entry.url or entry.path}", level=1)
+    _say(
+        "necro",
+        f"banished {len(result.removed)} graves, kept {len(result.kept)}, "
+        f"reclaimed {fsutil.human_size(result.reclaimed_bytes)}",
+    )
+
+
+# --------------------------------------------------------------------------
+# easter egg
+# --------------------------------------------------------------------------
+
+
+@app.command("boo", hidden=True)
+def boo_cmd() -> None:
+    """A random piece from the gallery."""
+    if not state.plain:
+        state.console.print(f"[eldritch]{art.boo()}[/eldritch]", highlight=False)
+
+
+def run() -> None:  # pragma: no cover - console-script shim
+    app()

git_reaper/core/__init__.py

@@ -0,0 +1 @@
+"""The engine. Everything here returns models, never formatted strings."""

git_reaper/core/harvest.py

@@ -0,0 +1,81 @@
+"""The flagship: gather files matching patterns and prepare them for
+concatenation. Returns a HarvestResult; rendering lives in formatters/."""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+
+from git_reaper import fsutil, schemas
+from git_reaper.core.provenance import make_provenance
+from git_reaper.ignore import IgnoreMatcher, walk_files
+from git_reaper.models import FileEntry, HarvestResult, RepoRef
+
+DEFAULT_PATTERNS = ("*.md",)
+
+
+class CapExceeded(RuntimeError):
+    """The total size cap was hit. The message says exactly where."""
+
+
+def _matches(rel_path: str, patterns: tuple[str, ...]) -> bool:
+    name = rel_path.rsplit("/", 1)[-1]
+    return any(fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(rel_path, pat) for pat in patterns)
+
+
+def harvest(
+    repo: RepoRef,
+    patterns: tuple[str, ...] = DEFAULT_PATTERNS,
+    excludes: list[str] | None = None,
+    max_file_size: int | None = None,
+    max_total_size: int | None = None,
+    include_binary: bool = False,
+    invoked: str = "reaper harvest",
+    generated: str | None = None,
+) -> HarvestResult:
+    """Gather every matching file under the resolved source.
+
+    Skips are never silent: each skipped file is recorded with its reason
+    so the report can show exactly what was left in the ground.
+    """
+    root = Path(repo.path)
+    matcher = IgnoreMatcher(root, extra_excludes=excludes)
+    result = HarvestResult(
+        provenance=make_provenance(schemas.artifact_schema("harvest"), repo, invoked, generated),
+        root=str(root),
+    )
+
+    for path in walk_files(root, matcher):
+        rel = path.relative_to(root).as_posix()
+        if not _matches(rel, patterns):
+            continue
+        size = path.stat().st_size
+        if max_file_size is not None and size > max_file_size:
+            result.skipped.append(
+                FileEntry(
+                    path=rel,
+                    size_bytes=size,
+                    skipped=True,
+                    skip_reason=f"over size cap ({fsutil.human_size(size)})",
+                )
+            )
+            continue
+        if not include_binary and fsutil.is_binary(path):
+            result.skipped.append(
+                FileEntry(path=rel, size_bytes=size, skipped=True, skip_reason="binary")
+            )
+            continue
+        if max_total_size is not None and result.total_bytes + size > max_total_size:
+            raise CapExceeded(
+                f"total size cap {fsutil.human_size(max_total_size)} reached at {rel}; "
+                "raise --max-total-size or narrow the pattern"
+            )
+        entry = FileEntry(path=rel, size_bytes=size, line_count=fsutil.count_lines(path))
+        result.files.append(entry)
+        result.total_bytes += size
+        result.total_lines += entry.line_count
+
+    result.token_estimate = fsutil.estimate_tokens(result.total_bytes)
+    result.provenance.files = len(result.files)
+    result.provenance.token_estimate = result.token_estimate
+    return result

git_reaper/core/provenance.py

@@ -0,0 +1,27 @@
+"""Provenance stamps: every combined artifact says where it came from."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from git_reaper import __version__
+from git_reaper.models import Provenance, RepoRef
+
+
+def make_provenance(
+    schema: str,
+    repo: RepoRef,
+    invoked: str,
+    generated: str | None = None,
+) -> Provenance:
+    """Build the stamp. `generated` is injectable so tests stay deterministic;
+    it is the only wall-clock value allowed anywhere in an artifact."""
+    return Provenance(
+        schema=schema,
+        source=repo.source,
+        ref=repo.ref,
+        sha=repo.sha,
+        generated=generated or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        tool_version=__version__,
+        invoked=invoked,
+    )