codebase-receipts-cli 1.0.0__py3-none-any.whl

receipts/ingest/git_source.py

@@ -0,0 +1,159 @@
+"""Clone a git repo to a temporary directory for ingestion.
+
+Windows facts verified against the installed GitPython 3.1.50: the clone's
+``.git/objects/pack`` files are read-only (plain ``shutil.rmtree`` fails with
+WinError 5) and mmap'd pack handles keep files locked until ``repo.close()``
+runs. ``git.util.rmtree`` handles the read-only case, so cleanup is always
+``repo.close()`` then ``git.util.rmtree``.
+
+Phase 19: remote repos are first-class. A URL gets a durable KB identity
+(``kb_dir_for_url``) derived from its canonical form — the same URL always
+maps to the same KB directory, so re-ingesting reuses the incremental sync
+instead of orphaning a new KB per clone. Private HTTPS repos authenticate
+via the ``GITHUB_TOKEN`` env var (BYOK-style: read at clone time, injected
+into the URL for that one process, never stored, and scrubbed from every
+error message).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+from git import Repo
+from git.exc import GitCommandError, GitCommandNotFound
+from git.util import rmtree as _git_rmtree  # NOT `git.index.util` — see module doc
+
+from receipts.errors import ReceiptsError
+
+_GIT_URL_PREFIXES = ("http://", "https://", "git@", "ssh://", "file://")
+
+
+def looks_like_git_url(source: str) -> bool:
+    """Heuristic: is this ingest source a git URL rather than a local folder?"""
+    return source.startswith(_GIT_URL_PREFIXES) or source.endswith(".git")
+
+
+def canonical_git_url(url: str) -> str:
+    """Reduce a git URL to a canonical ``host/owner/repo`` form.
+
+    ``https://github.com/Owner/Repo.git``, ``git@github.com:Owner/Repo`` and
+    ``https://github.com/Owner/Repo/`` all canonicalize identically, so they
+    share one KB. Credentials embedded in the URL are stripped — they must
+    never leak into KB names or the manifest.
+    """
+    s = url.strip().rstrip("/")
+    if s.endswith(".git"):
+        s = s[: -len(".git")]
+    if s.startswith("git@"):
+        host_path = s[len("git@") :].replace(":", "/", 1)
+    else:
+        for prefix in ("ssh://git@", "https://", "http://", "ssh://", "file://"):
+            if s.startswith(prefix):
+                host_path = s[len(prefix) :]
+                break
+        else:
+            host_path = s
+    first, sep, rest = host_path.partition("/")
+    if "@" in first:  # user:token@host — drop the credentials
+        first = first.rsplit("@", 1)[-1]
+    host_path = f"{first.lower()}{sep}{rest}"
+    return "/".join(p for p in host_path.split("/") if p)
+
+
+def repo_slug(url: str) -> str:
+    """Short human label for a repo URL — ``owner/repo`` where possible."""
+    parts = canonical_git_url(url).split("/")
+    return "/".join(parts[-2:]) if len(parts) >= 3 else parts[-1]
+
+
+def kb_dir_for_url(url: str) -> Path:
+    """Durable per-URL KB directory under ``~/.receipts/kb/``.
+
+    Keyed by the canonical URL, not the (random) clone directory — the same
+    repo always resolves to the same KB, which is what lets verify/ama/
+    dossier find it again and lets re-ingest hit the incremental sync.
+    """
+    canon = canonical_git_url(url)
+    short_hash = hashlib.sha256(canon.encode()).hexdigest()[:8]
+    safe_slug = re.sub(r"[^A-Za-z0-9._-]", "_", repo_slug(url))
+    return Path.home() / ".receipts" / "kb" / f"{safe_slug}_{short_hash}"
+
+
+def _with_token(url: str, token: str) -> str:
+    """Inject a token into an HTTPS clone URL (no-op otherwise).
+
+    Only applies when the URL carries no credentials already. The result is
+    used solely as the clone argument — never logged, never stored.
+    """
+    if not token or not url.startswith("https://"):
+        return url
+    rest = url[len("https://") :]
+    if "@" in rest.split("/", 1)[0]:
+        return url  # URL already carries credentials — leave it alone
+    return f"https://x-access-token:{token}@{rest}"
+
+
+def _scrub(text: str, token: str) -> str:
+    """Remove a token from text destined for the terminal or an exception."""
+    return text.replace(token, "***") if token else text
+
+
+@contextmanager
+def cloned_repo(
+    url: str,
+    *,
+    branch: str | None = None,
+    full_history: bool = False,
+) -> Iterator[Path]:
+    """Clone ``url`` into a temp dir, yield its path, always clean up.
+
+    Shallow (depth=1) by default — verification only needs the current tree.
+    ``full_history=True`` clones everything so git-history evidence (commit
+    timespan, contributors) works on remote repos too. ``branch`` selects a
+    branch or tag. A ``GITHUB_TOKEN`` env var, when present, authenticates
+    HTTPS clones of private repos.
+    """
+    token = os.environ.get("GITHUB_TOKEN", "").strip()
+    clone_url = _with_token(url, token)
+
+    kwargs: dict = {"env": {"GIT_TERMINAL_PROMPT": "0"}}  # fail fast, no prompts
+    if not full_history:
+        kwargs["depth"] = 1
+    if branch:
+        kwargs["branch"] = branch
+
+    tmp = Path(tempfile.mkdtemp(prefix="receipts-clone-"))
+    repo = None
+    try:
+        try:
+            repo = Repo.clone_from(clone_url, tmp, **kwargs)
+        except GitCommandNotFound as exc:
+            raise ReceiptsError(
+                "The `git` executable was not found. Install git from"
+                " https://git-scm.com/downloads and try again."
+            ) from exc
+        except GitCommandError as exc:
+            stderr = _scrub((exc.stderr or "").strip(), token)
+            hint = ""
+            if not token and url.startswith("https://"):
+                hint = (
+                    "\n  Private repo? Set GITHUB_TOKEN=<personal access token>"
+                    " and retry (the token is read from the environment, never"
+                    " stored)."
+                )
+            raise ReceiptsError(
+                f"Could not clone '{url}'.\n"
+                f"  Check the URL and that you have access. git said:\n"
+                f"{stderr}{hint}"
+            ) from None  # never chain the raw exception — it embeds the URL+token
+        yield tmp
+    finally:
+        if repo is not None:
+            repo.close()  # release mmap'd pack handles so Windows can delete
+        _git_rmtree(tmp)

receipts/ingest/manifest.py

@@ -0,0 +1,114 @@
+"""Source manifest — which KB belongs to which ingested source.
+
+A small JSON file (``~/.receipts/kb/manifest.json``) recording every source
+that was ever ingested: local folders and remote URLs alike, each with the
+KB directory it produced and a short alias. This is what lets every command
+accept ``receipts verify resume.tex owner/repo`` (or the full URL, or a
+folder name) anywhere a codebase folder is accepted — the KB outlives the
+temp clone.
+
+The manifest never stores credentials; remote entries hold the canonical
+URL with any embedded tokens already stripped.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+_DEFAULT_MANIFEST_PATH = Path.home() / ".receipts" / "kb" / "manifest.json"
+
+
+@dataclass(frozen=True)
+class SourceEntry:
+    source: str  # canonical URL or resolved local path
+    kb_dir: str
+    alias: str  # short label: "owner/repo" or the folder name
+    kind: str  # "remote" | "local"
+    last_ingested: float
+
+
+def _load(manifest_path: Path) -> dict:
+    if not manifest_path.is_file():
+        return {"version": 1, "sources": {}}
+    try:
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {"version": 1, "sources": {}}
+    if not isinstance(data.get("sources"), dict):
+        data["sources"] = {}
+    return data
+
+
+def record_source(
+    source: str,
+    kb_dir: Path,
+    *,
+    alias: str,
+    kind: str,
+    manifest_path: Path | None = None,
+) -> None:
+    """Record (or refresh) a source → KB mapping after a successful ingest."""
+    path = manifest_path or _DEFAULT_MANIFEST_PATH
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = _load(path)
+    data["sources"][source] = asdict(
+        SourceEntry(
+            source=source,
+            kb_dir=str(kb_dir),
+            alias=alias,
+            kind=kind,
+            last_ingested=time.time(),
+        )
+    )
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    tmp.replace(path)
+
+
+def lookup_source(
+    spec: str,
+    *,
+    manifest_path: Path | None = None,
+) -> SourceEntry | None:
+    """Find a recorded source by exact source key or by alias (case-insensitive).
+
+    Alias lookups prefer the most recently ingested entry when two sources
+    share an alias — the newest ingest is almost always the one meant.
+    """
+    path = manifest_path or _DEFAULT_MANIFEST_PATH
+    data = _load(path)
+    sources = data["sources"]
+
+    if spec in sources:
+        return SourceEntry(**sources[spec])
+
+    wanted = spec.strip().lower()
+    matches = [
+        SourceEntry(**e)
+        for e in sources.values()
+        if e.get("alias", "").lower() == wanted
+    ]
+    if not matches:
+        return None
+    return max(matches, key=lambda e: e.last_ingested)
+
+
+def known_aliases(*, manifest_path: Path | None = None) -> list[str]:
+    """All recorded aliases, newest first — for friendly error messages."""
+    path = manifest_path or _DEFAULT_MANIFEST_PATH
+    data = _load(path)
+    entries = sorted(
+        data["sources"].values(),
+        key=lambda e: -float(e.get("last_ingested", 0)),
+    )
+    seen: set[str] = set()
+    out: list[str] = []
+    for e in entries:
+        alias = e.get("alias", "")
+        if alias and alias not in seen:
+            seen.add(alias)
+            out.append(alias)
+    return out

receipts/ingest/scanner.py

@@ -0,0 +1,141 @@
+"""Tree-style structural scan of a local folder.
+
+Respects the repo's root ``.gitignore`` (nested .gitignore files are a known,
+documented limitation for now), always skips ``.git`` and cache directories,
+detects virtualenv directories without descending into them (they get flagged
+by the secrets scanner instead of ingested), and classifies files as text or
+binary so downstream stages only ever read text.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pathspec
+
+_ALWAYS_SKIP_DIRS = {".git", "__pycache__", ".pytest_cache", ".ruff_cache"}
+# Bulk dependency dirs: skipped for ingestion but recorded, never silent.
+_DEPENDENCY_DIRS = {"node_modules"}
+_VENV_DIR_NAMES = {".venv", "venv"}
+
+#: Per-file content read cap — protects against pathological giant files.
+MAX_CONTENT_BYTES = 2_000_000
+
+
+@dataclass(frozen=True)
+class ScannedFile:
+    rel_path: str  # posix-style, relative to the scan root
+    size_bytes: int
+    is_text: bool
+
+
+@dataclass
+class ScanResult:
+    root: Path
+    files: list[ScannedFile] = field(default_factory=list)
+    venv_dirs: list[str] = field(default_factory=list)
+    skipped_dirs: list[str] = field(default_factory=list)  # e.g. node_modules
+    ignored_count: int = 0  # files/dirs excluded by .gitignore
+
+    @property
+    def text_files(self) -> list[ScannedFile]:
+        return [f for f in self.files if f.is_text]
+
+    def render_tree(self, max_entries: int = 200) -> str:
+        """ASCII tree of scanned files (ASCII so Windows consoles render it)."""
+        lines = [f"{self.root.name}/"]
+        shown = 0
+        current_dir = None
+        for file in self.files:
+            if shown >= max_entries:
+                lines.append(f"  ... and {len(self.files) - shown} more files")
+                break
+            parent = file.rel_path.rsplit("/", 1)[0] if "/" in file.rel_path else ""
+            if parent and parent != current_dir:
+                lines.append(f"  {parent}/")
+            current_dir = parent or current_dir
+            name = file.rel_path.rsplit("/", 1)[-1]
+            indent = "    " if parent else "  "
+            suffix = "" if file.is_text else "  [binary]"
+            lines.append(f"{indent}{name}{suffix}")
+            shown += 1
+        return "\n".join(lines)
+
+
+def _looks_like_text(path: Path) -> bool:
+    try:
+        with open(path, "rb") as handle:
+            chunk = handle.read(8192)
+    except OSError:
+        return False
+    return b"\x00" not in chunk
+
+
+def _is_venv_dir(path: Path, name: str) -> bool:
+    return name in _VENV_DIR_NAMES or (path / "pyvenv.cfg").is_file()
+
+
+def scan(root: Path) -> ScanResult:
+    """Walk ``root`` and return the structural scan."""
+    root = Path(root).resolve()
+    if not root.is_dir():
+        raise NotADirectoryError(f"Not a folder: {root}")
+
+    gitignore = root / ".gitignore"
+    lines = (
+        gitignore.read_text(encoding="utf-8", errors="replace").splitlines()
+        if gitignore.is_file()
+        else []
+    )
+    spec = pathspec.GitIgnoreSpec.from_lines(lines)
+
+    result = ScanResult(root=root)
+    for dirpath, dirnames, filenames in os.walk(root):
+        rel_dir = Path(dirpath).relative_to(root).as_posix()
+
+        kept: list[str] = []
+        for name in sorted(dirnames):
+            rel = name if rel_dir == "." else f"{rel_dir}/{name}"
+            full = Path(dirpath) / name
+            if name in _ALWAYS_SKIP_DIRS:
+                continue
+            # .gitignore first: a properly ignored venv/.env is the CORRECT
+            # setup and must not be flagged — only content git would pick up
+            # counts as committed-by-mistake.
+            if spec.match_file(rel + "/"):  # dir patterns need a trailing slash
+                result.ignored_count += 1
+                continue
+            if name in _DEPENDENCY_DIRS:
+                result.skipped_dirs.append(rel)
+                continue
+            if _is_venv_dir(full, name):
+                result.venv_dirs.append(rel)
+                continue
+            kept.append(name)
+        dirnames[:] = kept
+
+        for name in sorted(filenames):
+            rel = name if rel_dir == "." else f"{rel_dir}/{name}"
+            if spec.match_file(rel):
+                result.ignored_count += 1
+                continue
+            full = Path(dirpath) / name
+            try:
+                size = full.stat().st_size
+            except OSError:
+                continue
+            result.files.append(
+                ScannedFile(
+                    rel_path=rel, size_bytes=size, is_text=_looks_like_text(full)
+                )
+            )
+    return result
+
+
+def read_text(root: Path, rel_path: str, max_bytes: int = MAX_CONTENT_BYTES) -> str:
+    """Read a scanned file's text content, capped at ``max_bytes``."""
+    with open(Path(root) / rel_path, "rb") as handle:
+        data = handle.read(max_bytes)
+    return data.decode("utf-8", errors="replace")

receipts/ingest/secrets_scanner.py

@@ -0,0 +1,225 @@
+"""Pre-embedding secrets scan.
+
+This runs BEFORE any repo content is embedded or sent to a provider (ground
+rule: security hygiene is a real feature). It detects likely secrets with
+named regex patterns plus a Shannon-entropy check, and it calls out two
+specific, common mistakes loudly: a real ``.env`` file and a committed
+virtualenv directory inside the scanned content.
+
+``redact()`` strips matched secrets so downstream code can embed or send
+content without leaking them. Entropy-only findings are warnings and are NOT
+redacted automatically — high-entropy strings are often legitimate (hashes,
+IDs), so mangling them silently would corrupt code content.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+HIGH = "high"
+WARNING = "warning"
+
+# Named patterns for well-known credential formats. Each match is a HIGH
+# finding and is redactable.
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    ("aws-access-key-id", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("google-api-key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")),
+    # Covers OpenAI (sk-...) and Anthropic (sk-ant-...) style keys.
+    ("sk-style-api-key", re.compile(r"\bsk-[A-Za-z0-9_\-]{16,}\b")),
+    (
+        "github-token",
+        re.compile(
+            r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}\b|\bgithub_pat_[A-Za-z0-9_]{22,}\b"
+        ),
+    ),
+    ("slack-token", re.compile(r"\bxox[baprs]-[A-Za-z0-9\-]{10,}\b")),
+    ("private-key-block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
+    (
+        "credential-assignment",
+        re.compile(
+            r"(?i)\b(?:api[_-]?key|secret|token|password|passwd)\s*[:=]\s*"
+            r"['\"](?P<value>[^'\"\s]{8,})['\"]"
+        ),
+    ),
+]
+
+# Candidate tokens for the entropy check: long runs of base64/url-safe chars.
+_ENTROPY_CANDIDATE = re.compile(r"\b[A-Za-z0-9+/_\-]{24,}\b")
+_ENTROPY_THRESHOLD = 4.2  # pure hex tops out at 4.0, so this skips plain hashes
+
+# Machine-generated dependency manifests are wall-to-wall hashes and URLs —
+# the classic entropy false-positive source. Named patterns still run on them;
+# only the entropy heuristic is skipped.
+_LOCKFILE_NAMES = {
+    "uv.lock",
+    "poetry.lock",
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "Cargo.lock",
+    "Gemfile.lock",
+    "composer.lock",
+    "go.sum",
+}
+
+# Past this many entropy findings in one file, collapse the rest into a single
+# summary finding — a flood of them means machine-generated content, and 900
+# identical warnings would bury the real ones.
+_MAX_ENTROPY_FINDINGS_PER_FILE = 15
+
+_SAFE_ENV_EXAMPLE_NAMES = {".env.example", ".env.sample", ".env.template"}
+
+
+@dataclass(frozen=True)
+class SecretFinding:
+    rel_path: str
+    line: int  # 1-based; 0 for file/directory-level findings
+    kind: str
+    severity: str
+    message: str
+
+
+@dataclass
+class SecretsReport:
+    findings: list[SecretFinding] = field(default_factory=list)
+
+    @property
+    def high_severity(self) -> list[SecretFinding]:
+        return [f for f in self.findings if f.severity == HIGH]
+
+    @property
+    def has_high_severity(self) -> bool:
+        return bool(self.high_severity)
+
+    def render(self) -> str:
+        if not self.findings:
+            return "Secrets scan: nothing suspicious found."
+        lines = [f"Secrets scan: {len(self.findings)} finding(s)."]
+        for f in sorted(self.findings, key=lambda f: (f.severity != HIGH, f.rel_path)):
+            location = f"{f.rel_path}:{f.line}" if f.line else f.rel_path
+            marker = "!!" if f.severity == HIGH else " !"
+            lines.append(f"  {marker} [{f.kind}] {location} - {f.message}")
+        return "\n".join(lines)
+
+
+def _shannon_entropy(token: str) -> float:
+    counts = Counter(token)
+    length = len(token)
+    return -sum(
+        (count / length) * math.log2(count / length) for count in counts.values()
+    )
+
+
+def _is_env_file(rel_path: str) -> bool:
+    name = rel_path.rsplit("/", 1)[-1]
+    if name in _SAFE_ENV_EXAMPLE_NAMES:
+        return False
+    return name == ".env" or name.startswith(".env.")
+
+
+def scan_text(rel_path: str, text: str) -> list[SecretFinding]:
+    """Scan one file's text for likely secrets."""
+    findings: list[SecretFinding] = []
+    entropy_enabled = rel_path.rsplit("/", 1)[-1] not in _LOCKFILE_NAMES
+    entropy_findings = 0
+    entropy_overflow = 0
+    for line_no, line in enumerate(text.splitlines(), start=1):
+        matched_spans: list[tuple[int, int]] = []
+        for kind, pattern in _PATTERNS:
+            for match in pattern.finditer(line):
+                matched_spans.append(match.span())
+                findings.append(
+                    SecretFinding(
+                        rel_path=rel_path,
+                        line=line_no,
+                        kind=kind,
+                        severity=HIGH,
+                        message="Likely credential; it will be redacted before "
+                        "any content leaves this machine.",
+                    )
+                )
+        if not entropy_enabled or "://" in line:
+            continue  # lockfile or URL line: named patterns only
+        for match in _ENTROPY_CANDIDATE.finditer(line):
+            span = match.span()
+            if any(s <= span[0] and span[1] <= e for s, e in matched_spans):
+                continue  # already reported by a named pattern
+            token = match.group()
+            if _shannon_entropy(token) >= _ENTROPY_THRESHOLD:
+                if entropy_findings >= _MAX_ENTROPY_FINDINGS_PER_FILE:
+                    entropy_overflow += 1
+                    continue
+                entropy_findings += 1
+                findings.append(
+                    SecretFinding(
+                        rel_path=rel_path,
+                        line=line_no,
+                        kind="high-entropy-string",
+                        severity=WARNING,
+                        message="High-entropy string - double-check this is not "
+                        "a real secret.",
+                    )
+                )
+    if entropy_overflow:
+        findings.append(
+            SecretFinding(
+                rel_path=rel_path,
+                line=0,
+                kind="high-entropy-string",
+                severity=WARNING,
+                message=f"...plus {entropy_overflow} more high-entropy strings "
+                "in this file - it looks machine-generated; review it once "
+                "rather than string by string.",
+            )
+        )
+    return findings
+
+
+def flag_special_paths(
+    file_rel_paths: Iterable[str], venv_dirs: Iterable[str]
+) -> list[SecretFinding]:
+    """File/directory-level red flags: real .env files and committed venvs."""
+    findings: list[SecretFinding] = []
+    for rel_path in file_rel_paths:
+        if _is_env_file(rel_path):
+            findings.append(
+                SecretFinding(
+                    rel_path=rel_path,
+                    line=0,
+                    kind="env-file",
+                    severity=HIGH,
+                    message="A real .env file is part of the scanned content. "
+                    "If this repo is shared or public, rotate every key in it "
+                    "and add .env to .gitignore.",
+                )
+            )
+    for rel_path in venv_dirs:
+        findings.append(
+            SecretFinding(
+                rel_path=rel_path,
+                line=0,
+                kind="venv-directory",
+                severity=HIGH,
+                message="A virtualenv is part of the scanned content - it "
+                "doesn't belong in a repo. Add it to .gitignore and remove it "
+                "from version control.",
+            )
+        )
+    return findings
+
+
+def redact(text: str) -> tuple[str, int]:
+    """Replace every named-pattern match with a [REDACTED:<kind>] marker.
+
+    Returns the redacted text and the number of replacements. Entropy-only
+    findings are intentionally left alone (see module docstring).
+    """
+    total = 0
+    for kind, pattern in _PATTERNS:
+        text, count = pattern.subn(f"[REDACTED:{kind}]", text)
+        total += count
+    return text, total

receipts/interactive/__init__.py

@@ -0,0 +1 @@
+"""Interactive slash-command REPL for Receipts."""