patchground 0.1.0__py3-none-any.whl

patchground/packs/registry.py

@@ -0,0 +1,318 @@
+"""Installed pack lifecycle: add, list, update and remove, plus re-exported resolution APIs.
+
+Installation only fetches, copies and structurally validates; it never builds or runs. Content is
+captured once (hashed while copied), validated in that captured copy and published atomically into
+the immutable cache, then the catalog is replaced atomically. A failure at any point leaves the
+previous catalog and active version untouched.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+from patchground.packs.execution import (
+    Inspection,
+    InstalledChallengeSummary,
+    InstalledDiscoveryResult,
+    discover_installed,
+    inspect_pack,
+    make_review,
+    prepare_execution,
+    resolve_challenge,
+    show_reference,
+)
+from patchground.packs.models import (
+    ConsentRequired,
+    PackError,
+    PackInfo,
+    PackListing,
+    PackProblem,
+    PackRemoval,
+    PackUpdateResult,
+    SourceSnapshot,
+)
+from patchground.packs.source import fetch_pack
+from patchground.packs.store import (
+    Capture,
+    Entry,
+    PublishedStateError,
+    State,
+    Version,
+    cache_path,
+    cache_problem,
+    capture_tree,
+    ensure_dirs,
+    hold_stage,
+    identity_of,
+    load_state,
+    locked,
+    make_identity,
+    packs_dir,
+    remove_tree,
+    save_state,
+    state_root,
+)
+from patchground.safeio import open_directory
+
+__all__ = [
+    "InstalledChallengeSummary",
+    "InstalledDiscoveryResult",
+    "add_pack",
+    "discover_installed",
+    "list_packs",
+    "prepare_execution",
+    "remove_pack",
+    "resolve_challenge",
+    "show_reference",
+    "update_pack",
+]
+
+
+class _Staged:
+    """A fetched revision captured into a private stage and validated, not yet published."""
+
+    def __init__(
+        self,
+        snap: SourceSnapshot,
+        path: Path,
+        identity: tuple[int, int],
+        capture: Capture,
+        inspection: Inspection,
+    ):
+        self.snap = snap
+        self.path = path
+        self.filesystem_identity = identity
+        self.capture = capture
+        self.inspection = inspection
+        self.name = inspection.manifest.name
+        self.version = Version(
+            version=inspection.manifest.version,
+            commit=snap.commit,
+            digest=capture.digest,
+            identity=make_identity(
+                self.name,
+                inspection.manifest.version,
+                snap.source_kind,
+                snap.source,
+                snap.commit,
+                capture.digest,
+            ),
+            challenges=tuple(s.model_copy(update={"path": s.id}) for s in inspection.summaries),
+            folders=capture.folder_digests(tuple(s.id for s in inspection.summaries)),
+        )
+
+
+@contextmanager
+def _stage(source: str | Path, ref: str, root: Path) -> Iterator[_Staged]:
+    with fetch_pack(source, ref) as snap:
+        ensure_dirs(root)
+        path = Path(tempfile.mkdtemp(prefix=".stage-", dir=packs_dir(root))).resolve()
+        # The descriptor stays open until cleanup is done, so the identity cannot be recycled.
+        with hold_stage(path) as identity:
+            try:
+                capture = capture_tree(snap.path.resolve(), path, seal=True)
+                yield _Staged(snap, path, identity, capture, inspect_pack(path))
+            finally:
+                # Only the directory created here: a replacement at this path is not ours to delete.
+                remove_tree(path, expect=identity)
+
+
+def _publish(root: Path, staged: _Staged, entry: Entry | None) -> bool:
+    """Place the staged tree in the cache (caller holds the lock); True when newly created.
+
+    An existing destination is never deleted or replaced. It is accepted only when its actual
+    bytes match what would be published (registered or not, e.g. after an interrupted add);
+    anything else fails closed.
+    """
+    final = cache_path(root, staged.name, staged.snap.commit)
+    if final.exists() or final.is_symlink():
+        known = entry.versions.get(staged.snap.commit) if entry is not None else None
+        if known is not None and cache_problem(root, staged.name, known) is not None:
+            raise PackError("registered pack cache failed integrity verification")
+        try:
+            actual = capture_tree(final, None, seal=False).digest
+        except PackError:
+            actual = None
+        if actual != staged.version.digest or (known and known.digest != actual):
+            raise PackError("pack cache destination exists with different content; not replaced")
+        return False
+    try:
+        # The stage must still be exactly what was validated, not something swapped in since.
+        if capture_tree(staged.path, None, seal=False).digest != staged.version.digest:
+            raise PackError("staged pack changed before publication")
+        staged.path.rename(final)
+        with open_directory(final.parent, error=PackError, label="pack cache") as directory:
+            os.fsync(directory)
+    except OSError as exc:
+        raise PackError("pack could not be published") from exc
+    return True
+
+
+def _info(root: Path, name: str, entry: Entry, commit: str) -> PackInfo:
+    ver = entry.versions[commit]
+    return PackInfo(
+        pack=identity_of(name, entry, ver),
+        path=cache_path(root, name, commit),
+        ref=entry.ref,
+        consented=ver.consented,
+        active=commit == entry.active,
+    )
+
+
+def add_pack(source: str | Path, *, ref: str = "HEAD", root: Path | None = None) -> PackInfo:
+    """Install a pack at an exact commit. Idempotent; never replaces an existing installation."""
+    base = state_root(root)
+    with _stage(source, ref, base) as staged, locked(base):
+        state = load_state(base)
+        entry = state.packs.get(staged.name)
+        if entry is not None:
+            if entry.source != staged.snap.source:
+                raise PackError(f"pack '{staged.name}' is already installed from another source")
+            if staged.snap.commit not in entry.versions:
+                raise PackError(
+                    f"pack '{staged.name}' is already installed at another revision; "
+                    "use pack update"
+                )
+            _publish(base, staged, entry)  # verifies identical content
+            return _info(base, staged.name, entry, staged.snap.commit)
+        created = _publish(base, staged, None)
+        new = Entry(
+            source=staged.snap.source,
+            source_kind=staged.snap.source_kind,
+            ref=ref,
+            active=staged.snap.commit,
+            versions={staged.snap.commit: staged.version},
+        )
+        updated = State(packs={**state.packs, staged.name: new})
+        try:
+            save_state(base, updated)
+        except PackError as exc:
+            if created and not isinstance(exc, PublishedStateError):
+                remove_tree(
+                    cache_path(base, staged.name, staged.snap.commit),
+                    expect=staged.filesystem_identity,
+                )
+            raise
+        return _info(base, staged.name, new, staged.snap.commit)
+
+
+def list_packs(*, root: Path | None = None) -> PackListing:
+    """Every registered cached version. Reads only; missing state is an empty listing."""
+    base = state_root(root)
+    packs: list[PackInfo] = []
+    invalid: list[PackProblem] = []
+    for name, entry in sorted(load_state(base).packs.items()):
+        for commit in entry.versions:
+            problem = cache_problem(base, name, entry.versions[commit])
+            if problem is not None:
+                invalid.append(PackProblem(name=f"{name}@{commit[:12]}", reason=problem))
+                continue
+            packs.append(_info(base, name, entry, commit))
+    return PackListing(packs=tuple(packs), invalid=tuple(invalid))
+
+
+def update_pack(
+    name: str,
+    *,
+    ref: str | None = None,
+    yes: bool = False,
+    expected_identity: str | None = None,
+    root: Path | None = None,
+) -> PackUpdateResult:
+    """Activate a new pinned revision of ``name`` only with explicit approval.
+
+    Without approval ``ConsentRequired`` carries the change review and nothing changes. Older
+    registered versions stay in the cache.
+    """
+    base = state_root(root)
+    entry = load_state(base).packs.get(name)
+    if entry is None:
+        raise PackError("pack is not installed")
+    start = entry.active
+    with _stage(entry.source, ref or entry.ref, base) as staged, locked(base):
+        state = load_state(base)
+        entry = state.packs.get(name)
+        if entry is None or entry.active != start:
+            raise PackError(f"pack '{name}' changed concurrently")
+        if staged.name != name or staged.snap.source != entry.source:
+            raise PackError("fetched pack does not match the installed pack")
+        commit = staged.snap.commit
+        pack = identity_of(name, entry, staged.version)
+        if expected_identity is not None and expected_identity != pack.identity:
+            raise PackError("pack identity differs from the reviewed version")
+        if commit == entry.active:
+            _publish(base, staged, entry)
+            if ref is not None and ref != entry.ref:
+                updated = entry.model_copy(update={"ref": ref})
+                save_state(base, State(packs={**state.packs, name: updated}))
+                entry = updated
+            return PackUpdateResult(
+                current=_info(base, name, entry, commit), previous_commit=commit, changed=False
+            )
+        previous = entry.versions[entry.active]
+        review = make_review(pack, staged.inspection.recipes, previous, staged.version.folders)
+        if not yes:
+            raise ConsentRequired(review)
+        created = _publish(base, staged, entry)
+        version = staged.version.model_copy(update={"consented": True})
+        updated = entry.model_copy(
+            update={
+                "ref": ref or entry.ref,
+                "active": commit,
+                "versions": {**entry.versions, commit: version},
+            }
+        )
+        new_state = State(packs={**state.packs, name: updated})
+        try:
+            save_state(base, new_state)
+        except PackError as exc:
+            if created and not isinstance(exc, PublishedStateError):
+                remove_tree(cache_path(base, name, commit), expect=staged.filesystem_identity)
+            raise
+        return PackUpdateResult(
+            current=_info(base, name, updated, commit),
+            previous_commit=previous.commit,
+            changed=True,
+        )
+
+
+def remove_pack(
+    name: str,
+    *,
+    yes: bool = False,
+    expected_identity: str | None = None,
+    root: Path | None = None,
+) -> PackRemoval:
+    """Remove every cached version and consent of ``name``; sessions are never touched."""
+    if not yes:
+        raise PackError("removing a pack requires confirmation")
+    base = state_root(root)
+    with locked(base):
+        state = load_state(base)
+        entry = state.packs.get(name)
+        if entry is None:
+            raise PackError("pack is not installed")
+        if (
+            expected_identity is not None
+            and expected_identity != entry.versions[entry.active].identity
+        ):
+            raise PackError("pack identity differs from the confirmed version")
+        remaining = {n: e for n, e in state.packs.items() if n != name}
+        save_state(base, State(packs=remaining))
+        # Unregistered from here on: no rollback is possible or attempted, only honest reporting.
+        failed = 0
+        for commit in entry.versions:
+            try:
+                remove_tree(cache_path(base, name, commit))
+            except PackError:
+                failed += 1
+        if failed:
+            raise PackError(
+                f"pack '{name}' was unregistered, but {failed} of {len(entry.versions)} cached "
+                "versions could not be removed and remain on disk unregistered"
+            )
+        return PackRemoval(name=name, removed_versions=len(entry.versions))

patchground/packs/source.py

@@ -0,0 +1,230 @@
+"""Import raw Git objects through isolated metadata, never a source checkout."""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import signal
+import stat
+import subprocess
+import tempfile
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager, suppress
+from pathlib import Path
+from typing import Literal
+from urllib.parse import urlsplit, urlunsplit
+
+from patchground import safeio
+from patchground.packs.models import PackError, SourceSnapshot
+
+MAX_BYTES = 100 * 1024 * 1024
+MAX_FILES = 10_000
+TIMEOUT = 300
+
+
+def _run_process(
+    command: list[str], *, cwd: Path, env: dict[str, str], timeout: float
+) -> subprocess.CompletedProcess[bytes]:
+    """Cancel Git and its HTTPS helpers together on timeout or interruption."""
+    with subprocess.Popen(
+        command,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+    ) as process:
+        try:
+            stdout, _ = process.communicate(timeout=timeout)
+        except BaseException:
+            with suppress(ProcessLookupError):
+                os.killpg(process.pid, signal.SIGKILL)
+            process.communicate()
+            raise
+        return subprocess.CompletedProcess(command, process.returncode, stdout)
+
+
+def _source(source: str | Path) -> tuple[str, Literal["https", "local"], Path | None]:
+    value = str(source)
+    if "://" in value or value.startswith(("git@", "ssh:", "file:")):
+        try:
+            url = urlsplit(value)
+            if (
+                url.scheme != "https"
+                or not url.hostname
+                or url.username is not None
+                or url.password is not None
+                or url.query
+                or url.fragment
+                or "%" in url.netloc
+                or any(c.isspace() for c in value)
+            ):
+                raise ValueError
+            port = url.port
+            host = url.hostname.lower()
+            if ":" in host:
+                host = f"[{host}]"
+            netloc = host + (f":{port}" if port and port != 443 else "")
+            return urlunsplit(("https", netloc, url.path.rstrip("/"), "", "")), "https", None
+        except ValueError:
+            raise PackError("pack source must be public HTTPS without credentials") from None
+    path = Path(value).absolute()
+    safeio.no_links(path, error=PackError, label="pack source")
+    return str(path.resolve()), "local", path
+
+
+def _copy_tree(parent: int, name: str, destination: Path, deadline: float) -> None:
+    if time.monotonic() > deadline:
+        raise PackError("pack transport timed out")
+    with safeio.open_entry(parent, name, error=PackError, label="Git data") as fd:
+        mode = os.fstat(fd).st_mode
+        if stat.S_ISDIR(mode):
+            destination.mkdir()
+            for child in os.listdir(fd):  # noqa: PTH208 - descriptor-pinned listing
+                # Alternates and commit graphs are unnecessary and may reference outside data.
+                if destination.name == "objects" and child == "info":
+                    continue
+                _copy_tree(fd, child, destination / child, deadline)
+        else:
+            with destination.open("xb") as output:
+                while data := os.read(fd, 1024 * 1024):
+                    if time.monotonic() > deadline:
+                        raise PackError("pack transport timed out")
+                    output.write(data)
+
+
+@contextmanager
+def fetch_pack(source: str | Path, ref: str = "HEAD") -> Iterator[SourceSnapshot]:
+    """Yield a private, temporary exact-commit regular-file tree with provenance."""
+    try:
+        canonical, kind, local = _source(source)
+        if not ref or ref.startswith("-") or any(c in ref for c in "\x00\r\n"):
+            raise PackError("invalid pack revision")
+        git = safeio.resolve_git(*((local,) if local else ()), error=PackError)
+        deadline = time.monotonic() + TIMEOUT
+        with tempfile.TemporaryDirectory(prefix="patchground-source-") as temporary:
+            root = Path(temporary)
+            metadata = root / "repo"
+            home = root / "home"
+            home.mkdir()
+            env = {
+                "PATH": str(Path(git).parent) + os.pathsep + os.defpath,
+                "HOME": str(home),
+                "XDG_CONFIG_HOME": str(home),
+                "LC_ALL": "C",
+                "GIT_CONFIG_NOSYSTEM": "1",
+                "GIT_CONFIG_GLOBAL": os.devnull,
+                "GIT_TERMINAL_PROMPT": "0",
+                "GIT_NO_REPLACE_OBJECTS": "1",
+                "GIT_ALLOW_PROTOCOL": "https",
+                "GIT_ATTR_NOSYSTEM": "1",
+            }
+
+            def run(*args: str) -> bytes:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    raise PackError("pack transport timed out")
+                result = _run_process(
+                    [
+                        git,
+                        "-c",
+                        "core.hooksPath=/dev/null",
+                        "-c",
+                        "credential.helper=",
+                        "-c",
+                        "http.followRedirects=false",
+                        "-c",
+                        "gc.auto=0",
+                        "-c",
+                        "maintenance.auto=false",
+                        "--git-dir",
+                        str(metadata),
+                        *args,
+                    ],
+                    cwd=root,
+                    env=env,
+                    timeout=remaining,
+                )
+                if result.returncode:
+                    raise PackError("cannot read pack source or revision")
+                return result.stdout
+
+            if local is not None:
+                safeio.safe_path(local, directory=True, error=PackError, label="pack source")
+                source_git = local / ".git" if (local / ".git").exists() else local
+                with safeio.open_directory(source_git, error=PackError, label="Git data") as fd:
+                    object_format = "sha1"
+                    with safeio.open_entry(fd, "config", error=PackError) as config:
+                        config_data = os.read(config, 1024 * 1024 + 1)
+                    if len(config_data) > 1024 * 1024:
+                        raise PackError("Git configuration is too large")
+                    if re.search(rb"^\s*objectformat\s*=\s*sha256\s*$", config_data, re.M | re.I):
+                        object_format = "sha256"
+                    run("init", "--bare", f"--object-format={object_format}", "--template=")
+                    for name in ("objects", "refs", "HEAD", "packed-refs", "shallow"):
+                        if name not in os.listdir(fd):  # noqa: PTH208 - descriptor-pinned listing
+                            continue
+                        target = metadata / name
+                        if target.is_dir():
+                            shutil.rmtree(target)
+                        elif target.exists():
+                            target.unlink()
+                        _copy_tree(fd, name, target, deadline)
+                revision = ref
+            else:
+                run("init", "--bare", "--template=")
+                run("fetch", "--no-tags", "--no-recurse-submodules", "--depth=1", canonical, ref)
+                revision = "FETCH_HEAD"
+            commit = run(
+                "rev-parse", "--verify", "--end-of-options", revision + "^{commit}"
+            ).strip()
+            if not re.fullmatch(rb"(?:[0-9a-f]{40}|[0-9a-f]{64})", commit):
+                raise PackError("invalid pack commit")
+            # Git normally trusts loose-object names; verify hashes before provenance is asserted.
+            run("fsck", "--full", "--no-reflogs", "--no-dangling", commit.decode())
+            entries = run("ls-tree", "-r", "-l", "-z", commit.decode()).split(b"\0")
+            files: list[tuple[Path, str, int, bool]] = []
+            total = 0
+            for entry in entries:
+                if not entry:
+                    continue
+                header, raw_path = entry.split(b"\t", 1)
+                mode, obj_type, oid, size = header.split()
+                parts = raw_path.decode("utf-8").split("/")
+                if (
+                    mode not in (b"100644", b"100755")
+                    or obj_type != b"blob"
+                    or any(
+                        p in ("", ".", "..") or p.lower() == ".git" or "\\" in p or ":" in p
+                        for p in parts
+                    )
+                ):
+                    raise PackError("pack contains an unsupported or unsafe Git entry")
+                length = int(size)
+                total += length
+                files.append((Path(*parts), oid.decode(), length, mode == b"100755"))
+                if total > MAX_BYTES or len(files) > MAX_FILES:
+                    raise PackError("pack exceeds exported size or file limit")
+            tree = root / "tree"
+            tree.mkdir()
+            for relative, blob_id, length, executable in files:
+                data = run("cat-file", "blob", blob_id)
+                if len(data) != length:
+                    raise PackError("pack blob changed during retrieval")
+                if data.startswith(b"version https://git-lfs.github.com/spec/v1"):
+                    raise PackError("Git LFS content is unsupported")
+                target = tree / relative
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_bytes(data)
+                target.chmod(0o755 if executable else 0o644)
+            yield SourceSnapshot(
+                path=tree, source=canonical, source_kind=kind, commit=commit.decode()
+            )
+    except PackError:
+        raise
+    except subprocess.TimeoutExpired:
+        raise PackError("pack transport timed out") from None
+    except (OSError, ValueError, UnicodeError, RuntimeError):
+        raise PackError("cannot safely retrieve pack source") from None