gitpane 1.0.0__py3-none-any.whl

gitpane/history_rewrite/prepare.py

@@ -0,0 +1,378 @@
+"""Prepare history removal in storage isolated from the source repository."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import tempfile
+from dataclasses import replace
+from typing import TYPE_CHECKING
+
+from gitpane.config import (
+    EXACT_PATH_CASE_ARGUMENTS,
+    GIT_ALTERNATE_OBJECT_ENVIRONMENT_NAME,
+    GIT_MUTATION_TIMEOUT_MS,
+    HISTORY_REWRITE_DIAGNOSTIC_LIMIT_CHARS,
+    HISTORY_REWRITE_SPOOL_LIMIT_BYTES,
+    HISTORY_STREAM_TIMEOUT_MS,
+    PROCESS_CHUNK_BYTES,
+    SUBMODULE_RECURSION_OFF_ARGUMENTS,
+)
+from gitpane.history.repository_value import validate_oid
+from gitpane.history_rewrite.graph import analyze_history
+from gitpane.history_rewrite.guard import capture_rewrite_state, validate_branch_state, validate_history_protection, validate_initial_state, validate_rewrite_capture
+from gitpane.history_rewrite.index import prepare_candidate_index
+from gitpane.history_rewrite.protocol import check_cancelled, read_export, read_line, read_nul_records, read_raw_commits, write_import
+from gitpane.history_rewrite.value import RewritePreparation, RewriteRefusal, RewriteTargetKind
+from gitpane.process.process_runner import LOCAL_WRITE_PROFILE, READ_PROFILE
+from gitpane.system.private_tree_removal import remove_private_tree
+from gitpane.tool.git_version import GIT_VERSION_PREFIX, git_version_answer_line, git_version_numbers
+from gitpane.worktree.command_sequence import FileTask, GitCommand
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Generator
+    from typing import BinaryIO
+
+    from gitpane.history_rewrite.value import ExportCommit, HistoryAnalysis, RawCommit, RewriteCapture, RewriteTarget
+
+PRIVATE_REF = b"refs/heads/gitpane-rewrite"
+PRIVATE_REPOSITORY_BASENAME = "repository.git"
+OWNER_BASENAME = "gitpane-history-rewrite"
+OWNER_CONTENT = b"GitPane private history preparation\n"
+# First Git release whose fast-export knows --signed-commits. Older releases refuse the option, but their export
+# already drops commit signatures, which is what the option asks for.
+SIGNED_COMMITS_EXPORT_MINIMUM_VERSION = (2, 50, 0)
+SIGNED_COMMITS_STRIP_OPTION = "--signed-commits=strip"
+UNREADABLE_GIT_VERSION_CAUSE = "The version of this Git could not be read, so the export options cannot be chosen; check the Git executable in Settings"
+# Settings accepts a Git whose version line has no release numbers, so this cause does not send the user there.
+NUMBERLESS_GIT_VERSION_CAUSE = "This Git reports its version without release numbers, so the export options cannot be chosen"
+
+
+def export_arguments(git_version: str) -> list[str]:
+    """Build a new fast-export argument list for the Git whose `git --version` line is given.
+
+    --signed-commits=strip is only passed to a Git that knows it: an older Git refuses the option, and drops the
+    signatures without it, so both forms give the same stream. Two versions are refused rather than given a default
+    form: one that is not a version line at all, which Settings refuses too, and a version line without release
+    numbers, which Settings accepts. The options are never guessed: a recent Git exported without the option follows
+    Git's own default for signed commits, which Git may change, and rewritten commits that kept their old signatures
+    would carry invalid ones.
+    """
+    if not git_version.startswith(GIT_VERSION_PREFIX):
+        raise ValueError(UNREADABLE_GIT_VERSION_CAUSE)
+    version_numbers = git_version_numbers(git_version)
+    if version_numbers is None:
+        raise ValueError(NUMBERLESS_GIT_VERSION_CAUSE)
+    arguments = ["fast-export", "--no-data", "--no-renames", "--show-original-ids", "--reencode=no"]
+    if version_numbers >= SIGNED_COMMITS_EXPORT_MINIMUM_VERSION:
+        arguments.append(SIGNED_COMMITS_STRIP_OPTION)
+    arguments.extend(["--use-done-feature", "--stdin"])
+    return arguments
+
+
+def with_git_version(capture: RewriteCapture) -> Generator[GitCommand, object, RewriteCapture]:
+    """Return the capture with a Git version, reading `git --version` of its Git only when the version is empty.
+
+    A non-empty version comes from the check of the chosen Git, at start or in Settings, and is never read again. An
+    empty one means that check has not answered yet, or failed; the export options depend on the version, so they are
+    never guessed: the preparation reads the version itself, once, before the first export, by the rule Settings
+    applies to accept a Git.
+    """
+    if capture.git_version:
+        return capture
+    output = yield GitCommand(["--version"])
+    return replace(capture, git_version=git_version_answer_line(output))
+
+
+def _create_storage(capture: RewriteCapture, storage_root_dir: str) -> tuple[str, tuple[int, int]]:
+    """Keep private files outside all source locations, including linked-worktree storage."""
+    storage_root_dir = os.path.realpath(storage_root_dir)
+    for source_dir in (capture.context.repository_dir, capture.context.git_dir, capture.context.common_git_dir):
+        source_dir = os.path.realpath(source_dir)
+        try:
+            inside = os.path.commonpath((source_dir, storage_root_dir)) == source_dir
+        except ValueError:
+            inside = False
+        if inside:
+            raise ValueError("History preparation storage must be outside the source repository and its Git storage; choose another storage folder")
+    os.makedirs(storage_root_dir, mode=0o700, exist_ok=True)
+    operation_dir = tempfile.mkdtemp(prefix="rewrite-", dir=storage_root_dir)
+    try:
+        with open(os.path.join(operation_dir, OWNER_BASENAME), "xb") as stream:
+            stream.write(OWNER_CONTENT)
+        os.mkdir(os.path.join(operation_dir, "hooks"))
+        os.mkdir(os.path.join(operation_dir, PRIVATE_REPOSITORY_BASENAME))
+        information = os.stat(operation_dir)
+        return operation_dir, (information.st_dev, information.st_ino)
+    except OSError:
+        remove_private_tree(operation_dir)
+        raise
+
+
+def discard_preparation(preparation: RewritePreparation) -> None:
+    """Remove only owned private storage after the sequence and its processes have drained."""
+    operation_dir = preparation.operation_dir
+    if not operation_dir or not os.path.lexists(operation_dir):
+        return
+    if os.path.islink(operation_dir):
+        raise ValueError("Private preparation directory was replaced; cleanup refused")
+    information = os.stat(operation_dir)
+    if preparation.operation_identity != (information.st_dev, information.st_ino):
+        raise ValueError("Private preparation directory identity changed; cleanup refused")
+    with open(os.path.join(operation_dir, OWNER_BASENAME), "rb") as stream:
+        if stream.read(len(OWNER_CONTENT) + 1) != OWNER_CONTENT:
+            raise ValueError("Private preparation ownership does not match; cleanup refused")
+    remove_private_tree(operation_dir)
+
+
+def _write_requests(operation_dir: str, records: list[ExportCommit], target: RewriteTarget, cancelled: Callable[[], bool] | None = None) -> None:
+    """Create batch requests on disk so argument and input queues do not grow with history."""
+    with open(os.path.join(operation_dir, "commits.in"), "wb") as commits, open(os.path.join(operation_dir, "presence.in"), "wb") as presence:
+        for record in records:
+            check_cancelled(cancelled)
+            commits.write(record.oid.encode("ascii") + b"\n")
+            presence.write(record.oid.encode("ascii") + b":" + target.path + b"\0")
+
+
+def _read_presence(filename: str, records: list[ExportCommit], target: RewriteTarget, cancelled: Callable[[], bool] | None = None) -> dict[str, bool]:
+    """Check literal file or subtree presence without confusing historical directory entries with files."""
+    present_by_oid = {}
+    with open(filename, "rb") as stream:
+        for record, part in zip(records, read_nul_records(stream, cancelled), strict=True):
+            absent = part == record.oid.encode("ascii") + b":" + target.path + b" missing"
+            if not absent:
+                oid, object_type, size = part.split(b" ")
+                validate_oid(oid.decode("ascii"))
+                if object_type not in (b"blob", b"tree", b"commit") or not size.isdigit():
+                    raise ValueError("Invalid history path check; source unchanged")
+                if target.kind == RewriteTargetKind.FILE and object_type == b"tree":
+                    absent = True
+            present_by_oid[record.oid] = not absent
+    return present_by_oid
+
+
+def _analyze(capture: RewriteCapture, target: RewriteTarget, operation_dir: str, records: list[ExportCommit], cancelled: Callable[[], bool] | None = None) -> HistoryAnalysis:
+    """Calculate graph decisions with raw messages left in their private spool."""
+    with open(os.path.join(operation_dir, "raw.out"), "rb") as raw_source:
+        raw_by_oid = read_raw_commits(raw_source, cancelled)
+        validate_history_protection(capture, target, records, raw_by_oid, cancelled)
+        analysis = analyze_history(capture, target, records, raw_by_oid, raw_source, cancelled)
+        with open(os.path.join(operation_dir, "import.in"), "wb") as destination:
+            write_import(destination, analysis.commits, raw_source, PRIVATE_REF, cancelled)
+    return analysis
+
+
+def _resolve_mapping(operation_dir: str, analysis: HistoryAnalysis, cancelled: Callable[[], bool] | None = None) -> dict[str, str | None]:
+    """Resolve import marks into actual object IDs before any proposal is complete."""
+    oid_by_mark = {}
+    with open(os.path.join(operation_dir, "marks.out"), "rb") as stream:
+        while line := read_line(stream, cancelled):
+            mark, oid = line.rstrip(b"\n").split(b" ")
+            oid_by_mark[mark.decode("ascii")] = validate_oid(oid.decode("ascii"))
+    replacement_by_oid = {}
+    for old_oid, replacement in analysis.replacement_by_oid.items():
+        check_cancelled(cancelled)
+        if replacement is not None and replacement.startswith(":"):
+            if replacement not in oid_by_mark:
+                raise ValueError(f"Import omitted replacement for {old_oid}; source unchanged")
+            replacement = oid_by_mark[replacement]
+        replacement_by_oid[old_oid] = replacement
+    return replacement_by_oid
+
+
+def _spool_command(command: GitCommand, filename: str, cancelled: Callable[[], bool] | None) -> Generator[GitCommand, object]:
+    """Bound each output write and total private disk use while keeping stdout out of the console.
+
+    Every spooled command reads its input from the private folder, which a refusal or a cancellation deletes as soon
+    as the command ends, so each one says so to the runner.
+    """
+    size = 0
+    with open(filename, "wb") as stream:
+        def consume(chunk: bytes) -> None:
+            """Stop the producer if its private output becomes incomplete or too large."""
+            nonlocal size
+            check_cancelled(cancelled)
+            size += len(chunk)
+            if size > HISTORY_REWRITE_SPOOL_LIMIT_BYTES:
+                raise ValueError("History preparation exceeds its private file size limit; source unchanged")
+            if stream.write(chunk) != len(chunk):
+                raise OSError("Incomplete private history spool write")
+
+        yield replace(command, consume=consume, stdout_limit_bytes=None, before_start=lambda: check_cancelled(cancelled), opens_files_deleted_at_end=True)
+
+
+def _message_digest(stream: BinaryIO, raw: RawCommit, cancelled: Callable[[], bool] | None) -> bytes:
+    """Compare binary messages using bounded reads instead of retaining their content."""
+    stream.seek(raw.message_offset)
+    remaining = raw.message_length
+    digest = hashlib.sha256()
+    while remaining:
+        check_cancelled(cancelled)
+        chunk = stream.read(min(remaining, PROCESS_CHUNK_BYTES))
+        if not chunk:
+            raise ValueError(f"Truncated verification message for {raw.oid}")
+        digest.update(chunk)
+        remaining -= len(chunk)
+    return digest.digest()
+
+
+def _verify_import(operation_dir: str, analysis: HistoryAnalysis, replacement_by_oid: dict[str, str | None], cancelled: Callable[[], bool] | None) -> None:
+    """Verify original metadata, every message byte, ordered parents and retained tree deltas."""
+    with open(os.path.join(operation_dir, "verified-raw.out"), "rb") as verified_source, open(os.path.join(operation_dir, "raw.out"), "rb") as raw_source:
+        verified_by_oid = read_raw_commits(verified_source, cancelled)
+        expected_by_oid = {}
+        for commit in analysis.commits:
+            check_cancelled(cancelled)
+            raw = commit.raw
+            new_oid = replacement_by_oid[raw.oid]
+            verified = verified_by_oid.get(new_oid)
+            parents = []
+            for parent_oid in raw.parent_oids:
+                replacement = replacement_by_oid[parent_oid]
+                if replacement is not None:
+                    parents.append(replacement)
+            if (verified is None or verified.author != raw.author or verified.committer != raw.committer or verified.encoding != raw.encoding
+                    or verified.parent_oids != tuple(parents) or verified.message_length != raw.message_length or verified.signed):
+                raise ValueError(f"Private commit metadata verification failed for {raw.oid}; source unchanged")
+            if _message_digest(raw_source, raw, cancelled) != _message_digest(verified_source, verified, cancelled):
+                raise ValueError(f"Private commit message verification failed for {raw.oid}; source unchanged")
+            expected_by_oid[new_oid] = commit
+    with open(os.path.join(operation_dir, "verified-export.out"), "rb") as source:
+        verified_records = read_export(source, cancelled)
+    verified_oid_set = set()
+    for record in verified_records:
+        check_cancelled(cancelled)
+        expected = expected_by_oid.get(record.oid)
+        if expected is None:
+            continue
+        if set(record.changes) != set(expected.changes):
+            raise ValueError(f"Private commit tree verification failed for {expected.raw.oid}; source unchanged")
+        verified_oid_set.add(record.oid)
+    if verified_oid_set != set(expected_by_oid):
+        raise ValueError("Private verification omitted rewritten commits; source unchanged")
+
+
+def _verification_request(operation_dir: str, analysis: HistoryAnalysis, records: list[ExportCommit], replacement_by_oid: dict[str, str | None],
+                          target: RewriteTarget, cancelled: Callable[[], bool] | None) -> list[ExportCommit]:
+    """Prepare deduplicated verification queries off the GUI thread."""
+    verification_records = []
+    replacement_set = set()
+    for record in records:
+        check_cancelled(cancelled)
+        new_oid = replacement_by_oid.get(record.oid)
+        if new_oid is not None and new_oid not in replacement_set:
+            verification_records.append(replace(record, oid=new_oid))
+            replacement_set.add(new_oid)
+    _write_requests(operation_dir, verification_records, target, cancelled)
+    with open(os.path.join(operation_dir, "verified-tips.in"), "wb") as stream:
+        for branch in analysis.branches:
+            check_cancelled(cancelled)
+            stream.write(replacement_by_oid[branch.old_oid].encode("ascii") + b"\n")
+    return verification_records
+
+
+def _write_tips(capture: RewriteCapture, operation_dir: str, cancelled: Callable[[], bool] | None) -> None:
+    """Keep export local while collecting known remote ancestry in one separate batch."""
+    with open(os.path.join(operation_dir, "tips.in"), "wb") as stream:
+        for oid in sorted(set(capture.local_ref_by_name.values())):
+            check_cancelled(cancelled)
+            stream.write(validate_oid(oid).encode("ascii") + b"\n")
+    with open(os.path.join(operation_dir, "ancestry.in"), "wb") as stream:
+        for oid in sorted(set(capture.local_ref_by_name.values()) | set(capture.remote_ref_by_name.values())):
+            check_cancelled(cancelled)
+            stream.write(validate_oid(oid).encode("ascii") + b"\n")
+
+
+def prepare_rewrite(capture: RewriteCapture, target: RewriteTarget, storage_root_dir: str,
+                    *, cancelled: Callable[[], bool] | None = None) -> Generator[GitCommand | FileTask, object, RewritePreparation]:
+    """Export, filter and verify a private rewrite without modifying source objects or files."""
+    preparation = RewritePreparation(capture, target)
+    try:
+        check_cancelled(cancelled)
+        observed = yield from capture_rewrite_state(capture)
+        if capture.complete:
+            validate_rewrite_capture(capture, observed)
+        elif capture.local_ref_by_name and (capture.local_ref_by_name != observed.local_ref_by_name or (capture.head_ref is not None and capture.head_ref != observed.head_ref)):
+            raise ValueError("Local branches changed before preparation; refresh and retry")
+        capture = observed
+        capture = yield from with_git_version(capture)
+        preparation = replace(preparation, capture=capture)
+        # Both exports, of the source and of the private result, take the options of the same Git. A version that
+        # gives none can only end in a refusal, so it is refused here, before the private storage is created and
+        # before any further Git read.
+        export_argument_list = export_arguments(capture.git_version)
+        yield from validate_initial_state(capture, target)
+        operation_dir, operation_identity = yield FileTask(lambda: _create_storage(capture, storage_root_dir))
+        preparation = replace(preparation, operation_dir=operation_dir, operation_identity=operation_identity, stream_filename=os.path.join(operation_dir, "import.in"))
+        yield FileTask(lambda: _write_tips(capture, operation_dir, cancelled))
+        object_format, shallow = (yield GitCommand(["rev-parse", "--show-object-format", "--is-shallow-repository"])).decode("ascii").splitlines()
+        if shallow != "false":
+            raise ValueError("Cannot rewrite a shallow repository; obtain its complete history before retrying")
+        if object_format not in ("sha1", "sha256"):
+            raise ValueError("Unsupported Git object format; source unchanged")
+        tips_filename = os.path.join(operation_dir, "tips.in")
+        export_filename = os.path.join(operation_dir, "export.out")
+        yield from _spool_command(GitCommand(export_argument_list, stdin_filename=tips_filename, timeout_ms=HISTORY_STREAM_TIMEOUT_MS, profile=READ_PROFILE), export_filename, cancelled)
+
+        def parse_export() -> list[ExportCommit]:
+            """Parse the finished export on the file worker, not the GUI thread."""
+            with open(export_filename, "rb") as source:
+                return read_export(source, cancelled)
+
+        records = yield FileTask(parse_export)
+        yield FileTask(lambda: _write_requests(operation_dir, records, target, cancelled))
+        yield from _spool_command(GitCommand(["cat-file", "--batch"], stdin_filename=os.path.join(operation_dir, "commits.in"), timeout_ms=HISTORY_STREAM_TIMEOUT_MS),
+                                  os.path.join(operation_dir, "raw.out"), cancelled)
+        analysis = yield FileTask(lambda: _analyze(capture, target, operation_dir, records, cancelled))
+        selected_records = []
+        for record in records:
+            if record.oid in analysis.replacement_by_oid:
+                selected_records.append(record)
+        yield FileTask(lambda: _write_requests(operation_dir, selected_records, target, cancelled))
+        yield from _spool_command(GitCommand(["cat-file", "--batch-check", "-Z"], stdin_filename=os.path.join(operation_dir, "presence.in"), timeout_ms=HISTORY_STREAM_TIMEOUT_MS),
+                                  os.path.join(operation_dir, "presence.out"), cancelled)
+        present_by_oid = yield FileTask(lambda: _read_presence(os.path.join(operation_dir, "presence.out"), selected_records, target, cancelled))
+        ancestry_filename = os.path.join(operation_dir, "ancestry.out")
+        yield from _spool_command(GitCommand(["rev-list", "--parents", "--stdin"], stdin_filename=os.path.join(operation_dir, "ancestry.in"),
+                                            timeout_ms=HISTORY_STREAM_TIMEOUT_MS), ancestry_filename, cancelled)
+        analysis = yield FileTask(lambda: validate_branch_state(capture, analysis, ancestry_filename, cancelled))
+        private_dir = os.path.join(operation_dir, PRIVATE_REPOSITORY_BASENAME)
+        private_arguments = [*SUBMODULE_RECURSION_OFF_ARGUMENTS, *EXACT_PATH_CASE_ARGUMENTS, "-C", private_dir, "-c", "core.hooksPath=" + os.path.join(operation_dir, "hooks")]
+        yield GitCommand([*private_arguments, "init", "--bare", "--template=", "--initial-branch=private", "--object-format=" + object_format], profile=LOCAL_WRITE_PROFILE,
+                         before_start=lambda: check_cancelled(cancelled))
+        environment_by_name = {GIT_ALTERNATE_OBJECT_ENVIRONMENT_NAME: os.path.join(capture.context.common_git_dir, "objects")}
+        yield GitCommand([*private_arguments, "fast-import", "--quiet", "--export-marks=" + os.path.join(operation_dir, "marks.out")],
+                         stdin_filename=preparation.stream_filename, environment_by_name=environment_by_name, profile=LOCAL_WRITE_PROFILE, timeout_ms=GIT_MUTATION_TIMEOUT_MS,
+                         before_start=lambda: check_cancelled(cancelled))
+        replacement_by_oid = yield FileTask(lambda: _resolve_mapping(operation_dir, analysis, cancelled))
+        verification_records = yield FileTask(lambda: _verification_request(operation_dir, analysis, records, replacement_by_oid, target, cancelled))
+        yield from _spool_command(GitCommand([*private_arguments, "cat-file", "--batch-check", "-Z"], stdin_filename=os.path.join(operation_dir, "presence.in"),
+                                            timeout_ms=HISTORY_STREAM_TIMEOUT_MS, environment_by_name=environment_by_name, profile=READ_PROFILE),
+                                  os.path.join(operation_dir, "verified-presence.out"), cancelled)
+        verified_by_oid = yield FileTask(lambda: _read_presence(os.path.join(operation_dir, "verified-presence.out"), verification_records, target, cancelled))
+        if any(verified_by_oid.values()):
+            raise ValueError("The private result still contains the target; source unchanged, report this failure")
+        yield from _spool_command(GitCommand([*private_arguments, "cat-file", "--batch"], stdin_filename=os.path.join(operation_dir, "commits.in"),
+                                            timeout_ms=HISTORY_STREAM_TIMEOUT_MS, environment_by_name=environment_by_name, profile=READ_PROFILE), os.path.join(operation_dir, "verified-raw.out"), cancelled)
+        yield from _spool_command(GitCommand([*private_arguments, *export_argument_list], stdin_filename=os.path.join(operation_dir, "verified-tips.in"),
+                                            timeout_ms=HISTORY_STREAM_TIMEOUT_MS, environment_by_name=environment_by_name, profile=READ_PROFILE), os.path.join(operation_dir, "verified-export.out"), cancelled)
+        yield FileTask(lambda: _verify_import(operation_dir, analysis, replacement_by_oid, cancelled))
+        index = yield from prepare_candidate_index(capture, target, operation_dir, private_dir, cancelled)
+        observed = yield from capture_rewrite_state(capture)
+        validate_rewrite_capture(capture, observed)
+        check_cancelled(cancelled)
+        branches = []
+        for branch in analysis.branches:
+            branches.append(replace(branch, new_oid=replacement_by_oid[branch.old_oid]))
+        return replace(preparation, replacement_by_oid=replacement_by_oid, branches=tuple(branches), rows=analysis.rows, signed_count=analysis.signed_count,
+                       current_absent=not present_by_oid[capture.local_ref_by_name[capture.head_ref]], complete=True, index=index,
+                       message_filename=os.path.join(operation_dir, "raw.out"))
+    except (OSError, ValueError, RuntimeError) as error:
+        detail_limit = HISTORY_REWRITE_DIAGNOSTIC_LIMIT_CHARS // 4
+        refusal = (f"Cannot prepare removal of {repr(target.path)[:detail_limit]} from {str(capture.head_ref)[:detail_limit]}: {str(error)[:detail_limit]}. "
+                   "Source unchanged; resolve the cause and prepare again.")
+        try:
+            yield FileTask(lambda: discard_preparation(preparation))
+        except (OSError, ValueError, RuntimeError):
+            refusal += f" Private cleanup failed at {preparation.operation_dir[:detail_limit]}; remove this owned preparation after all processes stop."
+        return replace(preparation, refusal=refusal, refusal_kind=error.kind if isinstance(error, RewriteRefusal) else None)