git-rewrite 0.5.0__py3-none-any.whl

git_rewrite/flatten/tree.py

@@ -0,0 +1,213 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   tree.py
+#   ───────
+#
+#   Tree manipulation and submodule expansion logic.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+from ..common import GITLINK_MODE
+from ..common import TREE_MODE
+from ..common import get_gitmodules_at_commit
+from ..common import get_tree_sha
+from ..common import hash_object
+from ..common import ls_tree
+from ..common import mktree
+from ..common import object_exists
+from ..common import read_blob
+
+if TYPE_CHECKING:
+    from ..common import SubmoduleInfo
+    from ..common import TreeEntry
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Types
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+SubmoduleRepos = dict[str, Path]
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Tree Expansion
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+def expand_tree(
+    source_repo: Path,
+    output_repo: Path,
+    tree_sha: str,
+    submodule_info: dict[str, SubmoduleInfo],
+    submodule_repos: SubmoduleRepos,
+    current_path: str = "",
+    verbose: bool = False,
+) -> str:
+    """
+    Recursively expand a tree, replacing submodule gitlinks with actual content.
+
+    Args:
+        source_repo: Path to the source repository
+        output_repo: Path to the output repository (where we create new objects)
+        tree_sha: SHA of the tree to expand
+        submodule_info: Submodule configuration from .gitmodules
+        submodule_repos: Mapping of submodule paths to their cloned repository paths
+        current_path: Current path within the tree (for nested submodules)
+        verbose: Whether to print verbose output
+
+    Returns:
+        SHA of the new expanded tree
+    """
+    entries = ls_tree(source_repo, tree_sha)
+    new_entries: list[TreeEntry] = []
+
+    for entry in entries:
+        full_path = f"{current_path}/{entry['name']}" if current_path else entry["name"]
+
+        if entry["mode"] == GITLINK_MODE:
+            # This is a submodule - expand it
+            new_entry = expand_submodule(
+                output_repo=output_repo,
+                submodule_path=full_path,
+                submodule_commit=entry["sha"],
+                submodule_info=submodule_info,
+                submodule_repos=submodule_repos,
+                verbose=verbose,
+            )
+            if new_entry:
+                new_entries.append(new_entry)
+            else:
+                # Could not expand submodule, skip it
+                if verbose:
+                    print(f"  Warning: Could not expand submodule at {full_path}")
+
+        elif entry["type"] == "tree":
+            # Recurse into subdirectory
+            expanded_sha = expand_tree(
+                source_repo=source_repo,
+                output_repo=output_repo,
+                tree_sha=entry["sha"],
+                submodule_info=submodule_info,
+                submodule_repos=submodule_repos,
+                current_path=full_path,
+                verbose=verbose,
+            )
+            new_entries.append(
+                {
+                    "mode": entry["mode"],
+                    "type": "tree",
+                    "sha": expanded_sha,
+                    "name": entry["name"],
+                }
+            )
+
+        else:
+            # Blob or other - copy to output repo
+            copy_object(source_repo, output_repo, entry["sha"])
+            new_entries.append(entry)
+
+    return mktree(output_repo, new_entries)
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def expand_submodule(
+    output_repo: Path,
+    submodule_path: str,
+    submodule_commit: str,
+    submodule_info: dict[str, SubmoduleInfo],
+    submodule_repos: SubmoduleRepos,
+    verbose: bool = False,
+) -> TreeEntry | None:
+    """
+    Expand a submodule by fetching its content at the specified commit.
+
+    Returns a tree entry for the expanded submodule, or None if expansion fails.
+    """
+    if submodule_path not in submodule_repos:
+        if verbose:
+            print(f"  Submodule {submodule_path} not in cloned repos")
+        return None
+
+    submodule_repo = submodule_repos[submodule_path]
+
+    # Check if the commit exists in the submodule repo
+    if not object_exists(submodule_repo, submodule_commit):
+        if verbose:
+            print(f"  Commit {submodule_commit[:8]} not found in {submodule_path}")
+        return None
+
+    # Get the tree SHA for this commit in the submodule
+    try:
+        submodule_tree = get_tree_sha(submodule_repo, submodule_commit)
+    except Exception:
+        if verbose:
+            print(f"  Could not get tree for {submodule_path}@{submodule_commit[:8]}")
+        return None
+
+    # Check if the submodule itself has submodules (nested submodules)
+    nested_submodule_info = get_gitmodules_at_commit(submodule_repo, submodule_commit)
+
+    # Get or create nested submodule repos if needed
+    nested_submodule_repos: SubmoduleRepos = {}
+    if nested_submodule_info:
+        # For nested submodules, we need to resolve them relative to the parent submodule
+        for nested_path in nested_submodule_info:
+            full_nested_path = f"{submodule_path}/{nested_path}"
+            if full_nested_path in submodule_repos:
+                nested_submodule_repos[nested_path] = submodule_repos[full_nested_path]
+
+    # Recursively expand the submodule's tree
+    expanded_sha = expand_tree(
+        source_repo=submodule_repo,
+        output_repo=output_repo,
+        tree_sha=submodule_tree,
+        submodule_info=nested_submodule_info,
+        submodule_repos=nested_submodule_repos,
+        verbose=verbose,
+    )
+
+    return {
+        "mode": TREE_MODE,
+        "type": "tree",
+        "sha": expanded_sha,
+        "name": submodule_path.split("/")[-1],
+    }
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def copy_object(source_repo: Path, output_repo: Path, sha: str) -> None:
+    """Copy a git object from source to output repository if it doesn't exist."""
+    if object_exists(output_repo, sha):
+        return
+
+    # Read object from source and write to output
+    try:
+        data = read_blob(source_repo, sha)
+        hash_object(output_repo, data, "blob")
+    except Exception:
+        # Object might be a tree or commit, which we handle differently
+        pass
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def copy_tree_objects(source_repo: Path, output_repo: Path, tree_sha: str) -> None:
+    """Recursively copy all objects in a tree from source to output repository."""
+    entries = ls_tree(source_repo, tree_sha)
+
+    for entry in entries:
+        if entry["type"] == "blob":
+            copy_object(source_repo, output_repo, entry["sha"])
+        elif entry["type"] == "tree":
+            copy_tree_objects(source_repo, output_repo, entry["sha"])

git_rewrite/remap/__init__.py

@@ -0,0 +1,21 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   remap/__init__.py
+#   ──────────────────
+#
+#   Remap subcommand - remap submodule commit references.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+from .remapper import remap_repository
+from .run import run
+
+__all__ = ["run", "remap_repository"]

git_rewrite/remap/remapper.py

@@ -0,0 +1,361 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   remapper.py
+#   ───────────
+#
+#   Core history rewriting logic for submodule remapping.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+import re
+import subprocess
+from typing import TYPE_CHECKING
+from ..common import GITLINK_MODE
+from ..common import get_all_branches
+from ..common import git
+from ..common import init_repo
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Constants
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+M_SHA_PATTERN = re.compile(r"^M (\d+) ([0-9a-f]{40}) (.+)$")
+M_ANY_PATTERN = re.compile(r"^M (\d+) (\S+) (.+)$")
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Mapping Loading
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def load_mapping_file(filepath: Path) -> dict[str, str]:
+    """
+    Load commit mapping from a file.
+
+    Format: old_sha new_sha (one per line, as output by sanitise)
+    """
+    mapping: dict[str, str] = {}
+    with filepath.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split()
+            if len(parts) >= 2:
+                old_sha = parts[0]
+                new_sha = parts[1]
+                mapping[old_sha] = new_sha
+    return mapping
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Main Remap Function
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+def remap_repository(
+    source_repo: Path,
+    output_repo: Path,
+    mapping_file: Path,
+    submodule_path: str | None = None,
+    url_rewrites: dict[str, str] | None = None,
+    verbose: bool = False,
+) -> dict[str, int]:
+    """
+    Remap submodule commit references in repository history.
+
+    Returns statistics about the remapping operation.
+    """
+    if url_rewrites is None:
+        url_rewrites = {}
+
+    print(f"Source: {source_repo}")
+    print(f"Output: {output_repo}")
+    if submodule_path:
+        print(f"Submodule path: {submodule_path}")
+    else:
+        print("Submodule path: (all matching gitlinks)")
+    if url_rewrites:
+        print(f"URL rewrites: {len(url_rewrites)}")
+        for old_url, new_url in url_rewrites.items():
+            print(f"  {old_url} -> {new_url}")
+
+    # Load mapping
+    print(f"Loading mapping from {mapping_file}...")
+    mapping = load_mapping_file(mapping_file)
+    print(f"Loaded {len(mapping)} commit mappings")
+
+    # Initialise output repository
+    print("Initialising output repository...")
+    init_repo(output_repo)
+
+    # Run the remapping
+    print("Processing history...")
+    stats = remap_history(
+        source_repo, output_repo, mapping, submodule_path, url_rewrites, verbose
+    )
+
+    print("Done!")
+    print(f"Processed {stats['commits']} commits")
+    print(f"Remapped {stats['remapped']} gitlink references")
+    if stats["unmapped"] > 0:
+        print(f"Warning: {stats['unmapped']} gitlinks had no mapping (kept original)")
+    if stats["gitmodules_updated"] > 0:
+        print(f"Updated .gitmodules URL in {stats['gitmodules_updated']} commits")
+
+    return stats
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def remap_history(
+    source_repo: Path,
+    output_repo: Path,
+    mapping: dict[str, str],
+    submodule_path: str | None,
+    url_rewrites: dict[str, str],
+    verbose: bool,
+) -> dict[str, int]:
+    """Process history through fast-export/fast-import, remapping gitlinks."""
+
+    # Start fast-export
+    export_cmd = [
+        "git",
+        "-C",
+        str(source_repo),
+        "fast-export",
+        "--all",
+        "--signed-tags=strip",
+        "--tag-of-filtered-object=rewrite",
+    ]
+    export_proc = subprocess.Popen(
+        export_cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    # Start fast-import
+    import_cmd = [
+        "git",
+        "-C",
+        str(output_repo),
+        "fast-import",
+        "--quiet",
+    ]
+    import_proc = subprocess.Popen(
+        import_cmd,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    assert export_proc.stdout is not None
+    assert import_proc.stdin is not None
+
+    # Statistics
+    stats = {
+        "commits": 0,
+        "remapped": 0,
+        "unmapped": 0,
+        "gitmodules_updated": 0,
+    }
+
+    # State tracking
+    in_data = False
+    in_blob_data = False
+    data_remaining = 0
+    current_blob_mark: str | None = None
+    current_blob_data = b""
+    blob_contents: dict[str, bytes] = {}  # mark -> content
+
+    # Helper to write to import
+    def write_import(data: bytes) -> None:
+        assert import_proc.stdin is not None
+        try:
+            import_proc.stdin.write(data)
+        except BrokenPipeError as e:
+            import_proc.wait()
+            stderr = import_proc.stderr.read().decode() if import_proc.stderr else ""
+            raise RuntimeError(f"fast-import failed: {stderr}") from e
+
+    # Process the stream
+    while True:
+        line = export_proc.stdout.readline()
+        if not line:
+            break
+
+        # Handle blob data - capture content while passing through
+        if in_blob_data:
+            current_blob_data += line
+            write_import(line)
+            data_remaining -= len(line)
+            if data_remaining <= 0:
+                in_blob_data = False
+                if current_blob_mark:
+                    blob_contents[current_blob_mark] = current_blob_data
+                current_blob_data = b""
+            continue
+
+        # Handle regular data blocks (commit messages, etc.)
+        if in_data:
+            write_import(line)
+            data_remaining -= len(line)
+            if data_remaining <= 0:
+                in_data = False
+            continue
+
+        line_str = line.decode("utf-8", errors="replace")
+
+        # Check for blob start
+        if line_str.strip() == "blob":
+            current_blob_mark = None
+            current_blob_data = b""
+            write_import(line)
+            continue
+
+        # Check for mark (track for blob content)
+        if line_str.startswith("mark :"):
+            current_blob_mark = line_str.strip().split()[1]  # e.g., ":3"
+            write_import(line)
+            continue
+
+        # Check for data command
+        if line_str.startswith("data "):
+            data_size = int(line_str[5:].strip())
+            data_remaining = data_size
+            write_import(line)
+
+            # If we're tracking a blob, capture its data
+            if current_blob_mark is not None:
+                in_blob_data = data_size > 0
+            else:
+                in_data = data_size > 0
+            continue
+
+        # Check for commit
+        if line_str.startswith("commit "):
+            stats["commits"] += 1
+            current_blob_mark = None  # Reset blob tracking
+            if stats["commits"] % 50 == 0:
+                print(f"  Processing commit {stats['commits']}...", flush=True)
+            write_import(line)
+            continue
+
+        # Check for file modification with SHA (gitlinks)
+        m_match = M_SHA_PATTERN.match(line_str)
+        if m_match:
+            mode = m_match.group(1)
+            sha = m_match.group(2)
+            filepath = m_match.group(3)
+
+            # Check if this is a gitlink we should remap
+            if mode == GITLINK_MODE and (
+                submodule_path is None or filepath == submodule_path
+            ):
+                if sha in mapping:
+                    new_sha = mapping[sha]
+                    write_import(f"M {mode} {new_sha} {filepath}\n".encode())
+                    stats["remapped"] += 1
+                    if verbose:
+                        print(f"  Remapped: {sha[:8]} -> {new_sha[:8]}")
+                else:
+                    write_import(line)
+                    stats["unmapped"] += 1
+                    if verbose:
+                        print(f"  Warning: No mapping for {sha[:8]}")
+                continue
+
+            write_import(line)
+            continue
+
+        # Check for file modification with mark (for .gitmodules)
+        m_any_match = M_ANY_PATTERN.match(line_str)
+        if m_any_match:
+            mode = m_any_match.group(1)
+            ref = m_any_match.group(2)
+            filepath = m_any_match.group(3)
+
+            # Check if this is .gitmodules - rewrite with modified content
+            if filepath == ".gitmodules" and url_rewrites and ref.startswith(":"):
+                if ref in blob_contents:
+                    original = blob_contents[ref]
+                    modified = modify_gitmodules(original, url_rewrites)
+                    if modified != original:
+                        stats["gitmodules_updated"] += 1
+                        # Emit as inline data with modified content
+                        write_import(f"M {mode} inline {filepath}\n".encode())
+                        write_import(f"data {len(modified)}\n".encode())
+                        write_import(modified)
+                        continue
+            write_import(line)
+            continue
+
+        # Pass through everything else
+        write_import(line)
+
+    # Close streams
+    import_proc.stdin.close()
+    export_proc.wait()
+    import_proc.wait()
+
+    if export_proc.returncode != 0:
+        stderr = export_proc.stderr.read().decode() if export_proc.stderr else ""
+        raise RuntimeError(f"fast-export failed: {stderr}")
+
+    if import_proc.returncode != 0:
+        stderr = import_proc.stderr.read().decode() if import_proc.stderr else ""
+        raise RuntimeError(f"fast-import failed: {stderr}")
+
+    # Checkout the default branch to clean up the working directory
+    # fast-import leaves things in a detached HEAD state with staged files
+    branches = get_all_branches(output_repo)
+    if branches:
+        # Prefer main, then master, then first available
+        if "main" in branches:
+            default_branch = "main"
+        elif "master" in branches:
+            default_branch = "master"
+        else:
+            default_branch = branches[0]
+        git(output_repo, "checkout", "-f", default_branch)
+
+    return stats
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def modify_gitmodules(content: bytes, url_rewrites: dict[str, str]) -> bytes:
+    """Modify .gitmodules content to replace URLs based on url_rewrites mapping."""
+    try:
+        text = content.decode("utf-8")
+    except UnicodeDecodeError:
+        return content
+
+    lines = text.split("\n")
+    result_lines: list[str] = []
+
+    for line in lines:
+        # Replace matching URL lines
+        if line.strip().startswith("url = "):
+            current_url = line.strip()[6:].strip()
+            if current_url in url_rewrites:
+                result_lines.append(f"\turl = {url_rewrites[current_url]}")
+            else:
+                result_lines.append(line)
+        else:
+            result_lines.append(line)
+
+    return "\n".join(result_lines).encode("utf-8")

git_rewrite/remap/run.py

@@ -0,0 +1,85 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   remap/run.py
+#   ────────────
+#
+#   CLI runner for the remap subcommand.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+import shutil
+import sys
+from typing import TYPE_CHECKING
+from ..common import copy_remotes
+from .remapper import remap_repository
+
+if TYPE_CHECKING:
+    import argparse
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Runner
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def run(args: argparse.Namespace) -> int:
+    """Run the remap subcommand."""
+    # Validate paths
+    if not args.repo.exists():
+        print(f"Error: Source repository not found: {args.repo}", file=sys.stderr)
+        return 1
+    if not args.commit_map.exists():
+        print(f"Error: Mapping file not found: {args.commit_map}", file=sys.stderr)
+        return 1
+
+    # Handle output directory
+    if args.output.exists():
+        if args.delete:
+            print(f"Removing existing output directory: {args.output}")
+            shutil.rmtree(args.output)
+        else:
+            print(
+                f"Error: Output directory already exists: {args.output}",
+                file=sys.stderr,
+            )
+            print("Use --delete to remove it first", file=sys.stderr)
+            return 1
+
+    # Build URL rewrite mapping
+    url_rewrites: dict[str, str] = {}
+    if args.url_rewrite:
+        for old_url, new_url in args.url_rewrite:
+            url_rewrites[old_url] = new_url
+
+    remap_repository(
+        source_repo=args.repo,
+        output_repo=args.output,
+        mapping_file=args.commit_map,
+        submodule_path=args.submodule_path,
+        url_rewrites=url_rewrites,
+        verbose=args.verbose,
+    )
+
+    # Copy remotes if requested
+    if args.copy_remotes:
+        print("Copying remotes from source repository...")
+        count = copy_remotes(args.repo, args.output, verbose=args.verbose)
+        if count > 0:
+            print(f"  Copied {count} remote(s)")
+        else:
+            print("  No remotes found in source repository")
+
+    print(f"\nOutput repository created at: {args.output}")
+    return 0