git-rewrite 0.5.0__py3-none-any.whl

git_rewrite/sanitise/run.py

@@ -0,0 +1,145 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   sanitise/run.py
+#   ────────────────
+#
+#   CLI runner for the sanitise subcommand.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+import shutil
+import sys
+from typing import TYPE_CHECKING
+from ..common import copy_remotes
+from .config import load_config
+from .config import load_submodule_mapping
+from .rewriter import rewrite_repository
+
+if TYPE_CHECKING:
+    import argparse
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Constants
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+SAMPLE_CONFIG = """{
+    "words": ["secretword", "internalname", "sensitiveid"],
+    "word_mapping": {
+        "secretword": "publicword",
+        "internalname": "externalname"
+    },
+    "email_mapping": {
+        "john smith": "jsmith@example.com",
+        "jane doe": "jdoe@example.com"
+    },
+    "exclude_files": [
+        "package-lock.json",
+        "yarn.lock"
+    ]
+}"""
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Runner
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def run(args: argparse.Namespace) -> int:
+    """Run the sanitise subcommand."""
+    # Print sample config if requested
+    if args.sample_config:
+        print(SAMPLE_CONFIG)
+        return 0
+
+    # Validate required arguments
+    if not args.repo:
+        print("Error: -r/--repo is required", file=sys.stderr)
+        return 1
+    if not args.output:
+        print("Error: -o/--output is required", file=sys.stderr)
+        return 1
+    if not args.config:
+        print("Error: -c/--config is required", file=sys.stderr)
+        return 1
+
+    # Validate paths exist
+    if not args.repo.exists():
+        print(f"Error: Repository not found: {args.repo}", file=sys.stderr)
+        return 1
+    if not args.config.exists():
+        print(f"Error: Config file not found: {args.config}", file=sys.stderr)
+        return 1
+
+    # Handle output directory
+    if args.output.exists():
+        if args.delete:
+            print(f"Removing existing output directory: {args.output}")
+            shutil.rmtree(args.output)
+        else:
+            print(
+                f"Error: Output directory already exists: {args.output}",
+                file=sys.stderr,
+            )
+            print("Use --delete to remove it first", file=sys.stderr)
+            return 1
+
+    # Load config
+    config = load_config(args.config)
+    words = config.get("words", [])
+    word_mapping = config.get("word_mapping", {})
+    email_mapping = config.get("email_mapping", {})
+    exclude_patterns = config.get("exclude_files", [])
+
+    if not words:
+        print("Error: No words specified in config file", file=sys.stderr)
+        return 1
+
+    # Load submodule mapping if provided
+    submodule_mapping: dict[str, str] = {}
+    if args.submodule_map:
+        if not args.submodule_map.exists():
+            print(
+                f"Error: Submodule mapping file not found: {args.submodule_map}",
+                file=sys.stderr,
+            )
+            return 1
+        submodule_mapping = load_submodule_mapping(args.submodule_map)
+        print(f"Loaded {len(submodule_mapping)} submodule mappings")
+
+    print(f"Sanitising {len(words)} dirty words...")
+    rewrite_repository(
+        repo_path=args.repo,
+        output_path=args.output,
+        dirty_words=words,
+        word_mapping=word_mapping,
+        default_replacement=args.default,
+        commit_map_path=args.commit_map,
+        exclude_patterns=exclude_patterns,
+        email_mapping=email_mapping,
+        submodule_mapping=submodule_mapping,
+        verbose=args.verbose,
+    )
+
+    # Copy remotes if requested
+    if args.copy_remotes:
+        print("Copying remotes from source repository...")
+        count = copy_remotes(args.repo, args.output, verbose=args.verbose)
+        if count > 0:
+            print(f"  Copied {count} remote(s)")
+        else:
+            print("  No remotes found in source repository")
+
+    print(f"\nOutput repository created at: {args.output}")
+    return 0

git_rewrite/scan/__init__.py

@@ -0,0 +1,21 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   scan/__init__.py
+#   ─────────────────
+#
+#   Scan subcommand - scan for sensitive words in git history.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+from .run import run
+from .scanner import scan_repository
+
+__all__ = ["run", "scan_repository"]

git_rewrite/scan/run.py

@@ -0,0 +1,100 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   scan/run.py
+#   ───────────
+#
+#   CLI runner for the scan subcommand.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+import sys
+from typing import TYPE_CHECKING
+from ..sanitise.config import load_config
+from .scanner import find_excluded_files
+from .scanner import scan_repository
+
+if TYPE_CHECKING:
+    import argparse
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Runner
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def run(args: argparse.Namespace) -> int:
+    """Run the scan subcommand."""
+    # Validate paths
+    if not args.repo.exists():
+        print(f"Error: Repository not found: {args.repo}", file=sys.stderr)
+        return 1
+    if not args.config.exists():
+        print(f"Error: Config file not found: {args.config}", file=sys.stderr)
+        return 1
+
+    # Load config
+    config = load_config(args.config)
+    words = config.get("words", [])
+    exclude_patterns = config.get("exclude_files", [])
+
+    if not words:
+        print("Error: No words specified in config file", file=sys.stderr)
+        return 1
+
+    print(f"Scanning for {len(words)} dirty words...")
+    blob_matches, commit_matches = scan_repository(args.repo, words, args.verbose)
+
+    # Find files that would be excluded
+    excluded_files: list[str] = []
+    if exclude_patterns:
+        print(f"\nChecking {len(exclude_patterns)} exclude patterns...")
+        excluded_files = find_excluded_files(args.repo, exclude_patterns, args.verbose)
+
+    # Report results
+    total_blob = sum(blob_matches.values())
+    total_commit = sum(commit_matches.values())
+
+    print("\n" + "=" * 60)
+    print("SCAN RESULTS")
+    print("=" * 60)
+
+    if blob_matches:
+        print("\nMatches in file contents:")
+        for word, count in sorted(blob_matches.items(), key=lambda x: -x[1]):
+            print(f"  {word}: {count}")
+
+    if commit_matches:
+        print("\nMatches in commit messages:")
+        for word, count in sorted(commit_matches.items(), key=lambda x: -x[1]):
+            print(f"  {word}: {count}")
+
+    print(f"\nTotal: {total_blob} in files, {total_commit} in commits")
+
+    if total_blob == 0 and total_commit == 0:
+        print("\nNo dirty words found!")
+
+    # Report excluded files
+    if exclude_patterns:
+        print("\n" + "-" * 60)
+        print("FILES THAT WOULD BE EXCLUDED")
+        print("-" * 60)
+        if excluded_files:
+            print(f"\n{len(excluded_files)} file(s) match exclude patterns:")
+            for filepath in excluded_files:
+                print(f"  {filepath}")
+        else:
+            print("\nNo files match the exclude patterns.")
+
+    return 0

git_rewrite/scan/scanner.py

@@ -0,0 +1,192 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   scanner.py
+#   ──────────
+#
+#   Repository scanning for dirty words using git fast-export.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Imports
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+import subprocess
+from collections import defaultdict
+from pathlib import PurePath
+from typing import TYPE_CHECKING
+from ..sanitise.patterns import build_combined_pattern
+
+if TYPE_CHECKING:
+    import re
+    from pathlib import Path
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Functions
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def scan_content_fast(
+    content: str,
+    pattern: re.Pattern[str],
+) -> dict[str, int]:
+    """Scan content with combined pattern and return match counts by word."""
+    matches: defaultdict[str, int] = defaultdict(int)
+    for match in pattern.finditer(content):
+        word = match.group(0).lower()
+        matches[word] += 1
+    return dict(matches)
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def scan_repository(
+    repo_path: Path,
+    dirty_words: list[str],
+    verbose: bool = False,
+) -> tuple[dict[str, int], dict[str, int]]:
+    """
+    Scan repository using fast-export streaming for speed.
+
+    Returns:
+        Tuple of (blob_matches, commit_matches) where each is a dict
+        mapping dirty word to count of occurrences.
+    """
+    blob_matches: dict[str, int] = defaultdict(int)
+    commit_matches: dict[str, int] = defaultdict(int)
+
+    pattern = build_combined_pattern(dirty_words)
+
+    print("Scanning repository...")
+
+    export_proc = subprocess.Popen(
+        ["git", "fast-export", "--all"],
+        cwd=repo_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+    )
+
+    assert export_proc.stdout is not None
+
+    commit_count = 0
+    blob_count = 0
+    in_data = False
+    data_remaining = 0
+    data_buffer = b""
+    in_commit_message = False
+    last_blob_update = 0
+    in_blobs_phase = True
+
+    for line in export_proc.stdout:
+        if in_data:
+            data_buffer += line
+            data_remaining -= len(line)
+
+            if data_remaining <= 0:
+                in_data = False
+                try:
+                    text = data_buffer.decode("utf-8")
+                except UnicodeDecodeError:
+                    try:
+                        text = data_buffer.decode("latin-1")
+                    except UnicodeDecodeError:
+                        data_buffer = b""
+                        in_commit_message = False
+                        continue
+
+                matches = scan_content_fast(text, pattern)
+                target = commit_matches if in_commit_message else blob_matches
+                for word, count in matches.items():
+                    target[word] += count
+                    if verbose:
+                        location = "commit message" if in_commit_message else "blob"
+                        print(f"  Found '{word}' x{count} in {location}")
+
+                if not in_commit_message:
+                    blob_count += 1
+                    if blob_count - last_blob_update >= 100:
+                        print(f"  Scanned {blob_count} blobs...")
+                        last_blob_update = blob_count
+
+                data_buffer = b""
+                in_commit_message = False
+
+        elif line.startswith(b"data "):
+            size_str = line[5:].strip().decode("utf-8")
+            data_remaining = int(size_str)
+            in_data = True
+            data_buffer = b""
+
+        elif line.startswith(b"commit "):
+            if in_blobs_phase:
+                in_blobs_phase = False
+                print(
+                    f"  Finished scanning {blob_count} blobs, now scanning commits..."
+                )
+            commit_count += 1
+            in_commit_message = True
+            if commit_count % 100 == 0:
+                print(f"  Scanned {commit_count} commits...")
+
+        elif line.startswith(b"blob"):
+            in_commit_message = False
+
+    export_proc.wait()
+    print(f"  Done: {commit_count} commits, {blob_count} blobs")
+
+    return dict(blob_matches), dict(commit_matches)
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def find_excluded_files(
+    repo_path: Path,
+    exclude_patterns: list[str],
+    verbose: bool = False,
+) -> list[str]:
+    """
+    Find all files in the repository that match the exclude patterns.
+
+    Scans all files across all branches to find matches.
+    Returns a sorted list of unique file paths that would be excluded.
+    """
+    if not exclude_patterns:
+        return []
+
+    # Get all unique file paths across all commits
+    result = subprocess.run(
+        ["git", "log", "--all", "--pretty=format:", "--name-only", "--diff-filter=A"],
+        cwd=repo_path,
+        capture_output=True,
+        text=True,
+    )
+
+    if result.returncode != 0:
+        return []
+
+    # Collect unique paths
+    all_paths: set[str] = set()
+    for line in result.stdout.splitlines():
+        line = line.strip()
+        if line:
+            all_paths.add(line)
+
+    # Check which paths match exclude patterns
+    excluded: set[str] = set()
+    for filepath in all_paths:
+        path = PurePath(filepath)
+        for pattern in exclude_patterns:
+            if path.match(pattern):
+                excluded.add(filepath)
+                if verbose:
+                    print(f"  Pattern '{pattern}' matches: {filepath}")
+                break
+
+    return sorted(excluded)

git_rewrite/version.py

@@ -0,0 +1,36 @@
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   version.py
+#   ──────────
+#
+#   Version string handling - imports from generated _version.py at build time,
+#   falls back to "dev" when not built.
+#
+#   (c) 2026 Cyber Assessment Labs — MIT License; see LICENSE in the project root.
+#
+#   Authors
+#   ───────
+#   bena (via Claude)
+#
+#   Version History
+#   ───────────────
+#   Jan 2026 - Created
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+#   Version
+# ────────────────────────────────────────────────────────────────────────────────────────
+
+
+# ────────────────────────────────────────────────────────────────────────────────────────
+def _get_version() -> str:
+    try:
+        # fmt: off
+        from ._version import __version__ as _v  # pyright: ignore[reportMissingImports,reportUnknownVariableType]  # noqa: I001
+        # fmt: on
+
+        return str(_v)  # pyright: ignore[reportUnknownArgumentType]
+    except ImportError:
+        return "dev"
+
+
+VERSION_STR: str = _get_version()

git_rewrite-0.5.0.dist-info/METADATA

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: git-rewrite
+Version: 0.5.0
+Summary: Git history rewriting tools - sanitise, flatten submodules, and remap references
+Project-URL: Repository, https://gitlab.com/cyberassessmentlabs/public/tools/git-rewrite
+Project-URL: Documentation, https://cyberassessmentlabs.gitlab.io/public/docs/git-rewrite/latest
+Author: Cyber Assessment Labs
+License-Expression: MIT
+License-File: LICENSE
+Keywords: git,history,rewrite,sanitise,submodules
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Version Control :: Git
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+
+# git-rewrite
+
+Git history rewriting tools - sanitise, flatten submodules, and remap references.
+
+## Installation
+
+```bash
+uv sync
+```
+
+## Usage
+
+### Scan for Sensitive Words
+
+Scan a repository for dirty words without making any changes:
+
+```bash
+uv run git-rewrite scan -r /path/to/repo -c config.json
+```
+
+Options:
+- `-r, --repo` - Path to the git repository
+- `-c, --config` - Path to JSON config file
+- `-v, --verbose` - Show verbose output
+
+### Sanitise Repository
+
+Rewrite history to remove sensitive words:
+
+```bash
+uv run git-rewrite sanitise -r /path/to/repo -o /path/to/output -c config.json
+```
+
+Options:
+- `-r, --repo` - Path to source repository
+- `-o, --output` - Path for output repository
+- `-c, --config` - Path to JSON config file
+- `-d, --default` - Default replacement word (default: REDACTED)
+- `-m, --commit-map` - Path to write commit mapping file
+- `-s, --submodule-map` - Path to submodule commit mapping file
+- `--delete` - Delete output directory if exists
+- `--sample-config` - Print sample config and exit
+- `-v, --verbose` - Show verbose output
+
+#### Sample Configuration
+
+```json
+{
+    "words": ["secretword", "internalname", "sensitiveid"],
+    "word_mapping": {
+        "secretword": "publicword",
+        "internalname": "externalname"
+    },
+    "email_mapping": {
+        "john smith": "jsmith@example.com",
+        "jane doe": "jdoe@example.com"
+    },
+    "exclude_files": [
+        "package-lock.json",
+        "yarn.lock"
+    ]
+}
+```
+
+### Flatten Submodules
+
+Flatten submodules into the main repository:
+
+```bash
+# Scan for submodules only
+uv run git-rewrite flatten -r /path/to/repo --scan-only
+
+# Flatten submodules
+uv run git-rewrite flatten -r /path/to/repo -o /path/to/output
+```
+
+Options:
+- `-r, --repo` - Path to source repository
+- `-o, --output` - Path for output repository
+- `-m, --commit-map` - Path to write commit mapping file
+- `--scan-only` - Only scan and list submodules
+- `--delete` - Delete output directory if exists
+- `-v, --verbose` - Show verbose output
+
+### Remap Submodule References
+
+Update submodule commit references in a repository:
+
+```bash
+uv run git-rewrite remap -r /path/to/repo -o /path/to/output -m mapping.txt
+```
+
+Options:
+- `-r, --repo` - Path to source repository
+- `-o, --output` - Path for output repository
+- `-m, --commit-map` - Path to commit mapping file
+- `--submodule-path` - Specific submodule path to remap
+- `--url-rewrite OLD NEW` - Rewrite URL in .gitmodules (can repeat)
+- `--delete` - Delete output directory if exists
+- `-v, --verbose` - Show verbose output
+
+## Workflow Example
+
+A typical workflow for cleaning up and flattening a repository:
+
+```bash
+# 1. Scan for sensitive words
+uv run git-rewrite scan -r ./myrepo -c dirty-words.json
+
+# 2. Sanitise the repository
+uv run git-rewrite sanitise -r ./myrepo -o ./myrepo-clean -c dirty-words.json \
+    -m commit-map.txt
+
+# 3. Flatten submodules in the cleaned repo
+uv run git-rewrite flatten -r ./myrepo-clean -o ./myrepo-flat -m submodule-map.txt
+
+# 4. Update consumer repositories to use new submodule commits
+uv run git-rewrite remap -r ./consumer-repo -o ./consumer-updated \
+    -m submodule-map.txt --submodule-path libs/mylib
+```
+
+## Development
+
+```bash
+# Set up development environment
+make dev
+
+# Run linting and type checking
+make check
+
+# Auto-format code
+make format
+
+# Build wheel and docs
+make build
+```
+
+## Publishing
+
+Publishing requires `cal-publish-python` configuration. See the [cal-publish-python documentation](https://cyberassessmentlabs.gitlab.io/public/docs/cal-publish-python/latest/) for setup.
+
+```bash
+# Build first
+make build
+
+# Publish wheel to PyPI and docs to GitLab Pages
+make publish
+```
+
+## Requirements
+
+- Python 3.14+
+- Git
+
+## Licence
+
+MIT