PyHardLinkBackup 1.0.0__py3-none-any.whl

PyHardLinkBackup/__init__.py

@@ -0,0 +1,7 @@
+"""DocWrite: README.md # PyHardLinkBackup
+    HardLink/Deduplication Backups with Python
+"""
+
+# See https://packaging.python.org/en/latest/specifications/version-specifiers/
+__version__ = '1.0.0'
+__author__ = 'Jens Diemer <PyHardLinkBackup@jensdiemer.de>'

PyHardLinkBackup/__main__.py

@@ -0,0 +1,10 @@
+"""
+    Allow PyHardLinkBackup to be executable
+    through `python -m PyHardLinkBackup`.
+"""
+
+from PyHardLinkBackup.cli_app import main
+
+
+if __name__ == '__main__':
+    main()

PyHardLinkBackup/backup.py

@@ -0,0 +1,203 @@
+import dataclasses
+import logging
+import os
+import shutil
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.constants import CHUNK_SIZE
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    copy_and_hash,
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+    read_and_hash_file,
+)
+from PyHardLinkBackup.utilities.humanize import human_filesize
+from PyHardLinkBackup.utilities.rich_utils import BackupProgress
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class BackupResult:
+    backup_dir: Path
+    backup_count: int
+    backup_size: int
+    symlink_files: int
+    hardlinked_files: int
+    hardlinked_size: int
+    copied_files: int
+    copied_size: int
+    copied_small_files: int
+    copied_small_size: int
+
+
+def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> BackupResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    if not backup_root.is_dir():
+        print('Error: Backup directory does not exist!')
+        print(f'Please create "{backup_root}" directory first and start again!\n')
+        sys.exit(1)
+
+    # Step 1: Scan source directory:
+    src_file_count, src_total_size = humanized_fs_scan(src_root, excludes)
+
+    phlb_conf_dir = backup_root / '.phlb'
+    phlb_conf_dir.mkdir(parents=False, exist_ok=True)
+
+    backup_dir = backup_root / src_root.name / datetime.now().strftime('%Y%m%d_%H%M%S')
+    logger.info('Backup %s to %s', src_root, backup_dir)
+    backup_dir.mkdir(parents=True, exist_ok=False)
+
+    print(f'\nBackup to {backup_dir}...\n')
+
+    with BackupProgress(src_file_count, src_total_size) as progress:
+        # "Databases" for deduplication
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        backup_count = 0
+        backup_size = 0
+
+        symlink_files = 0
+        hardlinked_files = 0
+        hardlinked_size = 0
+        copied_files = 0
+        copied_size = 0
+        copied_small_files = 0
+        copied_small_size = 0
+
+        next_update = 0
+        for entry in iter_scandir_files(src_root, excludes=excludes):
+            backup_count += 1
+            src_path = Path(entry.path)
+
+            dst_path = backup_dir / src_path.relative_to(src_root)
+            dst_dir_path = dst_path.parent
+            if not dst_dir_path.exists():
+                dst_dir_path.mkdir(parents=True, exist_ok=False)
+
+            try:
+                size = entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: Handle broken symlink
+                target = os.readlink(src_path)
+                dst_path.symlink_to(target)
+                symlink_files += 1
+                continue
+
+            backup_size += size
+
+            now = time.monotonic()
+            if now >= next_update:
+                progress.update(backup_count=backup_count, backup_size=backup_size)
+                next_update = now + 0.5
+
+            if entry.is_symlink():
+                logger.debug('Copy symlink: %s to %s', src_path, dst_path)
+                target = os.readlink(src_path)
+                dst_path.symlink_to(target)
+                symlink_files += 1
+                continue
+
+            # Process regular files
+            assert entry.is_file(follow_symlinks=False), f'Unexpected non-file: {src_path}'
+
+            # Deduplication logic
+
+            if size < size_db.MIN_SIZE:
+                # Small file -> always copy without deduplication
+                logger.info('Copy small file: %s to %s', src_path, dst_path)
+                shutil.copy2(src_path, dst_path)
+                copied_files += 1
+                copied_size += size
+                copied_small_files += 1
+                copied_small_size += size
+                continue
+
+            if size in size_db:
+                logger.debug('File with size %iBytes found before -> hash: %s', size, src_path)
+
+                if size <= CHUNK_SIZE:
+                    # File can be read complete into memory
+                    logger.debug(
+                        'File size %iBytes <= CHUNK_SIZE (%iBytes) -> read complete into memory', size, CHUNK_SIZE
+                    )
+                    file_content, file_hash = read_and_hash_file(src_path)
+                    if existing_path := hash_db.get(file_hash):
+                        logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                        os.link(existing_path, dst_path)
+                        hardlinked_files += 1
+                        hardlinked_size += size
+                    else:
+                        logger.info('Store unique file: %s to %s', src_path, dst_path)
+                        dst_path.write_bytes(file_content)
+                        hash_db[file_hash] = dst_path
+                        copied_files += 1
+                        copied_size += size
+
+                else:
+                    # Large file
+                    file_hash = hash_file(src_path)  # Calculate hash without copying
+
+                    if existing_path := hash_db.get(file_hash):
+                        logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                        os.link(existing_path, dst_path)
+                        hardlinked_files += 1
+                        hardlinked_size += size
+                    else:
+                        logger.info('Copy unique file: %s to %s', src_path, dst_path)
+                        hash_db[file_hash] = dst_path
+                        copied_files += 1
+                        copied_size += size
+
+                # Keep original file metadata (permission bits, time stamps, and flags)
+                shutil.copy2(src_path, dst_path)
+            else:
+                # A file with this size not backuped before -> Can't be duplicate -> copy and hash
+                file_hash = copy_and_hash(src_path, dst_path)
+                size_db.add(size)
+                hash_db[file_hash] = dst_path
+                copied_files += 1
+                copied_size += size
+
+        # Finalize progress indicator values:
+        progress.update(backup_count=backup_count, backup_size=backup_size)
+
+    print(f'\nBackup complete: {backup_dir} (total size {human_filesize(backup_size)})\n')
+    print(f'  Total files processed: {backup_count}')
+    print(f'   * Symlinked files: {symlink_files}')
+    print(f'   * Hardlinked files: {hardlinked_files} (saved {human_filesize(hardlinked_size)})')
+    print(f'   * Copied files: {copied_files} (total {human_filesize(copied_size)})')
+    print(
+        f'     of which small (<{size_db.MIN_SIZE} Bytes) files: {copied_small_files}'
+        f' (total {human_filesize(copied_small_size)})'
+    )
+    print()
+
+    return BackupResult(
+        backup_dir=backup_dir,
+        backup_count=backup_count,
+        backup_size=backup_size,
+        symlink_files=symlink_files,
+        hardlinked_files=hardlinked_files,
+        hardlinked_size=hardlinked_size,
+        copied_files=copied_files,
+        copied_size=copied_size,
+        copied_small_files=copied_small_files,
+        copied_small_size=copied_small_size,
+    )

PyHardLinkBackup/cli_app/__init__.py

@@ -0,0 +1,41 @@
+"""
+    CLI for usage
+"""
+
+import logging
+import sys
+from collections.abc import Sequence
+
+from cli_base.autodiscover import import_all_files
+from cli_base.cli_tools.version_info import print_version
+from rich import print  # noqa
+from tyro.extras import SubcommandApp
+
+import PyHardLinkBackup
+from PyHardLinkBackup import constants
+
+
+logger = logging.getLogger(__name__)
+
+app = SubcommandApp()
+
+# Register all CLI commands, just by import all files in this package:
+import_all_files(package=__package__, init_file=__file__)
+
+
+@app.command
+def version():
+    """Print version and exit"""
+    # Pseudo command, because the version always printed on every CLI call ;)
+    sys.exit(0)
+
+
+def main(args: Sequence[str] | None = None):
+    print_version(PyHardLinkBackup)
+    app.cli(
+        prog='./cli.py',
+        description=constants.CLI_EPILOG,
+        use_underscores=False,  # use hyphens instead of underscores
+        sort_subcommands=True,
+        args=args,
+    )

PyHardLinkBackup/cli_app/phlb.py

@@ -0,0 +1,50 @@
+import logging
+from pathlib import Path
+from typing import Annotated
+
+import tyro
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.backup import backup_tree
+from PyHardLinkBackup.cli_app import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def backup(
+    src: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='source',
+            help='Source directory to back up.',
+        ),
+    ],
+    dst: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='destination',
+            help='Destination directory for the backup.',
+        ),
+    ],
+    /,
+    excludes: Annotated[
+        tuple,
+        tyro.conf.arg(
+            help='List of directory or file names to exclude from backup.',
+        ),
+    ] = ('__pycache__', '.cache', '.temp', '.tmp', '.tox', '.nox'),
+    verbosity: TyroVerbosityArgType = 2,
+) -> None:
+    """
+    Backup the source directory to the destination directory using hard links for deduplication.
+    """
+    setup_logging(verbosity=verbosity)
+    backup_tree(
+        src_root=src,
+        backup_root=dst,
+        excludes=set(excludes),
+    )

PyHardLinkBackup/cli_dev/__init__.py

@@ -0,0 +1,70 @@
+"""
+    CLI for development
+"""
+
+import importlib
+import logging
+import sys
+from collections.abc import Sequence
+
+from bx_py_utils.path import assert_is_file
+from cli_base.autodiscover import import_all_files
+from cli_base.cli_tools.dev_tools import run_coverage, run_nox, run_unittest_cli
+from cli_base.cli_tools.version_info import print_version
+from typeguard import install_import_hook
+from tyro.extras import SubcommandApp
+
+import PyHardLinkBackup
+from PyHardLinkBackup import constants
+
+
+# Check type annotations via typeguard in all tests.
+# Sadly we must activate this here and can't do this in ./tests/__init__.py
+install_import_hook(packages=('PyHardLinkBackup',))
+
+# reload the module, after the typeguard import hook is activated:
+importlib.reload(PyHardLinkBackup)
+
+
+logger = logging.getLogger(__name__)
+
+
+PACKAGE_ROOT = constants.BASE_PATH.parent
+assert_is_file(PACKAGE_ROOT / 'pyproject.toml')  # Exists only in cloned git repo
+
+
+app = SubcommandApp()
+
+
+# Register all CLI commands, just by import all files in this package:
+import_all_files(package=__package__, init_file=__file__)
+
+
+@app.command
+def version():
+    """Print version and exit"""
+    # Pseudo command, because the version always printed on every CLI call ;)
+    sys.exit(0)
+
+
+def main(args: Sequence[str] | None = None):
+    print_version(PyHardLinkBackup)
+
+    if len(sys.argv) >= 2:
+        # Check if we can just pass a command call to origin CLI:
+        command = sys.argv[1]
+        command_map = {
+            'test': run_unittest_cli,
+            'nox': run_nox,
+            'coverage': run_coverage,
+        }
+        if real_func := command_map.get(command):
+            real_func(argv=sys.argv, exit_after_run=True)
+
+    app.cli(
+        prog='./dev-cli.py',
+        description=constants.CLI_EPILOG,
+        use_underscores=False,  # use hyphens instead of underscores
+        sort_subcommands=True,
+        args=args,
+    )

PyHardLinkBackup/cli_dev/benchmark.py

@@ -0,0 +1,119 @@
+import collections
+import hashlib
+import logging
+import time
+from pathlib import Path
+
+from bx_py_utils.path import assert_is_dir
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def benchmark_hashes(
+    base_path: Path,
+    /,
+    max_duration: int = 30,  # in seconds
+    min_file_size: int = 15 * 1024,  # 15 KiB
+    max_file_size: int = 100 * 1024 * 1024,  # 100 MiB
+    verbosity: TyroVerbosityArgType = 1,
+) -> None:
+    """
+    Benchmark different file hashing algorithms on the given path.
+    """
+    # Example output:
+    #
+    # Total files hashed: 220, total size: 1187.7 MiB
+    #
+    # Results:
+    # Total file content read time: 1.7817s
+    #
+    # sha1       | Total: 0.6827s | 0.4x hash/read
+    # sha256     | Total: 0.7189s | 0.4x hash/read
+    # sha224     | Total: 0.7375s | 0.4x hash/read
+    # sha384     | Total: 1.6552s | 0.9x hash/read
+    # blake2b    | Total: 1.6708s | 0.9x hash/read
+    # md5        | Total: 1.6870s | 0.9x hash/read
+    # sha512     | Total: 1.7269s | 1.0x hash/read
+    # shake_128  | Total: 1.9834s | 1.1x hash/read
+    # sha3_224   | Total: 2.3006s | 1.3x hash/read
+    # sha3_256   | Total: 2.3856s | 1.3x hash/read
+    # shake_256  | Total: 2.4375s | 1.4x hash/read
+    # blake2s    | Total: 2.5219s | 1.4x hash/read
+    # sha3_384   | Total: 3.2596s | 1.8x hash/read
+    # sha3_512   | Total: 4.5328s | 2.5x hash/read
+    setup_logging(verbosity=verbosity)
+    assert_is_dir(base_path)
+    print(f'Benchmarking file hashes under: {base_path}')
+
+    print(f'Min file size: {min_file_size} bytes')
+    print(f'Max file size: {max_file_size} bytes')
+    print(f'Max duration: {max_duration} seconds')
+
+    algorithms = sorted(hashlib.algorithms_guaranteed)
+    print(f'\nUsing {len(algorithms)} guaranteed algorithms: {algorithms}')
+    print('-' * 80)
+
+    file_count = 0
+    total_size = 0
+    total_read_time = 0.0
+    results = collections.defaultdict(set)
+
+    start_time = time.time()
+    stop_time = start_time + max_duration
+    next_update = start_time + 2
+
+    for dir_entry in iter_scandir_files(base_path):
+        entry_stat = dir_entry.stat()
+        file_size = entry_stat.st_size
+        if not (min_file_size <= file_size <= max_file_size):
+            continue
+
+        start_time = time.perf_counter()
+        file_content = Path(dir_entry.path).read_bytes()
+        duration = time.perf_counter() - start_time
+        total_read_time += duration
+
+        for algo in algorithms:
+            # Actual measurement:
+            start_time = time.perf_counter()
+            hashlib.new(algo, file_content)
+            duration = time.perf_counter() - start_time
+
+            results[algo].add(duration)
+
+        file_count += 1
+        total_size += entry_stat.st_size
+
+        now = time.time()
+        if now >= stop_time:
+            print('Reached max duration limit, stopping benchmark...')
+            break
+
+        if now >= next_update:
+            percent = (now - (stop_time - max_duration)) / max_duration * 100
+            print(
+                f'{int(percent)}% Processed {file_count} files so far,'
+                f' total size: {total_size / 1024 / 1024:.1f} MiB...'
+            )
+            next_update = now + 2
+
+    print(f'\nTotal files hashed: {file_count}, total size: {total_size / 1024 / 1024:.1f} MiB')
+
+    print('\nResults:')
+    print(f'Total file content read time: {total_read_time:.4f}s\n')
+
+    sorted_results = sorted(
+        ((algo, sum(durations)) for algo, durations in results.items()),
+        key=lambda x: x[1],  # Sort by total_duration
+    )
+    for algo, total_duration in sorted_results:
+        ratio = total_duration / total_read_time
+        print(f'{algo:10} | Total: {total_duration:.4f}s | {ratio:.1f}x hash/read')

PyHardLinkBackup/cli_dev/code_style.py

@@ -0,0 +1,12 @@
+from cli_base.cli_tools.code_style import assert_code_style
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def lint(verbosity: TyroVerbosityArgType = 1):
+    """
+    Check/fix code style by run: "ruff check --fix"
+    """
+    assert_code_style(package_root=PACKAGE_ROOT, verbose=bool(verbosity), sys_exit=True)

PyHardLinkBackup/cli_dev/packaging.py

@@ -0,0 +1,65 @@
+import logging
+
+from cli_base.cli_tools.dev_tools import run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import ToolsExecutor
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.run_pip_audit import run_pip_audit
+from cli_base.tyro_commands import TyroVerbosityArgType
+from manageprojects.utilities.publish import publish_package
+
+import PyHardLinkBackup
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def install():
+    """
+    Install requirements and 'PyHardLinkBackup' via pip as editable.
+    """
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+    tools_executor.verbose_check_call('uv', 'sync')
+    tools_executor.verbose_check_call('pip', 'install', '--no-deps', '-e', '.')
+
+
+@app.command
+def pip_audit(verbosity: TyroVerbosityArgType):
+    """
+    Run pip-audit check against current requirements files
+    """
+    setup_logging(verbosity=verbosity)
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+
+@app.command
+def update(verbosity: TyroVerbosityArgType):
+    """
+    Update dependencies (uv.lock) and git pre-commit hooks
+    """
+    setup_logging(verbosity=verbosity)
+
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+
+    tools_executor.verbose_check_call('pip', 'install', '-U', 'pip')
+    tools_executor.verbose_check_call('pip', 'install', '-U', 'uv')
+    tools_executor.verbose_check_call('uv', 'lock', '--upgrade')
+
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+    # Install new dependencies in current .venv:
+    tools_executor.verbose_check_call('uv', 'sync')
+
+    # Update git pre-commit hooks:
+    tools_executor.verbose_check_call('pre-commit', 'autoupdate')
+
+
+@app.command
+def publish():
+    """
+    Build and upload this project to PyPi
+    """
+    run_unittest_cli(verbose=False, exit_after_run=False)  # Don't publish a broken state
+
+    publish_package(module=PyHardLinkBackup, package_path=PACKAGE_ROOT)

PyHardLinkBackup/cli_dev/shell_completion.py

@@ -0,0 +1,23 @@
+import logging
+
+from cli_base.cli_tools.shell_completion import setup_tyro_shell_completion
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def shell_completion(verbosity: TyroVerbosityArgType = 1, remove: bool = False) -> None:
+    """
+    Setup shell completion for this CLI (Currently only for bash shell)
+    """
+    setup_logging(verbosity=verbosity)
+    setup_tyro_shell_completion(
+        prog_name='PyHardLinkBackup_dev_cli',
+        remove=remove,
+    )

PyHardLinkBackup/cli_dev/testing.py

@@ -0,0 +1,52 @@
+from cli_base.cli_tools.dev_tools import run_coverage, run_nox, run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import verbose_check_call
+from cli_base.cli_tools.test_utils.snapshot import UpdateTestSnapshotFiles
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def mypy(verbosity: TyroVerbosityArgType):
+    """Run Mypy (configured in pyproject.toml)"""
+    verbose_check_call('mypy', '.', cwd=PACKAGE_ROOT, verbose=verbosity > 0, exit_on_error=True)
+
+
+@app.command
+def update_test_snapshot_files(verbosity: TyroVerbosityArgType):
+    """
+    Update all test snapshot files (by remove and recreate all snapshot files)
+    """
+    with UpdateTestSnapshotFiles(root_path=PACKAGE_ROOT, verbose=verbosity > 0):
+        # Just recreate them by running tests:
+        run_unittest_cli(
+            extra_env=dict(
+                RAISE_SNAPSHOT_ERRORS='0',  # Recreate snapshot files without error
+            ),
+            verbose=verbosity > 1,
+            exit_after_run=False,
+        )
+
+
+@app.command  # Dummy command
+def test():
+    """
+    Run unittests
+    """
+    run_unittest_cli()
+
+
+@app.command  # Dummy command
+def coverage():
+    """
+    Run tests and show coverage report.
+    """
+    run_coverage()
+
+
+@app.command  # Dummy "nox" command
+def nox():
+    """
+    Run nox
+    """
+    run_nox()

PyHardLinkBackup/cli_dev/update_readme_history.py

@@ -0,0 +1,33 @@
+import logging
+import sys
+from pathlib import Path
+
+from cli_base.cli_tools import git_history
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def update_readme_history(verbosity: TyroVerbosityArgType):
+    """
+    Update project history base on git commits/tags in README.md
+
+    Will be exited with 1 if the README.md was updated otherwise with 0.
+
+    Also, callable via e.g.:
+        python -m cli_base update-readme-history -v
+    """
+    setup_logging(verbosity=verbosity)
+
+    logger.debug('%s called. CWD: %s', __name__, Path.cwd())
+    updated = git_history.update_readme_history(verbosity=verbosity)
+    exit_code = 1 if updated else 0
+    if verbosity:
+        print(f'{exit_code=}')
+    sys.exit(exit_code)