PyHardLinkBackup 1.5.0__py3-none-any.whl

PyHardLinkBackup/__init__.py

@@ -0,0 +1,7 @@
+"""DocWrite: README.md # PyHardLinkBackup
+    HardLink/Deduplication Backups with Python
+"""
+
+# See https://packaging.python.org/en/latest/specifications/version-specifiers/
+__version__ = '1.5.0'
+__author__ = 'Jens Diemer <PyHardLinkBackup@jensdiemer.de>'

PyHardLinkBackup/__main__.py

@@ -0,0 +1,10 @@
+"""
+    Allow PyHardLinkBackup to be executable
+    through `python -m PyHardLinkBackup`.
+"""
+
+from PyHardLinkBackup.cli_app import main
+
+
+if __name__ == '__main__':
+    main()

PyHardLinkBackup/backup.py

@@ -0,0 +1,261 @@
+import dataclasses
+import datetime
+import logging
+import os
+import shutil
+import sys
+import time
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.constants import CHUNK_SIZE
+from PyHardLinkBackup.logging_setup import LoggingManager
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    copy_and_hash,
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+    read_and_hash_file,
+    supports_hardlinks,
+)
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
+from PyHardLinkBackup.utilities.sha256sums import store_hash
+from PyHardLinkBackup.utilities.tee import TeeStdoutContext
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class BackupResult:
+    backup_dir: Path
+    log_file: Path
+    #
+    backup_count: int = 0
+    backup_size: int = 0
+    #
+    symlink_files: int = 0
+    hardlinked_files: int = 0
+    hardlinked_size: int = 0
+    #
+    copied_files: int = 0
+    copied_size: int = 0
+    #
+    copied_small_files: int = 0
+    copied_small_size: int = 0
+    #
+    error_count: int = 0
+
+
+def backup_one_file(
+    *,
+    src_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    backup_dir: Path,
+    backup_result: BackupResult,
+) -> None:
+    backup_result.backup_count += 1
+    src_path = Path(entry.path)
+
+    dst_path = backup_dir / src_path.relative_to(src_root)
+    dst_dir_path = dst_path.parent
+    if not dst_dir_path.exists():
+        dst_dir_path.mkdir(parents=True, exist_ok=False)
+
+    try:
+        size = entry.stat().st_size
+    except FileNotFoundError:
+        # e.g.: Handle broken symlink
+        target = os.readlink(src_path)
+        dst_path.symlink_to(target)
+        backup_result.symlink_files += 1
+        return
+
+    backup_result.backup_size += size
+
+    if entry.name == 'SHA256SUMS':
+        # Skip existing SHA256SUMS files in source tree,
+        # because we create our own SHA256SUMS files.
+        logger.debug('Skip existing SHA256SUMS file: %s', src_path)
+        return
+
+    if entry.is_symlink():
+        logger.debug('Copy symlink: %s to %s', src_path, dst_path)
+        target = os.readlink(src_path)
+        dst_path.symlink_to(target)
+        backup_result.symlink_files += 1
+        return
+
+    # Process regular files
+    assert entry.is_file(follow_symlinks=False), f'Unexpected non-file: {src_path}'
+
+    # Deduplication logic
+
+    if size < size_db.MIN_SIZE:
+        # Small file -> always copy without deduplication
+        logger.info('Copy small file: %s to %s', src_path, dst_path)
+        file_hash = copy_and_hash(src_path, dst_path)
+        backup_result.copied_files += 1
+        backup_result.copied_size += size
+        backup_result.copied_small_files += 1
+        backup_result.copied_small_size += size
+        store_hash(dst_path, file_hash)
+        return
+
+    if size in size_db:
+        logger.debug('File with size %iBytes found before -> hash: %s', size, src_path)
+
+        if size <= CHUNK_SIZE:
+            # File can be read complete into memory
+            logger.debug('File size %iBytes <= CHUNK_SIZE (%iBytes) -> read complete into memory', size, CHUNK_SIZE)
+            file_content, file_hash = read_and_hash_file(src_path)
+            if existing_path := hash_db.get(file_hash):
+                logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                os.link(existing_path, dst_path)
+                backup_result.hardlinked_files += 1
+                backup_result.hardlinked_size += size
+            else:
+                logger.info('Store unique file: %s to %s', src_path, dst_path)
+                dst_path.write_bytes(file_content)
+                hash_db[file_hash] = dst_path
+                backup_result.copied_files += 1
+                backup_result.copied_size += size
+
+        else:
+            # Large file
+            file_hash = hash_file(src_path)  # Calculate hash without copying
+
+            if existing_path := hash_db.get(file_hash):
+                logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                os.link(existing_path, dst_path)
+                backup_result.hardlinked_files += 1
+                backup_result.hardlinked_size += size
+            else:
+                logger.info('Copy unique file: %s to %s', src_path, dst_path)
+                file_hash = copy_and_hash(src_path, dst_path)
+                hash_db[file_hash] = dst_path
+                backup_result.copied_files += 1
+                backup_result.copied_size += size
+
+        # Keep original file metadata (permission bits, time stamps, and flags)
+        shutil.copystat(src_path, dst_path)
+    else:
+        # A file with this size not backuped before -> Can't be duplicate -> copy and hash
+        file_hash = copy_and_hash(src_path, dst_path)
+        size_db.add(size)
+        hash_db[file_hash] = dst_path
+        backup_result.copied_files += 1
+        backup_result.copied_size += size
+
+    store_hash(dst_path, file_hash)
+
+
+def backup_tree(
+    *,
+    src_root: Path,
+    backup_root: Path,
+    excludes: tuple[str, ...],
+    log_manager: LoggingManager,
+) -> BackupResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    if not backup_root.is_dir():
+        print('Error: Backup directory does not exist!')
+        print(f'Please create "{backup_root}" directory first and start again!\n')
+        sys.exit(1)
+
+    if not os.access(backup_root, os.W_OK):
+        print('Error: No write access to backup directory!')
+        print(f'Please check permissions for backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    if not supports_hardlinks(backup_root):
+        print('Error: Filesystem for backup directory does not support hardlinks!')
+        print(f'Please check backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    # Step 1: Scan source directory:
+    excludes: set = set(excludes)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        src_file_count, src_total_size = humanized_fs_scan(src_root, excludes=excludes)
+
+    phlb_conf_dir = backup_root / '.phlb'
+    phlb_conf_dir.mkdir(parents=False, exist_ok=True)
+
+    timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    backup_main_dir = backup_root / src_root.name
+    backup_dir = backup_main_dir / timestamp
+    backup_dir.mkdir(parents=True, exist_ok=False)
+
+    log_file = backup_main_dir / f'{timestamp}-backup.log'
+    log_manager.start_file_logging(log_file)
+
+    logger.info('Backup %s to %s', src_root, backup_dir)
+
+    print(f'\nBackup to {backup_dir}...\n')
+
+    with DisplayFileTreeProgress(src_file_count, src_total_size) as progress:
+        # "Databases" for deduplication
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        backup_result = BackupResult(backup_dir=backup_dir, log_file=log_file)
+
+        next_update = 0
+        for entry in iter_scandir_files(src_root, excludes=excludes):
+            try:
+                backup_one_file(
+                    src_root=src_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    backup_dir=backup_dir,
+                    backup_result=backup_result,
+                )
+            except Exception as err:
+                logger.exception(f'Backup {entry.path} {err.__class__.__name__}: {err}')
+                backup_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(
+                        completed_file_count=backup_result.backup_count, completed_size=backup_result.backup_size
+                    )
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(completed_file_count=backup_result.backup_count, completed_size=backup_result.backup_size)
+
+    summary_file = backup_main_dir / f'{timestamp}-summary.txt'
+    with TeeStdoutContext(summary_file):
+        print(f'\nBackup complete: {backup_dir} (total size {human_filesize(backup_result.backup_size)})\n')
+        print(f'  Total files processed: {backup_result.backup_count}')
+        print(f'   * Symlinked files: {backup_result.symlink_files}')
+        print(
+            f'   * Hardlinked files: {backup_result.hardlinked_files}'
+            f' (saved {human_filesize(backup_result.hardlinked_size)})'
+        )
+        print(f'   * Copied files: {backup_result.copied_files} (total {human_filesize(backup_result.copied_size)})')
+        print(
+            f'     of which small (<{size_db.MIN_SIZE} Bytes)'
+            f' files: {backup_result.copied_small_files}'
+            f' (total {human_filesize(backup_result.copied_small_size)})'
+        )
+        if backup_result.error_count > 0:
+            print(f'  Errors during backup: {backup_result.error_count} (see log for details)')
+        print()
+
+    logger.info('Backup completed. Summary created: %s', summary_file)
+
+    return backup_result

PyHardLinkBackup/cli_app/__init__.py

@@ -0,0 +1,41 @@
+"""
+    CLI for usage
+"""
+
+import logging
+import sys
+from collections.abc import Sequence
+
+from cli_base.autodiscover import import_all_files
+from cli_base.cli_tools.version_info import print_version
+from rich import print  # noqa
+from tyro.extras import SubcommandApp
+
+import PyHardLinkBackup
+from PyHardLinkBackup import constants
+
+
+logger = logging.getLogger(__name__)
+
+app = SubcommandApp()
+
+# Register all CLI commands, just by import all files in this package:
+import_all_files(package=__package__, init_file=__file__)
+
+
+@app.command
+def version():
+    """Print version and exit"""
+    # Pseudo command, because the version always printed on every CLI call ;)
+    sys.exit(0)
+
+
+def main(args: Sequence[str] | None = None):
+    print_version(PyHardLinkBackup)
+    app.cli(
+        prog='./cli.py',
+        description=constants.CLI_EPILOG,
+        use_underscores=False,  # use hyphens instead of underscores
+        sort_subcommands=True,
+        args=args,
+    )

PyHardLinkBackup/cli_app/phlb.py

@@ -0,0 +1,123 @@
+import logging
+from pathlib import Path
+from typing import Annotated
+
+import tyro
+from rich import print  # noqa
+
+from PyHardLinkBackup import compare_backup, rebuild_databases
+from PyHardLinkBackup.backup import backup_tree
+from PyHardLinkBackup.cli_app import app
+from PyHardLinkBackup.logging_setup import (
+    DEFAULT_CONSOLE_LOG_LEVEL,
+    DEFAULT_LOG_FILE_LEVEL,
+    LoggingManager,
+    TyroConsoleLogLevelArgType,
+    TyroLogFileLevelArgType,
+)
+from PyHardLinkBackup.utilities.tyro_cli_shared_args import (
+    DEFAULT_EXCLUDE_DIRECTORIES,
+    TyroExcludeDirectoriesArgType,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def backup(
+    src: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='source',
+            help='Source directory to back up.',
+        ),
+    ],
+    dst: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='destination',
+            help='Destination directory for the backup.',
+        ),
+    ],
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
+    log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
+) -> None:
+    """
+    Backup the source directory to the destination directory using hard links for deduplication.
+    """
+    log_manager = LoggingManager(
+        console_level=verbosity,
+        file_level=log_file_level,
+    )
+    backup_tree(
+        src_root=src,
+        backup_root=dst,
+        excludes=excludes,
+        log_manager=log_manager,
+    )
+
+
+@app.command
+def compare(
+    src: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='source',
+            help='Source directory that should be compared with the last backup.',
+        ),
+    ],
+    dst: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='destination',
+            help='Destination directory with the backups. Will pick the last backup for comparison.',
+        ),
+    ],
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
+    log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
+) -> None:
+    """
+    Compares a source tree with the last backup and validates all known file hashes.
+    """
+    log_manager = LoggingManager(
+        console_level=verbosity,
+        file_level=log_file_level,
+    )
+    compare_backup.compare_tree(
+        src_root=src,
+        backup_root=dst,
+        excludes=excludes,
+        log_manager=log_manager,
+    )
+
+
+@app.command
+def rebuild(
+    backup_root: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='backup-directory',
+            help='Root directory of the the backups.',
+        ),
+    ],
+    /,
+    verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
+    log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
+) -> None:
+    """
+    Rebuild the file hash and size database by scanning all backup files. And also verify SHA256SUMS
+    and/or store missing hashes in SHA256SUMS files.
+    """
+    log_manager = LoggingManager(
+        console_level=verbosity,
+        file_level=log_file_level,
+    )
+    rebuild_databases.rebuild(
+        backup_root=backup_root,
+        log_manager=log_manager,
+    )

PyHardLinkBackup/cli_dev/__init__.py

@@ -0,0 +1,70 @@
+"""
+    CLI for development
+"""
+
+import importlib
+import logging
+import sys
+from collections.abc import Sequence
+
+from bx_py_utils.path import assert_is_file
+from cli_base.autodiscover import import_all_files
+from cli_base.cli_tools.dev_tools import run_coverage, run_nox, run_unittest_cli
+from cli_base.cli_tools.version_info import print_version
+from typeguard import install_import_hook
+from tyro.extras import SubcommandApp
+
+import PyHardLinkBackup
+from PyHardLinkBackup import constants
+
+
+# Check type annotations via typeguard in all tests.
+# Sadly we must activate this here and can't do this in ./tests/__init__.py
+install_import_hook(packages=('PyHardLinkBackup',))
+
+# reload the module, after the typeguard import hook is activated:
+importlib.reload(PyHardLinkBackup)
+
+
+logger = logging.getLogger(__name__)
+
+
+PACKAGE_ROOT = constants.BASE_PATH.parent
+assert_is_file(PACKAGE_ROOT / 'pyproject.toml')  # Exists only in cloned git repo
+
+
+app = SubcommandApp()
+
+
+# Register all CLI commands, just by import all files in this package:
+import_all_files(package=__package__, init_file=__file__)
+
+
+@app.command
+def version():
+    """Print version and exit"""
+    # Pseudo command, because the version always printed on every CLI call ;)
+    sys.exit(0)
+
+
+def main(args: Sequence[str] | None = None):
+    print_version(PyHardLinkBackup)
+
+    if len(sys.argv) >= 2:
+        # Check if we can just pass a command call to origin CLI:
+        command = sys.argv[1]
+        command_map = {
+            'test': run_unittest_cli,
+            'nox': run_nox,
+            'coverage': run_coverage,
+        }
+        if real_func := command_map.get(command):
+            real_func(argv=sys.argv, exit_after_run=True)
+
+    app.cli(
+        prog='./dev-cli.py',
+        description=constants.CLI_EPILOG,
+        use_underscores=False,  # use hyphens instead of underscores
+        sort_subcommands=True,
+        args=args,
+    )

PyHardLinkBackup/cli_dev/benchmark.py

@@ -0,0 +1,138 @@
+import collections
+import hashlib
+import logging
+import time
+from pathlib import Path
+
+from bx_py_utils.path import assert_is_dir
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+from PyHardLinkBackup.utilities.filesystem import humanized_fs_scan, iter_scandir_files
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager
+from PyHardLinkBackup.utilities.tyro_cli_shared_args import DEFAULT_EXCLUDE_DIRECTORIES, TyroExcludeDirectoriesArgType
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def scan_benchmark(
+    base_path: Path,
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroVerbosityArgType = 1,
+) -> None:
+    """
+    Benchmark our filesystem scan routine.
+    """
+    setup_logging(verbosity=verbosity)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        humanized_fs_scan(path=base_path, excludes=set(excludes))
+
+
+@app.command
+def benchmark_hashes(
+    base_path: Path,
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    max_duration: int = 30,  # in seconds
+    min_file_size: int = 15 * 1024,  # 15 KiB
+    max_file_size: int = 100 * 1024 * 1024,  # 100 MiB
+    verbosity: TyroVerbosityArgType = 1,
+) -> None:
+    """
+    Benchmark different file hashing algorithms on the given path.
+    """
+    # Example output:
+    #
+    # Total files hashed: 220, total size: 1187.7 MiB
+    #
+    # Results:
+    # Total file content read time: 1.7817s
+    #
+    # sha1       | Total: 0.6827s | 0.4x hash/read
+    # sha256     | Total: 0.7189s | 0.4x hash/read
+    # sha224     | Total: 0.7375s | 0.4x hash/read
+    # sha384     | Total: 1.6552s | 0.9x hash/read
+    # blake2b    | Total: 1.6708s | 0.9x hash/read
+    # md5        | Total: 1.6870s | 0.9x hash/read
+    # sha512     | Total: 1.7269s | 1.0x hash/read
+    # shake_128  | Total: 1.9834s | 1.1x hash/read
+    # sha3_224   | Total: 2.3006s | 1.3x hash/read
+    # sha3_256   | Total: 2.3856s | 1.3x hash/read
+    # shake_256  | Total: 2.4375s | 1.4x hash/read
+    # blake2s    | Total: 2.5219s | 1.4x hash/read
+    # sha3_384   | Total: 3.2596s | 1.8x hash/read
+    # sha3_512   | Total: 4.5328s | 2.5x hash/read
+    setup_logging(verbosity=verbosity)
+    assert_is_dir(base_path)
+    print(f'Benchmarking file hashes under: {base_path}')
+
+    print(f'Min file size: {min_file_size} bytes')
+    print(f'Max file size: {max_file_size} bytes')
+    print(f'Max duration: {max_duration} seconds')
+
+    algorithms = sorted(hashlib.algorithms_guaranteed)
+    print(f'\nUsing {len(algorithms)} guaranteed algorithms: {algorithms}')
+    print('-' * 80)
+
+    file_count = 0
+    total_size = 0
+    total_read_time = 0.0
+    results = collections.defaultdict(set)
+
+    start_time = time.time()
+    stop_time = start_time + max_duration
+    next_update = start_time + 2
+
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        for dir_entry in iter_scandir_files(path=base_path, excludes=set(excludes)):
+            entry_stat = dir_entry.stat()
+            file_size = entry_stat.st_size
+            if not (min_file_size <= file_size <= max_file_size):
+                continue
+
+            start_time = time.perf_counter()
+            file_content = Path(dir_entry.path).read_bytes()
+            duration = time.perf_counter() - start_time
+            total_read_time += duration
+
+            for algo in algorithms:
+                # Actual measurement:
+                start_time = time.perf_counter()
+                hashlib.new(algo, file_content)
+                duration = time.perf_counter() - start_time
+
+                results[algo].add(duration)
+
+            file_count += 1
+            total_size += entry_stat.st_size
+
+            now = time.time()
+            if now >= stop_time:
+                print('Reached max duration limit, stopping benchmark...')
+                break
+
+            if now >= next_update:
+                percent = (now - (stop_time - max_duration)) / max_duration * 100
+                print(
+                    f'{int(percent)}% Processed {file_count} files so far,'
+                    f' total size: {total_size / 1024 / 1024:.1f} MiB...'
+                )
+                next_update = now + 2
+
+    print(f'\nTotal files hashed: {file_count}, total size: {total_size / 1024 / 1024:.1f} MiB')
+
+    print('\nResults:')
+    print(f'Total file content read time: {total_read_time:.4f}s\n')
+
+    sorted_results = sorted(
+        ((algo, sum(durations)) for algo, durations in results.items()),
+        key=lambda x: x[1],  # Sort by total_duration
+    )
+    for algo, total_duration in sorted_results:
+        ratio = total_duration / total_read_time
+        print(f'{algo:10} | Total: {total_duration:.4f}s | {ratio:.1f}x hash/read')

PyHardLinkBackup/cli_dev/code_style.py

@@ -0,0 +1,12 @@
+from cli_base.cli_tools.code_style import assert_code_style
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def lint(verbosity: TyroVerbosityArgType = 1):
+    """
+    Check/fix code style by run: "ruff check --fix"
+    """
+    assert_code_style(package_root=PACKAGE_ROOT, verbose=bool(verbosity), sys_exit=True)