PyHardLinkBackup 1.8.1__py3-none-any.whl

PyHardLinkBackup/cli_dev/benchmark.py

@@ -0,0 +1,138 @@
+import collections
+import hashlib
+import logging
+import time
+from pathlib import Path
+
+from bx_py_utils.path import assert_is_dir
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+from PyHardLinkBackup.utilities.filesystem import humanized_fs_scan, iter_scandir_files
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager
+from PyHardLinkBackup.utilities.tyro_cli_shared_args import DEFAULT_EXCLUDE_DIRECTORIES, TyroExcludeDirectoriesArgType
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def scan_benchmark(
+    base_path: Path,
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroVerbosityArgType = 1,
+) -> None:
+    """
+    Benchmark our filesystem scan routine.
+    """
+    setup_logging(verbosity=verbosity)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        humanized_fs_scan(path=base_path, excludes=set(excludes))
+
+
+@app.command
+def benchmark_hashes(
+    base_path: Path,
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    max_duration: int = 30,  # in seconds
+    min_file_size: int = 15 * 1024,  # 15 KiB
+    max_file_size: int = 100 * 1024 * 1024,  # 100 MiB
+    verbosity: TyroVerbosityArgType = 1,
+) -> None:
+    """
+    Benchmark different file hashing algorithms on the given path.
+    """
+    # Example output:
+    #
+    # Total files hashed: 220, total size: 1187.7 MiB
+    #
+    # Results:
+    # Total file content read time: 1.7817s
+    #
+    # sha1       | Total: 0.6827s | 0.4x hash/read
+    # sha256     | Total: 0.7189s | 0.4x hash/read
+    # sha224     | Total: 0.7375s | 0.4x hash/read
+    # sha384     | Total: 1.6552s | 0.9x hash/read
+    # blake2b    | Total: 1.6708s | 0.9x hash/read
+    # md5        | Total: 1.6870s | 0.9x hash/read
+    # sha512     | Total: 1.7269s | 1.0x hash/read
+    # shake_128  | Total: 1.9834s | 1.1x hash/read
+    # sha3_224   | Total: 2.3006s | 1.3x hash/read
+    # sha3_256   | Total: 2.3856s | 1.3x hash/read
+    # shake_256  | Total: 2.4375s | 1.4x hash/read
+    # blake2s    | Total: 2.5219s | 1.4x hash/read
+    # sha3_384   | Total: 3.2596s | 1.8x hash/read
+    # sha3_512   | Total: 4.5328s | 2.5x hash/read
+    setup_logging(verbosity=verbosity)
+    assert_is_dir(base_path)
+    print(f'Benchmarking file hashes under: {base_path}')
+
+    print(f'Min file size: {min_file_size} bytes')
+    print(f'Max file size: {max_file_size} bytes')
+    print(f'Max duration: {max_duration} seconds')
+
+    algorithms = sorted(hashlib.algorithms_guaranteed)
+    print(f'\nUsing {len(algorithms)} guaranteed algorithms: {algorithms}')
+    print('-' * 80)
+
+    file_count = 0
+    total_size = 0
+    total_read_time = 0.0
+    results = collections.defaultdict(set)
+
+    start_time = time.time()
+    stop_time = start_time + max_duration
+    next_update = start_time + 2
+
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        for dir_entry in iter_scandir_files(path=base_path, excludes=set(excludes)):
+            entry_stat = dir_entry.stat()
+            file_size = entry_stat.st_size
+            if not (min_file_size <= file_size <= max_file_size):
+                continue
+
+            start_time = time.perf_counter()
+            file_content = Path(dir_entry.path).read_bytes()
+            duration = time.perf_counter() - start_time
+            total_read_time += duration
+
+            for algo in algorithms:
+                # Actual measurement:
+                start_time = time.perf_counter()
+                hashlib.new(algo, file_content)
+                duration = time.perf_counter() - start_time
+
+                results[algo].add(duration)
+
+            file_count += 1
+            total_size += entry_stat.st_size
+
+            now = time.time()
+            if now >= stop_time:
+                print('Reached max duration limit, stopping benchmark...')
+                break
+
+            if now >= next_update:
+                percent = (now - (stop_time - max_duration)) / max_duration * 100
+                print(
+                    f'{int(percent)}% Processed {file_count} files so far,'
+                    f' total size: {total_size / 1024 / 1024:.1f} MiB...'
+                )
+                next_update = now + 2
+
+    print(f'\nTotal files hashed: {file_count}, total size: {total_size / 1024 / 1024:.1f} MiB')
+
+    print('\nResults:')
+    print(f'Total file content read time: {total_read_time:.4f}s\n')
+
+    sorted_results = sorted(
+        ((algo, sum(durations)) for algo, durations in results.items()),
+        key=lambda x: x[1],  # Sort by total_duration
+    )
+    for algo, total_duration in sorted_results:
+        ratio = total_duration / total_read_time
+        print(f'{algo:10} | Total: {total_duration:.4f}s | {ratio:.1f}x hash/read')

PyHardLinkBackup/cli_dev/code_style.py

@@ -0,0 +1,12 @@
+from cli_base.cli_tools.code_style import assert_code_style
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def lint(verbosity: TyroVerbosityArgType = 1):
+    """
+    Check/fix code style by run: "ruff check --fix"
+    """
+    assert_code_style(package_root=PACKAGE_ROOT, verbose=bool(verbosity), sys_exit=True)

PyHardLinkBackup/cli_dev/debugging.py

@@ -0,0 +1,47 @@
+import logging
+from pathlib import Path
+
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files, verbose_path_stat
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager
+from PyHardLinkBackup.utilities.tyro_cli_shared_args import (
+    DEFAULT_EXCLUDE_DIRECTORIES,
+    TyroExcludeDirectoriesArgType,
+    TyroOneFileSystemArgType,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def fs_info(
+    path: Path,
+    /,
+    one_file_system: TyroOneFileSystemArgType = True,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroVerbosityArgType = 2,
+) -> None:
+    """
+    Display information about the filesystem under the given path.
+    """
+    setup_logging(verbosity=verbosity)
+    exclude_set = set(excludes)
+
+    src_path_stat = verbose_path_stat(path)
+    src_device_id = src_path_stat.st_dev
+
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        for entry in iter_scandir_files(
+            path=path,
+            one_file_system=one_file_system,
+            src_device_id=src_device_id,
+            excludes=exclude_set,
+        ):
+            entry_path = Path(entry.path)
+            entry_stat = verbose_path_stat(entry_path)
+            print(f'Size: {entry_stat.st_size} bytes, Inode: {entry_stat.st_ino}, File: {entry_path}')

PyHardLinkBackup/cli_dev/packaging.py

@@ -0,0 +1,62 @@
+import logging
+
+from cli_base.cli_tools.dev_tools import run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import ToolsExecutor
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.run_pip_audit import run_pip_audit
+from cli_base.tyro_commands import TyroVerbosityArgType
+from manageprojects.utilities.publish import publish_package
+
+import PyHardLinkBackup
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def install():
+    """
+    Install requirements and 'PyHardLinkBackup' via pip as editable.
+    """
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+    tools_executor.verbose_check_call('uv', 'sync')
+    tools_executor.verbose_check_call('uv', 'pip', 'install', '--no-deps', '-e', '.')
+
+
+@app.command
+def pip_audit(verbosity: TyroVerbosityArgType):
+    """
+    Run pip-audit check against current requirements files
+    """
+    setup_logging(verbosity=verbosity)
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+
+@app.command
+def update(verbosity: TyroVerbosityArgType):
+    """
+    Update dependencies (uv.lock) and git pre-commit hooks
+    """
+    setup_logging(verbosity=verbosity)
+
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+    tools_executor.verbose_check_call('uv', 'lock', '--upgrade')
+
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+    # Install new dependencies in current .venv:
+    tools_executor.verbose_check_call('uv', 'sync')
+
+    # Update git pre-commit hooks:
+    tools_executor.verbose_check_call('pre-commit', 'autoupdate')
+
+
+@app.command
+def publish():
+    """
+    Build and upload this project to PyPi
+    """
+    run_unittest_cli(verbose=False, exit_after_run=False)  # Don't publish a broken state
+
+    publish_package(module=PyHardLinkBackup, package_path=PACKAGE_ROOT)

PyHardLinkBackup/cli_dev/shell_completion.py

@@ -0,0 +1,23 @@
+import logging
+
+from cli_base.cli_tools.shell_completion import setup_tyro_shell_completion
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def shell_completion(verbosity: TyroVerbosityArgType = 1, remove: bool = False) -> None:
+    """
+    Setup shell completion for this CLI (Currently only for bash shell)
+    """
+    setup_logging(verbosity=verbosity)
+    setup_tyro_shell_completion(
+        prog_name='PyHardLinkBackup_dev_cli',
+        remove=remove,
+    )

PyHardLinkBackup/cli_dev/testing.py

@@ -0,0 +1,52 @@
+from cli_base.cli_tools.dev_tools import run_coverage, run_nox, run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import verbose_check_call
+from cli_base.cli_tools.test_utils.snapshot import UpdateTestSnapshotFiles
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def mypy(verbosity: TyroVerbosityArgType):
+    """Run Mypy (configured in pyproject.toml)"""
+    verbose_check_call('mypy', '.', cwd=PACKAGE_ROOT, verbose=verbosity > 0, exit_on_error=True)
+
+
+@app.command
+def update_test_snapshot_files(verbosity: TyroVerbosityArgType):
+    """
+    Update all test snapshot files (by remove and recreate all snapshot files)
+    """
+    with UpdateTestSnapshotFiles(root_path=PACKAGE_ROOT, verbose=verbosity > 0):
+        # Just recreate them by running tests:
+        run_unittest_cli(
+            extra_env=dict(
+                RAISE_SNAPSHOT_ERRORS='0',  # Recreate snapshot files without error
+            ),
+            verbose=verbosity > 1,
+            exit_after_run=False,
+        )
+
+
+@app.command  # Dummy command
+def test():
+    """
+    Run unittests
+    """
+    run_unittest_cli()
+
+
+@app.command  # Dummy command
+def coverage():
+    """
+    Run tests and show coverage report.
+    """
+    run_coverage()
+
+
+@app.command  # Dummy "nox" command
+def nox():
+    """
+    Run nox
+    """
+    run_nox()

PyHardLinkBackup/cli_dev/update_readme_history.py

@@ -0,0 +1,33 @@
+import logging
+import sys
+from pathlib import Path
+
+from cli_base.cli_tools import git_history
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def update_readme_history(verbosity: TyroVerbosityArgType):
+    """
+    Update project history base on git commits/tags in README.md
+
+    Will be exited with 1 if the README.md was updated otherwise with 0.
+
+    Also, callable via e.g.:
+        python -m cli_base update-readme-history -v
+    """
+    setup_logging(verbosity=verbosity)
+
+    logger.debug('%s called. CWD: %s', __name__, Path.cwd())
+    updated = git_history.update_readme_history(verbosity=verbosity)
+    exit_code = 1 if updated else 0
+    if verbosity:
+        print(f'{exit_code=}')
+    sys.exit(exit_code)

PyHardLinkBackup/compare_backup.py

@@ -0,0 +1,259 @@
+import dataclasses
+import datetime
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.logging_setup import LoggingManager
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+    verbose_path_stat,
+)
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
+from PyHardLinkBackup.utilities.tee import TeeStdoutContext
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class CompareResult:
+    last_timestamp: str
+    compare_dir: Path
+    log_file: Path
+    #
+    total_file_count: int = 0
+    total_size: int = 0
+    #
+    src_file_new_count: int = 0
+    file_size_missmatch: int = 0
+    file_hash_missmatch: int = 0
+    #
+    small_file_count: int = 0
+    size_db_missing_count: int = 0
+    hash_db_missing_count: int = 0
+    #
+    successful_file_count: int = 0
+    error_count: int = 0
+
+
+def compare_one_file(
+    *,
+    src_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    compare_dir: Path,
+    compare_result: CompareResult,
+    progress: DisplayFileTreeProgress,
+) -> None:
+    if entry.is_file():
+        # For the progress bars:
+        compare_result.total_file_count += 1
+
+    try:
+        src_size = entry.stat().st_size
+    except FileNotFoundError as err:
+        logger.warning(f'Broken symlink {entry.path}: {err.__class__.__name__}: {err}')
+        return
+
+    # For the progress bars:
+    compare_result.total_size += src_size
+
+    src_path = Path(entry.path)
+
+    dst_path = compare_dir / src_path.relative_to(src_root)
+
+    if not dst_path.exists():
+        logger.warning('Source file %s not found in compare %s', src_path, dst_path)
+        compare_result.src_file_new_count += 1
+        return
+
+    if src_path.is_dir():
+        if not src_path.is_symlink():
+            raise RuntimeError(f'Internal error - Directory found: {src_path=}')
+
+        # compare directory symlink targets:
+        src_target = src_path.readlink()
+        dst_target = dst_path.readlink()
+        if src_target != dst_target:
+            logger.warning(
+                'Source directory symlink %s target %s differs from compare symlink %s target %s',
+                src_path,
+                src_target,
+                dst_path,
+                dst_target,
+            )
+            compare_result.error_count += 1
+        return
+
+    dst_size = dst_path.stat().st_size
+    if src_size != dst_size:
+        logger.warning(
+            'Source file %s size (%i Bytes) differs from compare file %s size (%iBytes)',
+            src_path,
+            src_size,
+            dst_path,
+            dst_size,
+        )
+        compare_result.file_size_missmatch += 1
+        return
+
+    src_hash = hash_file(src_path, progress=progress, total_size=src_size)
+    dst_hash = hash_file(dst_path, progress=progress, total_size=dst_size)
+
+    if src_hash != dst_hash:
+        logger.warning(
+            'Source file %s hash %r differs from compare file %s hash (%s)',
+            src_path,
+            src_hash,
+            dst_path,
+            dst_hash,
+        )
+        compare_result.file_hash_missmatch += 1
+        return
+
+    if src_size < size_db.MIN_SIZE:
+        # Small file -> Not in deduplication database
+        compare_result.small_file_count += 1
+    else:
+        if src_size not in size_db:
+            logger.warning(
+                'Source file %s size (%i Bytes) not found in deduplication database',
+                src_path,
+                src_size,
+            )
+            compare_result.size_db_missing_count += 1
+
+        if src_hash not in hash_db:
+            logger.warning(
+                'Source file %s hash %r not found in deduplication database',
+                src_path,
+                src_hash,
+            )
+            compare_result.hash_db_missing_count += 1
+
+    # Everything is ok
+    compare_result.successful_file_count += 1
+
+
+def compare_tree(
+    *,
+    src_root: Path,
+    backup_root: Path,
+    one_file_system: bool,
+    excludes: tuple[str, ...],
+    log_manager: LoggingManager,
+) -> CompareResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    phlb_conf_dir = backup_root / '.phlb'
+    if not phlb_conf_dir.is_dir():
+        print('Error: Compare directory seems to be wrong! (No .phlb configuration directory found)')
+        print(f'Please check backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    compare_main_dir = backup_root / src_root.name
+    timestamps = sorted(
+        path.name for path in compare_main_dir.iterdir() if path.is_dir() and path.name.startswith('20')
+    )
+    print(f'Found {len(timestamps)} compare(s) in {compare_main_dir}:')
+    for timestamp in timestamps:
+        print(f' * {timestamp}')
+    last_timestamp = timestamps[-1]
+    compare_dir = compare_main_dir / last_timestamp
+    print(f'\nComparing source tree {src_root} with {last_timestamp} compare:')
+    print(f'  {compare_dir}\n')
+
+    now_timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    log_file = compare_main_dir / f'{now_timestamp}-compare.log'
+    log_manager.start_file_logging(log_file)
+
+    src_device_id = verbose_path_stat(src_root).st_dev
+
+    excludes: set = set(excludes)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        src_file_count, src_total_size = humanized_fs_scan(
+            path=src_root,
+            one_file_system=one_file_system,
+            src_device_id=src_device_id,
+            excludes=excludes,
+        )
+
+    with DisplayFileTreeProgress(
+        description=f'Compare {src_root}...',
+        total_file_count=src_file_count,
+        total_size=src_total_size,
+    ) as progress:
+        # init "databases":
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        compare_result = CompareResult(last_timestamp=last_timestamp, compare_dir=compare_dir, log_file=log_file)
+
+        next_update = 0
+        for entry in iter_scandir_files(
+            path=src_root,
+            one_file_system=one_file_system,
+            src_device_id=src_device_id,
+            excludes=excludes,
+        ):
+            try:
+                compare_one_file(
+                    src_root=src_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    compare_dir=compare_dir,
+                    compare_result=compare_result,
+                    progress=progress,
+                )
+            except Exception as err:
+                logger.exception(f'Compare {entry.path} {err.__class__.__name__}: {err}')
+                compare_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(
+                        completed_file_count=compare_result.total_file_count,
+                        advance_size=compare_result.total_size,
+                    )
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(completed_file_count=compare_result.total_file_count, advance_size=compare_result.total_size)
+
+    summary_file = compare_main_dir / f'{now_timestamp}-summary.txt'
+    with TeeStdoutContext(summary_file):
+        print(f'\nCompare complete: {compare_dir} (total size {human_filesize(compare_result.total_size)})\n')
+        print(f'  Total files processed: {compare_result.total_file_count}')
+        print(f'   * Successful compared files: {compare_result.successful_file_count}')
+        print(f'   * New source files: {compare_result.src_file_new_count}')
+        print(f'   * File size missmatch: {compare_result.file_size_missmatch}')
+        print(f'   * File hash missmatch: {compare_result.file_hash_missmatch}')
+
+        print(f'   * Small (<{size_db.MIN_SIZE} Bytes) files: {compare_result.small_file_count}')
+        print(f'   * Missing in size DB: {compare_result.size_db_missing_count}')
+        print(f'   * Missing in hash DB: {compare_result.hash_db_missing_count}')
+
+        if compare_result.error_count > 0:
+            print(f'  Errors during compare: {compare_result.error_count} (see log for details)')
+        print()
+
+    logger.info('Compare completed. Summary created: %s', summary_file)
+
+    return compare_result

PyHardLinkBackup/constants.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+
+import PyHardLinkBackup
+
+
+CLI_EPILOG = 'Project Homepage: https://github.com/jedie/PyHardLinkBackup'
+
+BASE_PATH = Path(PyHardLinkBackup.__file__).parent
+
+
+##########################################################################
+# "Settings" for PyHardLinkBackup:
+
+HASH_ALGO = 'sha256'
+SMALL_FILE_THRESHOLD = 1000  # bytes
+CHUNK_SIZE = 64 * 1024 * 1024  # 64 MB
+LAGE_FILE_PROGRESS_MIN_SIZE = CHUNK_SIZE * 3
+