PyHardLinkBackup 1.5.0__py3-none-any.whl

PyHardLinkBackup/cli_dev/packaging.py

@@ -0,0 +1,65 @@
+import logging
+
+from cli_base.cli_tools.dev_tools import run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import ToolsExecutor
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.run_pip_audit import run_pip_audit
+from cli_base.tyro_commands import TyroVerbosityArgType
+from manageprojects.utilities.publish import publish_package
+
+import PyHardLinkBackup
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def install():
+    """
+    Install requirements and 'PyHardLinkBackup' via pip as editable.
+    """
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+    tools_executor.verbose_check_call('uv', 'sync')
+    tools_executor.verbose_check_call('pip', 'install', '--no-deps', '-e', '.')
+
+
+@app.command
+def pip_audit(verbosity: TyroVerbosityArgType):
+    """
+    Run pip-audit check against current requirements files
+    """
+    setup_logging(verbosity=verbosity)
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+
+@app.command
+def update(verbosity: TyroVerbosityArgType):
+    """
+    Update dependencies (uv.lock) and git pre-commit hooks
+    """
+    setup_logging(verbosity=verbosity)
+
+    tools_executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+
+    tools_executor.verbose_check_call('pip', 'install', '-U', 'pip')
+    tools_executor.verbose_check_call('pip', 'install', '-U', 'uv')
+    tools_executor.verbose_check_call('uv', 'lock', '--upgrade')
+
+    run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
+
+    # Install new dependencies in current .venv:
+    tools_executor.verbose_check_call('uv', 'sync')
+
+    # Update git pre-commit hooks:
+    tools_executor.verbose_check_call('pre-commit', 'autoupdate')
+
+
+@app.command
+def publish():
+    """
+    Build and upload this project to PyPi
+    """
+    run_unittest_cli(verbose=False, exit_after_run=False)  # Don't publish a broken state
+
+    publish_package(module=PyHardLinkBackup, package_path=PACKAGE_ROOT)

PyHardLinkBackup/cli_dev/shell_completion.py

@@ -0,0 +1,23 @@
+import logging
+
+from cli_base.cli_tools.shell_completion import setup_tyro_shell_completion
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def shell_completion(verbosity: TyroVerbosityArgType = 1, remove: bool = False) -> None:
+    """
+    Setup shell completion for this CLI (Currently only for bash shell)
+    """
+    setup_logging(verbosity=verbosity)
+    setup_tyro_shell_completion(
+        prog_name='PyHardLinkBackup_dev_cli',
+        remove=remove,
+    )

PyHardLinkBackup/cli_dev/testing.py

@@ -0,0 +1,52 @@
+from cli_base.cli_tools.dev_tools import run_coverage, run_nox, run_unittest_cli
+from cli_base.cli_tools.subprocess_utils import verbose_check_call
+from cli_base.cli_tools.test_utils.snapshot import UpdateTestSnapshotFiles
+from cli_base.tyro_commands import TyroVerbosityArgType
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT, app
+
+
+@app.command
+def mypy(verbosity: TyroVerbosityArgType):
+    """Run Mypy (configured in pyproject.toml)"""
+    verbose_check_call('mypy', '.', cwd=PACKAGE_ROOT, verbose=verbosity > 0, exit_on_error=True)
+
+
+@app.command
+def update_test_snapshot_files(verbosity: TyroVerbosityArgType):
+    """
+    Update all test snapshot files (by remove and recreate all snapshot files)
+    """
+    with UpdateTestSnapshotFiles(root_path=PACKAGE_ROOT, verbose=verbosity > 0):
+        # Just recreate them by running tests:
+        run_unittest_cli(
+            extra_env=dict(
+                RAISE_SNAPSHOT_ERRORS='0',  # Recreate snapshot files without error
+            ),
+            verbose=verbosity > 1,
+            exit_after_run=False,
+        )
+
+
+@app.command  # Dummy command
+def test():
+    """
+    Run unittests
+    """
+    run_unittest_cli()
+
+
+@app.command  # Dummy command
+def coverage():
+    """
+    Run tests and show coverage report.
+    """
+    run_coverage()
+
+
+@app.command  # Dummy "nox" command
+def nox():
+    """
+    Run nox
+    """
+    run_nox()

PyHardLinkBackup/cli_dev/update_readme_history.py

@@ -0,0 +1,33 @@
+import logging
+import sys
+from pathlib import Path
+
+from cli_base.cli_tools import git_history
+from cli_base.cli_tools.verbosity import setup_logging
+from cli_base.tyro_commands import TyroVerbosityArgType
+from rich import print  # noqa
+
+from PyHardLinkBackup.cli_dev import app
+
+
+logger = logging.getLogger(__name__)
+
+
+@app.command
+def update_readme_history(verbosity: TyroVerbosityArgType):
+    """
+    Update project history base on git commits/tags in README.md
+
+    Will be exited with 1 if the README.md was updated otherwise with 0.
+
+    Also, callable via e.g.:
+        python -m cli_base update-readme-history -v
+    """
+    setup_logging(verbosity=verbosity)
+
+    logger.debug('%s called. CWD: %s', __name__, Path.cwd())
+    updated = git_history.update_readme_history(verbosity=verbosity)
+    exit_code = 1 if updated else 0
+    if verbosity:
+        print(f'{exit_code=}')
+    sys.exit(exit_code)

PyHardLinkBackup/compare_backup.py

@@ -0,0 +1,212 @@
+import dataclasses
+import datetime
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.logging_setup import LoggingManager
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+)
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
+from PyHardLinkBackup.utilities.tee import TeeStdoutContext
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class CompareResult:
+    compare_dir: Path
+    log_file: Path
+    #
+    total_file_count: int = 0
+    total_size: int = 0
+    #
+    src_file_new_count: int = 0
+    file_size_missmatch: int = 0
+    file_hash_missmatch: int = 0
+    #
+    small_file_count: int = 0
+    size_db_missing_count: int = 0
+    hash_db_missing_count: int = 0
+    #
+    successful_file_count: int = 0
+    error_count: int = 0
+
+
+def compare_one_file(
+    *,
+    src_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    compare_dir: Path,
+    compare_result: CompareResult,
+) -> None:
+    src_size = entry.stat().st_size
+
+    # For the progress bars:
+    compare_result.total_file_count += 1
+    compare_result.total_size += src_size
+
+    src_path = Path(entry.path)
+    dst_path = compare_dir / src_path.relative_to(src_root)
+
+    if not dst_path.exists():
+        logger.warning('Source file %s not found in compare %s', src_path, dst_path)
+        compare_result.src_file_new_count += 1
+        return
+
+    dst_size = dst_path.stat().st_size
+    if src_size != dst_size:
+        logger.warning(
+            'Source file %s size (%i Bytes) differs from compare file %s size (%iBytes)',
+            src_path,
+            src_size,
+            dst_path,
+            dst_size,
+        )
+        compare_result.file_size_missmatch += 1
+        return
+
+    src_hash = hash_file(src_path)
+    dst_hash = hash_file(dst_path)
+
+    if src_hash != dst_hash:
+        logger.warning(
+            'Source file %s hash %r differs from compare file %s hash (%s)',
+            src_path,
+            src_hash,
+            dst_path,
+            dst_hash,
+        )
+        compare_result.file_hash_missmatch += 1
+        return
+
+    if src_size < size_db.MIN_SIZE:
+        # Small file -> Not in deduplication database
+        compare_result.small_file_count += 1
+    else:
+        if src_size not in size_db:
+            logger.warning(
+                'Source file %s size (%i Bytes) not found in deduplication database',
+                src_path,
+                src_size,
+            )
+            compare_result.size_db_missing_count += 1
+
+        if src_hash not in hash_db:
+            logger.warning(
+                'Source file %s hash %r not found in deduplication database',
+                src_path,
+                src_hash,
+            )
+            compare_result.hash_db_missing_count += 1
+
+    # Everything is ok
+    compare_result.successful_file_count += 1
+
+
+def compare_tree(
+    *,
+    src_root: Path,
+    backup_root: Path,
+    excludes: tuple[str, ...],
+    log_manager: LoggingManager,
+) -> CompareResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    phlb_conf_dir = backup_root / '.phlb'
+    if not phlb_conf_dir.is_dir():
+        print('Error: Compare directory seems to be wrong! (No .phlb configuration directory found)')
+        print(f'Please check backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    compare_main_dir = backup_root / src_root.name
+    timestamps = sorted(
+        path.name for path in compare_main_dir.iterdir() if path.is_dir() and path.name.startswith('20')
+    )
+    print(f'Found {len(timestamps)} compare(s) in {compare_main_dir}:')
+    for timestamp in timestamps:
+        print(f' * {timestamp}')
+    last_timestamp = timestamps[-1]
+    compare_dir = compare_main_dir / last_timestamp
+    print(f'\nComparing source tree {src_root} with {last_timestamp} compare:')
+    print(f'  {compare_dir}\n')
+
+    now_timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    log_file = compare_main_dir / f'{now_timestamp}-compare.log'
+    log_manager.start_file_logging(log_file)
+
+    excludes: set = set(excludes)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        src_file_count, src_total_size = humanized_fs_scan(src_root, excludes=excludes)
+
+    with DisplayFileTreeProgress(src_file_count, src_total_size) as progress:
+        # init "databases":
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        compare_result = CompareResult(compare_dir=compare_dir, log_file=log_file)
+
+        next_update = 0
+        for entry in iter_scandir_files(src_root, excludes=excludes):
+            try:
+                compare_one_file(
+                    src_root=src_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    compare_dir=compare_dir,
+                    compare_result=compare_result,
+                )
+            except Exception as err:
+                logger.exception(f'Compare {entry.path} {err.__class__.__name__}: {err}')
+                compare_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(
+                        completed_file_count=compare_result.total_file_count,
+                        completed_size=compare_result.total_size,
+                    )
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(completed_file_count=compare_result.total_file_count, completed_size=compare_result.total_size)
+
+    summary_file = compare_main_dir / f'{now_timestamp}-summary.txt'
+    with TeeStdoutContext(summary_file):
+        print(f'\nCompare complete: {compare_dir} (total size {human_filesize(compare_result.total_size)})\n')
+        print(f'  Total files processed: {compare_result.total_file_count}')
+        print(f'   * Successful compared files: {compare_result.successful_file_count}')
+        print(f'   * New source files: {compare_result.src_file_new_count}')
+        print(f'   * File size missmatch: {compare_result.file_size_missmatch}')
+        print(f'   * File hash missmatch: {compare_result.file_hash_missmatch}')
+
+        print(f'   * Small (<{size_db.MIN_SIZE} Bytes) files: {compare_result.small_file_count}')
+        print(f'   * Missing in size DB: {compare_result.size_db_missing_count}')
+        print(f'   * Missing in hash DB: {compare_result.hash_db_missing_count}')
+
+        if compare_result.error_count > 0:
+            print(f'  Errors during compare: {compare_result.error_count} (see log for details)')
+        print()
+
+    logger.info('Compare completed. Summary created: %s', summary_file)
+
+    return compare_result

PyHardLinkBackup/constants.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+import PyHardLinkBackup
+
+
+CLI_EPILOG = 'Project Homepage: https://github.com/jedie/PyHardLinkBackup'
+
+BASE_PATH = Path(PyHardLinkBackup.__file__).parent
+
+
+##########################################################################
+# "Settings" for PyHardLinkBackup:
+
+CHUNK_SIZE = 64 * 1024 * 1024  # 64 MB
+SMALL_FILE_THRESHOLD = 1000  # bytes
+HASH_ALGO = 'sha256'

PyHardLinkBackup/logging_setup.py

@@ -0,0 +1,124 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Annotated, Literal
+
+import tyro
+from bx_py_utils.path import assert_is_dir
+from rich import (
+    get_console,
+    print,  # noqa
+)
+from rich.logging import RichHandler
+
+
+logger = logging.getLogger(__name__)
+
+LogLevelLiteral = Literal['debug', 'info', 'warning', 'error']
+
+
+TyroConsoleLogLevelArgType = Annotated[
+    LogLevelLiteral,
+    tyro.conf.arg(
+        help='Log level for console logging.',
+    ),
+]
+DEFAULT_CONSOLE_LOG_LEVEL: TyroConsoleLogLevelArgType = 'warning'
+
+
+TyroLogFileLevelArgType = Annotated[
+    LogLevelLiteral,
+    tyro.conf.arg(
+        help='Log level for the log file',
+    ),
+]
+DEFAULT_LOG_FILE_LEVEL: TyroLogFileLevelArgType = 'info'
+
+
+def log_level_name2int(level_name: str) -> int:
+    level_name = level_name.upper()
+    level_mapping = logging.getLevelNamesMapping()
+    try:
+        return level_mapping[level_name]
+    except KeyError as err:
+        raise ValueError(f'Invalid log level name: {level_name}') from err
+
+
+console = get_console()
+
+
+class LoggingManager:
+    def __init__(
+        self,
+        *,
+        console_level: TyroConsoleLogLevelArgType,
+        file_level: TyroLogFileLevelArgType,
+    ):
+        self.console_level_name = console_level
+        self.console_level: int = log_level_name2int(console_level)
+        self.file_level_name = file_level
+        self.file_level: int = log_level_name2int(file_level)
+
+        self.lowest_level = min(self.console_level, self.file_level)
+
+        if console_level == logging.DEBUG:
+            log_format = '(%(name)s) %(message)s'
+        else:
+            log_format = '%(message)s'
+
+        console.print(
+            f'(Set [bold]console[bold] log level: [cyan]{self.console_level_name}[/cyan])',
+            justify='right',
+        )
+        handler = RichHandler(console=console, omit_repeated_times=False)
+        handler.setLevel(self.console_level)
+        logging.basicConfig(
+            level=self.lowest_level,
+            format=log_format,
+            datefmt='[%x %X.%f]',
+            handlers=[handler],
+            force=True,
+        )
+        sys.excepthook = self.log_unhandled_exception
+
+    def start_file_logging(self, log_file: Path):
+        console.print(
+            f'(initialize log file [bold]{log_file}[/bold] with level: [cyan]{self.file_level_name}[/cyan])',
+            justify='right',
+        )
+
+        assert_is_dir(log_file.parent)
+
+        root_logger = logging.getLogger()
+
+        file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
+        file_handler.setLevel(self.file_level)
+
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S',
+        )
+        file_handler.setFormatter(formatter)
+
+        root_logger.addHandler(file_handler)
+
+    def log_unhandled_exception(self, exc_type, exc_value, exc_traceback):
+        if issubclass(exc_type, KeyboardInterrupt):
+            logger.info('Program interrupted by user (KeyboardInterrupt). Exiting...')
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+        else:
+            logger.exception(
+                'Unhandled exception occurred:',
+                exc_info=(exc_type, exc_value, exc_traceback),
+            )
+
+
+class NoopLoggingManager(LoggingManager):
+    """
+    Only for tests: A logging manager that does nothing.
+    """
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def start_file_logging(self, log_file: Path):
+        pass