PyHardLinkBackup 1.8.1__py3-none-any.whl

PyHardLinkBackup/logging_setup.py

@@ -0,0 +1,124 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Annotated, Literal
+
+import tyro
+from bx_py_utils.path import assert_is_dir
+from rich import (
+    get_console,
+    print,  # noqa
+)
+from rich.logging import RichHandler
+
+
+logger = logging.getLogger(__name__)
+
+LogLevelLiteral = Literal['debug', 'info', 'warning', 'error']
+
+
+TyroConsoleLogLevelArgType = Annotated[
+    LogLevelLiteral,
+    tyro.conf.arg(
+        help='Log level for console logging.',
+    ),
+]
+DEFAULT_CONSOLE_LOG_LEVEL: TyroConsoleLogLevelArgType = 'warning'
+
+
+TyroLogFileLevelArgType = Annotated[
+    LogLevelLiteral,
+    tyro.conf.arg(
+        help='Log level for the log file',
+    ),
+]
+DEFAULT_LOG_FILE_LEVEL: TyroLogFileLevelArgType = 'info'
+
+
+def log_level_name2int(level_name: str) -> int:
+    level_name = level_name.upper()
+    level_mapping = logging.getLevelNamesMapping()
+    try:
+        return level_mapping[level_name]
+    except KeyError as err:
+        raise ValueError(f'Invalid log level name: {level_name}') from err
+
+
+console = get_console()
+
+
+class LoggingManager:
+    def __init__(
+        self,
+        *,
+        console_level: TyroConsoleLogLevelArgType,
+        file_level: TyroLogFileLevelArgType,
+    ):
+        self.console_level_name = console_level
+        self.console_level: int = log_level_name2int(console_level)
+        self.file_level_name = file_level
+        self.file_level: int = log_level_name2int(file_level)
+
+        self.lowest_level = min(self.console_level, self.file_level)
+
+        if console_level == logging.DEBUG:
+            log_format = '(%(name)s) %(message)s'
+        else:
+            log_format = '%(message)s'
+
+        console.print(
+            f'(Set [bold]console[bold] log level: [cyan]{self.console_level_name}[/cyan])',
+            justify='right',
+        )
+        handler = RichHandler(console=console, omit_repeated_times=False)
+        handler.setLevel(self.console_level)
+        logging.basicConfig(
+            level=self.lowest_level,
+            format=log_format,
+            datefmt='[%x %X.%f]',
+            handlers=[handler],
+            force=True,
+        )
+        sys.excepthook = self.log_unhandled_exception
+
+    def start_file_logging(self, log_file: Path):
+        console.print(
+            f'(initialize log file [bold]{log_file}[/bold] with level: [cyan]{self.file_level_name}[/cyan])',
+            justify='right',
+        )
+
+        assert_is_dir(log_file.parent)
+
+        root_logger = logging.getLogger()
+
+        file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
+        file_handler.setLevel(self.file_level)
+
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S',
+        )
+        file_handler.setFormatter(formatter)
+
+        root_logger.addHandler(file_handler)
+
+    def log_unhandled_exception(self, exc_type, exc_value, exc_traceback):
+        if issubclass(exc_type, KeyboardInterrupt):
+            logger.info('Program interrupted by user (KeyboardInterrupt). Exiting...')
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+        else:
+            logger.exception(
+                'Unhandled exception occurred:',
+                exc_info=(exc_type, exc_value, exc_traceback),
+            )
+
+
+class NoopLoggingManager(LoggingManager):
+    """
+    Only for tests: A logging manager that does nothing.
+    """
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def start_file_logging(self, log_file: Path):
+        pass

PyHardLinkBackup/rebuild_databases.py

@@ -0,0 +1,217 @@
+import dataclasses
+import datetime
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+from PyHardLinkBackup.logging_setup import LoggingManager
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import hash_file, humanized_fs_scan, iter_scandir_files
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
+from PyHardLinkBackup.utilities.sha256sums import check_sha256sums, store_hash
+from PyHardLinkBackup.utilities.tee import TeeStdoutContext
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class RebuildResult:
+    process_count: int = 0
+    process_size: int = 0
+    #
+    added_size_count: int = 0
+    added_hash_count: int = 0
+    #
+    error_count: int = 0
+    #
+    hash_verified_count: int = 0
+    hash_mismatch_count: int = 0
+    hash_not_found_count: int = 0
+    #
+    unique_inode_count: int = 0
+    skip_by_inode_count: int = 0
+
+
+def rebuild_one_file(
+    *,
+    backup_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    seen_inodes: set,
+    skip_same_inode: bool,
+    rebuild_result: RebuildResult,
+    progress: DisplayFileTreeProgress,
+):
+    inode = entry.inode()
+    if inode not in seen_inodes:
+        seen_inodes.add(inode)
+    else:
+        if skip_same_inode:
+            rebuild_result.skip_by_inode_count += 1
+            # Update counters used in progress display:
+            rebuild_result.process_size += entry.stat().st_size
+            rebuild_result.process_count += 1
+            return
+
+    file_path = Path(entry.path)
+
+    # We should ignore all files in the root backup directory itself
+    # e.g.: Our *-summary.txt and *.log files
+    if file_path.parent == backup_root:
+        return
+
+    rebuild_result.process_count += 1
+
+    if entry.name == 'SHA256SUMS':
+        # Skip existing SHA256SUMS files
+        return
+
+    size = entry.stat().st_size
+    rebuild_result.process_size += size
+    if size < size_db.MIN_SIZE:
+        # Small files will never deduplicate, skip them
+        return
+
+    file_hash = hash_file(file_path, progress=progress, total_size=size)
+
+    if size not in size_db:
+        size_db.add(size)
+        rebuild_result.added_size_count += 1
+
+    if file_hash not in hash_db:
+        hash_db[file_hash] = file_path
+        rebuild_result.added_hash_count += 1
+
+    # We have calculated the current hash of the file,
+    # Let's check if we can verify it, too:
+    file_path = Path(entry.path)
+    compare_result = check_sha256sums(
+        file_path=file_path,
+        file_hash=file_hash,
+    )
+    if compare_result is True:
+        rebuild_result.hash_verified_count += 1
+    elif compare_result is False:
+        rebuild_result.hash_mismatch_count += 1
+    elif compare_result is None:
+        rebuild_result.hash_not_found_count += 1
+        store_hash(
+            file_path=file_path,
+            file_hash=file_hash,
+        )
+
+
+def rebuild(
+    backup_root: Path,
+    skip_same_inode: bool,
+    log_manager: LoggingManager,
+) -> RebuildResult:
+    backup_root = backup_root.resolve()
+    if not backup_root.is_dir():
+        print(f'Error: Backup directory "{backup_root}" does not exist!')
+        sys.exit(1)
+
+    phlb_conf_dir = backup_root / '.phlb'
+    if not phlb_conf_dir.is_dir():
+        print(
+            f'Error: Backup directory "{backup_root}" seems to be wrong:'
+            f' Our hidden ".phlb" configuration directory is missing!'
+        )
+        sys.exit(1)
+
+    timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    log_manager.start_file_logging(log_file=backup_root / f'{timestamp}-rebuild.log')
+
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        file_count, total_size = humanized_fs_scan(
+            path=backup_root,
+            one_file_system=False,
+            src_device_id=None,
+            excludes={'.phlb'},
+        )
+
+        # We should ignore all files in the root backup directory itself
+        # e.g.: Our *-summary.txt and *.log files
+        for file in backup_root.iterdir():
+            if file.is_file():
+                file_count -= 1
+                total_size -= file.stat().st_size
+
+    with DisplayFileTreeProgress(
+        description=f'Rebuild {backup_root}...',
+        total_file_count=file_count,
+        total_size=total_size,
+    ) as progress:
+        # "Databases" for deduplication
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        seen_inodes = set()
+
+        rebuild_result = RebuildResult()
+
+        next_update = 0
+        for entry in iter_scandir_files(
+            path=backup_root,
+            one_file_system=False,
+            src_device_id=None,
+            excludes={'.phlb'},
+        ):
+            try:
+                rebuild_one_file(
+                    backup_root=backup_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    seen_inodes=seen_inodes,
+                    skip_same_inode=skip_same_inode,
+                    rebuild_result=rebuild_result,
+                    progress=progress,
+                )
+            except Exception as err:
+                logger.exception(f'Backup {entry.path} {err.__class__.__name__}: {err}')
+                rebuild_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(
+                        completed_file_count=rebuild_result.process_count, completed_size=rebuild_result.process_size
+                    )
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(completed_file_count=rebuild_result.process_count, completed_size=rebuild_result.process_size)
+
+    rebuild_result.unique_inode_count = len(seen_inodes)
+
+    summary_file = backup_root / f'{timestamp}-rebuild-summary.txt'
+    with TeeStdoutContext(summary_file):
+        print(f'\nRebuild "{backup_root}" completed:')
+        print(f'  Total files processed: {rebuild_result.process_count}')
+        print(f'  Total size processed: {human_filesize(rebuild_result.process_size)}')
+
+        print(f'  Unique inodes count: {rebuild_result.unique_inode_count}')
+        print(f'  Skipped files by inode: {rebuild_result.skip_by_inode_count}')
+
+        print(f'  Added file size information entries: {rebuild_result.added_size_count}')
+        print(f'  Added file hash entries: {rebuild_result.added_hash_count}')
+
+        if rebuild_result.error_count > 0:
+            print(f'  Errors during rebuild: {rebuild_result.error_count} (see log for details)')
+
+        print('\nSHA256SUMS verification results:')
+        print(f'  Successfully verified files: {rebuild_result.hash_verified_count}')
+        print(f'  File hash mismatches: {rebuild_result.hash_mismatch_count}')
+        print(f'  File hashes not found, newly stored: {rebuild_result.hash_not_found_count}')
+
+        print()
+
+    logger.info('Rebuild completed. Summary created: %s', summary_file)
+
+    return rebuild_result

PyHardLinkBackup/tests/__init__.py

@@ -0,0 +1,36 @@
+import os
+import unittest.util
+from pathlib import Path
+
+from bx_py_utils.test_utils.deny_requests import deny_any_real_request
+from cli_base.cli_tools.verbosity import MAX_LOG_LEVEL, setup_logging
+from rich import print  # noqa
+from typeguard import install_import_hook
+
+
+# Check type annotations via typeguard in all tests:
+install_import_hook(packages=('PyHardLinkBackup',))
+
+
+def pre_configure_tests() -> None:
+    print(f'Configure unittests via "load_tests Protocol" from {Path(__file__).relative_to(Path.cwd())}')
+
+    # Hacky way to display more "assert"-Context in failing tests:
+    _MIN_MAX_DIFF = unittest.util._MAX_LENGTH - unittest.util._MIN_DIFF_LEN
+    unittest.util._MAX_LENGTH = int(os.environ.get('UNITTEST_MAX_LENGTH', 2000))
+    unittest.util._MIN_DIFF_LEN = unittest.util._MAX_LENGTH - _MIN_MAX_DIFF
+
+    # Deny any request via docket/urllib3 because tests they should mock all requests:
+    deny_any_real_request()
+
+    # Display DEBUG logs in tests:
+    setup_logging(verbosity=MAX_LOG_LEVEL)
+
+
+def load_tests(loader, tests, pattern):
+    """
+    Use unittest "load_tests Protocol" as a hook to setup test environment before running tests.
+    https://docs.python.org/3/library/unittest.html#load-tests-protocol
+    """
+    pre_configure_tests()
+    return loader.discover(start_dir=Path(__file__).parent, pattern=pattern)