PyHardLinkBackup 1.4.1__tar.gz → 1.5.0__tar.gz

{pyhardlinkbackup-1.4.1 → pyhardlinkbackup-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyHardLinkBackup
-Version: 1.4.1
+Version: 1.5.0
 Summary: HardLink/Deduplication Backups with Python
 Project-URL: Documentation, https://github.com/jedie/PyHardLinkBackup
 Project-URL: Source, https://github.com/jedie/PyHardLinkBackup
@@ -80,7 +80,7 @@ complete help for main CLI app:
 
 [comment]: <> (✂✂✂ auto generated main help start ✂✂✂)
 ```
-usage: phlb [-h] {backup,rebuild,version}
+usage: phlb [-h] {backup,compare,rebuild,version}
 
 
 
@@ -90,6 +90,7 @@ usage: phlb [-h] {backup,rebuild,version}
 ╭─ subcommands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮
 │ (required)                                                                                                           │
 │   • backup   Backup the source directory to the destination directory using hard links for deduplication.            │
+│   • compare  Compares a source tree with the last backup and validates all known file hashes.                        │
 │   • rebuild  Rebuild the file hash and size database by scanning all backup files. And also verify SHA256SUMS and/or │
 │              store missing hashes in SHA256SUMS files.                                                               │
 │   • version  Print version and exit                                                                                  │
@@ -231,6 +232,8 @@ Overview of main changes:
 
 [comment]: <> (✂✂✂ auto generated history start ✂✂✂)
 
+* [v1.5.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.4.1...v1.5.0)
+  * 2026-01-17 - NEW: Compare command to verify source tree with last backup
 * [v1.4.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.4.0...v1.4.1)
   * 2026-01-16 - Bugfix large file handling
 * [v1.4.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.3.0...v1.4.0)
@@ -243,15 +246,15 @@ Overview of main changes:
   * 2026-01-15 - Add tests for rebuild
   * 2026-01-15 - Add command to "rebuld" the size and hash filesystem database
   * 2026-01-15 - Add screenshots in the README
+
+<details><summary>Expand older history entries ...</summary>
+
 * [v1.2.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.1.0...v1.2.0)
   * 2026-01-15 - Add error handling: Log exception but continue with the backup
   * 2026-01-15 - Check permission and hadlink support on destination path
   * 2026-01-14 - Enhance progress bars
   * 2026-01-14 - A a note to rsync --link-dest
   * 2026-01-14 - Use cli_base.cli_tools.test_utils.base_testcases
-
-<details><summary>Expand older history entries ...</summary>
-
 * [v1.1.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.1...v1.1.0)
   * 2026-01-14 - Change backup timestamp directory to old schema: '%Y-%m-%d-%H%M%S'
   * 2026-01-14 - Add "Overview of main changes" to README

{pyhardlinkbackup-1.4.1 → pyhardlinkbackup-1.5.0}/PyHardLinkBackup/__init__.py

@@ -3,5 +3,5 @@
 """
 
 # See https://packaging.python.org/en/latest/specifications/version-specifiers/
-__version__ = '1.4.1'
+__version__ = '1.5.0'
 __author__ = 'Jens Diemer <PyHardLinkBackup@jensdiemer.de>'

{pyhardlinkbackup-1.4.1 → pyhardlinkbackup-1.5.0}/PyHardLinkBackup/cli_app/phlb.py

@@ -5,7 +5,7 @@ from typing import Annotated
 import tyro
 from rich import print  # noqa
 
-from PyHardLinkBackup import rebuild_databases
+from PyHardLinkBackup import compare_backup, rebuild_databases
 from PyHardLinkBackup.backup import backup_tree
 from PyHardLinkBackup.cli_app import app
 from PyHardLinkBackup.logging_setup import (
@@ -60,6 +60,42 @@ def backup(
     )
 
 
+@app.command
+def compare(
+    src: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='source',
+            help='Source directory that should be compared with the last backup.',
+        ),
+    ],
+    dst: Annotated[
+        Path,
+        tyro.conf.arg(
+            metavar='destination',
+            help='Destination directory with the backups. Will pick the last backup for comparison.',
+        ),
+    ],
+    /,
+    excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
+    verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
+    log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
+) -> None:
+    """
+    Compares a source tree with the last backup and validates all known file hashes.
+    """
+    log_manager = LoggingManager(
+        console_level=verbosity,
+        file_level=log_file_level,
+    )
+    compare_backup.compare_tree(
+        src_root=src,
+        backup_root=dst,
+        excludes=excludes,
+        log_manager=log_manager,
+    )
+
+
 @app.command
 def rebuild(
     backup_root: Annotated[

pyhardlinkbackup-1.5.0/PyHardLinkBackup/compare_backup.py

@@ -0,0 +1,212 @@
+import dataclasses
+import datetime
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.logging_setup import LoggingManager
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+)
+from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
+from PyHardLinkBackup.utilities.tee import TeeStdoutContext
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class CompareResult:
+    compare_dir: Path
+    log_file: Path
+    #
+    total_file_count: int = 0
+    total_size: int = 0
+    #
+    src_file_new_count: int = 0
+    file_size_missmatch: int = 0
+    file_hash_missmatch: int = 0
+    #
+    small_file_count: int = 0
+    size_db_missing_count: int = 0
+    hash_db_missing_count: int = 0
+    #
+    successful_file_count: int = 0
+    error_count: int = 0
+
+
+def compare_one_file(
+    *,
+    src_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    compare_dir: Path,
+    compare_result: CompareResult,
+) -> None:
+    src_size = entry.stat().st_size
+
+    # For the progress bars:
+    compare_result.total_file_count += 1
+    compare_result.total_size += src_size
+
+    src_path = Path(entry.path)
+    dst_path = compare_dir / src_path.relative_to(src_root)
+
+    if not dst_path.exists():
+        logger.warning('Source file %s not found in compare %s', src_path, dst_path)
+        compare_result.src_file_new_count += 1
+        return
+
+    dst_size = dst_path.stat().st_size
+    if src_size != dst_size:
+        logger.warning(
+            'Source file %s size (%i Bytes) differs from compare file %s size (%iBytes)',
+            src_path,
+            src_size,
+            dst_path,
+            dst_size,
+        )
+        compare_result.file_size_missmatch += 1
+        return
+
+    src_hash = hash_file(src_path)
+    dst_hash = hash_file(dst_path)
+
+    if src_hash != dst_hash:
+        logger.warning(
+            'Source file %s hash %r differs from compare file %s hash (%s)',
+            src_path,
+            src_hash,
+            dst_path,
+            dst_hash,
+        )
+        compare_result.file_hash_missmatch += 1
+        return
+
+    if src_size < size_db.MIN_SIZE:
+        # Small file -> Not in deduplication database
+        compare_result.small_file_count += 1
+    else:
+        if src_size not in size_db:
+            logger.warning(
+                'Source file %s size (%i Bytes) not found in deduplication database',
+                src_path,
+                src_size,
+            )
+            compare_result.size_db_missing_count += 1
+
+        if src_hash not in hash_db:
+            logger.warning(
+                'Source file %s hash %r not found in deduplication database',
+                src_path,
+                src_hash,
+            )
+            compare_result.hash_db_missing_count += 1
+
+    # Everything is ok
+    compare_result.successful_file_count += 1
+
+
+def compare_tree(
+    *,
+    src_root: Path,
+    backup_root: Path,
+    excludes: tuple[str, ...],
+    log_manager: LoggingManager,
+) -> CompareResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    phlb_conf_dir = backup_root / '.phlb'
+    if not phlb_conf_dir.is_dir():
+        print('Error: Compare directory seems to be wrong! (No .phlb configuration directory found)')
+        print(f'Please check backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    compare_main_dir = backup_root / src_root.name
+    timestamps = sorted(
+        path.name for path in compare_main_dir.iterdir() if path.is_dir() and path.name.startswith('20')
+    )
+    print(f'Found {len(timestamps)} compare(s) in {compare_main_dir}:')
+    for timestamp in timestamps:
+        print(f' * {timestamp}')
+    last_timestamp = timestamps[-1]
+    compare_dir = compare_main_dir / last_timestamp
+    print(f'\nComparing source tree {src_root} with {last_timestamp} compare:')
+    print(f'  {compare_dir}\n')
+
+    now_timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    log_file = compare_main_dir / f'{now_timestamp}-compare.log'
+    log_manager.start_file_logging(log_file)
+
+    excludes: set = set(excludes)
+    with PrintTimingContextManager('Filesystem scan completed in'):
+        src_file_count, src_total_size = humanized_fs_scan(src_root, excludes=excludes)
+
+    with DisplayFileTreeProgress(src_file_count, src_total_size) as progress:
+        # init "databases":
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        compare_result = CompareResult(compare_dir=compare_dir, log_file=log_file)
+
+        next_update = 0
+        for entry in iter_scandir_files(src_root, excludes=excludes):
+            try:
+                compare_one_file(
+                    src_root=src_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    compare_dir=compare_dir,
+                    compare_result=compare_result,
+                )
+            except Exception as err:
+                logger.exception(f'Compare {entry.path} {err.__class__.__name__}: {err}')
+                compare_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(
+                        completed_file_count=compare_result.total_file_count,
+                        completed_size=compare_result.total_size,
+                    )
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(completed_file_count=compare_result.total_file_count, completed_size=compare_result.total_size)
+
+    summary_file = compare_main_dir / f'{now_timestamp}-summary.txt'
+    with TeeStdoutContext(summary_file):
+        print(f'\nCompare complete: {compare_dir} (total size {human_filesize(compare_result.total_size)})\n')
+        print(f'  Total files processed: {compare_result.total_file_count}')
+        print(f'   * Successful compared files: {compare_result.successful_file_count}')
+        print(f'   * New source files: {compare_result.src_file_new_count}')
+        print(f'   * File size missmatch: {compare_result.file_size_missmatch}')
+        print(f'   * File hash missmatch: {compare_result.file_hash_missmatch}')
+
+        print(f'   * Small (<{size_db.MIN_SIZE} Bytes) files: {compare_result.small_file_count}')
+        print(f'   * Missing in size DB: {compare_result.size_db_missing_count}')
+        print(f'   * Missing in hash DB: {compare_result.hash_db_missing_count}')
+
+        if compare_result.error_count > 0:
+            print(f'  Errors during compare: {compare_result.error_count} (see log for details)')
+        print()
+
+    logger.info('Compare completed. Summary created: %s', summary_file)
+
+    return compare_result

pyhardlinkbackup-1.5.0/PyHardLinkBackup/tests/test_compare_backup.py

@@ -0,0 +1,86 @@
+import shutil
+import tempfile
+from pathlib import Path
+from unittest import TestCase
+
+from bx_py_utils.test_utils.redirect import RedirectOut
+from cli_base.cli_tools.test_utils.base_testcases import OutputMustCapturedTestCaseMixin
+
+from PyHardLinkBackup.compare_backup import CompareResult, LoggingManager, compare_tree
+from PyHardLinkBackup.logging_setup import DEFAULT_LOG_FILE_LEVEL
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import hash_file
+
+
+class CompareBackupTestCase(OutputMustCapturedTestCaseMixin, TestCase):
+    def test_happy_path(self):
+        with tempfile.TemporaryDirectory() as src_dir, tempfile.TemporaryDirectory() as backup_dir:
+            src_root = Path(src_dir).resolve()
+            backup_root = Path(backup_dir).resolve()
+
+            # Setup backup structure
+            phlb_conf_dir = backup_root / '.phlb'
+            phlb_conf_dir.mkdir()
+
+            compare_main_dir = backup_root / src_root.name
+            compare_main_dir.mkdir()
+
+            timestamp = '2026-01-17-120000'
+            compare_dir = compare_main_dir / timestamp
+            compare_dir.mkdir()
+
+            # Create source files
+            (src_root / 'small_file.txt').write_text('hello world')
+            (src_root / 'large_file_missing.txt').write_bytes(b'X' * FileSizeDatabase.MIN_SIZE)
+            large_file_in_dbs = src_root / 'large_file_in_dbs.txt'
+            large_file_in_dbs.write_bytes(b'Y' * (FileSizeDatabase.MIN_SIZE + 1))
+
+            # Copy files to backup
+            total_size = 0
+            total_file_count = 0
+            for file_path in src_root.iterdir():
+                shutil.copy2(file_path, compare_dir / file_path.name)
+                total_size += file_path.stat().st_size
+                total_file_count += 1
+            self.assertEqual(total_file_count, 3)
+            self.assertEqual(total_size, 2012)
+
+            # Create databases and add values from 'large_file_in_dbs.txt'
+            size_db = FileSizeDatabase(phlb_conf_dir)
+            size_db.add(FileSizeDatabase.MIN_SIZE + 1)
+            hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+            src_hash = hash_file(large_file_in_dbs)
+            hash_db[src_hash] = compare_dir / 'large_file_in_dbs.txt'
+
+            # Run compare_tree
+            with RedirectOut() as redirected_out:
+                result = compare_tree(
+                    src_root=src_root,
+                    backup_root=backup_root,
+                    excludes=(),
+                    log_manager=LoggingManager(
+                        console_level='info',
+                        file_level=DEFAULT_LOG_FILE_LEVEL,
+                    ),
+                )
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertIn('Compare completed.', redirected_out.stdout)
+            self.assertEqual(
+                result,
+                CompareResult(
+                    compare_dir=compare_dir,
+                    log_file=result.log_file,
+                    total_file_count=total_file_count,
+                    total_size=total_size,
+                    src_file_new_count=0,
+                    file_size_missmatch=0,
+                    file_hash_missmatch=0,
+                    small_file_count=1,
+                    size_db_missing_count=1,
+                    hash_db_missing_count=1,
+                    successful_file_count=total_file_count,
+                    error_count=0,
+                ),
+                redirected_out.stdout,
+            )

{pyhardlinkbackup-1.4.1 → pyhardlinkbackup-1.5.0}/README.md

@@ -65,7 +65,7 @@ complete help for main CLI app:
 
 [comment]: <> (✂✂✂ auto generated main help start ✂✂✂)
 ```
-usage: phlb [-h] {backup,rebuild,version}
+usage: phlb [-h] {backup,compare,rebuild,version}
 
 
 
@@ -75,6 +75,7 @@ usage: phlb [-h] {backup,rebuild,version}
 ╭─ subcommands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮
 │ (required)                                                                                                           │
 │   • backup   Backup the source directory to the destination directory using hard links for deduplication.            │
+│   • compare  Compares a source tree with the last backup and validates all known file hashes.                        │
 │   • rebuild  Rebuild the file hash and size database by scanning all backup files. And also verify SHA256SUMS and/or │
 │              store missing hashes in SHA256SUMS files.                                                               │
 │   • version  Print version and exit                                                                                  │
@@ -216,6 +217,8 @@ Overview of main changes:
 
 [comment]: <> (✂✂✂ auto generated history start ✂✂✂)
 
+* [v1.5.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.4.1...v1.5.0)
+  * 2026-01-17 - NEW: Compare command to verify source tree with last backup
 * [v1.4.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.4.0...v1.4.1)
   * 2026-01-16 - Bugfix large file handling
 * [v1.4.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.3.0...v1.4.0)
@@ -228,15 +231,15 @@ Overview of main changes:
   * 2026-01-15 - Add tests for rebuild
   * 2026-01-15 - Add command to "rebuld" the size and hash filesystem database
   * 2026-01-15 - Add screenshots in the README
+
+<details><summary>Expand older history entries ...</summary>
+
 * [v1.2.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.1.0...v1.2.0)
   * 2026-01-15 - Add error handling: Log exception but continue with the backup
   * 2026-01-15 - Check permission and hadlink support on destination path
   * 2026-01-14 - Enhance progress bars
   * 2026-01-14 - A a note to rsync --link-dest
   * 2026-01-14 - Use cli_base.cli_tools.test_utils.base_testcases
-
-<details><summary>Expand older history entries ...</summary>
-
 * [v1.1.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.1...v1.1.0)
   * 2026-01-14 - Change backup timestamp directory to old schema: '%Y-%m-%d-%H%M%S'
   * 2026-01-14 - Add "Overview of main changes" to README