PyHardLinkBackup 1.5.0__py3-none-any.whl

PyHardLinkBackup/tests/test_compare_backup.py

@@ -0,0 +1,86 @@
+import shutil
+import tempfile
+from pathlib import Path
+from unittest import TestCase
+
+from bx_py_utils.test_utils.redirect import RedirectOut
+from cli_base.cli_tools.test_utils.base_testcases import OutputMustCapturedTestCaseMixin
+
+from PyHardLinkBackup.compare_backup import CompareResult, LoggingManager, compare_tree
+from PyHardLinkBackup.logging_setup import DEFAULT_LOG_FILE_LEVEL
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import hash_file
+
+
+class CompareBackupTestCase(OutputMustCapturedTestCaseMixin, TestCase):
+    def test_happy_path(self):
+        with tempfile.TemporaryDirectory() as src_dir, tempfile.TemporaryDirectory() as backup_dir:
+            src_root = Path(src_dir).resolve()
+            backup_root = Path(backup_dir).resolve()
+
+            # Setup backup structure
+            phlb_conf_dir = backup_root / '.phlb'
+            phlb_conf_dir.mkdir()
+
+            compare_main_dir = backup_root / src_root.name
+            compare_main_dir.mkdir()
+
+            timestamp = '2026-01-17-120000'
+            compare_dir = compare_main_dir / timestamp
+            compare_dir.mkdir()
+
+            # Create source files
+            (src_root / 'small_file.txt').write_text('hello world')
+            (src_root / 'large_file_missing.txt').write_bytes(b'X' * FileSizeDatabase.MIN_SIZE)
+            large_file_in_dbs = src_root / 'large_file_in_dbs.txt'
+            large_file_in_dbs.write_bytes(b'Y' * (FileSizeDatabase.MIN_SIZE + 1))
+
+            # Copy files to backup
+            total_size = 0
+            total_file_count = 0
+            for file_path in src_root.iterdir():
+                shutil.copy2(file_path, compare_dir / file_path.name)
+                total_size += file_path.stat().st_size
+                total_file_count += 1
+            self.assertEqual(total_file_count, 3)
+            self.assertEqual(total_size, 2012)
+
+            # Create databases and add values from 'large_file_in_dbs.txt'
+            size_db = FileSizeDatabase(phlb_conf_dir)
+            size_db.add(FileSizeDatabase.MIN_SIZE + 1)
+            hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+            src_hash = hash_file(large_file_in_dbs)
+            hash_db[src_hash] = compare_dir / 'large_file_in_dbs.txt'
+
+            # Run compare_tree
+            with RedirectOut() as redirected_out:
+                result = compare_tree(
+                    src_root=src_root,
+                    backup_root=backup_root,
+                    excludes=(),
+                    log_manager=LoggingManager(
+                        console_level='info',
+                        file_level=DEFAULT_LOG_FILE_LEVEL,
+                    ),
+                )
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertIn('Compare completed.', redirected_out.stdout)
+            self.assertEqual(
+                result,
+                CompareResult(
+                    compare_dir=compare_dir,
+                    log_file=result.log_file,
+                    total_file_count=total_file_count,
+                    total_size=total_size,
+                    src_file_new_count=0,
+                    file_size_missmatch=0,
+                    file_hash_missmatch=0,
+                    small_file_count=1,
+                    size_db_missing_count=1,
+                    hash_db_missing_count=1,
+                    successful_file_count=total_file_count,
+                    error_count=0,
+                ),
+                redirected_out.stdout,
+            )

PyHardLinkBackup/tests/test_doc_write.py

@@ -0,0 +1,26 @@
+from unittest import TestCase
+
+from bx_py_utils.doc_write.api import GeneratedInfo, generate
+from bx_py_utils.path import assert_is_file
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT
+
+
+class DocuWriteApiTestCase(TestCase):
+    def test_up2date_docs(self):
+        """DocWrite: about-docs.md # generate Doc-Write
+
+        These documentation files are generated automatically with the "Doc-Write" tool.
+        They updated automatically by unittests.
+
+        More information about Doc-Write can be found here:
+
+        https://github.com/boxine/bx_py_utils/tree/master/bx_py_utils/doc_write
+        """
+        assert_is_file(PACKAGE_ROOT / 'pyproject.toml')
+
+        with self.assertLogs():
+            info: GeneratedInfo = generate(base_path=PACKAGE_ROOT)
+        self.assertGreaterEqual(len(info.paths), 1)
+        self.assertEqual(info.update_count, 0, 'No files should be updated, commit the changes')
+        self.assertEqual(info.remove_count, 0, 'No files should be removed, commit the changes')

PyHardLinkBackup/tests/test_doctests.py

@@ -0,0 +1,10 @@
+from bx_py_utils.test_utils.unittest_utils import BaseDocTests
+
+import PyHardLinkBackup
+
+
+class DocTests(BaseDocTests):
+    def test_doctests(self):
+        self.run_doctests(
+            modules=(PyHardLinkBackup,),
+        )

PyHardLinkBackup/tests/test_project_setup.py

@@ -0,0 +1,46 @@
+import subprocess
+from unittest import TestCase
+
+from bx_py_utils.path import assert_is_file
+from cli_base.cli_tools.code_style import assert_code_style
+from cli_base.cli_tools.subprocess_utils import ToolsExecutor
+from manageprojects.test_utils.project_setup import check_editor_config, get_py_max_line_length
+from packaging.version import Version
+
+from PyHardLinkBackup import __version__
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT
+
+
+class ProjectSetupTestCase(TestCase):
+    def test_version(self):
+        self.assertIsNotNone(__version__)
+
+        version = Version(__version__)  # Will raise InvalidVersion() if wrong formatted
+        self.assertEqual(str(version), __version__)
+
+        cli_bin = PACKAGE_ROOT / 'cli.py'
+        assert_is_file(cli_bin)
+
+        output = subprocess.check_output([cli_bin, 'version'], text=True)
+        self.assertIn(f'PyHardLinkBackup v{__version__}', output)
+
+        dev_cli_bin = PACKAGE_ROOT / 'dev-cli.py'
+        assert_is_file(dev_cli_bin)
+
+        output = subprocess.check_output([dev_cli_bin, 'version'], text=True)
+        self.assertIn(f'PyHardLinkBackup v{__version__}', output)
+
+    def test_code_style(self):
+        return_code = assert_code_style(package_root=PACKAGE_ROOT)
+        self.assertEqual(return_code, 0, 'Code style error, see output above!')
+
+    def test_check_editor_config(self):
+        check_editor_config(package_root=PACKAGE_ROOT)
+
+        max_line_length = get_py_max_line_length(package_root=PACKAGE_ROOT)
+        self.assertEqual(max_line_length, 119)
+
+    def test_pre_commit_hooks(self):
+        executor = ToolsExecutor(cwd=PACKAGE_ROOT)
+        for command in ('migrate-config', 'validate-config', 'validate-manifest'):
+            executor.verbose_check_call('pre-commit', command, exit_on_error=True)

PyHardLinkBackup/tests/test_readme.py

@@ -0,0 +1,75 @@
+from bx_py_utils.auto_doc import assert_readme_block
+from bx_py_utils.path import assert_is_file
+from cli_base.cli_tools.test_utils.rich_test_utils import NoColorEnvRich, invoke
+from manageprojects.tests.base import BaseTestCase
+
+from PyHardLinkBackup import constants
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT
+
+
+def assert_cli_help_in_readme(text_block: str, marker: str):
+    README_PATH = PACKAGE_ROOT / 'README.md'
+    assert_is_file(README_PATH)
+
+    text_block = text_block.replace(constants.CLI_EPILOG, '')
+    text_block = f'```\n{text_block.strip()}\n```'
+    assert_readme_block(
+        readme_path=README_PATH,
+        text_block=text_block,
+        start_marker_line=f'[comment]: <> (✂✂✂ auto generated {marker} start ✂✂✂)',
+        end_marker_line=f'[comment]: <> (✂✂✂ auto generated {marker} end ✂✂✂)',
+    )
+
+
+class ReadmeTestCase(BaseTestCase):
+
+    def test_main_help(self):
+        with NoColorEnvRich():
+            stdout = invoke(cli_bin=PACKAGE_ROOT / 'cli.py', args=['--help'], strip_line_prefix='usage: ')
+
+        self.assert_in_content(
+            got=stdout,
+            parts=(
+                'usage: ./cli.py [-h]',
+                ' version ',
+                'Print version and exit',
+                constants.CLI_EPILOG,
+            ),
+        )
+
+        # Installed via pipx is called 'phlb', not 'cli.py':
+        stdout = stdout.replace('./cli.py', 'phlb')
+
+        assert_cli_help_in_readme(text_block=stdout, marker='main help')
+
+    def test_backup_help(self):
+        with NoColorEnvRich():
+            stdout = invoke(cli_bin=PACKAGE_ROOT / 'cli.py', args=['backup', '--help'], strip_line_prefix='usage: ')
+        self.assert_in_content(
+            got=stdout,
+            parts=(
+                'usage: ./cli.py backup [-h] ',
+                'Backup the source directory to the destination',
+            ),
+        )
+
+        # Installed via pipx is called 'phlb', not 'cli.py':
+        stdout = stdout.replace('./cli.py', 'phlb')
+
+        assert_cli_help_in_readme(text_block=stdout, marker='backup help')
+
+    def test_dev_help(self):
+        with NoColorEnvRich():
+            stdout = invoke(cli_bin=PACKAGE_ROOT / 'dev-cli.py', args=['--help'], strip_line_prefix='usage: ')
+        self.assert_in_content(
+            got=stdout,
+            parts=(
+                'usage: ./dev-cli.py [-h]',
+                ' lint ',
+                ' coverage ',
+                ' update-readme-history ',
+                ' publish ',
+                constants.CLI_EPILOG,
+            ),
+        )
+        assert_cli_help_in_readme(text_block=stdout, marker='dev help')

PyHardLinkBackup/tests/test_readme_history.py

@@ -0,0 +1,9 @@
+from unittest import TestCase
+
+from cli_base.cli_tools.git_history import update_readme_history
+
+
+class ReadmeHistoryTestCase(TestCase):
+    def test_readme_history(self):
+        with self.assertLogs():
+            update_readme_history(raise_update_error=True)

PyHardLinkBackup/tests/test_rebuild_database.py

@@ -0,0 +1,224 @@
+import logging
+import tempfile
+import textwrap
+from pathlib import Path
+from unittest.mock import patch
+
+from bx_py_utils.test_utils.redirect import RedirectOut
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+from freezegun import freeze_time
+
+from PyHardLinkBackup import rebuild_databases
+from PyHardLinkBackup.logging_setup import NoopLoggingManager
+from PyHardLinkBackup.rebuild_databases import RebuildResult, rebuild, rebuild_one_file
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+
+
+def sorted_rglob_paths(path: Path):
+    return sorted([str(p.relative_to(path)) for p in path.rglob('*')])
+
+
+def sorted_rglob_files(path: Path):
+    return sorted([str(p.relative_to(path)) for p in path.rglob('*') if p.is_file()])
+
+
+class RebuildDatabaseTestCase(BaseTestCase):
+    maxDiff = None
+
+    def test_happy_path(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir).resolve()
+
+            backup_root = temp_path / 'backup'
+
+            with self.assertRaises(SystemExit), RedirectOut() as redirected_out:
+                rebuild(backup_root, log_manager=NoopLoggingManager())
+
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertEqual(redirected_out.stdout, f'Error: Backup directory "{backup_root}" does not exist!\n')
+
+            backup_root.mkdir()
+
+            with self.assertRaises(SystemExit), RedirectOut() as redirected_out:
+                rebuild(backup_root, log_manager=NoopLoggingManager())
+
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertIn('hidden ".phlb" configuration directory is missing', redirected_out.stdout)
+
+            phlb_conf_dir = backup_root / '.phlb'
+            phlb_conf_dir.mkdir()
+
+            #######################################################################################
+            # Run on empty backup directory:
+
+            self.assertEqual(sorted_rglob_paths(backup_root), ['.phlb'])
+
+            with (
+                self.assertLogs('PyHardLinkBackup', level=logging.DEBUG),
+                RedirectOut() as redirected_out,
+                freeze_time('2026-01-16T12:34:56Z', auto_tick_seconds=0),
+            ):
+                rebuild_result = rebuild(backup_root, log_manager=NoopLoggingManager())
+            self.assertEqual(
+                rebuild_result,
+                RebuildResult(
+                    process_count=0,
+                    process_size=0,
+                    added_size_count=0,
+                    added_hash_count=0,
+                    error_count=0,
+                ),
+            )
+            self.assertEqual(
+                sorted_rglob_paths(backup_root),
+                [
+                    '.phlb',
+                    '.phlb/hash-lookup',
+                    '.phlb/size-lookup',
+                    '2026-01-16-123456-rebuild-summary.txt',
+                ],
+            )
+            self.assertEqual(redirected_out.stderr, '')
+
+            #######################################################################################
+            # Add one backuped file and run again:
+
+            snapshot_path = backup_root / 'source-name' / '2026-01-15-181709'
+            snapshot_path.mkdir(parents=True)
+
+            minimum_file_content = 'X' * FileSizeDatabase.MIN_SIZE
+            (snapshot_path / 'file1.txt').write_text(minimum_file_content)
+
+            with (
+                self.assertLogs('PyHardLinkBackup', level=logging.DEBUG),
+                RedirectOut() as redirected_out,
+                freeze_time('2026-01-16T12:34:56Z', auto_tick_seconds=0),
+            ):
+                rebuild_result = rebuild(backup_root, log_manager=NoopLoggingManager())
+            self.assertEqual(
+                sorted_rglob_paths(backup_root),
+                [
+                    '.phlb',
+                    '.phlb/hash-lookup',
+                    '.phlb/hash-lookup/bb',
+                    '.phlb/hash-lookup/bb/c4',
+                    '.phlb/hash-lookup/bb/c4/bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8',
+                    '.phlb/size-lookup',
+                    '.phlb/size-lookup/10',
+                    '.phlb/size-lookup/10/00',
+                    '.phlb/size-lookup/10/00/1000',
+                    '2026-01-16-123456-rebuild-summary.txt',
+                    'source-name',
+                    'source-name/2026-01-15-181709',
+                    'source-name/2026-01-15-181709/SHA256SUMS',
+                    'source-name/2026-01-15-181709/file1.txt',
+                ],
+            )
+            self.assertEqual(
+                sorted_rglob_files(backup_root),
+                [
+                    '.phlb/hash-lookup/bb/c4/bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8',
+                    '.phlb/size-lookup/10/00/1000',
+                    '2026-01-16-123456-rebuild-summary.txt',
+                    'source-name/2026-01-15-181709/SHA256SUMS',
+                    'source-name/2026-01-15-181709/file1.txt',
+                ],
+            )
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertEqual(
+                rebuild_result,
+                RebuildResult(
+                    process_count=1,
+                    process_size=1000,
+                    added_size_count=1,
+                    added_hash_count=1,
+                    error_count=0,
+                    hash_verified_count=0,
+                    hash_mismatch_count=0,
+                    hash_not_found_count=1,
+                ),
+            )
+            self.assertEqual(
+                (snapshot_path / 'SHA256SUMS').read_text(),
+                'bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8  file1.txt\n',
+            )
+
+            #######################################################################################
+            # Add a file with same content and run again:
+
+            minimum_file_content = 'X' * FileSizeDatabase.MIN_SIZE
+            (snapshot_path / 'same_content.txt').write_text(minimum_file_content)
+
+            with (
+                self.assertLogs('PyHardLinkBackup', level=logging.DEBUG) as logs,
+                RedirectOut() as redirected_out,
+                freeze_time('2026-01-16T12:34:56Z', auto_tick_seconds=0),
+            ):
+                rebuild_result = rebuild(backup_root, log_manager=NoopLoggingManager())
+            # No new hash of size entries, just the new file:
+            self.assertEqual(
+                sorted_rglob_files(backup_root),
+                [
+                    '.phlb/hash-lookup/bb/c4/bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8',
+                    '.phlb/size-lookup/10/00/1000',
+                    '2026-01-16-123456-rebuild-summary.txt',
+                    'source-name/2026-01-15-181709/SHA256SUMS',
+                    'source-name/2026-01-15-181709/file1.txt',
+                    'source-name/2026-01-15-181709/same_content.txt',
+                ],
+            )
+            self.assertEqual(redirected_out.stderr, '')
+            self.assertEqual(
+                rebuild_result,
+                RebuildResult(
+                    process_count=3,
+                    process_size=2000,
+                    added_size_count=0,
+                    added_hash_count=0,
+                    error_count=0,
+                    hash_verified_count=1,  # Existing file verified successfully
+                    hash_mismatch_count=0,
+                    hash_not_found_count=1,  # One file added
+                ),
+                '\n'.join(logs.output) + redirected_out.stdout,
+            )
+            self.assertEqual(
+                (snapshot_path / 'SHA256SUMS').read_text(),
+                textwrap.dedent("""\
+                    bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8  file1.txt
+                    bbc4de2ca238d1ec41fb622b75b5cf7d31a6d2ac92405043dd8f8220364fefc8  same_content.txt
+                """),
+            )
+
+            #######################################################################################
+            # Test error handling
+
+            def rebuild_one_file_mock(*, entry, **kwargs):
+                if entry.name == 'file1.txt':
+                    raise IOError('Bam!')
+                return rebuild_one_file(entry=entry, **kwargs)
+
+            with (
+                self.assertLogs('PyHardLinkBackup', level=logging.ERROR) as logs,
+                RedirectOut() as redirected_out,
+                patch.object(rebuild_databases, 'rebuild_one_file', rebuild_one_file_mock),
+            ):
+                rebuild_result = rebuild(backup_root, log_manager=NoopLoggingManager())
+            logs = ''.join(logs.output)
+            self.assertIn(f'Backup {snapshot_path}/file1.txt OSError: Bam!\n', logs)
+            self.assertIn('\nTraceback (most recent call last):\n', logs)
+            self.assertEqual(redirected_out.stderr, '')
+
+            self.assertEqual(
+                rebuild_result,
+                RebuildResult(
+                    process_count=2,
+                    process_size=1000,
+                    added_size_count=0,
+                    added_hash_count=0,
+                    error_count=1,  # <<< one file caused error
+                    hash_verified_count=1,
+                    hash_mismatch_count=0,
+                    hash_not_found_count=0,
+                ),
+            )

PyHardLinkBackup/utilities/file_hash_database.py

@@ -0,0 +1,62 @@
+import logging
+from pathlib import Path
+
+
+class HashAlreadyExistsError(ValueError):
+    pass
+
+
+class FileHashDatabase:
+    """DocWrite: README.md ## FileHashDatabase
+    A simple "database" to store file content hash <-> relative path mappings.
+    Uses a directory structure to avoid too many files in a single directory.
+    Path structure:
+            {base_dst}/.phlb/hash-lookup/{XX}/{YY}/{hash}
+    e.g.:
+        hash '12ab000a1b2c3...' results in: {base_dst}/.phlb/hash-lookup/12/ab/12ab000a1b2c3...
+
+    Notes:
+      * Hash length will be not validated, so it can be used with any hash algorithm.
+      * The "relative path" that will be stored is not validated, so it can be any string.
+      * We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    """
+
+    def __init__(self, backup_root: Path, phlb_conf_dir: Path):
+        self.backup_root = backup_root
+        self.base_path = phlb_conf_dir / 'hash-lookup'
+        self.base_path.mkdir(parents=False, exist_ok=True)
+
+    def _get_hash_path(self, hash: str) -> Path:
+        first_dir_name = hash[:2]
+        second_dir_name = hash[2:4]
+        hash_path = self.base_path / first_dir_name / second_dir_name / hash
+        return hash_path
+
+    def __contains__(self, hash: str) -> bool:
+        hash_path = self._get_hash_path(hash)
+        return hash_path.exists()
+
+    def get(self, hash: str) -> Path | None:
+        hash_path = self._get_hash_path(hash)
+        try:
+            rel_file_path = hash_path.read_text()
+        except FileNotFoundError:
+            return None
+        else:
+            abs_file_path = self.backup_root / rel_file_path
+            if not abs_file_path.is_file():
+                logging.warning('Hash database entry found, but file does not exist: %s', abs_file_path)
+                hash_path.unlink()
+                return None
+            return abs_file_path
+
+    def __setitem__(self, hash: str, abs_file_path: Path):
+        hash_path = self._get_hash_path(hash)
+        hash_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # File should be found before and results in hardlink creation!
+        # So deny change of existing hashes:
+        if hash_path.exists():
+            raise HashAlreadyExistsError(f'Hash {hash} already exists in the database!')
+
+        hash_path.write_text(str(abs_file_path.relative_to(self.backup_root)))

PyHardLinkBackup/utilities/file_size_database.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+
+class FileSizeDatabase:
+    """DocWrite: README.md ## FileSizeDatabase
+    A simple "database" to track which file sizes have been seen.
+
+    Uses a directory structure to avoid too many files in a single directory.
+    We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    """
+
+    MIN_SIZE = 1000  # no padding is made, so the min size is 1000 bytes!
+
+    def __init__(self, phlb_conf_dir: Path):
+        self.base_path = phlb_conf_dir / 'size-lookup'
+        self.base_path.mkdir(parents=False, exist_ok=True)
+
+    def _get_size_path(self, size: int) -> Path:
+        assert size >= self.MIN_SIZE, f'Size must be at least {self.MIN_SIZE} bytes'
+        size_str = str(size)
+
+        """DocWrite: README.md ## FileSizeDatabase
+        Path structure:
+         * `{base_dst}/.phlb/size-lookup/{XX}/{YY}/{size}`
+
+        e.g.:
+
+         * `1234567890` results in: `{base_dst}/.phlb/size-lookup/12/34/1234567890`
+        """
+        first_dir_name = size_str[:2]
+        second_dir_name = size_str[2:4]
+        size_path = self.base_path / first_dir_name / second_dir_name / size_str
+        return size_path
+
+    def __contains__(self, size: int) -> bool:
+        size_path = self._get_size_path(size)
+        return size_path.exists()
+
+    def add(self, size: int):
+        size_path = self._get_size_path(size)
+        if not size_path.exists():
+            size_path.parent.mkdir(parents=True, exist_ok=True)
+
+            """DocWrite: README.md ## FileSizeDatabase
+            All files are created empty, as we only care about their existence."""
+            size_path.touch(exist_ok=False)