PyHardLinkBackup 1.8.1__py3-none-any.whl

PyHardLinkBackup/utilities/tests/test_file_hash_database.py

@@ -0,0 +1,153 @@
+import logging
+import tempfile
+import textwrap
+from pathlib import Path
+
+from bx_py_utils.path import assert_is_dir
+from bx_py_utils.test_utils.assertion import assert_text_equal
+from bx_py_utils.test_utils.log_utils import NoLogs
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase, HashAlreadyExistsError
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileHashDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileHashDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir).resolve()
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        hash_db = FileHashDatabase(backup_root=backup_root, phlb_conf_dir=phlb_conf_dir)
+        return hash_db
+
+
+def get_hash_db_filenames(hash_db: FileHashDatabase) -> list[str]:
+    # with NoLogs('PyHardLinkBackup.utilities.filesystem'):
+    return sorted(
+        str(Path(entry.path).relative_to(hash_db.base_path))
+        for entry in iter_scandir_files(
+            path=hash_db.base_path,
+            one_file_system=False,
+            src_device_id=None,
+            excludes=set(),
+        )
+    )
+
+
+def get_hash_db_info(backup_root: Path) -> str:
+    db_base_path = backup_root / '.phlb' / 'hash-lookup'
+    assert_is_dir(db_base_path)
+
+    with NoLogs(logger_name='XY'):
+        lines = []
+        for entry in iter_scandir_files(
+            path=db_base_path,
+            one_file_system=False,
+            src_device_id=None,
+            excludes=set(),
+        ):
+            hash_path = Path(entry.path)
+            rel_path = hash_path.relative_to(db_base_path)
+            rel_file_path = hash_path.read_text()
+            lines.append(f'{str(rel_path)[:20]}… -> {rel_file_path}')
+    return '\n'.join(sorted(lines))
+
+
+def assert_hash_db_info(backup_root: Path, expected: str):
+    expected = textwrap.dedent(expected).strip()
+    actual = get_hash_db_info(backup_root)
+    assert_text_equal(
+        actual,
+        expected,
+        msg=f'FileHashDatabase info does not match as expected.\n\n{actual}\n\n',
+    )
+
+
+class FileHashDatabaseTestCase(BaseTestCase):
+    def test_happy_path(self):
+        with TemporaryFileHashDatabase() as hash_db:
+            self.assertIsInstance(hash_db, FileHashDatabase)
+
+            backup_root_path = hash_db.backup_root
+            assert_is_dir(backup_root_path)
+
+            test_path = hash_db._get_hash_path('12345678abcdef')
+            self.assertEqual(test_path, hash_db.base_path / '12' / '34' / '12345678abcdef')
+
+            file_a_path = backup_root_path / 'rel/path/to/file-A'
+            file_a_path.parent.mkdir(parents=True, exist_ok=True)
+            file_a_path.touch()
+
+            self.assertIs(hash_db.get('12345678abcdef'), None)
+            self.assertIs('12345678abcdef' in hash_db, False)
+            hash_db['12345678abcdef'] = file_a_path
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            self.assertIs('12345678abcdef' in hash_db, True)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_hash_db_filenames(hash_db),
+                    ['12/34/12345678abcdef'],
+                )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_hash_db = FileHashDatabase(
+                backup_root=hash_db.backup_root,
+                phlb_conf_dir=hash_db.base_path.parent,
+            )
+            self.assertEqual(another_hash_db.get('12345678abcdef'), file_a_path)
+            self.assertIs(another_hash_db.get('12abcd345678abcdef'), None)
+
+            file_b_path = backup_root_path / 'rel/path/to/file-B'
+            file_b_path.parent.mkdir(parents=True, exist_ok=True)
+            file_b_path.touch()
+
+            another_hash_db['12abcd345678abcdef'] = file_b_path
+            self.assertEqual(another_hash_db.get('12abcd345678abcdef'), file_b_path)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_hash_db_filenames(another_hash_db),
+                    [
+                        '12/34/12345678abcdef',
+                        '12/ab/12abcd345678abcdef',
+                    ],
+                )
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                assert_hash_db_info(
+                    backup_root=hash_db.backup_root,
+                    expected="""
+                        12/34/12345678abcdef… -> rel/path/to/file-A
+                        12/ab/12abcd345678ab… -> rel/path/to/file-B
+                    """,
+                )
+
+            ########################################################################################
+            # Deny "overwrite" of existing hash:
+
+            with self.assertRaises(HashAlreadyExistsError):
+                hash_db['12abcd345678abcdef'] = 'foo/bar/baz'  # already exists!
+
+            ########################################################################################
+            # Don't use stale entries pointing to missing files:
+
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            file_a_path.unlink()
+
+            """DocWrite: README.md ## FileHashDatabase - Missing hardlink target file
+            We check if the hardlink source file still exists. If not, we remove the hash entry from the database.
+            A warning is logged in this case."""
+            with self.assertLogs(level=logging.WARNING) as logs:
+                self.assertIs(hash_db.get('12345678abcdef'), None)
+            self.assertIn('Hash database entry found, but file does not exist', ''.join(logs.output))
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                assert_hash_db_info(
+                    backup_root=hash_db.backup_root,
+                    expected="""
+                        12/ab/12abcd345678ab… -> rel/path/to/file-B
+                    """,
+                )

PyHardLinkBackup/utilities/tests/test_file_size_database.py

@@ -0,0 +1,151 @@
+import logging
+import tempfile
+from collections.abc import Iterable
+from pathlib import Path
+
+from bx_py_utils.test_utils.log_utils import NoLogs
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileSizeDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileSizeDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir).resolve()
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        size_db = FileSizeDatabase(phlb_conf_dir=phlb_conf_dir)
+        return size_db
+
+
+def get_size_db_filenames(size_db: FileSizeDatabase) -> Iterable[str]:
+    return sorted(
+        str(Path(entry.path).relative_to(size_db.base_path))
+        for entry in iter_scandir_files(
+            path=size_db.base_path,
+            one_file_system=False,
+            src_device_id=None,
+            excludes=set(),
+        )
+    )
+
+
+def get_sizes(size_db: FileSizeDatabase) -> Iterable[int]:
+    with NoLogs('PyHardLinkBackup.utilities.filesystem'):
+        return sorted(
+            int(entry.name)
+            for entry in iter_scandir_files(
+                path=size_db.base_path,
+                one_file_system=False,
+                src_device_id=None,
+                excludes=set(),
+            )
+        )
+
+
+class FileSizeDatabaseTestCase(BaseTestCase):
+    def test_happy_path(self):
+        with TemporaryFileSizeDatabase() as size_db:
+            self.assertIsInstance(size_db, FileSizeDatabase)
+
+            test_path1 = size_db._get_size_path(1234)
+            self.assertEqual(test_path1, size_db.base_path / '12' / '34' / '1234')
+
+            test_path2 = size_db._get_size_path(567890)
+            self.assertEqual(test_path2, size_db.base_path / '56' / '78' / '567890')
+
+            self.assertNotIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(1234)
+            self.assertIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(567890)
+            self.assertIn(1234, size_db)
+            self.assertIn(567890, size_db)
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(get_sizes(size_db), [1234, 567890])
+                self.assertEqual(
+                    get_size_db_filenames(size_db),
+                    [
+                        '12/34/1234',
+                        '56/78/567890',
+                    ],
+                )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_size_db = FileSizeDatabase(phlb_conf_dir=size_db.base_path.parent)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(get_sizes(another_size_db), [1234, 567890])
+                self.assertEqual(
+                    get_size_db_filenames(another_size_db),
+                    [
+                        '12/34/1234',
+                        '56/78/567890',
+                    ],
+                )
+
+            ########################################################################################
+            # "Share" directories:
+
+            for size in (123400001111, 123400002222, 128800003333, 129900004444):
+                self.assertNotIn(size, size_db)
+                size_db.add(size)
+                self.assertIn(size, size_db)
+
+            ########################################################################################
+            # Min size is 1000 bytes:
+
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The minimum file size that can be stored in the FileSizeDatabase is 1000 bytes.
+            This is because no padding is made for sizes below 1000 bytes, which would
+            break the directory structure.
+            """
+            self.assertEqual(FileSizeDatabase.MIN_SIZE, 1000)
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The idea is, that it's more efficient to backup small files directly, instead of
+            checking for duplicates via hardlinks. Therefore, small files below this size
+            are not tracked in the FileSizeDatabase.
+            """
+
+            with self.assertRaises(AssertionError):
+                size_db._get_size_path(999)
+            with self.assertRaises(AssertionError):
+                size_db.add(999)
+            with self.assertRaises(AssertionError):
+                999 in size_db
+
+            ########################################################################################
+            # Check final state:
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_size_db_filenames(size_db),
+                    [
+                        '12/34/1234',
+                        '12/34/123400001111',
+                        '12/34/123400002222',
+                        '12/88/128800003333',
+                        '12/99/129900004444',
+                        '56/78/567890',
+                    ],
+                )
+                self.assertEqual(
+                    get_sizes(size_db),
+                    [
+                        1234,
+                        567890,
+                        123400001111,
+                        123400002222,
+                        128800003333,
+                        129900004444,
+                    ],
+                )

PyHardLinkBackup/utilities/tests/test_filesystem.py

@@ -0,0 +1,167 @@
+import hashlib
+import logging
+import os
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+
+from PyHardLinkBackup.constants import HASH_ALGO
+from PyHardLinkBackup.utilities.filesystem import (
+    copy_and_hash,
+    hash_file,
+    iter_scandir_files,
+    read_and_hash_file,
+    supports_hardlinks,
+)
+from PyHardLinkBackup.utilities.rich_utils import NoopProgress
+from PyHardLinkBackup.utilities.tests.unittest_utilities import TemporaryDirectoryPath
+
+
+class TestHashFile(BaseTestCase):
+    maxDiff = None
+
+    def test_hash_file(self):
+        self.assertEqual(
+            hashlib.new(HASH_ALGO, b'test content').hexdigest(),
+            '6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72',
+        )
+        with tempfile.NamedTemporaryFile() as temp:
+            temp_file_path = Path(temp.name)
+            temp_file_path.write_bytes(b'test content')
+
+            with self.assertLogs(level='INFO') as logs:
+                file_hash = hash_file(temp_file_path, progress=NoopProgress(), total_size=123)
+        self.assertEqual(file_hash, '6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72')
+        self.assertIn(' sha256 hash: 6ae8a7', ''.join(logs.output))
+
+    def test_copy_and_hash(self):
+        with TemporaryDirectoryPath() as temp_path:
+            src_path = temp_path / 'source.txt'
+            dst_path = temp_path / 'dest.txt'
+
+            src_path.write_bytes(b'test content')
+
+            with self.assertLogs(level='INFO') as logs:
+                file_hash = copy_and_hash(src=src_path, dst=dst_path, progress=NoopProgress(), total_size=123)
+
+            self.assertEqual(dst_path.read_bytes(), b'test content')
+        self.assertEqual(file_hash, '6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72')
+        self.assertIn(' backup to ', ''.join(logs.output))
+
+    def test_read_and_hash_file(self):
+        with tempfile.NamedTemporaryFile() as temp:
+            temp_file_path = Path(temp.name)
+            temp_file_path.write_bytes(b'test content')
+
+            with self.assertLogs(level='INFO') as logs:
+                content, file_hash = read_and_hash_file(temp_file_path)
+        self.assertEqual(content, b'test content')
+        self.assertEqual(file_hash, '6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72')
+        self.assertIn(' sha256 hash: 6ae8a7', ''.join(logs.output))
+
+    def test_iter_scandir_files(self):
+        with TemporaryDirectoryPath() as temp_path:
+            (temp_path / 'file1.txt').write_bytes(b'content1')
+            (temp_path / 'file2.txt').write_bytes(b'content2')
+            subdir = temp_path / 'subdir'
+            subdir.mkdir()
+            (subdir / 'file3.txt').write_bytes(b'content3')
+
+            symlink_dir = temp_path / 'symlink_dir2subdir'
+            symlink_dir.symlink_to(subdir, target_is_directory=True)
+
+            # Add a symlink to file1.txt
+            (temp_path / 'symlink_to_file1.txt').symlink_to(temp_path / 'file1.txt')
+
+            # Add a hardlink to file2.txt
+            os.link(temp_path / 'file2.txt', temp_path / 'hardlink_to_file2.txt')
+
+            exclude_subdir = temp_path / '__pycache__'
+            exclude_subdir.mkdir()
+            (exclude_subdir / 'BAM.txt').write_bytes(b'foobar')
+
+            broken_symlink_path = temp_path / 'broken_symlink'
+            broken_symlink_path.symlink_to(temp_path / 'not/existing/file.txt')
+
+            with self.assertLogs(level='DEBUG') as logs:
+                files = list(
+                    iter_scandir_files(
+                        path=temp_path,
+                        one_file_system=False,
+                        src_device_id=None,
+                        excludes={'__pycache__'},
+                    )
+                )
+
+        file_names = sorted([Path(f.path).relative_to(temp_path).as_posix() for f in files])
+
+        self.assertEqual(
+            file_names,
+            [
+                'broken_symlink',
+                'file1.txt',
+                'file2.txt',
+                'hardlink_to_file2.txt',
+                'subdir/file3.txt',
+                'symlink_dir2subdir',
+                'symlink_to_file1.txt',
+            ],
+        )
+        logs = ''.join(logs.output)
+        self.assertIn('Scanning directory ', logs)
+        self.assertIn('Excluding directory ', logs)
+
+    def test_one_file_system(self):
+        def scan(temp_path, *, one_file_system, src_device_id):
+            with self.assertLogs(level='DEBUG') as logs:
+                files = list(
+                    iter_scandir_files(
+                        path=temp_path,
+                        one_file_system=one_file_system,
+                        src_device_id=src_device_id,
+                        excludes=set(),
+                    )
+                )
+            file_names = sorted([Path(f.path).relative_to(temp_path).as_posix() for f in files])
+            return file_names, '\n'.join(logs.output)
+
+        with TemporaryDirectoryPath() as temp_path:
+            (temp_path / 'file1.txt').touch()
+            subdir = temp_path / 'subdir'
+            subdir.mkdir()
+            (subdir / 'file2.txt').touch()
+
+            file_names, logs = scan(temp_path, one_file_system=False, src_device_id=None)
+            self.assertEqual(file_names, ['file1.txt', 'subdir/file2.txt'])
+            self.assertIn('Scanning directory ', logs)
+            self.assertNotIn('Skipping', logs)
+
+            file_names, logs = scan(temp_path, one_file_system=True, src_device_id='FooBar')
+            self.assertEqual(file_names, ['file1.txt'])
+            self.assertIn('Scanning directory ', logs)
+            self.assertIn('Skipping directory ', logs)
+            self.assertIn('different device ID', logs)
+            self.assertIn('(src device ID: FooBar)', logs)
+
+    def test_supports_hardlinks(self):
+        with TemporaryDirectoryPath() as temp_path:
+            with self.assertLogs(level=logging.INFO) as logs:
+                self.assertTrue(supports_hardlinks(temp_path))
+            self.assertEqual(
+                ''.join(logs.output),
+                f'INFO:PyHardLinkBackup.utilities.filesystem:Hardlink support in {temp_path}: True',
+            )
+
+            with (
+                self.assertLogs(level=logging.ERROR) as logs,
+                patch('PyHardLinkBackup.utilities.filesystem.os.link', side_effect=OSError),
+            ):
+                self.assertFalse(supports_hardlinks(temp_path))
+            logs = ''.join(logs.output)
+            self.assertIn(f'Hardlink test failed in {temp_path}:', logs)
+            self.assertIn('OSError', logs)
+
+        with self.assertLogs(level=logging.DEBUG), self.assertRaises(NotADirectoryError):
+            supports_hardlinks(Path('/not/existing/directory'))

PyHardLinkBackup/utilities/tests/unittest_utilities.py

@@ -0,0 +1,78 @@
+import pathlib
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+from bx_py_utils.test_utils.context_managers import MassContextManager
+
+
+class TemporaryDirectoryPath(tempfile.TemporaryDirectory):
+    """
+    Similar to tempfile.TemporaryDirectory,
+    but returns a resolved Path instance.
+    """
+
+    def __enter__(self) -> Path:
+        super().__enter__()
+        return Path(self.name).resolve()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return super().__exit__(exc_type, exc_val, exc_tb)
+
+
+class PyHardLinkBackupTestCaseMixin:
+    def setUp(self):
+        super().setUp()
+        self.temp_path_cm = TemporaryDirectoryPath()
+        self.temp_path = self.temp_path_cm.__enter__()
+
+        self.src_root = self.temp_path / 'source'
+        self.backup_root = self.temp_path / 'backups'
+
+        self.src_root.mkdir()
+        self.backup_root.mkdir()
+
+    def tearDown(self):
+        super().tearDown()
+        self.temp_path_cm.__exit__(None, None, None)
+
+
+class CollectOpenFiles(MassContextManager):
+    """
+    Context manager to collect opened files for read and write within a given root directory.
+    Raises an AssertionError if the same file is opened multiple times for read or write.
+
+    Works only for standard open() and pathlib.Path.open().
+    """
+    def __init__(self, root: Path):
+        self.root = root
+
+        self.origin_open = open
+        self.mocks = (
+            patch('builtins.open', self.open_mock),
+            patch.object(pathlib.Path, 'open', self.make_path_open_wrapper()),
+        )
+        self.opened_for_read = []
+        self.opened_for_write = []
+
+    def open_mock(self, file, mode='r', *args, **kwargs):
+        rel_path = Path(file).resolve().relative_to(self.root)
+
+        if 'r' in mode and '+' not in mode:
+            if file in self.opened_for_read:
+                raise AssertionError(f'File {rel_path} already opened for read')
+            self.opened_for_read.append(f'{mode} {rel_path}')
+        elif any(m in mode for m in 'wax+'):
+            if file in self.opened_for_write:
+                raise AssertionError(f'File {rel_path} already opened for write')
+            self.opened_for_write.append(f'{mode} {rel_path}')
+        else:
+            raise NotImplementedError(f'Unsupported file open {mode=}')
+
+        return self.origin_open(file, mode, *args, **kwargs)
+
+    def make_path_open_wrapper(self):
+        def open_wrapper(path_self, *args, **kwargs):
+            return self.open_mock(path_self, *args, **kwargs)
+
+        return open_wrapper

PyHardLinkBackup/utilities/tyro_cli_shared_args.py

@@ -0,0 +1,29 @@
+from typing import Annotated
+
+import tyro
+
+
+TyroExcludeDirectoriesArgType = Annotated[
+    tuple[str, ...],
+    tyro.conf.arg(
+        help='List of directories to exclude from backup.',
+    ),
+]
+DEFAULT_EXCLUDE_DIRECTORIES = ('__pycache__', '.cache', '.temp', '.tmp', '.tox', '.nox')
+
+TyroOneFileSystemArgType = Annotated[
+    bool,
+    tyro.conf.arg(
+        help='Do not cross filesystem boundaries.',
+    ),
+]
+
+TyroBackupNameArgType = Annotated[
+    str | None,
+    tyro.conf.arg(
+        help=(
+            'Optional name for the backup (used to create a subdirectory in the backup destination).'
+            ' If not provided, the name of the source directory is used.'
+        ),
+    ),
+]