PyHardLinkBackup 1.0.0__py3-none-any.whl

PyHardLinkBackup/utilities/file_hash_database.py

@@ -0,0 +1,58 @@
+import logging
+from pathlib import Path
+
+
+class HashAlreadyExistsError(ValueError):
+    pass
+
+
+class FileHashDatabase:
+    """DocWrite: README.md ## FileHashDatabase
+    A simple "database" to store file content hash <-> relative path mappings.
+    Uses a directory structure to avoid too many files in a single directory.
+    Path structure:
+            {base_dst}/.phlb/hash-lookup/{XX}/{YY}/{hash}
+    e.g.:
+        hash '12ab000a1b2c3...' results in: {base_dst}/.phlb/hash-lookup/12/ab/12ab000a1b2c3...
+
+    Notes:
+      * Hash length will be not validated, so it can be used with any hash algorithm.
+      * The "relative path" that will be stored is not validated, so it can be any string.
+      * We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    """
+
+    def __init__(self, backup_root: Path, phlb_conf_dir: Path):
+        self.backup_root = backup_root
+        self.base_path = phlb_conf_dir / 'hash-lookup'
+        self.base_path.mkdir(parents=False, exist_ok=True)
+
+    def _get_hash_path(self, hash: str) -> Path:
+        first_dir_name = hash[:2]
+        second_dir_name = hash[2:4]
+        hash_path = self.base_path / first_dir_name / second_dir_name / hash
+        return hash_path
+
+    def get(self, hash: str) -> Path | None:
+        hash_path = self._get_hash_path(hash)
+        try:
+            rel_file_path = hash_path.read_text()
+        except FileNotFoundError:
+            return None
+        else:
+            abs_file_path = self.backup_root / rel_file_path
+            if not abs_file_path.is_file():
+                logging.warning('Hash database entry found, but file does not exist: %s', abs_file_path)
+                hash_path.unlink()
+                return None
+            return abs_file_path
+
+    def __setitem__(self, hash: str, abs_file_path: Path):
+        hash_path = self._get_hash_path(hash)
+        hash_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # File should be found before and results in hardlink creation!
+        # So deny change of existing hashes:
+        if hash_path.exists():
+            raise HashAlreadyExistsError(f'Hash {hash} already exists in the database!')
+
+        hash_path.write_text(str(abs_file_path.relative_to(self.backup_root)))

PyHardLinkBackup/utilities/file_size_database.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+
+class FileSizeDatabase:
+    """DocWrite: README.md ## FileSizeDatabase
+    A simple "database" to track which file sizes have been seen.
+
+    Uses a directory structure to avoid too many files in a single directory.
+    We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    """
+
+    MIN_SIZE = 1000  # no padding is made, so the min size is 1000 bytes!
+
+    def __init__(self, phlb_conf_dir: Path):
+        self.base_path = phlb_conf_dir / 'size-lookup'
+        self.base_path.mkdir(parents=False, exist_ok=True)
+
+    def _get_size_path(self, size: int) -> Path:
+        assert size >= self.MIN_SIZE, f'Size must be at least {self.MIN_SIZE} bytes'
+        size_str = str(size)
+
+        """DocWrite: README.md ## FileSizeDatabase
+        Path structure:
+         * `{base_dst}/.phlb/size-lookup/{XX}/{YY}/{size}`
+
+        e.g.:
+
+         * `1234567890` results in: `{base_dst}/.phlb/size-lookup/12/34/1234567890`
+        """
+        first_dir_name = size_str[:2]
+        second_dir_name = size_str[2:4]
+        size_path = self.base_path / first_dir_name / second_dir_name / size_str
+        return size_path
+
+    def __contains__(self, size: int) -> bool:
+        size_path = self._get_size_path(size)
+        return size_path.exists()
+
+    def add(self, size: int):
+        size_path = self._get_size_path(size)
+        if not size_path.exists():
+            size_path.parent.mkdir(parents=True, exist_ok=True)
+
+            """DocWrite: README.md ## FileSizeDatabase
+            All files are created empty, as we only care about their existence."""
+            size_path.touch(exist_ok=False)

PyHardLinkBackup/utilities/filesystem.py

@@ -0,0 +1,133 @@
+import hashlib
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Iterable
+
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from PyHardLinkBackup.constants import CHUNK_SIZE, HASH_ALGO
+from PyHardLinkBackup.utilities.rich_utils import HumanFileSizeColumn
+
+
+logger = logging.getLogger(__name__)
+
+
+def hash_file(path: Path) -> str:
+    logger.debug('Hash file %s using %s', path, HASH_ALGO)
+    with path.open('rb') as f:
+        digest = hashlib.file_digest(f, HASH_ALGO)
+
+    file_hash = digest.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def copy_and_hash(src: Path, dst: Path) -> str:
+    logger.debug('Copy and hash file %s to %s using %s', src, dst, HASH_ALGO)
+    hasher = hashlib.new(HASH_ALGO)
+    with src.open('rb') as source_file, dst.open('wb') as dst_file:
+        while chunk := source_file.read(CHUNK_SIZE):
+            dst_file.write(chunk)
+            hasher.update(chunk)
+
+    # Keep original file metadata (permission bits, last access time, last modification time, and flags)
+    shutil.copystat(src, dst)
+
+    file_hash = hasher.hexdigest()
+    logger.info('%s backup to %s with %s hash: %s', src, dst, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def read_and_hash_file(path: Path) -> tuple[bytes, str]:
+    logger.debug('Read and hash file %s using %s into RAM', path, HASH_ALGO)
+    content = path.read_bytes()
+    hasher = hashlib.new(HASH_ALGO, content)
+    file_hash = hasher.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return content, file_hash
+
+
+def iter_scandir_files(path: Path, excludes: set[str]) -> Iterable[os.DirEntry]:
+    """
+    Recursively yield all files+symlinks in the given directory.
+    """
+    logger.debug('Scanning directory %s', path)
+    with os.scandir(path) as scandir_iterator:
+        for entry in scandir_iterator:
+            if entry.is_dir(follow_symlinks=True):
+                if entry.name in excludes:
+                    logger.debug('Excluding directory %s', entry.path)
+                    continue
+                yield from iter_scandir_files(Path(entry.path), excludes=excludes)
+            else:
+                # It's a file or symlink or broken symlink
+                yield entry
+
+
+def humanized_fs_scan(path: Path, excludes: set[str]) -> tuple[int, int]:
+    print(f'\nScanning filesystem at: {path}...')
+
+    progress = Progress(
+        TimeElapsedColumn(),
+        '{task.description}',
+        SpinnerColumn('simpleDots'),
+        TextColumn('[green]{task.fields[file_count]} Files'),
+        HumanFileSizeColumn(field_name='total_size'),
+        TextColumn('| [cyan]{task.fields[files_per_sec]} Files/sec'),
+    )
+
+    file_count = 0
+    total_size = 0
+    start_time = time.time()
+    scan_task_id = progress.add_task(
+        description='Scanning',
+        file_count=file_count,
+        total_size=total_size,
+        files_per_sec=0.0,
+        total=None,
+    )
+    next_update = 0
+    with progress:
+        for entry in iter_scandir_files(path, excludes=excludes):
+            file_count += 1
+            try:
+                total_size += entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: broken symlink
+                continue
+
+            now = time.time()
+            if now >= next_update:
+                elapsed = max(now - start_time, 1e-6)
+                files_per_sec = int(file_count / elapsed)
+                progress.update(
+                    scan_task_id,
+                    file_count=file_count,
+                    total_size=total_size,
+                    files_per_sec=files_per_sec,
+                )
+                next_update = now + 1
+
+        now = time.time()
+
+        elapsed = max(now - start_time, 1e-6)
+        files_per_sec = int(file_count / elapsed)
+        progress.stop_task(scan_task_id)
+        progress.update(
+            scan_task_id,
+            description='Completed',
+            completed=True,
+            file_count=file_count,
+            total_size=total_size,
+            files_per_sec=files_per_sec,
+        )
+
+    return file_count, total_size

PyHardLinkBackup/utilities/humanize.py

@@ -0,0 +1,22 @@
+
+
+def human_filesize(size: int | float) -> str:
+    """
+    >>> human_filesize(1024)
+    '1.00 KiB'
+    >>> human_filesize(2.2*1024)
+    '2.20 KiB'
+    >>> human_filesize(3.33*1024*1024)
+    '3.33 MiB'
+    >>> human_filesize(4.44*1024*1024*1024)
+    '4.44 GiB'
+    >>> human_filesize(5.55*1024*1024*1024*1024)
+    '5.55 TiB'
+    >>> human_filesize(6.66*1024*1024*1024*1024*1024)
+    '6.66 PiB'
+    """
+    for unit in ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']:
+        if size < 1024.0:
+            return f'{size:.2f} {unit}'
+        size /= 1024.0
+    return f'{size:.2f} PiB'

PyHardLinkBackup/utilities/rich_utils.py

@@ -0,0 +1,98 @@
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    Progress,
+    ProgressColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from PyHardLinkBackup.utilities.humanize import human_filesize
+
+
+class HumanFileSizeColumn(ProgressColumn):
+    def __init__(self, field_name: str | None = None, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.field_name = field_name
+
+    def render(self, task):
+        if self.field_name is None:
+            file_size = task.completed
+        else:
+            try:
+                file_size = task.fields[self.field_name]
+            except KeyError:
+                raise KeyError(f'Field {self.field_name=} not found in: {task.fields.keys()=}') from None
+        return Text(f'| {human_filesize(file_size)}')
+
+
+class BackupProgress:
+    def __init__(self, src_file_count: int, src_total_size: int):
+        self.overall_progress = Progress(
+            TaskProgressColumn(),
+            BarColumn(bar_width=50),
+            TextColumn('Elapsed:'),
+            TimeElapsedColumn(),
+            TextColumn('Remaining:'),
+            TimeRemainingColumn(),
+        )
+        self.overall_progress_task_id = self.overall_progress.add_task(description='', total=100)
+
+        self.file_count_progress = Progress(
+            TaskProgressColumn(),
+            BarColumn(bar_width=50),
+            TextColumn('{task.completed} Files'),
+        )
+        self.file_count_progress_task_id = self.file_count_progress.add_task(description='', total=src_file_count)
+        self.file_count_progress_task = self.file_count_progress.tasks[0]
+
+        self.file_size_progress = Progress(
+            TaskProgressColumn(),
+            BarColumn(bar_width=50),
+            HumanFileSizeColumn(),
+            '|',
+            TransferSpeedColumn(),
+        )
+        self.file_size_progress_task_id = self.file_size_progress.add_task(description='', total=src_total_size)
+        self.file_size_progress_task = self.file_size_progress.tasks[0]
+
+        progress_table = Table.grid()
+        progress_table.add_row(Panel(self.overall_progress, title='[b]Overall Backup Progress', border_style='green'))
+        progress_table.add_row(Panel(self.file_count_progress, title='Total files saved'))
+        progress_table.add_row(Panel(self.file_size_progress, title='Total file size processed'))
+
+        self.live = Live(progress_table, auto_refresh=False)
+
+    def __enter__(self):
+        self.live.__enter__()
+        return self
+
+    def update(self, backup_count: int, backup_size: int):
+        self.file_count_progress.update(
+            task_id=self.file_count_progress_task_id,
+            completed=backup_count,
+            refresh=True,
+        )
+        self.file_size_progress.update(
+            task_id=self.file_size_progress_task_id,
+            completed=backup_size,
+            refresh=True,
+        )
+        self.overall_progress.update(
+            task_id=self.overall_progress_task_id,
+            completed=(self.file_count_progress_task.percentage + self.file_size_progress_task.percentage) / 2,
+            refresh=True,
+        )
+        self.live.refresh()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.overall_progress.stop()
+        self.file_count_progress.stop()
+        self.file_size_progress.stop()
+        self.live.__exit__(exc_type, exc_value, traceback)

PyHardLinkBackup/utilities/tests/test_file_hash_database.py

@@ -0,0 +1,134 @@
+import logging
+import tempfile
+import textwrap
+from pathlib import Path
+from unittest import TestCase
+
+from bx_py_utils.path import assert_is_dir
+from bx_py_utils.test_utils.assertion import assert_text_equal
+
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase, HashAlreadyExistsError
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileHashDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileHashDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir)
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        hash_db = FileHashDatabase(backup_root=backup_root, phlb_conf_dir=phlb_conf_dir)
+        return hash_db
+
+
+def get_hash_db_filenames(hash_db: FileHashDatabase) -> list[str]:
+    return sorted(
+        str(Path(entry.path).relative_to(hash_db.base_path))
+        for entry in iter_scandir_files(hash_db.base_path, excludes=set())
+    )
+
+
+def get_hash_db_info(backup_root: Path) -> str:
+    db_base_path = backup_root / '.phlb' / 'hash-lookup'
+    assert_is_dir(db_base_path)
+
+    lines = []
+    for entry in iter_scandir_files(db_base_path, excludes=set()):
+        hash_path = Path(entry.path)
+        rel_path = hash_path.relative_to(db_base_path)
+        rel_file_path = hash_path.read_text()
+        lines.append(f'{str(rel_path)[:20]}… -> {rel_file_path}')
+    return '\n'.join(sorted(lines))
+
+
+def assert_hash_db_info(backup_root: Path, expected: str):
+    expected = textwrap.dedent(expected).strip()
+    actual = get_hash_db_info(backup_root)
+    assert_text_equal(
+        actual,
+        expected,
+        msg=f'FileHashDatabase info does not match as expected.\n\n{actual}\n\n',
+    )
+
+
+class FileHashDatabaseTestCase(TestCase):
+    def test_happy_path(self):
+        with TemporaryFileHashDatabase() as hash_db:
+            self.assertIsInstance(hash_db, FileHashDatabase)
+
+            backup_root_path = hash_db.backup_root
+            assert_is_dir(backup_root_path)
+
+            test_path = hash_db._get_hash_path('12345678abcdef')
+            self.assertEqual(test_path, hash_db.base_path / '12' / '34' / '12345678abcdef')
+
+            file_a_path = backup_root_path / 'rel/path/to/file-A'
+            file_a_path.parent.mkdir(parents=True, exist_ok=True)
+            file_a_path.touch()
+
+            self.assertIs(hash_db.get('12345678abcdef'), None)
+            hash_db['12345678abcdef'] = file_a_path
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            self.assertEqual(
+                get_hash_db_filenames(hash_db),
+                ['12/34/12345678abcdef'],
+            )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_hash_db = FileHashDatabase(
+                backup_root=hash_db.backup_root,
+                phlb_conf_dir=hash_db.base_path.parent,
+            )
+            self.assertEqual(another_hash_db.get('12345678abcdef'), file_a_path)
+            self.assertIs(another_hash_db.get('12abcd345678abcdef'), None)
+
+            file_b_path = backup_root_path / 'rel/path/to/file-B'
+            file_b_path.parent.mkdir(parents=True, exist_ok=True)
+            file_b_path.touch()
+
+            another_hash_db['12abcd345678abcdef'] = file_b_path
+            self.assertEqual(another_hash_db.get('12abcd345678abcdef'), file_b_path)
+            self.assertEqual(
+                get_hash_db_filenames(another_hash_db),
+                [
+                    '12/34/12345678abcdef',
+                    '12/ab/12abcd345678abcdef',
+                ],
+            )
+
+            assert_hash_db_info(
+                backup_root=hash_db.backup_root,
+                expected="""
+                    12/34/12345678abcdef… -> rel/path/to/file-A
+                    12/ab/12abcd345678ab… -> rel/path/to/file-B
+                """,
+            )
+
+            ########################################################################################
+            # Deny "overwrite" of existing hash:
+
+            with self.assertRaises(HashAlreadyExistsError):
+                hash_db['12abcd345678abcdef'] = 'foo/bar/baz'  # already exists!
+
+            ########################################################################################
+            # Don't use stale entries pointing to missing files:
+
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            file_a_path.unlink()
+
+            """DocWrite: README.md ## FileHashDatabase - Missing hardlink target file
+            We check if the hardlink source file still exists. If not, we remove the hash entry from the database.
+            A warning is logged in this case."""
+            with self.assertLogs(level=logging.WARNING) as logs:
+                self.assertIs(hash_db.get('12345678abcdef'), None)
+            self.assertIn('Hash database entry found, but file does not exist', ''.join(logs.output))
+            assert_hash_db_info(
+                backup_root=hash_db.backup_root,
+                expected="""
+                    12/ab/12abcd345678ab… -> rel/path/to/file-B
+                """,
+            )

PyHardLinkBackup/utilities/tests/test_file_size_database.py

@@ -0,0 +1,131 @@
+import tempfile
+from collections.abc import Iterable
+from pathlib import Path
+from unittest import TestCase
+
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileSizeDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileSizeDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir)
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        size_db = FileSizeDatabase(phlb_conf_dir=phlb_conf_dir)
+        return size_db
+
+
+def get_size_db_filenames(size_db: FileSizeDatabase) -> Iterable[str]:
+    return sorted(
+        str(Path(entry.path).relative_to(size_db.base_path))
+        for entry in iter_scandir_files(size_db.base_path, excludes=set())
+    )
+
+
+def get_sizes(size_db: FileSizeDatabase) -> Iterable[int]:
+    return sorted(int(entry.name) for entry in iter_scandir_files(size_db.base_path, excludes=set()))
+
+
+class FileSizeDatabaseTestCase(TestCase):
+    def test_happy_path(self):
+        with TemporaryFileSizeDatabase() as size_db:
+            self.assertIsInstance(size_db, FileSizeDatabase)
+
+            test_path1 = size_db._get_size_path(1234)
+            self.assertEqual(test_path1, size_db.base_path / '12' / '34' / '1234')
+
+            test_path2 = size_db._get_size_path(567890)
+            self.assertEqual(test_path2, size_db.base_path / '56' / '78' / '567890')
+
+            self.assertNotIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(1234)
+            self.assertIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(567890)
+            self.assertIn(1234, size_db)
+            self.assertIn(567890, size_db)
+
+            self.assertEqual(get_sizes(size_db), [1234, 567890])
+            self.assertEqual(
+                get_size_db_filenames(size_db),
+                [
+                    '12/34/1234',
+                    '56/78/567890',
+                ],
+            )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_size_db = FileSizeDatabase(phlb_conf_dir=size_db.base_path.parent)
+            self.assertEqual(get_sizes(another_size_db), [1234, 567890])
+            self.assertEqual(
+                get_size_db_filenames(another_size_db),
+                [
+                    '12/34/1234',
+                    '56/78/567890',
+                ],
+            )
+
+            ########################################################################################
+            # "Share" directories:
+
+            for size in (123400001111, 123400002222, 128800003333, 129900004444):
+                self.assertNotIn(size, size_db)
+                size_db.add(size)
+                self.assertIn(size, size_db)
+
+            ########################################################################################
+            # Min size is 1000 bytes:
+
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The minimum file size that can be stored in the FileSizeDatabase is 1000 bytes.
+            This is because no padding is made for sizes below 1000 bytes, which would
+            break the directory structure.
+            """
+            self.assertEqual(FileSizeDatabase.MIN_SIZE, 1000)
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The idea is, that it's more efficient to backup small files directly, instead of
+            checking for duplicates via hardlinks. Therefore, small files below this size
+            are not tracked in the FileSizeDatabase.
+            """
+
+            with self.assertRaises(AssertionError):
+                size_db._get_size_path(999)
+            with self.assertRaises(AssertionError):
+                size_db.add(999)
+            with self.assertRaises(AssertionError):
+                999 in size_db
+
+            ########################################################################################
+            # Check final state:
+
+            self.assertEqual(
+                get_size_db_filenames(size_db),
+                [
+                    '12/34/1234',
+                    '12/34/123400001111',
+                    '12/34/123400002222',
+                    '12/88/128800003333',
+                    '12/99/129900004444',
+                    '56/78/567890',
+                ],
+            )
+            self.assertEqual(
+                get_sizes(size_db),
+                [
+                    1234,
+                    567890,
+                    123400001111,
+                    123400002222,
+                    128800003333,
+                    129900004444,
+                ],
+            )