PyHardLinkBackup 1.5.0__py3-none-any.whl

PyHardLinkBackup/utilities/filesystem.py

@@ -0,0 +1,158 @@
+import hashlib
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Iterable
+
+from bx_py_utils.path import assert_is_dir
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from PyHardLinkBackup.constants import CHUNK_SIZE, HASH_ALGO
+from PyHardLinkBackup.utilities.rich_utils import HumanFileSizeColumn
+
+
+logger = logging.getLogger(__name__)
+
+
+def hash_file(path: Path) -> str:
+    logger.debug('Hash file %s using %s', path, HASH_ALGO)
+    with path.open('rb') as f:
+        digest = hashlib.file_digest(f, HASH_ALGO)
+
+    file_hash = digest.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def copy_and_hash(src: Path, dst: Path) -> str:
+    logger.debug('Copy and hash file %s to %s using %s', src, dst, HASH_ALGO)
+    hasher = hashlib.new(HASH_ALGO)
+    with src.open('rb') as source_file, dst.open('wb') as dst_file:
+        while chunk := source_file.read(CHUNK_SIZE):
+            dst_file.write(chunk)
+            hasher.update(chunk)
+
+    # Keep original file metadata (permission bits, last access time, last modification time, and flags)
+    shutil.copystat(src, dst)
+
+    file_hash = hasher.hexdigest()
+    logger.info('%s backup to %s with %s hash: %s', src, dst, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def read_and_hash_file(path: Path) -> tuple[bytes, str]:
+    logger.debug('Read and hash file %s using %s into RAM', path, HASH_ALGO)
+    content = path.read_bytes()
+    hasher = hashlib.new(HASH_ALGO, content)
+    file_hash = hasher.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return content, file_hash
+
+
+def iter_scandir_files(path: Path, excludes: set[str]) -> Iterable[os.DirEntry]:
+    """
+    Recursively yield all files+symlinks in the given directory.
+    """
+    logger.debug('Scanning directory %s', path)
+    with os.scandir(path) as scandir_iterator:
+        for entry in scandir_iterator:
+            if entry.is_dir(follow_symlinks=True):
+                if entry.name in excludes:
+                    logger.debug('Excluding directory %s', entry.path)
+                    continue
+                yield from iter_scandir_files(Path(entry.path), excludes=excludes)
+            else:
+                # It's a file or symlink or broken symlink
+                yield entry
+
+
+def humanized_fs_scan(path: Path, excludes: set[str]) -> tuple[int, int]:
+    print(f'\nScanning filesystem at: {path}...')
+
+    progress = Progress(
+        TimeElapsedColumn(),
+        '{task.description}',
+        SpinnerColumn('simpleDots'),
+        TextColumn('[green]{task.fields[file_count]} Files'),
+        '|',
+        HumanFileSizeColumn(field_name='total_size'),
+        '|',
+        TextColumn('[cyan]{task.fields[files_per_sec]} Files/sec'),
+    )
+
+    file_count = 0
+    total_size = 0
+    start_time = time.time()
+    scan_task_id = progress.add_task(
+        description='Scanning',
+        file_count=file_count,
+        total_size=total_size,
+        files_per_sec=0.0,
+        total=None,
+    )
+    next_update = 0
+    with progress:
+        for entry in iter_scandir_files(path, excludes=excludes):
+            file_count += 1
+            try:
+                total_size += entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: broken symlink
+                continue
+
+            now = time.time()
+            if now >= next_update:
+                elapsed = max(now - start_time, 1e-6)
+                files_per_sec = int(file_count / elapsed)
+                progress.update(
+                    scan_task_id,
+                    file_count=file_count,
+                    total_size=total_size,
+                    files_per_sec=files_per_sec,
+                )
+                next_update = now + 1
+
+        now = time.time()
+
+        elapsed = max(now - start_time, 1e-6)
+        files_per_sec = int(file_count / elapsed)
+        progress.stop_task(scan_task_id)
+        progress.update(
+            scan_task_id,
+            description='Completed',
+            completed=True,
+            file_count=file_count,
+            total_size=total_size,
+            files_per_sec=files_per_sec,
+        )
+
+    return file_count, total_size
+
+
+def supports_hardlinks(directory: Path) -> bool:
+    logger.debug('Checking hardlink support in %s', directory)
+    assert_is_dir(directory)
+    test_src_file = directory / '.phlb_test'
+    test_dst_file = directory / '.phlb_test_link'
+    hardlinks_supported = False
+    try:
+        test_src_file.write_text('test')
+        os.link(test_src_file, test_dst_file)
+        assert test_dst_file.read_text() == 'test'
+        hardlinks_supported = True
+    except OSError as err:
+        # e.g.: FAT/exFAT filesystems ;)
+        logger.exception('Hardlink test failed in %s: %s', directory, err)
+    finally:
+        test_src_file.unlink(missing_ok=True)
+        test_dst_file.unlink(missing_ok=True)
+
+    logger.info('Hardlink support in %s: %s', directory, hardlinks_supported)
+    return hardlinks_supported

PyHardLinkBackup/utilities/humanize.py

@@ -0,0 +1,39 @@
+import time
+
+from bx_py_utils.humanize.time import human_timedelta
+
+
+def human_filesize(size: int | float) -> str:
+    """
+    >>> human_filesize(1024)
+    '1.00 KiB'
+    >>> human_filesize(2.2*1024)
+    '2.20 KiB'
+    >>> human_filesize(3.33*1024*1024)
+    '3.33 MiB'
+    >>> human_filesize(4.44*1024*1024*1024)
+    '4.44 GiB'
+    >>> human_filesize(5.55*1024*1024*1024*1024)
+    '5.55 TiB'
+    >>> human_filesize(6.66*1024*1024*1024*1024*1024)
+    '6.66 PiB'
+    """
+    for unit in ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']:
+        if size < 1024.0:
+            return f'{size:.2f} {unit}'
+        size /= 1024.0
+    return f'{size:.2f} PiB'
+
+
+class PrintTimingContextManager:
+    def __init__(self, description: str):
+        self.description = description
+
+    def __enter__(self) -> None:
+        self.start_time = time.perf_counter()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        duration = time.perf_counter() - self.start_time
+        print(f'{self.description}: {human_timedelta(duration)}\n')
+        if exc_type:
+            return False  # Do not suppress exceptions

PyHardLinkBackup/utilities/rich_utils.py

@@ -0,0 +1,99 @@
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    Progress,
+    ProgressColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from PyHardLinkBackup.utilities.humanize import human_filesize
+
+
+class HumanFileSizeColumn(ProgressColumn):
+    def __init__(self, field_name: str | None = None, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.field_name = field_name
+
+    def render(self, task):
+        if self.field_name is None:
+            file_size = task.completed
+        else:
+            try:
+                file_size = task.fields[self.field_name]
+            except KeyError:
+                raise KeyError(f'Field {self.field_name=} not found in: {task.fields.keys()=}') from None
+        return Text(human_filesize(file_size))
+
+
+class DisplayFileTreeProgress:
+    def __init__(self, total_file_count: int, total_size: int):
+        percentage_format = '[progress.percentage]{task.percentage:>3.1f}%'
+        self.overall_progress = Progress(
+            TaskProgressColumn(text_format=percentage_format),
+            BarColumn(bar_width=50),
+            TextColumn('Elapsed:'),
+            TimeElapsedColumn(),
+            TextColumn('Remaining:'),
+            TimeRemainingColumn(),
+        )
+        self.overall_progress_task_id = self.overall_progress.add_task(description='', total=100)
+
+        self.file_count_progress = Progress(
+            TaskProgressColumn(text_format=percentage_format),
+            BarColumn(bar_width=50),
+            TextColumn('{task.completed}/{task.total} Files'),
+        )
+        self.file_count_progress_task_id = self.file_count_progress.add_task(description='', total=total_file_count)
+        self.file_count_progress_task = self.file_count_progress.tasks[0]
+
+        self.file_size_progress = Progress(
+            TaskProgressColumn(text_format=percentage_format),
+            BarColumn(bar_width=50),
+            HumanFileSizeColumn(),
+            '|',
+            TransferSpeedColumn(),
+        )
+        self.file_size_progress_task_id = self.file_size_progress.add_task(description='', total=total_size)
+        self.file_size_progress_task = self.file_size_progress.tasks[0]
+
+        progress_table = Table.grid()
+        progress_table.add_row(Panel(self.overall_progress, title='[b]Overall Progress', border_style='green'))
+        progress_table.add_row(Panel(self.file_count_progress, title='Total files saved'))
+        progress_table.add_row(Panel(self.file_size_progress, title='Total file size processed'))
+
+        self.live = Live(progress_table, auto_refresh=False)
+
+    def __enter__(self):
+        self.live.__enter__()
+        return self
+
+    def update(self, completed_file_count: int, completed_size: int):
+        self.file_count_progress.update(
+            task_id=self.file_count_progress_task_id,
+            completed=completed_file_count,
+            refresh=True,
+        )
+        self.file_size_progress.update(
+            task_id=self.file_size_progress_task_id,
+            completed=completed_size,
+            refresh=True,
+        )
+        self.overall_progress.update(
+            task_id=self.overall_progress_task_id,
+            completed=(self.file_count_progress_task.percentage + self.file_size_progress_task.percentage) / 2,
+            refresh=True,
+        )
+        self.live.refresh()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.overall_progress.stop()
+        self.file_count_progress.stop()
+        self.file_size_progress.stop()
+        self.live.__exit__(exc_type, exc_value, traceback)

PyHardLinkBackup/utilities/sha256sums.py

@@ -0,0 +1,61 @@
+import logging
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_sha256sums_path(file_path: Path):
+    """
+    >>> get_sha256sums_path(Path('foo/bar/baz.txt'))
+    PosixPath('foo/bar/SHA256SUMS')
+    """
+    hash_file_path = file_path.parent / 'SHA256SUMS'
+    return hash_file_path
+
+
+def store_hash(file_path: Path, file_hash: str):
+    """DocWrite: README.md ## SHA256SUMS
+    A `SHA256SUMS` file is stored in each backup directory containing the SHA256 hashes of all files in that directory.
+    It's the same format as e.g.: `sha256sum * > SHA256SUMS` command produces.
+    So it's possible to verify the integrity of the backup files later.
+    e.g.:
+    ```bash
+    cd .../your/backup/foobar/20240101_120000/
+    sha256sum -c SHA256SUMS
+    ```
+    """
+    hash_file_path = get_sha256sums_path(file_path)
+    with hash_file_path.open('a') as f:
+        f.write(f'{file_hash}  {file_path.name}\n')
+
+
+def check_sha256sums(
+    *,
+    file_path: Path,
+    file_hash: str,
+) -> bool | None:
+    hash_file_path = get_sha256sums_path(file_path=file_path)
+    if not hash_file_path.is_file():
+        return None  # Nothing to verify against
+
+    with hash_file_path.open('r') as f:
+        for line in f:
+            try:
+                expected_hash, filename = line.split(' ', maxsplit=1)
+            except ValueError:
+                logger.exception(f'Invalid line in "{hash_file_path}": {line!r}')
+            else:
+                filename = filename.strip()
+                if filename == file_path.name:
+                    if not expected_hash == file_hash:
+                        logger.error(
+                            f'Hash {file_hash} from file {file_path} does not match hash in {hash_file_path} !'
+                        )
+                        return False
+                    else:
+                        logger.debug(f'{file_path} hash verified successfully from {hash_file_path}.')
+                        return True
+
+    logger.info('No SHA256SUMS entry found for file: %s', file_path)
+    return None

PyHardLinkBackup/utilities/tee.py

@@ -0,0 +1,40 @@
+import re
+import sys
+from contextlib import redirect_stdout
+
+
+# Borrowed from click:
+_ansi_re = re.compile(r'\033\[[;?0-9]*[a-zA-Z]')
+
+
+def strip_ansi_codes(value: str) -> str:
+    return _ansi_re.sub('', value)
+
+
+class TeeStdout:
+    def __init__(self, file):
+        self.file = file
+        self.stdout = sys.stdout
+
+    def write(self, data):
+        self.stdout.write(data)
+        self.file.write(strip_ansi_codes(data))
+
+    def flush(self):
+        self.stdout.flush()
+        self.file.flush()
+
+
+class TeeStdoutContext:
+    def __init__(self, file_path):
+        self.file_path = file_path
+
+    def __enter__(self):
+        self.file = open(self.file_path, 'w')
+        self.redirect = redirect_stdout(TeeStdout(self.file))
+        self.redirect.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.redirect.__exit__(exc_type, exc_val, exc_tb)
+        self.file.close()

PyHardLinkBackup/utilities/tests/test_file_hash_database.py

@@ -0,0 +1,143 @@
+import logging
+import tempfile
+import textwrap
+from pathlib import Path
+
+from bx_py_utils.path import assert_is_dir
+from bx_py_utils.test_utils.assertion import assert_text_equal
+from bx_py_utils.test_utils.log_utils import NoLogs
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase, HashAlreadyExistsError
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileHashDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileHashDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir).resolve()
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        hash_db = FileHashDatabase(backup_root=backup_root, phlb_conf_dir=phlb_conf_dir)
+        return hash_db
+
+
+def get_hash_db_filenames(hash_db: FileHashDatabase) -> list[str]:
+    # with NoLogs('PyHardLinkBackup.utilities.filesystem'):
+    return sorted(
+        str(Path(entry.path).relative_to(hash_db.base_path))
+        for entry in iter_scandir_files(hash_db.base_path, excludes=set())
+    )
+
+
+def get_hash_db_info(backup_root: Path) -> str:
+    db_base_path = backup_root / '.phlb' / 'hash-lookup'
+    assert_is_dir(db_base_path)
+
+    with NoLogs(logger_name='XY'):
+        lines = []
+        for entry in iter_scandir_files(db_base_path, excludes=set()):
+            hash_path = Path(entry.path)
+            rel_path = hash_path.relative_to(db_base_path)
+            rel_file_path = hash_path.read_text()
+            lines.append(f'{str(rel_path)[:20]}… -> {rel_file_path}')
+    return '\n'.join(sorted(lines))
+
+
+def assert_hash_db_info(backup_root: Path, expected: str):
+    expected = textwrap.dedent(expected).strip()
+    actual = get_hash_db_info(backup_root)
+    assert_text_equal(
+        actual,
+        expected,
+        msg=f'FileHashDatabase info does not match as expected.\n\n{actual}\n\n',
+    )
+
+
+class FileHashDatabaseTestCase(BaseTestCase):
+    def test_happy_path(self):
+        with TemporaryFileHashDatabase() as hash_db:
+            self.assertIsInstance(hash_db, FileHashDatabase)
+
+            backup_root_path = hash_db.backup_root
+            assert_is_dir(backup_root_path)
+
+            test_path = hash_db._get_hash_path('12345678abcdef')
+            self.assertEqual(test_path, hash_db.base_path / '12' / '34' / '12345678abcdef')
+
+            file_a_path = backup_root_path / 'rel/path/to/file-A'
+            file_a_path.parent.mkdir(parents=True, exist_ok=True)
+            file_a_path.touch()
+
+            self.assertIs(hash_db.get('12345678abcdef'), None)
+            self.assertIs('12345678abcdef' in hash_db, False)
+            hash_db['12345678abcdef'] = file_a_path
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            self.assertIs('12345678abcdef' in hash_db, True)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_hash_db_filenames(hash_db),
+                    ['12/34/12345678abcdef'],
+                )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_hash_db = FileHashDatabase(
+                backup_root=hash_db.backup_root,
+                phlb_conf_dir=hash_db.base_path.parent,
+            )
+            self.assertEqual(another_hash_db.get('12345678abcdef'), file_a_path)
+            self.assertIs(another_hash_db.get('12abcd345678abcdef'), None)
+
+            file_b_path = backup_root_path / 'rel/path/to/file-B'
+            file_b_path.parent.mkdir(parents=True, exist_ok=True)
+            file_b_path.touch()
+
+            another_hash_db['12abcd345678abcdef'] = file_b_path
+            self.assertEqual(another_hash_db.get('12abcd345678abcdef'), file_b_path)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_hash_db_filenames(another_hash_db),
+                    [
+                        '12/34/12345678abcdef',
+                        '12/ab/12abcd345678abcdef',
+                    ],
+                )
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                assert_hash_db_info(
+                    backup_root=hash_db.backup_root,
+                    expected="""
+                        12/34/12345678abcdef… -> rel/path/to/file-A
+                        12/ab/12abcd345678ab… -> rel/path/to/file-B
+                    """,
+                )
+
+            ########################################################################################
+            # Deny "overwrite" of existing hash:
+
+            with self.assertRaises(HashAlreadyExistsError):
+                hash_db['12abcd345678abcdef'] = 'foo/bar/baz'  # already exists!
+
+            ########################################################################################
+            # Don't use stale entries pointing to missing files:
+
+            self.assertEqual(hash_db.get('12345678abcdef'), file_a_path)
+            file_a_path.unlink()
+
+            """DocWrite: README.md ## FileHashDatabase - Missing hardlink target file
+            We check if the hardlink source file still exists. If not, we remove the hash entry from the database.
+            A warning is logged in this case."""
+            with self.assertLogs(level=logging.WARNING) as logs:
+                self.assertIs(hash_db.get('12345678abcdef'), None)
+            self.assertIn('Hash database entry found, but file does not exist', ''.join(logs.output))
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                assert_hash_db_info(
+                    backup_root=hash_db.backup_root,
+                    expected="""
+                        12/ab/12abcd345678ab… -> rel/path/to/file-B
+                    """,
+                )

PyHardLinkBackup/utilities/tests/test_file_size_database.py

@@ -0,0 +1,138 @@
+import logging
+import tempfile
+from collections.abc import Iterable
+from pathlib import Path
+
+from bx_py_utils.test_utils.log_utils import NoLogs
+from cli_base.cli_tools.test_utils.base_testcases import BaseTestCase
+
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+
+
+class TemporaryFileSizeDatabase(tempfile.TemporaryDirectory):
+    def __enter__(self) -> FileSizeDatabase:
+        temp_dir = super().__enter__()
+        backup_root = Path(temp_dir).resolve()
+
+        phlb_conf_dir = backup_root / '.phlb'
+        phlb_conf_dir.mkdir()
+
+        size_db = FileSizeDatabase(phlb_conf_dir=phlb_conf_dir)
+        return size_db
+
+
+def get_size_db_filenames(size_db: FileSizeDatabase) -> Iterable[str]:
+    return sorted(
+        str(Path(entry.path).relative_to(size_db.base_path))
+        for entry in iter_scandir_files(size_db.base_path, excludes=set())
+    )
+
+
+def get_sizes(size_db: FileSizeDatabase) -> Iterable[int]:
+    with NoLogs('PyHardLinkBackup.utilities.filesystem'):
+        return sorted(int(entry.name) for entry in iter_scandir_files(size_db.base_path, excludes=set()))
+
+
+class FileSizeDatabaseTestCase(BaseTestCase):
+    def test_happy_path(self):
+        with TemporaryFileSizeDatabase() as size_db:
+            self.assertIsInstance(size_db, FileSizeDatabase)
+
+            test_path1 = size_db._get_size_path(1234)
+            self.assertEqual(test_path1, size_db.base_path / '12' / '34' / '1234')
+
+            test_path2 = size_db._get_size_path(567890)
+            self.assertEqual(test_path2, size_db.base_path / '56' / '78' / '567890')
+
+            self.assertNotIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(1234)
+            self.assertIn(1234, size_db)
+            self.assertNotIn(567890, size_db)
+
+            size_db.add(567890)
+            self.assertIn(1234, size_db)
+            self.assertIn(567890, size_db)
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(get_sizes(size_db), [1234, 567890])
+                self.assertEqual(
+                    get_size_db_filenames(size_db),
+                    [
+                        '12/34/1234',
+                        '56/78/567890',
+                    ],
+                )
+
+            ########################################################################################
+            # Another instance using the same directory:
+
+            another_size_db = FileSizeDatabase(phlb_conf_dir=size_db.base_path.parent)
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(get_sizes(another_size_db), [1234, 567890])
+                self.assertEqual(
+                    get_size_db_filenames(another_size_db),
+                    [
+                        '12/34/1234',
+                        '56/78/567890',
+                    ],
+                )
+
+            ########################################################################################
+            # "Share" directories:
+
+            for size in (123400001111, 123400002222, 128800003333, 129900004444):
+                self.assertNotIn(size, size_db)
+                size_db.add(size)
+                self.assertIn(size, size_db)
+
+            ########################################################################################
+            # Min size is 1000 bytes:
+
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The minimum file size that can be stored in the FileSizeDatabase is 1000 bytes.
+            This is because no padding is made for sizes below 1000 bytes, which would
+            break the directory structure.
+            """
+            self.assertEqual(FileSizeDatabase.MIN_SIZE, 1000)
+            """DocWrite: README.md ## FileSizeDatabase - minimum file size
+            The idea is, that it's more efficient to backup small files directly, instead of
+            checking for duplicates via hardlinks. Therefore, small files below this size
+            are not tracked in the FileSizeDatabase.
+            """
+
+            with self.assertRaises(AssertionError):
+                size_db._get_size_path(999)
+            with self.assertRaises(AssertionError):
+                size_db.add(999)
+            with self.assertRaises(AssertionError):
+                999 in size_db
+
+            ########################################################################################
+            # Check final state:
+
+            with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
+                self.assertEqual(
+                    get_size_db_filenames(size_db),
+                    [
+                        '12/34/1234',
+                        '12/34/123400001111',
+                        '12/34/123400002222',
+                        '12/88/128800003333',
+                        '12/99/129900004444',
+                        '56/78/567890',
+                    ],
+                )
+                self.assertEqual(
+                    get_sizes(size_db),
+                    [
+                        1234,
+                        567890,
+                        123400001111,
+                        123400002222,
+                        128800003333,
+                        129900004444,
+                    ],
+                )