PyHardLinkBackup 1.8.1__py3-none-any.whl

PyHardLinkBackup/utilities/file_size_database.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+
+class FileSizeDatabase:
+    """DocWrite: README.md ## FileSizeDatabase
+    A simple "database" to track which file sizes have been seen.
+
+    Uses a directory structure to avoid too many files in a single directory.
+    We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    """
+
+    MIN_SIZE = 1000  # no padding is made, so the min size is 1000 bytes!
+
+    def __init__(self, phlb_conf_dir: Path):
+        self.base_path = phlb_conf_dir / 'size-lookup'
+        self.base_path.mkdir(parents=False, exist_ok=True)
+
+    def _get_size_path(self, size: int) -> Path:
+        assert size >= self.MIN_SIZE, f'Size must be at least {self.MIN_SIZE} bytes'
+        size_str = str(size)
+
+        """DocWrite: README.md ## FileSizeDatabase
+        Path structure:
+         * `{base_dst}/.phlb/size-lookup/{XX}/{YY}/{size}`
+
+        e.g.:
+
+         * `1234567890` results in: `{base_dst}/.phlb/size-lookup/12/34/1234567890`
+        """
+        first_dir_name = size_str[:2]
+        second_dir_name = size_str[2:4]
+        size_path = self.base_path / first_dir_name / second_dir_name / size_str
+        return size_path
+
+    def __contains__(self, size: int) -> bool:
+        size_path = self._get_size_path(size)
+        return size_path.exists()
+
+    def add(self, size: int):
+        size_path = self._get_size_path(size)
+        if not size_path.exists():
+            size_path.parent.mkdir(parents=True, exist_ok=True)
+
+            """DocWrite: README.md ## FileSizeDatabase
+            All files are created empty, as we only care about their existence."""
+            size_path.touch(exist_ok=False)

PyHardLinkBackup/utilities/filesystem.py

@@ -0,0 +1,257 @@
+import hashlib
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Iterable
+
+from bx_py_utils.path import assert_is_dir
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from PyHardLinkBackup.constants import CHUNK_SIZE, HASH_ALGO
+from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress, HumanFileSizeColumn, LargeFileProgress
+
+
+logger = logging.getLogger(__name__)
+
+MIN_SIZE_FOR_PROGRESS_BAR = CHUNK_SIZE * 10
+
+
+def verbose_path_stat(path: Path) -> os.stat_result:
+    stat_result = path.stat()
+    stat_dict = {}
+    for key in dir(stat_result):
+        if key.startswith('st_'):
+            value = getattr(stat_result, key)
+            stat_dict[key] = value
+    logger.info('Stat for %s: %s', path, stat_dict)
+    return stat_result
+
+
+class RemoveFileOnError:
+    def __init__(self, file_path: Path):
+        self.file_path = file_path
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        if exc_type:
+            logger.info(
+                f'Removing incomplete file {self.file_path} due to error: {exc_value}',
+                exc_info=(exc_type, exc_value, exc_traceback),
+            )
+            self.file_path.unlink(missing_ok=True)
+            return False
+
+
+def hash_file(path: Path, progress: DisplayFileTreeProgress, total_size: int) -> str:
+    logger.debug('Hash file %s using %s', path, HASH_ALGO)
+    hasher = hashlib.new(HASH_ALGO)
+    with LargeFileProgress(
+        description=f'Hashing large file: "[yellow]{path}[/yellow]"',
+        parent_progress=progress,
+        total_size=total_size,
+    ) as progress_bar:
+        with path.open('rb') as f:
+            while chunk := f.read(CHUNK_SIZE):
+                hasher.update(chunk)
+                progress_bar.update(advance=len(chunk))
+    file_hash = hasher.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def copy_with_progress(src: Path, dst: Path, progress: DisplayFileTreeProgress, total_size: int) -> None:
+    logger.debug('Copy file %s to %s using %s', src, dst, HASH_ALGO)
+    with LargeFileProgress(
+        description=f'Copying large file: "[yellow]{src}[/yellow]"',
+        parent_progress=progress,
+        total_size=total_size,
+    ) as progress_bar:
+        with src.open('rb') as source_file, dst.open('wb') as dst_file:
+            while chunk := source_file.read(CHUNK_SIZE):
+                dst_file.write(chunk)
+                progress_bar.update(advance=len(chunk))
+
+    # Keep original file metadata (permission bits, last access time, last modification time, and flags)
+    shutil.copystat(src, dst)
+
+
+def copy_and_hash(src: Path, dst: Path, progress: DisplayFileTreeProgress, total_size: int) -> str:
+    logger.debug('Copy and hash file %s to %s using %s', src, dst, HASH_ALGO)
+    hasher = hashlib.new(HASH_ALGO)
+    with LargeFileProgress(
+        description=f'Copy and hash large file: "[yellow]{src}[/yellow]"',
+        parent_progress=progress,
+        total_size=total_size,
+    ) as progress_bar:
+        with src.open('rb') as source_file, dst.open('wb') as dst_file:
+            while chunk := source_file.read(CHUNK_SIZE):
+                dst_file.write(chunk)
+                hasher.update(chunk)
+                progress_bar.update(advance=len(chunk))
+
+    # Keep original file metadata (permission bits, last access time, last modification time, and flags)
+    shutil.copystat(src, dst)
+
+    file_hash = hasher.hexdigest()
+    logger.info('%s backup to %s with %s hash: %s', src, dst, HASH_ALGO, file_hash)
+    return file_hash
+
+
+def read_and_hash_file(path: Path) -> tuple[bytes, str]:
+    logger.debug('Read and hash file %s using %s into RAM', path, HASH_ALGO)
+    content = path.read_bytes()
+    hasher = hashlib.new(HASH_ALGO, content)
+    file_hash = hasher.hexdigest()
+    logger.info('%s %s hash: %s', path, HASH_ALGO, file_hash)
+    return content, file_hash
+
+
+def iter_scandir_files(
+    *,
+    path: Path,
+    one_file_system: bool,
+    src_device_id,
+    excludes: set[str],
+) -> Iterable[os.DirEntry]:
+    """
+    Recursively yield all files+symlinks in the given directory.
+    Note: Directory symlinks are treated as files (not recursed into).
+    """
+    logger.debug('Scanning directory %s', path)
+    with os.scandir(path) as scandir_iterator:
+        for entry in scandir_iterator:
+            if entry.is_dir(
+                follow_symlinks=False,  # Handle directory symlinks as files!
+            ):
+                if entry.name in excludes:
+                    logger.debug('Excluding directory %s', entry.path)
+                    continue
+
+                if one_file_system:
+                    try:
+                        entry_device_id = entry.stat(follow_symlinks=False).st_dev
+                    except OSError as err:
+                        # e.g.: broken symlink
+                        logger.debug('Skipping directory %s: %s', entry.path, err)
+                        continue
+                    if entry_device_id != src_device_id:
+                        logger.debug(
+                            'Skipping directory %s: different device ID %s (src device ID: %s)',
+                            entry.path,
+                            entry_device_id,
+                            src_device_id,
+                        )
+                        continue
+
+                yield from iter_scandir_files(
+                    path=Path(entry.path),
+                    one_file_system=one_file_system,
+                    src_device_id=src_device_id,
+                    excludes=excludes,
+                )
+            else:
+                # It's a file or symlink or broken symlink
+                yield entry
+
+
+def humanized_fs_scan(
+    *,
+    path: Path,
+    **iter_scandir_files_kwargs,
+) -> tuple[int, int]:
+    print(f'\nScanning filesystem at: {path}...')
+
+    progress = Progress(
+        TimeElapsedColumn(),
+        '{task.description}',
+        SpinnerColumn('simpleDots'),
+        TextColumn('[green]{task.fields[file_count]} Files'),
+        '|',
+        HumanFileSizeColumn(field_name='total_size'),
+        '|',
+        TextColumn('[cyan]{task.fields[files_per_sec]} Files/sec'),
+    )
+
+    file_count = 0
+    total_size = 0
+    start_time = time.time()
+    scan_task_id = progress.add_task(
+        description='Scanning',
+        file_count=file_count,
+        total_size=total_size,
+        files_per_sec=0.0,
+        total=None,
+    )
+    next_update = 0
+    with progress:
+        for entry in iter_scandir_files(path=path, **iter_scandir_files_kwargs):
+            if not entry.is_file():
+                # Ignore e.g.: directory symlinks
+                continue
+
+            file_count += 1
+
+            try:
+                total_size += entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: broken symlink
+                continue
+
+            now = time.time()
+            if now >= next_update:
+                elapsed = max(now - start_time, 1e-6)
+                files_per_sec = int(file_count / elapsed)
+                progress.update(
+                    scan_task_id,
+                    file_count=file_count,
+                    total_size=total_size,
+                    files_per_sec=files_per_sec,
+                )
+                next_update = now + 1
+
+        now = time.time()
+
+        elapsed = max(now - start_time, 1e-6)
+        files_per_sec = int(file_count / elapsed)
+        progress.stop_task(scan_task_id)
+        progress.update(
+            scan_task_id,
+            description='Completed',
+            completed=True,
+            file_count=file_count,
+            total_size=total_size,
+            files_per_sec=files_per_sec,
+        )
+
+    return file_count, total_size
+
+
+def supports_hardlinks(directory: Path) -> bool:
+    logger.debug('Checking hardlink support in %s', directory)
+    assert_is_dir(directory)
+    test_src_file = directory / '.phlb_test'
+    test_dst_file = directory / '.phlb_test_link'
+    hardlinks_supported = False
+    try:
+        test_src_file.write_text('test')
+        os.link(test_src_file, test_dst_file)
+        assert test_dst_file.read_text() == 'test'
+        hardlinks_supported = True
+    except OSError as err:
+        # e.g.: FAT/exFAT filesystems ;)
+        logger.exception('Hardlink test failed in %s: %s', directory, err)
+    finally:
+        test_src_file.unlink(missing_ok=True)
+        test_dst_file.unlink(missing_ok=True)
+
+    logger.info('Hardlink support in %s: %s', directory, hardlinks_supported)
+    return hardlinks_supported

PyHardLinkBackup/utilities/humanize.py

@@ -0,0 +1,39 @@
+import time
+
+from bx_py_utils.humanize.time import human_timedelta
+
+
+def human_filesize(size: int | float) -> str:
+    """
+    >>> human_filesize(1024)
+    '1.00 KiB'
+    >>> human_filesize(2.2*1024)
+    '2.20 KiB'
+    >>> human_filesize(3.33*1024*1024)
+    '3.33 MiB'
+    >>> human_filesize(4.44*1024*1024*1024)
+    '4.44 GiB'
+    >>> human_filesize(5.55*1024*1024*1024*1024)
+    '5.55 TiB'
+    >>> human_filesize(6.66*1024*1024*1024*1024*1024)
+    '6.66 PiB'
+    """
+    for unit in ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']:
+        if size < 1024.0:
+            return f'{size:.2f} {unit}'
+        size /= 1024.0
+    return f'{size:.2f} PiB'
+
+
+class PrintTimingContextManager:
+    def __init__(self, description: str):
+        self.description = description
+
+    def __enter__(self) -> None:
+        self.start_time = time.perf_counter()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        duration = time.perf_counter() - self.start_time
+        print(f'{self.description}: {human_timedelta(duration)}\n')
+        if exc_type:
+            return False  # Do not suppress exceptions

PyHardLinkBackup/utilities/rich_utils.py

@@ -0,0 +1,237 @@
+import time
+
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    Progress,
+    ProgressColumn,
+    Task,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
+from rich.style import Style
+from rich.table import Table
+from rich.text import Text
+
+from PyHardLinkBackup.constants import LAGE_FILE_PROGRESS_MIN_SIZE
+from PyHardLinkBackup.utilities.humanize import human_filesize
+
+
+class HumanFileSizeColumn(ProgressColumn):
+    def __init__(self, field_name: str | None = None, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.field_name = field_name
+
+    def render(self, task):
+        if self.field_name is None:
+            advance_size = task.completed
+            remaining_size = task.remaining
+            return (
+                f'[white][progress.elapsed]{human_filesize(advance_size)}[white]'
+                f' / [progress.remaining]{human_filesize(remaining_size)}'
+            )
+        else:
+            try:
+                advance_size = task.fields[self.field_name]
+            except KeyError:
+                raise KeyError(f'Field {self.field_name=} not found in: {task.fields.keys()=}') from None
+        return Text(human_filesize(advance_size))
+
+
+class TransferSpeedColumn2(ProgressColumn):
+    def __init__(self, *args, unit: str = 'it', **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.unit = unit
+
+    def render(self, task: Task) -> Text:
+        speed = task.finished_speed or task.speed
+        if speed is None:
+            return Text('?', style='grey50')
+        if speed < 0.1:
+            return Text('-', style='grey50')
+        return Text(f'{speed:.1f} {self.unit}/s', style='progress.data.speed')
+
+
+class DisplayFileTreeProgress:
+    def __init__(self, *, description: str, total_file_count: int, total_size: int):
+        percent_kwargs = dict(
+            text_format='[progress.percentage]{task.percentage:>3.1f}%',
+            justify='right',
+        )
+        self.overall_progress_bar = Progress(TaskProgressColumn(**percent_kwargs), BarColumn(bar_width=None))
+        self.file_count_progress_bar = Progress(TaskProgressColumn(**percent_kwargs), BarColumn(bar_width=None))
+        self.file_size_progress_bar = Progress(TaskProgressColumn(**percent_kwargs), BarColumn(bar_width=None))
+
+        self.overall_progress = Progress(
+            TimeElapsedColumn(),
+            '/',
+            TimeRemainingColumn(),
+        )
+        self.file_count_progress = Progress(
+            TextColumn('[white][progress.elapsed]{task.completed}[white] / [progress.remaining]{task.remaining}'),
+            '|',
+            TransferSpeedColumn2(unit='files'),
+            '|',
+            TimeRemainingColumn(),
+        )
+        self.file_size_progress = Progress(
+            HumanFileSizeColumn(),
+            '|',
+            TransferSpeedColumn(),
+            '|',
+            TimeRemainingColumn(),
+        )
+
+        self.overall_progress_task_bar = self.overall_progress_bar.add_task('', total=100)
+        self.file_count_progress_task_bar = self.file_count_progress_bar.add_task('', total=total_file_count)
+        self.file_size_progress_task_bar = self.file_size_progress_bar.add_task('', total=total_size)
+
+        self.overall_progress_task_time = self.overall_progress.add_task('', total=100)
+        self.file_count_progress_task_time = self.file_count_progress.add_task('', total=total_file_count)
+        self.file_size_progress_task_time = self.file_size_progress.add_task('', total=total_size)
+
+        progress_table = Table(box=None, expand=True, padding=(0, 2), show_header=False)
+        progress_table.add_row('[b]Overall', self.overall_progress_bar, self.overall_progress)
+        progress_table.add_row('Files', self.file_count_progress_bar, self.file_count_progress)
+        progress_table.add_row('Size', self.file_size_progress_bar, self.file_size_progress)
+
+        self.file_count_progress_task = self.file_count_progress.tasks[0]
+        self.file_size_progress_task = self.file_size_progress.tasks[0]
+
+        self.live = Live(
+            Panel(
+                progress_table,
+                title=Text(description, style='progress.data.speed'),
+                border_style=Style(color='white', bold=True),
+            ),
+            auto_refresh=False,
+        )
+
+    def __enter__(self):
+        self.live.__enter__()
+        return self
+
+    def update(
+        self,
+        completed_file_count: int | None = None,
+        advance_size: int | None = None,
+        completed_size: int | None = None,
+    ):
+        if completed_file_count is not None:
+            self.file_count_progress_bar.update(self.file_count_progress_task_bar, completed=completed_file_count)
+            self.file_count_progress.update(self.file_count_progress_task_time, completed=completed_file_count)
+
+        if completed_size is not None:
+            self.file_size_progress_bar.update(self.file_size_progress_task_bar, completed=completed_size)
+            self.file_size_progress.update(self.file_size_progress_task_time, completed=completed_size)
+        elif advance_size is not None:
+            self.file_size_progress_bar.update(self.file_size_progress_task_bar, advance=advance_size)
+            self.file_size_progress.update(self.file_size_progress_task_time, advance=advance_size)
+
+        overall_completed = (self.file_count_progress_task.percentage + self.file_size_progress_task.percentage) / 2
+
+        self.overall_progress_bar.update(self.overall_progress_task_bar, completed=overall_completed)
+        self.overall_progress.update(self.overall_progress_task_time, completed=overall_completed)
+
+        self.live.refresh()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.overall_progress.stop()
+        self.file_count_progress.stop()
+        self.file_size_progress.stop()
+        self.live.__exit__(exc_type, exc_value, traceback)
+
+
+class NoopProgress(DisplayFileTreeProgress):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __enter__(self):
+        return self
+
+    def update(self, *args, **kwargs):
+        pass
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return bool(exc_type)
+
+
+class LargeFileProgress:
+    def __init__(self, description: str, *, parent_progress: DisplayFileTreeProgress, total_size: int):
+        self.description = description
+        self.parent_progress = parent_progress
+        self.total_size = total_size
+
+        self.progress = None
+
+    def __enter__(self):
+        is_large_file = self.total_size > LAGE_FILE_PROGRESS_MIN_SIZE
+        if is_large_file:
+            self.start_time = time.monotonic()
+            self.next_update = self.start_time + 1
+            self.advance = 0
+        else:
+            # No progress indicator for small files
+            self.next_update = None
+        return self
+
+    def update(self, advance: int):
+        if not self.next_update:
+            # Small file -> no progress indicator
+            return
+
+        self.advance += advance
+
+        now = time.monotonic()
+        if now <= self.next_update:
+            return
+        self.next_update = now + 1
+
+        if not self.progress:
+            percent_done = self.advance / self.total_size
+            if percent_done >= 0.4:
+                # After 1 sec. is 40 % done, we are probably done soon!
+                # Avoid showing progress bar for fast operations
+                self.next_update = None  # No progress indicator
+                return
+
+            self.progress = Progress(
+                TaskProgressColumn(text_format='[progress.percentage]{task.percentage:>3.1f}%'),
+                BarColumn(bar_width=None, finished_style=Style(color='rgb(0,100,0)')),
+                HumanFileSizeColumn(),
+                '|',
+                TransferSpeedColumn(),
+                '|',
+                TimeElapsedColumn(),
+                TimeRemainingColumn(),
+            )
+            self.progress.log(f'Large file processing start: {self.description}')
+            self.task_id = self.progress.add_task(
+                description=self.description,
+                total=self.total_size,
+                start_time=self.start_time,
+            )
+            self.live = Live(
+                Panel(self.progress, title=self.description, border_style=Style(color='yellow', bold=True)),
+                auto_refresh=False,
+            )
+            self.live.__enter__()
+
+        self.parent_progress.update(advance_size=self.advance)
+        self.progress.update(task_id=self.task_id, advance=self.advance, refresh=True)
+        self.live.refresh()
+        self.advance = 0
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.progress:
+            self.progress.log(f'Large file processing finished: {self.description}')
+            self.progress.update(task_id=self.task_id, advance=self.advance, refresh=True)
+            self.progress.stop()
+            self.live.renderable.border_style = 'grey50'
+            self.live.refresh()
+            self.live.__exit__(exc_type, exc_value, traceback)
+            print('\n')  # Add spacing after progress bar

PyHardLinkBackup/utilities/sha256sums.py

@@ -0,0 +1,61 @@
+import logging
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_sha256sums_path(file_path: Path):
+    """
+    >>> get_sha256sums_path(Path('foo/bar/baz.txt'))
+    PosixPath('foo/bar/SHA256SUMS')
+    """
+    hash_file_path = file_path.parent / 'SHA256SUMS'
+    return hash_file_path
+
+
+def store_hash(file_path: Path, file_hash: str):
+    """DocWrite: README.md ## SHA256SUMS
+    A `SHA256SUMS` file is stored in each backup directory containing the SHA256 hashes of all files in that directory.
+    It's the same format as e.g.: `sha256sum * > SHA256SUMS` command produces.
+    So it's possible to verify the integrity of the backup files later.
+    e.g.:
+    ```bash
+    cd .../your/backup/foobar/20240101_120000/
+    sha256sum -c SHA256SUMS
+    ```
+    """
+    hash_file_path = get_sha256sums_path(file_path)
+    with hash_file_path.open('a') as f:
+        f.write(f'{file_hash}  {file_path.name}\n')
+
+
+def check_sha256sums(
+    *,
+    file_path: Path,
+    file_hash: str,
+) -> bool | None:
+    hash_file_path = get_sha256sums_path(file_path=file_path)
+    if not hash_file_path.is_file():
+        return None  # Nothing to verify against
+
+    with hash_file_path.open('r') as f:
+        for line in f:
+            try:
+                expected_hash, filename = line.split(' ', maxsplit=1)
+            except ValueError:
+                logger.exception(f'Invalid line in "{hash_file_path}": {line!r}')
+            else:
+                filename = filename.strip()
+                if filename == file_path.name:
+                    if not expected_hash == file_hash:
+                        logger.error(
+                            f'Hash {file_hash} from file {file_path} does not match hash in {hash_file_path} !'
+                        )
+                        return False
+                    else:
+                        logger.debug(f'{file_path} hash verified successfully from {hash_file_path}.')
+                        return True
+
+    logger.info('No SHA256SUMS entry found for file: %s', file_path)
+    return None

PyHardLinkBackup/utilities/tee.py

@@ -0,0 +1,40 @@
+import re
+import sys
+from contextlib import redirect_stdout
+
+
+# Borrowed from click:
+_ansi_re = re.compile(r'\033\[[;?0-9]*[a-zA-Z]')
+
+
+def strip_ansi_codes(value: str) -> str:
+    return _ansi_re.sub('', value)
+
+
+class TeeStdout:
+    def __init__(self, file):
+        self.file = file
+        self.stdout = sys.stdout
+
+    def write(self, data):
+        self.stdout.write(data)
+        self.file.write(strip_ansi_codes(data))
+
+    def flush(self):
+        self.stdout.flush()
+        self.file.flush()
+
+
+class TeeStdoutContext:
+    def __init__(self, file_path):
+        self.file_path = file_path
+
+    def __enter__(self):
+        self.file = open(self.file_path, 'w')
+        self.redirect = redirect_stdout(TeeStdout(self.file))
+        self.redirect.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.redirect.__exit__(exc_type, exc_val, exc_tb)
+        self.file.close()