PyHardLinkBackup 1.0.1__tar.gz → 1.2.0__tar.gz

{pyhardlinkbackup-1.0.1 → pyhardlinkbackup-1.2.0}/.pre-commit-config.yaml

@@ -2,6 +2,6 @@
 # See https://pre-commit.com for more information
 repos:
   - repo: https://github.com/jedie/cli-base-utilities
-    rev: v0.26.0
+    rev: v0.27.0
     hooks:
       - id: update-readme-history

pyhardlinkbackup-1.0.1/README.md → pyhardlinkbackup-1.2.0/PKG-INFO

@@ -1,3 +1,18 @@
+Metadata-Version: 2.4
+Name: PyHardLinkBackup
+Version: 1.2.0
+Summary: HardLink/Deduplication Backups with Python
+Project-URL: Documentation, https://github.com/jedie/PyHardLinkBackup
+Project-URL: Source, https://github.com/jedie/PyHardLinkBackup
+Author-email: Jens Diemer <PyHardLinkBackup@jensdiemer.de>
+License: GPL-3.0-or-later
+Requires-Python: >=3.12
+Requires-Dist: bx-py-utils
+Requires-Dist: cli-base-utilities>=0.27.0
+Requires-Dist: rich
+Requires-Dist: tyro
+Description-Content-Type: text/markdown
+
 # PyHardLinkBackup
 
 [![tests](https://github.com/jedie/PyHardLinkBackup/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/jedie/PyHardLinkBackup/actions/workflows/tests.yml)
@@ -10,6 +25,8 @@ HardLink/Deduplication Backups with Python
 
 **WIP:** v1.0.0 is a complete rewrite of PyHardLinkBackup.
 
+It's similar to `rsync --link-dest` but the deduplication is done globally for all backups and all paths.
+
 ## installation
 
 You can use [pipx](https://pipx.pypa.io/stable/installation/) to install and use PyHardLinkBackup, e.g.:
@@ -185,10 +202,27 @@ usage: ./dev-cli.py [-h] {benchmark-hashes,coverage,install,lint,mypy,nox,pip-au
 
 v1 is a complete rewrite of PyHardLinkBackup.
 
+Overview of main changes:
+
+* Remove Django dependency:
+  * No SQlite database anymore -> Data for deduplication stored in filesystem only
+  * No Django Admin, because we have no database anymore ;)
+* Change hash algorithm from SHA512 to SHA256, because it's faster and still secure enough
+* Don't store `*.sha512` for every file anymore -> We store one `SHA256SUMS` file in every backup directory
+
 ## History
 
 [comment]: <> (✂✂✂ auto generated history start ✂✂✂)
 
+* [v1.2.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.1.0...v1.2.0)
+  * 2026-01-15 - Add error handling: Log exception but continue with the backup
+  * 2026-01-15 - Check permission and hadlink support on destination path
+  * 2026-01-14 - Enhance progress bars
+  * 2026-01-14 - A a note to rsync --link-dest
+  * 2026-01-14 - Use cli_base.cli_tools.test_utils.base_testcases
+* [v1.1.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.1...v1.1.0)
+  * 2026-01-14 - Change backup timestamp directory to old schema: '%Y-%m-%d-%H%M%S'
+  * 2026-01-14 - Add "Overview of main changes" to README
 * [v1.0.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.0...v1.0.1)
   * 2026-01-13 - Store SHA256SUMS files in backup directories
 * [v1.0.0](https://github.com/jedie/PyHardLinkBackup/compare/v0.13.0...v1.0.0)
@@ -201,6 +235,9 @@ v1 is a complete rewrite of PyHardLinkBackup.
   * 2026-01-13 - Add DocWrite, handle broken symlinks, keep file meta, handle missing hardlink sources
   * 2026-01-12 - First working iteration with rich progess bar
   * 2026-01-08 - Rewrite everything
+
+<details><summary>Expand older history entries ...</summary>
+
 * [v0.13.0](https://github.com/jedie/PyHardLinkBackup/compare/v0.12.3...v0.13.0)
   * 2020-03-18 - release v0.13.0
   * 2020-03-17 - deactivate pypy tests in travis, because of SQLite errors, like:
@@ -228,9 +265,6 @@ v1 is a complete rewrite of PyHardLinkBackup.
   * 2020-03-16 - just warn if used directly (needfull for devlopment to call this directly ;)
   * 2020-03-16 - update requirements
   * 2020-03-16 - +pytest-randomly
-
-<details><summary>Expand older history entries ...</summary>
-
 * [v0.12.2](https://github.com/jedie/PyHardLinkBackup/compare/v0.12.1...v0.12.2)
   * 2020-03-06 - repare v0.12.2 release
   * 2020-03-06 - enhance log file content

{pyhardlinkbackup-1.0.1 → pyhardlinkbackup-1.2.0}/PyHardLinkBackup/__init__.py

@@ -3,5 +3,5 @@
 """
 
 # See https://packaging.python.org/en/latest/specifications/version-specifiers/
-__version__ = '1.0.1'
+__version__ = '1.2.0'
 __author__ = 'Jens Diemer <PyHardLinkBackup@jensdiemer.de>'

pyhardlinkbackup-1.2.0/PyHardLinkBackup/backup.py

@@ -0,0 +1,252 @@
+import dataclasses
+import logging
+import os
+import shutil
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+from rich import print  # noqa
+
+from PyHardLinkBackup.constants import CHUNK_SIZE
+from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import (
+    copy_and_hash,
+    hash_file,
+    humanized_fs_scan,
+    iter_scandir_files,
+    read_and_hash_file,
+    supports_hardlinks,
+)
+from PyHardLinkBackup.utilities.humanize import human_filesize
+from PyHardLinkBackup.utilities.rich_utils import BackupProgress
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class BackupResult:
+    backup_dir: Path
+    #
+    backup_count: int = 0
+    backup_size: int = 0
+    #
+    symlink_files: int = 0
+    hardlinked_files: int = 0
+    hardlinked_size: int = 0
+    #
+    copied_files: int = 0
+    copied_size: int = 0
+    #
+    copied_small_files: int = 0
+    copied_small_size: int = 0
+    #
+    error_count: int = 0
+
+
+def backup_one_file(
+    *,
+    src_root: Path,
+    entry: os.DirEntry,
+    size_db: FileSizeDatabase,
+    hash_db: FileHashDatabase,
+    backup_dir: Path,
+    backup_result: BackupResult,
+) -> None:
+    backup_result.backup_count += 1
+    src_path = Path(entry.path)
+
+    dst_path = backup_dir / src_path.relative_to(src_root)
+    dst_dir_path = dst_path.parent
+    if not dst_dir_path.exists():
+        dst_dir_path.mkdir(parents=True, exist_ok=False)
+
+    try:
+        size = entry.stat().st_size
+    except FileNotFoundError:
+        # e.g.: Handle broken symlink
+        target = os.readlink(src_path)
+        dst_path.symlink_to(target)
+        backup_result.symlink_files += 1
+        return
+
+    backup_result.backup_size += size
+
+    if entry.name == 'SHA256SUMS':
+        # Skip existing SHA256SUMS files in source tree,
+        # because we create our own SHA256SUMS files.
+        logger.debug('Skip existing SHA256SUMS file: %s', src_path)
+        return
+
+    if entry.is_symlink():
+        logger.debug('Copy symlink: %s to %s', src_path, dst_path)
+        target = os.readlink(src_path)
+        dst_path.symlink_to(target)
+        backup_result.symlink_files += 1
+        return
+
+    # Process regular files
+    assert entry.is_file(follow_symlinks=False), f'Unexpected non-file: {src_path}'
+
+    # Deduplication logic
+
+    if size < size_db.MIN_SIZE:
+        # Small file -> always copy without deduplication
+        logger.info('Copy small file: %s to %s', src_path, dst_path)
+        file_hash = copy_and_hash(src_path, dst_path)
+        backup_result.copied_files += 1
+        backup_result.copied_size += size
+        backup_result.copied_small_files += 1
+        backup_result.copied_small_size += size
+        store_hash(dst_path, file_hash)
+        return
+
+    if size in size_db:
+        logger.debug('File with size %iBytes found before -> hash: %s', size, src_path)
+
+        if size <= CHUNK_SIZE:
+            # File can be read complete into memory
+            logger.debug('File size %iBytes <= CHUNK_SIZE (%iBytes) -> read complete into memory', size, CHUNK_SIZE)
+            file_content, file_hash = read_and_hash_file(src_path)
+            if existing_path := hash_db.get(file_hash):
+                logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                os.link(existing_path, dst_path)
+                backup_result.hardlinked_files += 1
+                backup_result.hardlinked_size += size
+            else:
+                logger.info('Store unique file: %s to %s', src_path, dst_path)
+                dst_path.write_bytes(file_content)
+                hash_db[file_hash] = dst_path
+                backup_result.copied_files += 1
+                backup_result.copied_size += size
+
+        else:
+            # Large file
+            file_hash = hash_file(src_path)  # Calculate hash without copying
+
+            if existing_path := hash_db.get(file_hash):
+                logger.info('Hardlink duplicate file: %s to %s', dst_path, existing_path)
+                os.link(existing_path, dst_path)
+                backup_result.hardlinked_files += 1
+                backup_result.hardlinked_size += size
+            else:
+                logger.info('Copy unique file: %s to %s', src_path, dst_path)
+                hash_db[file_hash] = dst_path
+                backup_result.copied_files += 1
+                backup_result.copied_size += size
+
+        # Keep original file metadata (permission bits, time stamps, and flags)
+        shutil.copy2(src_path, dst_path)
+    else:
+        # A file with this size not backuped before -> Can't be duplicate -> copy and hash
+        file_hash = copy_and_hash(src_path, dst_path)
+        size_db.add(size)
+        hash_db[file_hash] = dst_path
+        backup_result.copied_files += 1
+        backup_result.copied_size += size
+
+    store_hash(dst_path, file_hash)
+
+
+def store_hash(file_path: Path, file_hash: str):
+    """DocWrite: README.md ## SHA256SUMS
+    A `SHA256SUMS` file is stored in each backup directory containing the SHA256 hashes of all files in that directory.
+    It's the same format as e.g.: `sha256sum * > SHA256SUMS` command produces.
+    So it's possible to verify the integrity of the backup files later.
+    e.g.:
+    ```bash
+    cd .../your/backup/foobar/20240101_120000/
+    sha256sum -c SHA256SUMS
+    ```
+    """
+    hash_file_path = file_path.parent / 'SHA256SUMS'
+    with hash_file_path.open('a') as f:
+        f.write(f'{file_hash}  {file_path.name}\n')
+
+
+def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> BackupResult:
+    src_root = src_root.resolve()
+    if not src_root.is_dir():
+        print('Error: Source directory does not exist!')
+        print(f'Please check source directory: "{src_root}"\n')
+        sys.exit(1)
+
+    backup_root = backup_root.resolve()
+    if not backup_root.is_dir():
+        print('Error: Backup directory does not exist!')
+        print(f'Please create "{backup_root}" directory first and start again!\n')
+        sys.exit(1)
+
+    if not os.access(backup_root, os.W_OK):
+        print('Error: No write access to backup directory!')
+        print(f'Please check permissions for backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    if not supports_hardlinks(backup_root):
+        print('Error: Filesystem for backup directory does not support hardlinks!')
+        print(f'Please check backup directory: "{backup_root}"\n')
+        sys.exit(1)
+
+    # Step 1: Scan source directory:
+    src_file_count, src_total_size = humanized_fs_scan(src_root, excludes)
+
+    phlb_conf_dir = backup_root / '.phlb'
+    phlb_conf_dir.mkdir(parents=False, exist_ok=True)
+
+    backup_dir = backup_root / src_root.name / datetime.now().strftime('%Y-%m-%d-%H%M%S')
+    logger.info('Backup %s to %s', src_root, backup_dir)
+    backup_dir.mkdir(parents=True, exist_ok=False)
+
+    print(f'\nBackup to {backup_dir}...\n')
+
+    with BackupProgress(src_file_count, src_total_size) as progress:
+        # "Databases" for deduplication
+        size_db = FileSizeDatabase(phlb_conf_dir)
+        hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
+
+        backup_result = BackupResult(backup_dir=backup_dir)
+
+        next_update = 0
+        for entry in iter_scandir_files(src_root, excludes=excludes):
+            try:
+                backup_one_file(
+                    src_root=src_root,
+                    entry=entry,
+                    size_db=size_db,
+                    hash_db=hash_db,
+                    backup_dir=backup_dir,
+                    backup_result=backup_result,
+                )
+            except Exception as err:
+                logger.exception(f'Backup {entry.path} {err.__class__.__name__}: {err}')
+                backup_result.error_count += 1
+            else:
+                now = time.monotonic()
+                if now >= next_update:
+                    progress.update(backup_count=backup_result.backup_count, backup_size=backup_result.backup_size)
+                    next_update = now + 0.5
+
+        # Finalize progress indicator values:
+        progress.update(backup_count=backup_result.backup_count, backup_size=backup_result.backup_size)
+
+    print(f'\nBackup complete: {backup_dir} (total size {human_filesize(backup_result.backup_size)})\n')
+    print(f'  Total files processed: {backup_result.backup_count}')
+    print(f'   * Symlinked files: {backup_result.symlink_files}')
+    print(
+        f'   * Hardlinked files: {backup_result.hardlinked_files}'
+        f' (saved {human_filesize(backup_result.hardlinked_size)})'
+    )
+    print(f'   * Copied files: {backup_result.copied_files} (total {human_filesize(backup_result.copied_size)})')
+    print(
+        f'     of which small (<{size_db.MIN_SIZE} Bytes)'
+        f' files: {backup_result.copied_small_files}'
+        f' (total {human_filesize(backup_result.copied_small_size)})'
+    )
+    if backup_result.error_count > 0:
+        print(f'  Errors during backup: {backup_result.error_count} (see log for details)')
+    print()
+
+    return backup_result