PyHardLinkBackup 1.0.0rc0__tar.gz → 1.0.0rc1__tar.gz

pyhardlinkbackup-1.0.0rc1/.run/only DocWrite.run.xml

@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="only DocWrite" type="tests" factoryName="Unittests">
+    <module name="PyHardLinkBackup" />
+    <option name="ENV_FILES" value="" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+      <env name="PYTHONWARNINGS" value="always" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="false" />
+    <option name="ADD_SOURCE_ROOTS" value="false" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="RUN_TOOL" value="" />
+    <option name="_new_pattern" value="&quot;&quot;" />
+    <option name="_new_additionalArguments" value="&quot;&quot;" />
+    <option name="_new_target" value="&quot;PyHardLinkBackup.tests.test_doc_write&quot;" />
+    <option name="_new_targetType" value="&quot;PYTHON&quot;" />
+    <method v="2" />
+  </configuration>
+</component>

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyHardLinkBackup
-Version: 1.0.0rc0
+Version: 1.0.0rc1
 Summary: HardLink/Deduplication Backups with Python
 Project-URL: Documentation, https://github.com/jedie/PyHardLinkBackup
 Project-URL: Source, https://github.com/jedie/PyHardLinkBackup
@@ -190,7 +190,9 @@ v1 is a complete rewrite of PyHardLinkBackup.
 
 [comment]: <> (✂✂✂ auto generated history start ✂✂✂)
 
-* [v1.0.0rc0](https://github.com/jedie/PyHardLinkBackup/compare/v0.13.0...v1.0.0rc0)
+* [v1.0.0rc1](https://github.com/jedie/PyHardLinkBackup/compare/v0.13.0...v1.0.0rc1)
+  * 2026-01-13 - Rename [project.scripts] hooks
+  * 2026-01-13 - Add DocWrite, handle broken symlinks, keep file meta, handle missing hardlink sources
   * 2026-01-12 - First working iteration with rich progess bar
   * 2026-01-08 - Rewrite everything
 * [v0.13.0](https://github.com/jedie/PyHardLinkBackup/compare/v0.12.3...v0.13.0)

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PyHardLinkBackup/__init__.py

@@ -1,8 +1,7 @@
-"""
-    PyHardLinkBackup
+"""DocWrite: README.md # PyHardLinkBackup
     HardLink/Deduplication Backups with Python
 """
 
 # See https://packaging.python.org/en/latest/specifications/version-specifiers/
-__version__ = '1.0.0rc0'
+__version__ = '1.0.0rc1'
 __author__ = 'Jens Diemer <PyHardLinkBackup@jensdiemer.de>'

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PyHardLinkBackup/backup.py

@@ -84,7 +84,22 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
         next_update = 0
         for entry in iter_scandir_files(src_root, excludes=excludes):
             backup_count += 1
-            size = entry.stat().st_size
+            src_path = Path(entry.path)
+
+            dst_path = backup_dir / src_path.relative_to(src_root)
+            dst_dir_path = dst_path.parent
+            if not dst_dir_path.exists():
+                dst_dir_path.mkdir(parents=True, exist_ok=False)
+
+            try:
+                size = entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: Handle broken symlink
+                target = os.readlink(src_path)
+                dst_path.symlink_to(target)
+                symlink_files += 1
+                continue
+
             backup_size += size
 
             now = time.monotonic()
@@ -92,14 +107,8 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
                 progress.update(backup_count=backup_count, backup_size=backup_size)
                 next_update = now + 0.5
 
-            src_path = Path(entry.path)
-            dst_path = backup_dir / src_path.relative_to(src_root)
-
-            dst_path.parent.mkdir(parents=True, exist_ok=True)
-
             if entry.is_symlink():
                 logger.debug('Copy symlink: %s to %s', src_path, dst_path)
-                # Copy symlinks as-is
                 target = os.readlink(src_path)
                 dst_path.symlink_to(target)
                 symlink_files += 1
@@ -152,10 +161,12 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
                         hardlinked_size += size
                     else:
                         logger.info('Copy unique file: %s to %s', src_path, dst_path)
-                        shutil.copy2(src_path, dst_path)
                         hash_db[file_hash] = dst_path
                         copied_files += 1
                         copied_size += size
+
+                # Keep original file metadata (permission bits, time stamps, and flags)
+                shutil.copy2(src_path, dst_path)
             else:
                 # A file with this size not backuped before -> Can't be duplicate -> copy and hash
                 file_hash = copy_and_hash(src_path, dst_path)
@@ -164,6 +175,7 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
                 copied_files += 1
                 copied_size += size
 
+        # Finalize progress indicator values:
         progress.update(backup_count=backup_count, backup_size=backup_size)
 
     print(f'\nBackup complete: {backup_dir} (total size {human_filesize(backup_size)})\n')

pyhardlinkbackup-1.0.0rc1/PyHardLinkBackup/tests/test_backup.py

@@ -0,0 +1,399 @@
+import datetime
+import os
+import tempfile
+import textwrap
+import zlib
+from pathlib import Path
+from unittest import TestCase
+
+from bx_py_utils.test_utils.assertion import assert_text_equal
+from bx_py_utils.test_utils.datetime import parse_dt
+from freezegun import freeze_time
+from tabulate import tabulate
+
+from PyHardLinkBackup.backup import BackupResult, backup_tree
+from PyHardLinkBackup.constants import CHUNK_SIZE
+from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
+from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
+from PyHardLinkBackup.utilities.tests.test_file_hash_database import assert_hash_db_info
+
+
+def set_file_times(path: Path, dt: datetime.datetime):
+    # move dt to UTC if it has timezone info:
+    if dt.tzinfo is not None:
+        dt = dt.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+    fixed_time = dt.timestamp()
+    for entry in iter_scandir_files(path, excludes=set()):
+        os.utime(entry.path, (fixed_time, fixed_time))
+
+
+def fs_tree_overview(root: Path) -> str:
+    lines = []
+    for entry in iter_scandir_files(root, excludes=set()):
+        file_path = Path(entry.path)
+        try:
+            file_stat = entry.stat()
+        except FileNotFoundError:
+            crc32 = '-'
+            nlink = '-'
+            size = '-'
+            birthtime = '-'
+        else:
+            crc32 = zlib.crc32(file_path.read_bytes())
+            crc32 = f'{crc32:08x}'
+            nlink = file_stat.st_nlink
+            size = file_stat.st_size
+            birthtime = getattr(file_stat, 'st_birthtime', file_stat.st_mtime)
+            birthtime = datetime.datetime.fromtimestamp(birthtime).strftime('%H:%M:%S')
+
+        if entry.is_symlink():
+            file_type = 'symlink'
+        elif nlink > 1:
+            file_type = 'hardlink'
+        else:
+            file_type = 'file'
+
+        lines.append(
+            [
+                str(file_path.relative_to(root)),
+                birthtime,
+                file_type,
+                nlink,
+                size,
+                crc32,
+            ]
+        )
+
+    result = tabulate(sorted(lines), headers=['path', 'birthtime', 'type', 'nlink', 'size', 'CRC32'], tablefmt='plain')
+    return result
+
+
+def assert_fs_tree_overview(root: Path, expected_overview: str):
+    expected_overview = textwrap.dedent(expected_overview).strip()
+    actual_overview = fs_tree_overview(root)
+    assert_text_equal(
+        actual_overview,
+        expected_overview,
+        msg=f'Filesystem tree overview does not match expected overview.\n\n{actual_overview}\n\n',
+    )
+
+
+class BackupTreeTestCase(TestCase):
+    def test_happy_path(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+
+            src_root = temp_path / 'source'
+            backup_root = temp_path / 'backup'
+
+            src_root.mkdir()
+            backup_root.mkdir()
+
+            file1_path = src_root / 'file2.txt'
+            file1_path.write_text('This is file 1')
+
+            (src_root / 'symlink2file1').symlink_to(file1_path)
+            os.link(file1_path, src_root / 'hardlink2file1')
+
+            sub_dir = src_root / 'subdir'
+            sub_dir.mkdir()
+            (sub_dir / 'file.txt').write_text('This is file in subdir')
+
+            # Only files bigger than MIN_SIZE will be considered for hardlinking:
+            size_db_min_file = src_root / 'min_sized_file1.bin'
+            size_db_min_file.write_bytes(b'X' * FileSizeDatabase.MIN_SIZE)
+
+            # Same content and big enough to be considered for hardlinking:
+            size_db_min_file = src_root / 'min_sized_file2.bin'
+            size_db_min_file.write_bytes(b'X' * FileSizeDatabase.MIN_SIZE)
+
+            # Larger then CHUNK_SIZE file will be handled differently:
+            large_file = src_root / 'large_file.bin'
+            large_file.write_bytes(b'Y' * (CHUNK_SIZE + 1))
+
+            excluded_dir = src_root / '.cache'
+            excluded_dir.mkdir()
+            (excluded_dir / 'tempfile.tmp').write_text('Temporary file that should be excluded')
+
+            # FIXME: freezegun doesn't handle this, see: https://github.com/spulec/freezegun/issues/392
+            # Set modification times to a fixed time for easier testing:
+            set_file_times(src_root, dt=parse_dt('2026-01-01T12:00:00+0000'))
+
+            #######################################################################################
+            # Create first backup:
+
+            with freeze_time('2026-01-01T12:34:56Z', auto_tick_seconds=0):
+                result = backup_tree(
+                    src_root=src_root,
+                    backup_root=backup_root,
+                    excludes={'.cache'},
+                )
+            backup_dir = result.backup_dir
+            self.assertEqual(
+                str(Path(backup_dir).relative_to(temp_path)),
+                'backup/source/20260101_123456',
+            )
+            self.assertEqual(
+                result,
+                BackupResult(
+                    backup_dir=backup_dir,
+                    backup_count=7,
+                    backup_size=67110929,
+                    symlink_files=1,
+                    hardlinked_files=1,
+                    hardlinked_size=1000,
+                    copied_files=5,
+                    copied_size=67109915,
+                    copied_small_files=3,
+                    copied_small_size=50,
+                ),
+            )
+
+            # The sources:
+            assert_fs_tree_overview(
+                root=src_root,
+                expected_overview="""
+                    path                 birthtime    type        nlink      size  CRC32
+                    .cache/tempfile.tmp  12:00:00     file            1        38  41d7a2c9
+                    file2.txt            12:00:00     hardlink        2        14  8a11514a
+                    hardlink2file1       12:00:00     hardlink        2        14  8a11514a
+                    large_file.bin       12:00:00     file            1  67108865  9671eaac
+                    min_sized_file1.bin  12:00:00     file            1      1000  f0d93de4
+                    min_sized_file2.bin  12:00:00     file            1      1000  f0d93de4
+                    subdir/file.txt      12:00:00     file            1        22  c0167e63
+                    symlink2file1        12:00:00     symlink         2        14  8a11514a
+                """,
+            )
+            # The backup:
+            # * /.cache/ -> excluded
+            # * min_sized_file1.bin and min_sized_file2.bin -> hardlinked
+            assert_fs_tree_overview(
+                root=backup_dir,
+                expected_overview="""
+                    path                 birthtime    type        nlink      size  CRC32
+                    file2.txt            12:00:00     file            1        14  8a11514a
+                    hardlink2file1       12:00:00     file            1        14  8a11514a
+                    large_file.bin       12:00:00     file            1  67108865  9671eaac
+                    min_sized_file1.bin  12:00:00     hardlink        2      1000  f0d93de4
+                    min_sized_file2.bin  12:00:00     hardlink        2      1000  f0d93de4
+                    subdir/file.txt      12:00:00     file            1        22  c0167e63
+                    symlink2file1        12:00:00     symlink         2        14  8a11514a
+                """,
+            )
+
+            # Let's check our FileHashDatabase:
+            assert_hash_db_info(
+                backup_root=backup_root,
+                expected="""
+                    bb/c4/bbc4de2ca238d1… -> source/20260101_123456/min_sized_file1.bin
+                    e6/37/e6374ac11d9049… -> source/20260101_123456/large_file.bin
+                """,
+            )
+
+            #######################################################################################
+            # Just backup again:
+
+            with freeze_time('2026-01-02T12:34:56Z', auto_tick_seconds=0):
+                result = backup_tree(
+                    src_root=src_root,
+                    backup_root=backup_root,
+                    excludes={'.cache'},
+                )
+            backup_dir = result.backup_dir
+            self.assertEqual(
+                str(Path(backup_dir).relative_to(temp_path)),
+                'backup/source/20260102_123456',
+            )
+            self.assertEqual(
+                result,
+                BackupResult(
+                    backup_dir=backup_dir,
+                    backup_count=7,
+                    backup_size=67110929,
+                    symlink_files=1,
+                    hardlinked_files=3,  # <<< More hardlinks this time!
+                    hardlinked_size=67110865,
+                    copied_files=3,
+                    copied_size=50,
+                    copied_small_files=3,
+                    copied_small_size=50,
+                ),
+            )
+            # The second backup:
+            # * /.cache/ -> excluded
+            # * min_sized_file1.bin and min_sized_file2.bin -> hardlinked
+            assert_fs_tree_overview(
+                root=backup_dir,
+                expected_overview="""
+                    path                 birthtime    type        nlink      size  CRC32
+                    file2.txt            12:00:00     file            1        14  8a11514a
+                    hardlink2file1       12:00:00     file            1        14  8a11514a
+                    large_file.bin       12:00:00     hardlink        2  67108865  9671eaac
+                    min_sized_file1.bin  12:00:00     hardlink        4      1000  f0d93de4
+                    min_sized_file2.bin  12:00:00     hardlink        4      1000  f0d93de4
+                    subdir/file.txt      12:00:00     file            1        22  c0167e63
+                    symlink2file1        12:00:00     symlink         2        14  8a11514a
+                """,
+            )
+
+            # The FileHashDatabase remains the same:
+            assert_hash_db_info(
+                backup_root=backup_root,
+                expected="""
+                    bb/c4/bbc4de2ca238d1… -> source/20260101_123456/min_sized_file1.bin
+                    e6/37/e6374ac11d9049… -> source/20260101_123456/large_file.bin
+                """,
+            )
+
+            #######################################################################################
+            # Don't create broken hardlinks!
+
+            """DocWrite: README.md ## FileHashDatabase - Missing hardlink target file
+            If a hardlink source from a old backup is missing, we cannot create a hardlink to it.
+            But it still works to hardlink same files within the current backup.
+            """
+
+            # Let's remove one of the files used for hardlinking from the first backup:
+            (backup_root / 'source/20260101_123456/min_sized_file1.bin').unlink()
+
+            # Backup again:
+            with freeze_time('2026-01-03T12:34:56Z', auto_tick_seconds=0):
+                result = backup_tree(
+                    src_root=src_root,
+                    backup_root=backup_root,
+                    excludes={'.cache'},
+                )
+            backup_dir = result.backup_dir
+            self.assertEqual(
+                result,
+                BackupResult(
+                    backup_dir=backup_dir,
+                    backup_count=7,
+                    backup_size=67110929,
+                    symlink_files=1,
+                    hardlinked_files=2,  # <<< Less hardlinks this time, because of missing link source!
+                    hardlinked_size=67109865,
+                    copied_files=4,
+                    copied_size=1050,
+                    copied_small_files=3,
+                    copied_small_size=50,
+                ),
+            )
+
+            # Note: min_sized_file1.bin and min_sized_file2.bin are hardlinked,
+            # but not with the first backup anymore! So it's only nlink=2 now!
+            assert_fs_tree_overview(
+                root=backup_dir,
+                expected_overview="""
+                    path                 birthtime    type        nlink      size  CRC32
+                    file2.txt            12:00:00     file            1        14  8a11514a
+                    hardlink2file1       12:00:00     file            1        14  8a11514a
+                    large_file.bin       12:00:00     hardlink        3  67108865  9671eaac
+                    min_sized_file1.bin  12:00:00     hardlink        2      1000  f0d93de4
+                    min_sized_file2.bin  12:00:00     hardlink        2      1000  f0d93de4
+                    subdir/file.txt      12:00:00     file            1        22  c0167e63
+                    symlink2file1        12:00:00     symlink         2        14  8a11514a
+                """,
+            )
+
+            # Note: min_sized_file1.bin is now from the 2026-01-03 backup!
+            self.assertEqual(backup_dir.name, '20260103_123456')  # Latest backup dir name
+            assert_hash_db_info(
+                backup_root=backup_root,
+                expected="""
+                    bb/c4/bbc4de2ca238d1… -> source/20260103_123456/min_sized_file1.bin
+                    e6/37/e6374ac11d9049… -> source/20260101_123456/large_file.bin
+                """,
+            )
+
+    def test_symlink(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+
+            src_root = temp_path / 'src'
+            backup_root = temp_path / 'bak'
+
+            src_root.mkdir()
+            backup_root.mkdir()
+
+            source_file_path = src_root / 'source_file.txt'
+            source_file_path.write_text('File in the "source" directory.')
+
+            symlink2source_file_path = src_root / 'symlink2source'
+            symlink2source_file_path.symlink_to(source_file_path)
+            self.assertEqual(symlink2source_file_path.read_text(), 'File in the "source" directory.')
+
+            outside_file_path = temp_path / 'outside_file.txt'
+            outside_file_path.write_text('File outside the "source" directory!')
+
+            symlink2outside_file_path = src_root / 'symlink2outside'
+            symlink2outside_file_path.symlink_to(outside_file_path)
+            self.assertEqual(symlink2outside_file_path.read_text(), 'File outside the "source" directory!')
+
+            # FIXME: freezegun doesn't handle this, see: https://github.com/spulec/freezegun/issues/392
+            # Set modification times to a fixed time for easier testing:
+            set_file_times(src_root, dt=parse_dt('2026-01-01T12:00:00+0000'))
+
+            broken_symlink_path = src_root / 'broken_symlink'
+            broken_symlink_path.symlink_to(temp_path / 'not/existing/file.txt')
+            broken_symlink_path.is_symlink()
+
+            #######################################################################################
+            # Create first backup:
+
+            with freeze_time('2026-01-01T12:34:56Z', auto_tick_seconds=0):
+                result = backup_tree(src_root=src_root, backup_root=backup_root, excludes=set())
+            backup_dir1 = result.backup_dir
+            self.assertEqual(
+                str(Path(backup_dir1).relative_to(temp_path)),
+                'bak/src/20260101_123456',
+            )
+
+            assert_fs_tree_overview(
+                root=temp_path,  # The complete overview os source + backup and outside file
+                expected_overview="""
+                    path                                     birthtime    type     nlink    size    CRC32
+                    bak/src/20260101_123456/broken_symlink   -            symlink  -        -       -
+                    bak/src/20260101_123456/source_file.txt  12:00:00     file     1        31      9309a10c
+                    bak/src/20260101_123456/symlink2outside  12:00:00     symlink  1        36      24b5bf4c
+                    bak/src/20260101_123456/symlink2source   12:00:00     symlink  1        31      9309a10c
+                    outside_file.txt                         12:00:00     file     1        36      24b5bf4c
+                    src/broken_symlink                       -            symlink  -        -       -
+                    src/source_file.txt                      12:00:00     file     1        31      9309a10c
+                    src/symlink2outside                      12:00:00     symlink  1        36      24b5bf4c
+                    src/symlink2source                       12:00:00     symlink  1        31      9309a10c
+                """,
+            )
+
+            self.assertEqual(
+                result,
+                BackupResult(
+                    backup_dir=backup_dir1,
+                    backup_count=4,
+                    backup_size=98,
+                    symlink_files=3,
+                    hardlinked_files=0,
+                    hardlinked_size=0,
+                    copied_files=1,
+                    copied_size=31,
+                    copied_small_files=1,
+                    copied_small_size=31,
+                ),
+            )
+
+            """DocWrite: README.md ## backup implementation - Symlinks
+            Symlinks are copied as symlinks in the backup."""
+            self.assertEqual(
+                (backup_dir1 / 'symlink2outside').read_text(),
+                'File outside the "source" directory!',
+            )
+            self.assertEqual(
+                (backup_dir1 / 'symlink2source').read_text(),
+                'File in the "source" directory.',
+            )
+            self.assertEqual((backup_dir1 / 'symlink2outside').readlink(), outside_file_path)
+            self.assertEqual((backup_dir1 / 'symlink2source').readlink(), source_file_path)
+
+            """DocWrite: README.md ## backup implementation - Symlinks
+            Symlinks are not stored in our FileHashDatabase, because they are not considered for hardlinking."""
+            assert_hash_db_info(backup_root=backup_root, expected='')

pyhardlinkbackup-1.0.0rc1/PyHardLinkBackup/tests/test_doc_write.py

@@ -0,0 +1,25 @@
+from unittest import TestCase
+
+from bx_py_utils.doc_write.api import GeneratedInfo, generate
+from bx_py_utils.path import assert_is_file
+
+from PyHardLinkBackup.cli_dev import PACKAGE_ROOT
+
+
+class DocuWriteApiTestCase(TestCase):
+    def test_up2date_docs(self):
+        """DocWrite: about-docs.md # generate Doc-Write
+
+        These documentation files are generated automatically with the "Doc-Write" tool.
+        They updated automatically by unittests.
+
+        More information about Doc-Write can be found here:
+
+        https://github.com/boxine/bx_py_utils/tree/master/bx_py_utils/doc_write
+        """
+        assert_is_file(PACKAGE_ROOT / 'pyproject.toml')
+
+        info: GeneratedInfo = generate(base_path=PACKAGE_ROOT)
+        self.assertGreaterEqual(len(info.paths), 1)
+        self.assertEqual(info.update_count, 0, 'No files should be updated, commit the changes')
+        self.assertEqual(info.remove_count, 0, 'No files should be removed, commit the changes')

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PyHardLinkBackup/utilities/file_hash_database.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 
 
@@ -6,8 +7,8 @@ class HashAlreadyExistsError(ValueError):
 
 
 class FileHashDatabase:
-    """
-    A simple database to store file content hash <-> relative path mappings.
+    """DocWrite: README.md ## FileHashDatabase
+    A simple "database" to store file content hash <-> relative path mappings.
     Uses a directory structure to avoid too many files in a single directory.
     Path structure:
             {base_dst}/.phlb/hash-lookup/{XX}/{YY}/{hash}
@@ -39,6 +40,10 @@ class FileHashDatabase:
             return None
         else:
             abs_file_path = self.backup_root / rel_file_path
+            if not abs_file_path.is_file():
+                logging.warning('Hash database entry found, but file does not exist: %s', abs_file_path)
+                hash_path.unlink()
+                return None
             return abs_file_path
 
     def __setitem__(self, hash: str, abs_file_path: Path):

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PyHardLinkBackup/utilities/file_size_database.py

@@ -2,17 +2,11 @@ from pathlib import Path
 
 
 class FileSizeDatabase:
-    """
-    A simple database to track which file sizes have been seen.
-    Uses a directory structure to avoid too many files in a single directory.
+    """DocWrite: README.md ## FileSizeDatabase
+    A simple "database" to track which file sizes have been seen.
 
-    Path structure:
-         {base_dst}/.phlb/size-lookup/{XX}/{YY}/{size}
-    e.g.:
-        1234567890 results in: {base_dst}/.phlb/size-lookup/12/34/1234567890
-
-    Notes:
-      * We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
+    Uses a directory structure to avoid too many files in a single directory.
+    We don't "cache" anything in Memory, to avoid high memory consumption for large datasets.
     """
 
     MIN_SIZE = 1000  # no padding is made, so the min size is 1000 bytes!
@@ -24,6 +18,15 @@ class FileSizeDatabase:
     def _get_size_path(self, size: int) -> Path:
         assert size >= self.MIN_SIZE, f'Size must be at least {self.MIN_SIZE} bytes'
         size_str = str(size)
+
+        """DocWrite: README.md ## FileSizeDatabase
+        Path structure:
+         * `{base_dst}/.phlb/size-lookup/{XX}/{YY}/{size}`
+
+        e.g.:
+
+         * `1234567890` results in: `{base_dst}/.phlb/size-lookup/12/34/1234567890`
+        """
         first_dir_name = size_str[:2]
         second_dir_name = size_str[2:4]
         size_path = self.base_path / first_dir_name / second_dir_name / size_str
@@ -37,4 +40,7 @@ class FileSizeDatabase:
         size_path = self._get_size_path(size)
         if not size_path.exists():
             size_path.parent.mkdir(parents=True, exist_ok=True)
+
+            """DocWrite: README.md ## FileSizeDatabase
+            All files are created empty, as we only care about their existence."""
             size_path.touch(exist_ok=False)

{pyhardlinkbackup-1.0.0rc0 → pyhardlinkbackup-1.0.0rc1}/PyHardLinkBackup/utilities/filesystem.py

@@ -1,6 +1,7 @@
 import hashlib
 import logging
 import os
+import shutil
 import time
 from pathlib import Path
 from typing import Iterable
@@ -36,6 +37,10 @@ def copy_and_hash(src: Path, dst: Path) -> str:
         while chunk := source_file.read(CHUNK_SIZE):
             dst_file.write(chunk)
             hasher.update(chunk)
+
+    # Keep original file metadata (permission bits, last access time, last modification time, and flags)
+    shutil.copystat(src, dst)
+
     file_hash = hasher.hexdigest()
     logger.info('%s backup to %s with %s hash: %s', src, dst, HASH_ALGO, file_hash)
     return file_hash
@@ -55,14 +60,16 @@ def iter_scandir_files(path: Path, excludes: set[str]) -> Iterable[os.DirEntry]:
     Recursively yield all files+symlinks in the given directory.
     """
     logger.debug('Scanning directory %s', path)
-    for entry in os.scandir(path):
-        if entry.is_file(follow_symlinks=True):
-            yield entry
-        elif entry.is_dir(follow_symlinks=True):
-            if entry.name in excludes:
-                logger.debug('Excluding directory %s', entry.path)
-                continue
-            yield from iter_scandir_files(Path(entry.path), excludes=excludes)
+    with os.scandir(path) as scandir_iterator:
+        for entry in scandir_iterator:
+            if entry.is_dir(follow_symlinks=True):
+                if entry.name in excludes:
+                    logger.debug('Excluding directory %s', entry.path)
+                    continue
+                yield from iter_scandir_files(Path(entry.path), excludes=excludes)
+            else:
+                # It's a file or symlink or broken symlink
+                yield entry
 
 
 def humanized_fs_scan(path: Path, excludes: set[str]) -> tuple[int, int]:
@@ -91,7 +98,11 @@ def humanized_fs_scan(path: Path, excludes: set[str]) -> tuple[int, int]:
     with progress:
         for entry in iter_scandir_files(path, excludes=excludes):
             file_count += 1
-            total_size += entry.stat().st_size
+            try:
+                total_size += entry.stat().st_size
+            except FileNotFoundError:
+                # e.g.: broken symlink
+                continue
 
             now = time.time()
             if now >= next_update: