PyHardLinkBackup 1.7.3__tar.gz → 1.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PKG-INFO +26 -17
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/__init__.py +1 -1
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/backup.py +25 -4
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_app/phlb.py +8 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/compare_backup.py +16 -2
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/rebuild_databases.py +12 -2
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_backup.py +61 -7
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_compare_backup.py +2 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/filesystem.py +67 -7
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/rich_utils.py +8 -25
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/test_file_hash_database.py +12 -2
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/test_file_size_database.py +15 -2
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/test_filesystem.py +40 -1
- pyhardlinkbackup-1.8.0/PyHardLinkBackup/utilities/tyro_cli_shared_args.py +29 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/README.md +25 -16
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/docs/README.md +5 -0
- pyhardlinkbackup-1.7.3/PyHardLinkBackup/utilities/tyro_cli_shared_args.py +0 -12
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.editorconfig +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.github/workflows/tests.yml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.gitignore +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.idea/.gitignore +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.pre-commit-config.yaml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.pre-commit-hooks.yaml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/Template Python tests.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/Unittests - __all__.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/cli.py --help.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/dev-cli update.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/only DocTests.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.run/only DocWrite.run.xml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/.venv-app/lib/python3.12/site-packages/cli_base/tests/shell_complete_snapshots/.gitignore +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/__main__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_app/__init__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/__init__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/benchmark.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/code_style.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/packaging.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/shell_completion.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/testing.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/update_readme_history.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/constants.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/logging_setup.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/__init__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_doc_write.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_doctests.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_project_setup.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_readme.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_readme_history.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_rebuild_database.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/__init__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/file_hash_database.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/file_size_database.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/humanize.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/sha256sums.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tee.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/__init__.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/unittest_utilities.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/cli.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/dev-cli.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/dist/.gitignore +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/docs/about-docs.md +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/noxfile.py +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/pyproject.toml +0 -0
- {pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyHardLinkBackup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0
|
|
4
4
|
Summary: HardLink/Deduplication Backups with Python
|
|
5
5
|
Project-URL: Documentation, https://github.com/jedie/PyHardLinkBackup
|
|
6
6
|
Project-URL: Source, https://github.com/jedie/PyHardLinkBackup
|
|
@@ -66,19 +66,23 @@ usage: phlb backup [-h] [BACKUP OPTIONS]
|
|
|
66
66
|
|
|
67
67
|
Backup the source directory to the destination directory using hard links for deduplication.
|
|
68
68
|
|
|
69
|
-
╭─ positional arguments
|
|
70
|
-
│ source
|
|
71
|
-
│ destination
|
|
72
|
-
|
|
73
|
-
╭─ options
|
|
74
|
-
│ -h, --help
|
|
75
|
-
│ --
|
|
76
|
-
│
|
|
77
|
-
│ --
|
|
78
|
-
│
|
|
79
|
-
│ --
|
|
80
|
-
│
|
|
81
|
-
|
|
69
|
+
╭─ positional arguments ───────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
70
|
+
│ source Source directory to back up. (required) │
|
|
71
|
+
│ destination Destination directory for the backup. (required) │
|
|
72
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
73
|
+
╭─ options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
74
|
+
│ -h, --help show this help message and exit │
|
|
75
|
+
│ --name {None}|STR Optional name for the backup (used to create a subdirectory in the backup destination). If not │
|
|
76
|
+
│ provided, the name of the source directory is used. (default: None) │
|
|
77
|
+
│ --one-file-system, --no-one-file-system │
|
|
78
|
+
│ Do not cross filesystem boundaries. (default: True) │
|
|
79
|
+
│ --excludes [STR [STR ...]] │
|
|
80
|
+
│ List of directories to exclude from backup. (default: __pycache__ .cache .temp .tmp .tox .nox) │
|
|
81
|
+
│ --verbosity {debug,info,warning,error} │
|
|
82
|
+
│ Log level for console logging. (default: warning) │
|
|
83
|
+
│ --log-file-level {debug,info,warning,error} │
|
|
84
|
+
│ Log level for the log file (default: info) │
|
|
85
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
82
86
|
```
|
|
83
87
|
[comment]: <> (✂✂✂ auto generated backup help end ✂✂✂)
|
|
84
88
|
|
|
@@ -285,12 +289,20 @@ Overview of main changes:
|
|
|
285
289
|
|
|
286
290
|
[comment]: <> (✂✂✂ auto generated history start ✂✂✂)
|
|
287
291
|
|
|
292
|
+
* [v1.8.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.3...v1.8.0)
|
|
293
|
+
* 2026-01-22 - Add optional "--name" to enforce a name for the backup sub directory
|
|
294
|
+
* 2026-01-22 - Do not cross filesystem boundaries as default
|
|
295
|
+
* 2026-01-22 - Display progress also for large unique file copy
|
|
296
|
+
* 2026-01-22 - Optimize progress bars for smaller screens
|
|
288
297
|
* [v1.7.3](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.2...v1.7.3)
|
|
289
298
|
* 2026-01-21 - Handle directory symlinks correct
|
|
290
299
|
* [v1.7.2](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.1...v1.7.2)
|
|
291
300
|
* 2026-01-21 - Display "Remaining time" to files and sizes, too.
|
|
292
301
|
* [v1.7.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.0...v1.7.1)
|
|
293
302
|
* 2026-01-19 - Update requirements to fix problems under Windows
|
|
303
|
+
|
|
304
|
+
<details><summary>Expand older history entries ...</summary>
|
|
305
|
+
|
|
294
306
|
* [v1.7.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.6.0...v1.7.0)
|
|
295
307
|
* 2026-01-19 - Speedup and enhance unittest
|
|
296
308
|
* 2026-01-17 - Remove unfinished copied files on errors
|
|
@@ -300,9 +312,6 @@ Overview of main changes:
|
|
|
300
312
|
* 2026-01-17 - simplify tests
|
|
301
313
|
* 2026-01-17 - Warn if broken symlink found
|
|
302
314
|
* 2026-01-17 - Update README
|
|
303
|
-
|
|
304
|
-
<details><summary>Expand older history entries ...</summary>
|
|
305
|
-
|
|
306
315
|
* [v1.6.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.5.0...v1.6.0)
|
|
307
316
|
* 2026-01-17 - Fix flaky test, because of terminal size
|
|
308
317
|
* 2026-01-17 - Bugfix: Don't hash new large files twice
|
|
@@ -16,11 +16,13 @@ from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
|
|
|
16
16
|
from PyHardLinkBackup.utilities.filesystem import (
|
|
17
17
|
RemoveFileOnError,
|
|
18
18
|
copy_and_hash,
|
|
19
|
+
copy_with_progress,
|
|
19
20
|
hash_file,
|
|
20
21
|
humanized_fs_scan,
|
|
21
22
|
iter_scandir_files,
|
|
22
23
|
read_and_hash_file,
|
|
23
24
|
supports_hardlinks,
|
|
25
|
+
verbose_path_stat,
|
|
24
26
|
)
|
|
25
27
|
from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
|
|
26
28
|
from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
|
|
@@ -148,7 +150,7 @@ def backup_one_file(
|
|
|
148
150
|
backup_result.hardlinked_size += size
|
|
149
151
|
else:
|
|
150
152
|
logger.info('Copy unique file: %s to %s', src_path, dst_path)
|
|
151
|
-
|
|
153
|
+
copy_with_progress(src_path, dst_path, progress=progress, total_size=size)
|
|
152
154
|
hash_db[file_hash] = dst_path
|
|
153
155
|
backup_result.copied_files += 1
|
|
154
156
|
backup_result.copied_size += size
|
|
@@ -170,6 +172,8 @@ def backup_tree(
|
|
|
170
172
|
*,
|
|
171
173
|
src_root: Path,
|
|
172
174
|
backup_root: Path,
|
|
175
|
+
backup_name: str | None,
|
|
176
|
+
one_file_system: bool,
|
|
173
177
|
excludes: tuple[str, ...],
|
|
174
178
|
log_manager: LoggingManager,
|
|
175
179
|
) -> BackupResult:
|
|
@@ -179,12 +183,17 @@ def backup_tree(
|
|
|
179
183
|
print(f'Please check source directory: "{src_root}"\n')
|
|
180
184
|
sys.exit(1)
|
|
181
185
|
|
|
186
|
+
src_stat = verbose_path_stat(src_root)
|
|
187
|
+
src_device_id = src_stat.st_dev
|
|
188
|
+
|
|
182
189
|
backup_root = backup_root.resolve()
|
|
183
190
|
if not backup_root.is_dir():
|
|
184
191
|
print('Error: Backup directory does not exist!')
|
|
185
192
|
print(f'Please create "{backup_root}" directory first and start again!\n')
|
|
186
193
|
sys.exit(1)
|
|
187
194
|
|
|
195
|
+
verbose_path_stat(backup_root)
|
|
196
|
+
|
|
188
197
|
if not os.access(backup_root, os.W_OK):
|
|
189
198
|
print('Error: No write access to backup directory!')
|
|
190
199
|
print(f'Please check permissions for backup directory: "{backup_root}"\n')
|
|
@@ -198,13 +207,20 @@ def backup_tree(
|
|
|
198
207
|
# Step 1: Scan source directory:
|
|
199
208
|
excludes: set = set(excludes)
|
|
200
209
|
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
201
|
-
src_file_count, src_total_size = humanized_fs_scan(
|
|
210
|
+
src_file_count, src_total_size = humanized_fs_scan(
|
|
211
|
+
path=src_root,
|
|
212
|
+
one_file_system=one_file_system,
|
|
213
|
+
src_device_id=src_device_id,
|
|
214
|
+
excludes=excludes,
|
|
215
|
+
)
|
|
202
216
|
|
|
203
217
|
phlb_conf_dir = backup_root / '.phlb'
|
|
204
218
|
phlb_conf_dir.mkdir(parents=False, exist_ok=True)
|
|
205
219
|
|
|
206
220
|
timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
|
|
207
|
-
|
|
221
|
+
if not backup_name:
|
|
222
|
+
backup_name = src_root.name
|
|
223
|
+
backup_main_dir = backup_root / backup_name
|
|
208
224
|
backup_dir = backup_main_dir / timestamp
|
|
209
225
|
backup_dir.mkdir(parents=True, exist_ok=False)
|
|
210
226
|
|
|
@@ -227,7 +243,12 @@ def backup_tree(
|
|
|
227
243
|
backup_result = BackupResult(backup_dir=backup_dir, log_file=log_file)
|
|
228
244
|
|
|
229
245
|
next_update = 0
|
|
230
|
-
for entry in iter_scandir_files(
|
|
246
|
+
for entry in iter_scandir_files(
|
|
247
|
+
path=src_root,
|
|
248
|
+
one_file_system=one_file_system,
|
|
249
|
+
src_device_id=src_device_id,
|
|
250
|
+
excludes=excludes,
|
|
251
|
+
):
|
|
231
252
|
try:
|
|
232
253
|
backup_one_file(
|
|
233
254
|
src_root=src_root,
|
|
@@ -17,7 +17,9 @@ from PyHardLinkBackup.logging_setup import (
|
|
|
17
17
|
)
|
|
18
18
|
from PyHardLinkBackup.utilities.tyro_cli_shared_args import (
|
|
19
19
|
DEFAULT_EXCLUDE_DIRECTORIES,
|
|
20
|
+
TyroBackupNameArgType,
|
|
20
21
|
TyroExcludeDirectoriesArgType,
|
|
22
|
+
TyroOneFileSystemArgType,
|
|
21
23
|
)
|
|
22
24
|
|
|
23
25
|
|
|
@@ -41,6 +43,8 @@ def backup(
|
|
|
41
43
|
),
|
|
42
44
|
],
|
|
43
45
|
/,
|
|
46
|
+
name: TyroBackupNameArgType = None,
|
|
47
|
+
one_file_system: TyroOneFileSystemArgType = True,
|
|
44
48
|
excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
|
|
45
49
|
verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
|
|
46
50
|
log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
|
|
@@ -55,6 +59,8 @@ def backup(
|
|
|
55
59
|
backup_tree(
|
|
56
60
|
src_root=src,
|
|
57
61
|
backup_root=dst,
|
|
62
|
+
backup_name=name,
|
|
63
|
+
one_file_system=one_file_system,
|
|
58
64
|
excludes=excludes,
|
|
59
65
|
log_manager=log_manager,
|
|
60
66
|
)
|
|
@@ -77,6 +83,7 @@ def compare(
|
|
|
77
83
|
),
|
|
78
84
|
],
|
|
79
85
|
/,
|
|
86
|
+
one_file_system: TyroOneFileSystemArgType = True,
|
|
80
87
|
excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
|
|
81
88
|
verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
|
|
82
89
|
log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
|
|
@@ -91,6 +98,7 @@ def compare(
|
|
|
91
98
|
compare_backup.compare_tree(
|
|
92
99
|
src_root=src,
|
|
93
100
|
backup_root=dst,
|
|
101
|
+
one_file_system=one_file_system,
|
|
94
102
|
excludes=excludes,
|
|
95
103
|
log_manager=log_manager,
|
|
96
104
|
)
|
|
@@ -15,6 +15,7 @@ from PyHardLinkBackup.utilities.filesystem import (
|
|
|
15
15
|
hash_file,
|
|
16
16
|
humanized_fs_scan,
|
|
17
17
|
iter_scandir_files,
|
|
18
|
+
verbose_path_stat,
|
|
18
19
|
)
|
|
19
20
|
from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
|
|
20
21
|
from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
|
|
@@ -149,6 +150,7 @@ def compare_tree(
|
|
|
149
150
|
*,
|
|
150
151
|
src_root: Path,
|
|
151
152
|
backup_root: Path,
|
|
153
|
+
one_file_system: bool,
|
|
152
154
|
excludes: tuple[str, ...],
|
|
153
155
|
log_manager: LoggingManager,
|
|
154
156
|
) -> CompareResult:
|
|
@@ -181,9 +183,16 @@ def compare_tree(
|
|
|
181
183
|
log_file = compare_main_dir / f'{now_timestamp}-compare.log'
|
|
182
184
|
log_manager.start_file_logging(log_file)
|
|
183
185
|
|
|
186
|
+
src_device_id = verbose_path_stat(src_root).st_dev
|
|
187
|
+
|
|
184
188
|
excludes: set = set(excludes)
|
|
185
189
|
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
186
|
-
src_file_count, src_total_size = humanized_fs_scan(
|
|
190
|
+
src_file_count, src_total_size = humanized_fs_scan(
|
|
191
|
+
path=src_root,
|
|
192
|
+
one_file_system=one_file_system,
|
|
193
|
+
src_device_id=src_device_id,
|
|
194
|
+
excludes=excludes,
|
|
195
|
+
)
|
|
187
196
|
|
|
188
197
|
with DisplayFileTreeProgress(
|
|
189
198
|
description=f'Compare {src_root}...',
|
|
@@ -197,7 +206,12 @@ def compare_tree(
|
|
|
197
206
|
compare_result = CompareResult(last_timestamp=last_timestamp, compare_dir=compare_dir, log_file=log_file)
|
|
198
207
|
|
|
199
208
|
next_update = 0
|
|
200
|
-
for entry in iter_scandir_files(
|
|
209
|
+
for entry in iter_scandir_files(
|
|
210
|
+
path=src_root,
|
|
211
|
+
one_file_system=one_file_system,
|
|
212
|
+
src_device_id=src_device_id,
|
|
213
|
+
excludes=excludes,
|
|
214
|
+
):
|
|
201
215
|
try:
|
|
202
216
|
compare_one_file(
|
|
203
217
|
src_root=src_root,
|
|
@@ -113,7 +113,12 @@ def rebuild(
|
|
|
113
113
|
log_manager.start_file_logging(log_file=backup_root / f'{timestamp}-rebuild.log')
|
|
114
114
|
|
|
115
115
|
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
116
|
-
file_count, total_size = humanized_fs_scan(
|
|
116
|
+
file_count, total_size = humanized_fs_scan(
|
|
117
|
+
path=backup_root,
|
|
118
|
+
one_file_system=False,
|
|
119
|
+
src_device_id=None,
|
|
120
|
+
excludes={'.phlb'},
|
|
121
|
+
)
|
|
117
122
|
|
|
118
123
|
# We should ignore all files in the root backup directory itself
|
|
119
124
|
# e.g.: Our *-summary.txt and *.log files
|
|
@@ -134,7 +139,12 @@ def rebuild(
|
|
|
134
139
|
rebuild_result = RebuildResult()
|
|
135
140
|
|
|
136
141
|
next_update = 0
|
|
137
|
-
for entry in iter_scandir_files(
|
|
142
|
+
for entry in iter_scandir_files(
|
|
143
|
+
path=backup_root,
|
|
144
|
+
one_file_system=False,
|
|
145
|
+
src_device_id=None,
|
|
146
|
+
excludes={'.phlb'},
|
|
147
|
+
):
|
|
138
148
|
try:
|
|
139
149
|
rebuild_one_file(
|
|
140
150
|
backup_root=backup_root,
|
|
@@ -36,15 +36,14 @@ class SortedIterScandirFiles:
|
|
|
36
36
|
This class wraps iter_scandir_files() and yields the entries sorted by name.
|
|
37
37
|
"""
|
|
38
38
|
|
|
39
|
-
def __init__(self,
|
|
40
|
-
self.
|
|
41
|
-
self.excludes = excludes
|
|
39
|
+
def __init__(self, **iter_scandir_files_kwargs):
|
|
40
|
+
self.iter_scandir_files_kwargs = iter_scandir_files_kwargs
|
|
42
41
|
|
|
43
42
|
def __enter__(self):
|
|
44
43
|
return self
|
|
45
44
|
|
|
46
45
|
def __iter__(self) -> Iterable[os.DirEntry]:
|
|
47
|
-
scandir_iterator = iter_scandir_files(self.
|
|
46
|
+
scandir_iterator = iter_scandir_files(**self.iter_scandir_files_kwargs)
|
|
48
47
|
yield from sorted(scandir_iterator, key=lambda e: e.name)
|
|
49
48
|
|
|
50
49
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
@@ -57,7 +56,12 @@ def set_file_times(path: Path, dt: datetime.datetime):
|
|
|
57
56
|
dt = dt.astimezone(datetime.timezone.utc).replace(tzinfo=None)
|
|
58
57
|
fixed_time = dt.timestamp()
|
|
59
58
|
with NoLogs(logger_name=''):
|
|
60
|
-
for entry in iter_scandir_files(
|
|
59
|
+
for entry in iter_scandir_files(
|
|
60
|
+
path=path,
|
|
61
|
+
one_file_system=False,
|
|
62
|
+
src_device_id=None,
|
|
63
|
+
excludes=set(),
|
|
64
|
+
):
|
|
61
65
|
try:
|
|
62
66
|
os.utime(entry.path, (fixed_time, fixed_time))
|
|
63
67
|
except FileNotFoundError:
|
|
@@ -67,7 +71,12 @@ def set_file_times(path: Path, dt: datetime.datetime):
|
|
|
67
71
|
|
|
68
72
|
def _fs_tree_overview(root: Path) -> str:
|
|
69
73
|
lines = []
|
|
70
|
-
for entry in iter_scandir_files(
|
|
74
|
+
for entry in iter_scandir_files(
|
|
75
|
+
path=root,
|
|
76
|
+
one_file_system=False,
|
|
77
|
+
src_device_id=None,
|
|
78
|
+
excludes=set(),
|
|
79
|
+
):
|
|
71
80
|
file_path = Path(entry.path)
|
|
72
81
|
crc32 = '-'
|
|
73
82
|
try:
|
|
@@ -143,6 +152,7 @@ class BackupTreeTestCase(
|
|
|
143
152
|
self,
|
|
144
153
|
*,
|
|
145
154
|
time_to_freeze: str,
|
|
155
|
+
backup_name=None,
|
|
146
156
|
log_file_level: LogLevelLiteral = DEFAULT_LOG_FILE_LEVEL,
|
|
147
157
|
):
|
|
148
158
|
# FIXME: freezegun doesn't handle this, see: https://github.com/spulec/freezegun/issues/392
|
|
@@ -157,6 +167,8 @@ class BackupTreeTestCase(
|
|
|
157
167
|
result = backup_tree(
|
|
158
168
|
src_root=self.src_root,
|
|
159
169
|
backup_root=self.backup_root,
|
|
170
|
+
backup_name=backup_name,
|
|
171
|
+
one_file_system=True,
|
|
160
172
|
excludes=('.cache',),
|
|
161
173
|
log_manager=LoggingManager(
|
|
162
174
|
console_level='info',
|
|
@@ -924,7 +936,6 @@ class BackupTreeTestCase(
|
|
|
924
936
|
f' to {self.temp_path}/backups/source/2026-02-22-123456/large_fileB.txt',
|
|
925
937
|
log_file_content,
|
|
926
938
|
)
|
|
927
|
-
|
|
928
939
|
with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
|
|
929
940
|
assert_fs_tree_overview(
|
|
930
941
|
root=self.backup_root / 'source',
|
|
@@ -1111,3 +1122,46 @@ class BackupTreeTestCase(
|
|
|
1111
1122
|
excpected_successful_file_count=1,
|
|
1112
1123
|
excpected_error_count=0,
|
|
1113
1124
|
)
|
|
1125
|
+
|
|
1126
|
+
def test_backup_name(self):
|
|
1127
|
+
"""DocWrite: README.md # PyHardLinkBackup - Backup Naming
|
|
1128
|
+
The backup name is optional.
|
|
1129
|
+
If not provided, the name of the source directory is used.
|
|
1130
|
+
"""
|
|
1131
|
+
(self.src_root / 'file.txt').touch()
|
|
1132
|
+
|
|
1133
|
+
redirected_out, result = self.create_backup(time_to_freeze='2026-01-01T12:34:56Z', backup_name=None)
|
|
1134
|
+
self.assertEqual(
|
|
1135
|
+
str(result.backup_dir.relative_to(self.temp_path)),
|
|
1136
|
+
'backups/source/2026-01-01-123456',
|
|
1137
|
+
)
|
|
1138
|
+
|
|
1139
|
+
redirected_out, result = self.create_backup(time_to_freeze='2026-01-01T12:34:56Z', backup_name='My-Backup')
|
|
1140
|
+
self.assertEqual(
|
|
1141
|
+
str(result.backup_dir.relative_to(self.temp_path)),
|
|
1142
|
+
'backups/My-Backup/2026-01-01-123456',
|
|
1143
|
+
)
|
|
1144
|
+
redirected_out, result = self.create_backup(time_to_freeze='2026-12-24T00:12:34Z', backup_name='My-Backup')
|
|
1145
|
+
self.assertEqual(
|
|
1146
|
+
str(result.backup_dir.relative_to(self.temp_path)),
|
|
1147
|
+
'backups/My-Backup/2026-12-24-001234',
|
|
1148
|
+
)
|
|
1149
|
+
with self.assertLogs('PyHardLinkBackup', level=logging.DEBUG):
|
|
1150
|
+
assert_fs_tree_overview(
|
|
1151
|
+
root=self.backup_root,
|
|
1152
|
+
expected_overview="""
|
|
1153
|
+
path birthtime type nlink size CRC32
|
|
1154
|
+
My-Backup/2026-01-01-123456-backup.log <mock> file 1 <mock> <mock>
|
|
1155
|
+
My-Backup/2026-01-01-123456-summary.txt <mock> file 1 <mock> <mock>
|
|
1156
|
+
My-Backup/2026-01-01-123456/SHA256SUMS <mock> file 1 75 43d11c57
|
|
1157
|
+
My-Backup/2026-01-01-123456/file.txt 12:00:00 file 1 0 00000000
|
|
1158
|
+
My-Backup/2026-12-24-001234-backup.log <mock> file 1 <mock> <mock>
|
|
1159
|
+
My-Backup/2026-12-24-001234-summary.txt <mock> file 1 <mock> <mock>
|
|
1160
|
+
My-Backup/2026-12-24-001234/SHA256SUMS <mock> file 1 75 43d11c57
|
|
1161
|
+
My-Backup/2026-12-24-001234/file.txt 12:00:00 file 1 0 00000000
|
|
1162
|
+
source/2026-01-01-123456-backup.log <mock> file 1 <mock> <mock>
|
|
1163
|
+
source/2026-01-01-123456-summary.txt <mock> file 1 <mock> <mock>
|
|
1164
|
+
source/2026-01-01-123456/SHA256SUMS <mock> file 1 75 43d11c57
|
|
1165
|
+
source/2026-01-01-123456/file.txt 12:00:00 file 1 0 00000000
|
|
1166
|
+
""",
|
|
1167
|
+
)
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_compare_backup.py
RENAMED
|
@@ -39,6 +39,7 @@ def assert_compare_backup(
|
|
|
39
39
|
result = compare_tree(
|
|
40
40
|
src_root=src_root,
|
|
41
41
|
backup_root=backup_root,
|
|
42
|
+
one_file_system=True,
|
|
42
43
|
excludes=excludes,
|
|
43
44
|
log_manager=LoggingManager(
|
|
44
45
|
console_level='info',
|
|
@@ -105,6 +106,7 @@ class CompareBackupTestCase(PyHardLinkBackupTestCaseMixin, TestCase):
|
|
|
105
106
|
result = compare_tree(
|
|
106
107
|
src_root=self.src_root,
|
|
107
108
|
backup_root=self.backup_root,
|
|
109
|
+
one_file_system=True,
|
|
108
110
|
excludes=(),
|
|
109
111
|
log_manager=LoggingManager(
|
|
110
112
|
console_level='info',
|
|
@@ -23,6 +23,17 @@ logger = logging.getLogger(__name__)
|
|
|
23
23
|
MIN_SIZE_FOR_PROGRESS_BAR = CHUNK_SIZE * 10
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
def verbose_path_stat(path: Path) -> os.stat_result:
|
|
27
|
+
stat_result = path.stat()
|
|
28
|
+
stat_dict = {}
|
|
29
|
+
for key in dir(stat_result):
|
|
30
|
+
if key.startswith('st_'):
|
|
31
|
+
value = getattr(stat_result, key)
|
|
32
|
+
stat_dict[key] = value
|
|
33
|
+
logger.info('Stat for %s: %s', path, stat_dict)
|
|
34
|
+
return stat_result
|
|
35
|
+
|
|
36
|
+
|
|
26
37
|
class RemoveFileOnError:
|
|
27
38
|
def __init__(self, file_path: Path):
|
|
28
39
|
self.file_path = file_path
|
|
@@ -32,7 +43,8 @@ class RemoveFileOnError:
|
|
|
32
43
|
|
|
33
44
|
def __exit__(self, exc_type, exc_value, exc_traceback):
|
|
34
45
|
if exc_type:
|
|
35
|
-
logger.info(
|
|
46
|
+
logger.info(
|
|
47
|
+
f'Removing incomplete file {self.file_path} due to error: {exc_value}',
|
|
36
48
|
exc_info=(exc_type, exc_value, exc_traceback),
|
|
37
49
|
)
|
|
38
50
|
self.file_path.unlink(missing_ok=True)
|
|
@@ -43,7 +55,7 @@ def hash_file(path: Path, progress: DisplayFileTreeProgress, total_size: int) ->
|
|
|
43
55
|
logger.debug('Hash file %s using %s', path, HASH_ALGO)
|
|
44
56
|
hasher = hashlib.new(HASH_ALGO)
|
|
45
57
|
with LargeFileProgress(
|
|
46
|
-
f'Hashing large file: {path
|
|
58
|
+
description=f'Hashing large file: "[yellow]{path}[/yellow]"',
|
|
47
59
|
parent_progress=progress,
|
|
48
60
|
total_size=total_size,
|
|
49
61
|
) as progress_bar:
|
|
@@ -56,11 +68,27 @@ def hash_file(path: Path, progress: DisplayFileTreeProgress, total_size: int) ->
|
|
|
56
68
|
return file_hash
|
|
57
69
|
|
|
58
70
|
|
|
71
|
+
def copy_with_progress(src: Path, dst: Path, progress: DisplayFileTreeProgress, total_size: int) -> None:
|
|
72
|
+
logger.debug('Copy file %s to %s using %s', src, dst, HASH_ALGO)
|
|
73
|
+
with LargeFileProgress(
|
|
74
|
+
description=f'Copying large file: "[yellow]{src}[/yellow]"',
|
|
75
|
+
parent_progress=progress,
|
|
76
|
+
total_size=total_size,
|
|
77
|
+
) as progress_bar:
|
|
78
|
+
with src.open('rb') as source_file, dst.open('wb') as dst_file:
|
|
79
|
+
while chunk := source_file.read(CHUNK_SIZE):
|
|
80
|
+
dst_file.write(chunk)
|
|
81
|
+
progress_bar.update(advance=len(chunk))
|
|
82
|
+
|
|
83
|
+
# Keep original file metadata (permission bits, last access time, last modification time, and flags)
|
|
84
|
+
shutil.copystat(src, dst)
|
|
85
|
+
|
|
86
|
+
|
|
59
87
|
def copy_and_hash(src: Path, dst: Path, progress: DisplayFileTreeProgress, total_size: int) -> str:
|
|
60
88
|
logger.debug('Copy and hash file %s to %s using %s', src, dst, HASH_ALGO)
|
|
61
89
|
hasher = hashlib.new(HASH_ALGO)
|
|
62
90
|
with LargeFileProgress(
|
|
63
|
-
f'
|
|
91
|
+
description=f'Copy and hash large file: "[yellow]{src}[/yellow]"',
|
|
64
92
|
parent_progress=progress,
|
|
65
93
|
total_size=total_size,
|
|
66
94
|
) as progress_bar:
|
|
@@ -87,7 +115,13 @@ def read_and_hash_file(path: Path) -> tuple[bytes, str]:
|
|
|
87
115
|
return content, file_hash
|
|
88
116
|
|
|
89
117
|
|
|
90
|
-
def iter_scandir_files(
|
|
118
|
+
def iter_scandir_files(
|
|
119
|
+
*,
|
|
120
|
+
path: Path,
|
|
121
|
+
one_file_system: bool,
|
|
122
|
+
src_device_id,
|
|
123
|
+
excludes: set[str],
|
|
124
|
+
) -> Iterable[os.DirEntry]:
|
|
91
125
|
"""
|
|
92
126
|
Recursively yield all files+symlinks in the given directory.
|
|
93
127
|
Note: Directory symlinks are treated as files (not recursed into).
|
|
@@ -101,13 +135,39 @@ def iter_scandir_files(path: Path, excludes: set[str]) -> Iterable[os.DirEntry]:
|
|
|
101
135
|
if entry.name in excludes:
|
|
102
136
|
logger.debug('Excluding directory %s', entry.path)
|
|
103
137
|
continue
|
|
104
|
-
|
|
138
|
+
|
|
139
|
+
if one_file_system:
|
|
140
|
+
try:
|
|
141
|
+
entry_device_id = entry.stat(follow_symlinks=False).st_dev
|
|
142
|
+
except OSError as err:
|
|
143
|
+
# e.g.: broken symlink
|
|
144
|
+
logger.debug('Skipping directory %s: %s', entry.path, err)
|
|
145
|
+
continue
|
|
146
|
+
if entry_device_id != src_device_id:
|
|
147
|
+
logger.debug(
|
|
148
|
+
'Skipping directory %s: different device ID %s (src device ID: %s)',
|
|
149
|
+
entry.path,
|
|
150
|
+
entry_device_id,
|
|
151
|
+
src_device_id,
|
|
152
|
+
)
|
|
153
|
+
continue
|
|
154
|
+
|
|
155
|
+
yield from iter_scandir_files(
|
|
156
|
+
path=Path(entry.path),
|
|
157
|
+
one_file_system=one_file_system,
|
|
158
|
+
src_device_id=src_device_id,
|
|
159
|
+
excludes=excludes,
|
|
160
|
+
)
|
|
105
161
|
else:
|
|
106
162
|
# It's a file or symlink or broken symlink
|
|
107
163
|
yield entry
|
|
108
164
|
|
|
109
165
|
|
|
110
|
-
def humanized_fs_scan(
|
|
166
|
+
def humanized_fs_scan(
|
|
167
|
+
*,
|
|
168
|
+
path: Path,
|
|
169
|
+
**iter_scandir_files_kwargs,
|
|
170
|
+
) -> tuple[int, int]:
|
|
111
171
|
print(f'\nScanning filesystem at: {path}...')
|
|
112
172
|
|
|
113
173
|
progress = Progress(
|
|
@@ -133,7 +193,7 @@ def humanized_fs_scan(path: Path, excludes: set[str]) -> tuple[int, int]:
|
|
|
133
193
|
)
|
|
134
194
|
next_update = 0
|
|
135
195
|
with progress:
|
|
136
|
-
for entry in iter_scandir_files(path,
|
|
196
|
+
for entry in iter_scandir_files(path=path, **iter_scandir_files_kwargs):
|
|
137
197
|
if not entry.is_file():
|
|
138
198
|
# Ignore e.g.: directory symlinks
|
|
139
199
|
continue
|
|
@@ -31,18 +31,9 @@ class HumanFileSizeColumn(ProgressColumn):
|
|
|
31
31
|
advance_size = task.completed
|
|
32
32
|
remaining_size = task.remaining
|
|
33
33
|
return (
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
human_filesize(advance_size),
|
|
37
|
-
style='progress.elapsed',
|
|
38
|
-
)
|
|
39
|
-
+ Text(' Remaining: ', style='white')
|
|
40
|
-
+ Text(
|
|
41
|
-
human_filesize(remaining_size),
|
|
42
|
-
style='progress.remaining',
|
|
43
|
-
)
|
|
34
|
+
f'[white][progress.elapsed]{human_filesize(advance_size)}[white]'
|
|
35
|
+
f' / [progress.remaining]{human_filesize(remaining_size)}'
|
|
44
36
|
)
|
|
45
|
-
|
|
46
37
|
else:
|
|
47
38
|
try:
|
|
48
39
|
advance_size = task.fields[self.field_name]
|
|
@@ -76,20 +67,15 @@ class DisplayFileTreeProgress:
|
|
|
76
67
|
self.file_size_progress_bar = Progress(TaskProgressColumn(**percent_kwargs), BarColumn(bar_width=None))
|
|
77
68
|
|
|
78
69
|
self.overall_progress = Progress(
|
|
79
|
-
TextColumn('Elapsed:'),
|
|
80
70
|
TimeElapsedColumn(),
|
|
81
|
-
|
|
71
|
+
'/',
|
|
82
72
|
TimeRemainingColumn(),
|
|
83
73
|
)
|
|
84
74
|
self.file_count_progress = Progress(
|
|
85
|
-
TextColumn(
|
|
86
|
-
'[white]Achieved: [progress.elapsed]{task.completed}[white]'
|
|
87
|
-
' Remaining: [progress.remaining]{task.remaining}'
|
|
88
|
-
),
|
|
75
|
+
TextColumn('[white][progress.elapsed]{task.completed}[white] / [progress.remaining]{task.remaining}'),
|
|
89
76
|
'|',
|
|
90
77
|
TransferSpeedColumn2(unit='files'),
|
|
91
78
|
'|',
|
|
92
|
-
TextColumn('Remaining:'),
|
|
93
79
|
TimeRemainingColumn(),
|
|
94
80
|
)
|
|
95
81
|
self.file_size_progress = Progress(
|
|
@@ -97,7 +83,6 @@ class DisplayFileTreeProgress:
|
|
|
97
83
|
'|',
|
|
98
84
|
TransferSpeedColumn(),
|
|
99
85
|
'|',
|
|
100
|
-
TextColumn('Remaining:'),
|
|
101
86
|
TimeRemainingColumn(),
|
|
102
87
|
)
|
|
103
88
|
|
|
@@ -110,9 +95,9 @@ class DisplayFileTreeProgress:
|
|
|
110
95
|
self.file_size_progress_task_time = self.file_size_progress.add_task('', total=total_size)
|
|
111
96
|
|
|
112
97
|
progress_table = Table(box=None, expand=True, padding=(0, 2), show_header=False)
|
|
113
|
-
progress_table.add_row('[b]Overall
|
|
114
|
-
progress_table.add_row('
|
|
115
|
-
progress_table.add_row('
|
|
98
|
+
progress_table.add_row('[b]Overall', self.overall_progress_bar, self.overall_progress)
|
|
99
|
+
progress_table.add_row('Files', self.file_count_progress_bar, self.file_count_progress)
|
|
100
|
+
progress_table.add_row('Size', self.file_size_progress_bar, self.file_size_progress)
|
|
116
101
|
|
|
117
102
|
self.file_count_progress_task = self.file_count_progress.tasks[0]
|
|
118
103
|
self.file_size_progress_task = self.file_size_progress.tasks[0]
|
|
@@ -121,7 +106,7 @@ class DisplayFileTreeProgress:
|
|
|
121
106
|
Panel(
|
|
122
107
|
progress_table,
|
|
123
108
|
title=Text(description, style='progress.data.speed'),
|
|
124
|
-
border_style=Style(color='
|
|
109
|
+
border_style=Style(color='white', bold=True),
|
|
125
110
|
),
|
|
126
111
|
auto_refresh=False,
|
|
127
112
|
)
|
|
@@ -221,9 +206,7 @@ class LargeFileProgress:
|
|
|
221
206
|
'|',
|
|
222
207
|
TransferSpeedColumn(),
|
|
223
208
|
'|',
|
|
224
|
-
TextColumn('Elapsed:'),
|
|
225
209
|
TimeElapsedColumn(),
|
|
226
|
-
TextColumn('Remaining:'),
|
|
227
210
|
TimeRemainingColumn(),
|
|
228
211
|
)
|
|
229
212
|
self.progress.log(f'Large file processing start: {self.description}')
|
|
@@ -28,7 +28,12 @@ def get_hash_db_filenames(hash_db: FileHashDatabase) -> list[str]:
|
|
|
28
28
|
# with NoLogs('PyHardLinkBackup.utilities.filesystem'):
|
|
29
29
|
return sorted(
|
|
30
30
|
str(Path(entry.path).relative_to(hash_db.base_path))
|
|
31
|
-
for entry in iter_scandir_files(
|
|
31
|
+
for entry in iter_scandir_files(
|
|
32
|
+
path=hash_db.base_path,
|
|
33
|
+
one_file_system=False,
|
|
34
|
+
src_device_id=None,
|
|
35
|
+
excludes=set(),
|
|
36
|
+
)
|
|
32
37
|
)
|
|
33
38
|
|
|
34
39
|
|
|
@@ -38,7 +43,12 @@ def get_hash_db_info(backup_root: Path) -> str:
|
|
|
38
43
|
|
|
39
44
|
with NoLogs(logger_name='XY'):
|
|
40
45
|
lines = []
|
|
41
|
-
for entry in iter_scandir_files(
|
|
46
|
+
for entry in iter_scandir_files(
|
|
47
|
+
path=db_base_path,
|
|
48
|
+
one_file_system=False,
|
|
49
|
+
src_device_id=None,
|
|
50
|
+
excludes=set(),
|
|
51
|
+
):
|
|
42
52
|
hash_path = Path(entry.path)
|
|
43
53
|
rel_path = hash_path.relative_to(db_base_path)
|
|
44
54
|
rel_file_path = hash_path.read_text()
|
|
@@ -25,13 +25,26 @@ class TemporaryFileSizeDatabase(tempfile.TemporaryDirectory):
|
|
|
25
25
|
def get_size_db_filenames(size_db: FileSizeDatabase) -> Iterable[str]:
|
|
26
26
|
return sorted(
|
|
27
27
|
str(Path(entry.path).relative_to(size_db.base_path))
|
|
28
|
-
for entry in iter_scandir_files(
|
|
28
|
+
for entry in iter_scandir_files(
|
|
29
|
+
path=size_db.base_path,
|
|
30
|
+
one_file_system=False,
|
|
31
|
+
src_device_id=None,
|
|
32
|
+
excludes=set(),
|
|
33
|
+
)
|
|
29
34
|
)
|
|
30
35
|
|
|
31
36
|
|
|
32
37
|
def get_sizes(size_db: FileSizeDatabase) -> Iterable[int]:
|
|
33
38
|
with NoLogs('PyHardLinkBackup.utilities.filesystem'):
|
|
34
|
-
return sorted(
|
|
39
|
+
return sorted(
|
|
40
|
+
int(entry.name)
|
|
41
|
+
for entry in iter_scandir_files(
|
|
42
|
+
path=size_db.base_path,
|
|
43
|
+
one_file_system=False,
|
|
44
|
+
src_device_id=None,
|
|
45
|
+
excludes=set(),
|
|
46
|
+
)
|
|
47
|
+
)
|
|
35
48
|
|
|
36
49
|
|
|
37
50
|
class FileSizeDatabaseTestCase(BaseTestCase):
|
|
@@ -86,7 +86,14 @@ class TestHashFile(BaseTestCase):
|
|
|
86
86
|
broken_symlink_path.symlink_to(temp_path / 'not/existing/file.txt')
|
|
87
87
|
|
|
88
88
|
with self.assertLogs(level='DEBUG') as logs:
|
|
89
|
-
files = list(
|
|
89
|
+
files = list(
|
|
90
|
+
iter_scandir_files(
|
|
91
|
+
path=temp_path,
|
|
92
|
+
one_file_system=False,
|
|
93
|
+
src_device_id=None,
|
|
94
|
+
excludes={'__pycache__'},
|
|
95
|
+
)
|
|
96
|
+
)
|
|
90
97
|
|
|
91
98
|
file_names = sorted([Path(f.path).relative_to(temp_path).as_posix() for f in files])
|
|
92
99
|
|
|
@@ -106,6 +113,38 @@ class TestHashFile(BaseTestCase):
|
|
|
106
113
|
self.assertIn('Scanning directory ', logs)
|
|
107
114
|
self.assertIn('Excluding directory ', logs)
|
|
108
115
|
|
|
116
|
+
def test_one_file_system(self):
|
|
117
|
+
def scan(temp_path, *, one_file_system, src_device_id):
|
|
118
|
+
with self.assertLogs(level='DEBUG') as logs:
|
|
119
|
+
files = list(
|
|
120
|
+
iter_scandir_files(
|
|
121
|
+
path=temp_path,
|
|
122
|
+
one_file_system=one_file_system,
|
|
123
|
+
src_device_id=src_device_id,
|
|
124
|
+
excludes=set(),
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
file_names = sorted([Path(f.path).relative_to(temp_path).as_posix() for f in files])
|
|
128
|
+
return file_names, '\n'.join(logs.output)
|
|
129
|
+
|
|
130
|
+
with TemporaryDirectoryPath() as temp_path:
|
|
131
|
+
(temp_path / 'file1.txt').touch()
|
|
132
|
+
subdir = temp_path / 'subdir'
|
|
133
|
+
subdir.mkdir()
|
|
134
|
+
(subdir / 'file2.txt').touch()
|
|
135
|
+
|
|
136
|
+
file_names, logs = scan(temp_path, one_file_system=False, src_device_id=None)
|
|
137
|
+
self.assertEqual(file_names, ['file1.txt', 'subdir/file2.txt'])
|
|
138
|
+
self.assertIn('Scanning directory ', logs)
|
|
139
|
+
self.assertNotIn('Skipping', logs)
|
|
140
|
+
|
|
141
|
+
file_names, logs = scan(temp_path, one_file_system=True, src_device_id='FooBar')
|
|
142
|
+
self.assertEqual(file_names, ['file1.txt'])
|
|
143
|
+
self.assertIn('Scanning directory ', logs)
|
|
144
|
+
self.assertIn('Skipping directory ', logs)
|
|
145
|
+
self.assertIn('different device ID', logs)
|
|
146
|
+
self.assertIn('(src device ID: FooBar)', logs)
|
|
147
|
+
|
|
109
148
|
def test_supports_hardlinks(self):
|
|
110
149
|
with TemporaryDirectoryPath() as temp_path:
|
|
111
150
|
with self.assertLogs(level=logging.INFO) as logs:
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
import tyro
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
TyroExcludeDirectoriesArgType = Annotated[
|
|
7
|
+
tuple[str, ...],
|
|
8
|
+
tyro.conf.arg(
|
|
9
|
+
help='List of directories to exclude from backup.',
|
|
10
|
+
),
|
|
11
|
+
]
|
|
12
|
+
DEFAULT_EXCLUDE_DIRECTORIES = ('__pycache__', '.cache', '.temp', '.tmp', '.tox', '.nox')
|
|
13
|
+
|
|
14
|
+
TyroOneFileSystemArgType = Annotated[
|
|
15
|
+
bool,
|
|
16
|
+
tyro.conf.arg(
|
|
17
|
+
help='Do not cross filesystem boundaries.',
|
|
18
|
+
),
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
TyroBackupNameArgType = Annotated[
|
|
22
|
+
str | None,
|
|
23
|
+
tyro.conf.arg(
|
|
24
|
+
help=(
|
|
25
|
+
'Optional name for the backup (used to create a subdirectory in the backup destination).'
|
|
26
|
+
' If not provided, the name of the source directory is used.'
|
|
27
|
+
),
|
|
28
|
+
),
|
|
29
|
+
]
|
|
@@ -51,19 +51,23 @@ usage: phlb backup [-h] [BACKUP OPTIONS]
|
|
|
51
51
|
|
|
52
52
|
Backup the source directory to the destination directory using hard links for deduplication.
|
|
53
53
|
|
|
54
|
-
╭─ positional arguments
|
|
55
|
-
│ source
|
|
56
|
-
│ destination
|
|
57
|
-
|
|
58
|
-
╭─ options
|
|
59
|
-
│ -h, --help
|
|
60
|
-
│ --
|
|
61
|
-
│
|
|
62
|
-
│ --
|
|
63
|
-
│
|
|
64
|
-
│ --
|
|
65
|
-
│
|
|
66
|
-
|
|
54
|
+
╭─ positional arguments ───────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
55
|
+
│ source Source directory to back up. (required) │
|
|
56
|
+
│ destination Destination directory for the backup. (required) │
|
|
57
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
58
|
+
╭─ options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
59
|
+
│ -h, --help show this help message and exit │
|
|
60
|
+
│ --name {None}|STR Optional name for the backup (used to create a subdirectory in the backup destination). If not │
|
|
61
|
+
│ provided, the name of the source directory is used. (default: None) │
|
|
62
|
+
│ --one-file-system, --no-one-file-system │
|
|
63
|
+
│ Do not cross filesystem boundaries. (default: True) │
|
|
64
|
+
│ --excludes [STR [STR ...]] │
|
|
65
|
+
│ List of directories to exclude from backup. (default: __pycache__ .cache .temp .tmp .tox .nox) │
|
|
66
|
+
│ --verbosity {debug,info,warning,error} │
|
|
67
|
+
│ Log level for console logging. (default: warning) │
|
|
68
|
+
│ --log-file-level {debug,info,warning,error} │
|
|
69
|
+
│ Log level for the log file (default: info) │
|
|
70
|
+
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
67
71
|
```
|
|
68
72
|
[comment]: <> (✂✂✂ auto generated backup help end ✂✂✂)
|
|
69
73
|
|
|
@@ -270,12 +274,20 @@ Overview of main changes:
|
|
|
270
274
|
|
|
271
275
|
[comment]: <> (✂✂✂ auto generated history start ✂✂✂)
|
|
272
276
|
|
|
277
|
+
* [v1.8.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.3...v1.8.0)
|
|
278
|
+
* 2026-01-22 - Add optional "--name" to enforce a name for the backup sub directory
|
|
279
|
+
* 2026-01-22 - Do not cross filesystem boundaries as default
|
|
280
|
+
* 2026-01-22 - Display progress also for large unique file copy
|
|
281
|
+
* 2026-01-22 - Optimize progress bars for smaller screens
|
|
273
282
|
* [v1.7.3](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.2...v1.7.3)
|
|
274
283
|
* 2026-01-21 - Handle directory symlinks correct
|
|
275
284
|
* [v1.7.2](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.1...v1.7.2)
|
|
276
285
|
* 2026-01-21 - Display "Remaining time" to files and sizes, too.
|
|
277
286
|
* [v1.7.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.7.0...v1.7.1)
|
|
278
287
|
* 2026-01-19 - Update requirements to fix problems under Windows
|
|
288
|
+
|
|
289
|
+
<details><summary>Expand older history entries ...</summary>
|
|
290
|
+
|
|
279
291
|
* [v1.7.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.6.0...v1.7.0)
|
|
280
292
|
* 2026-01-19 - Speedup and enhance unittest
|
|
281
293
|
* 2026-01-17 - Remove unfinished copied files on errors
|
|
@@ -285,9 +297,6 @@ Overview of main changes:
|
|
|
285
297
|
* 2026-01-17 - simplify tests
|
|
286
298
|
* 2026-01-17 - Warn if broken symlink found
|
|
287
299
|
* 2026-01-17 - Update README
|
|
288
|
-
|
|
289
|
-
<details><summary>Expand older history entries ...</summary>
|
|
290
|
-
|
|
291
300
|
* [v1.6.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.5.0...v1.6.0)
|
|
292
301
|
* 2026-01-17 - Fix flaky test, because of terminal size
|
|
293
302
|
* 2026-01-17 - Bugfix: Don't hash new large files twice
|
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
HardLink/Deduplication Backups with Python
|
|
4
4
|
|
|
5
|
+
# PyHardLinkBackup - Backup Naming
|
|
6
|
+
|
|
7
|
+
The backup name is optional.
|
|
8
|
+
If not provided, the name of the source directory is used.
|
|
9
|
+
|
|
5
10
|
# PyHardLinkBackup - Notes
|
|
6
11
|
|
|
7
12
|
A log file is stored in the backup directory. e.g.:
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
from typing import Annotated
|
|
2
|
-
|
|
3
|
-
import tyro
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
TyroExcludeDirectoriesArgType = Annotated[
|
|
7
|
-
tuple[str, ...],
|
|
8
|
-
tyro.conf.arg(
|
|
9
|
-
help='List of directories to exclude from backup.',
|
|
10
|
-
),
|
|
11
|
-
]
|
|
12
|
-
DEFAULT_EXCLUDE_DIRECTORIES = ('__pycache__', '.cache', '.temp', '.tmp', '.tox', '.nox')
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/shell_completion.py
RENAMED
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/cli_dev/update_readme_history.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_project_setup.py
RENAMED
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_readme_history.py
RENAMED
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/tests/test_rebuild_database.py
RENAMED
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/file_hash_database.py
RENAMED
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/file_size_database.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pyhardlinkbackup-1.7.3 → pyhardlinkbackup-1.8.0}/PyHardLinkBackup/utilities/tests/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|