PyHardLinkBackup 1.3.0__tar.gz → 1.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.github/workflows/tests.yml +2 -1
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PKG-INFO +27 -18
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/__init__.py +1 -1
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/backup.py +46 -24
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_app/phlb.py +30 -14
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/benchmark.py +51 -32
- pyhardlinkbackup-1.4.1/PyHardLinkBackup/logging_setup.py +124 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/rebuild_databases.py +45 -16
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_backup.py +168 -82
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_rebuild_database.py +30 -10
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/humanize.py +17 -0
- pyhardlinkbackup-1.4.1/PyHardLinkBackup/utilities/tee.py +40 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/tests/test_file_hash_database.py +1 -1
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/tests/test_file_size_database.py +1 -1
- pyhardlinkbackup-1.4.1/PyHardLinkBackup/utilities/tyro_cli_shared_args.py +12 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/README.md +26 -17
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/docs/README.md +8 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.editorconfig +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.gitignore +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.idea/.gitignore +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.pre-commit-config.yaml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.pre-commit-hooks.yaml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/Template Python tests.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/Unittests - __all__.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/cli.py --help.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/dev-cli update.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/only DocTests.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.run/only DocWrite.run.xml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/.venv-app/lib/python3.12/site-packages/cli_base/tests/shell_complete_snapshots/.gitignore +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/__main__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_app/__init__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/__init__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/code_style.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/packaging.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/shell_completion.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/testing.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/cli_dev/update_readme_history.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/constants.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/__init__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_doc_write.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_doctests.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_project_setup.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_readme.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/tests/test_readme_history.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/__init__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/file_hash_database.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/file_size_database.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/filesystem.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/rich_utils.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/sha256sums.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/tests/__init__.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/PyHardLinkBackup/utilities/tests/test_filesystem.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/cli.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/dev-cli.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/dist/.gitignore +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/docs/about-docs.md +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/noxfile.py +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/pyproject.toml +0 -0
- {pyhardlinkbackup-1.3.0 → pyhardlinkbackup-1.4.1}/uv.lock +0 -0
|
@@ -13,11 +13,12 @@ on:
|
|
|
13
13
|
|
|
14
14
|
jobs:
|
|
15
15
|
test:
|
|
16
|
-
runs-on: ubuntu-latest
|
|
17
16
|
strategy:
|
|
18
17
|
fail-fast: false
|
|
19
18
|
matrix:
|
|
20
19
|
python-version: ["3.14", "3.13", "3.12"]
|
|
20
|
+
os: [ubuntu-latest, macos-latest] # TODO: windows-latest
|
|
21
|
+
runs-on: ${{ matrix.os }}
|
|
21
22
|
steps:
|
|
22
23
|
- name: Checkout
|
|
23
24
|
run: |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyHardLinkBackup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.1
|
|
4
4
|
Summary: HardLink/Deduplication Backups with Python
|
|
5
5
|
Project-URL: Documentation, https://github.com/jedie/PyHardLinkBackup
|
|
6
6
|
Project-URL: Source, https://github.com/jedie/PyHardLinkBackup
|
|
@@ -42,22 +42,23 @@ The main command is `phlb backup <source> <destination>`:
|
|
|
42
42
|
|
|
43
43
|
[comment]: <> (✂✂✂ auto generated backup help start ✂✂✂)
|
|
44
44
|
```
|
|
45
|
-
usage: phlb backup [-h]
|
|
45
|
+
usage: phlb backup [-h] [BACKUP OPTIONS]
|
|
46
46
|
|
|
47
47
|
Backup the source directory to the destination directory using hard links for deduplication.
|
|
48
48
|
|
|
49
|
-
╭─ positional arguments
|
|
50
|
-
│ source Source directory to back up. (required)
|
|
51
|
-
│ destination Destination directory for the backup. (required)
|
|
52
|
-
|
|
53
|
-
╭─ options
|
|
54
|
-
│ -h, --help show this help message and exit
|
|
55
|
-
│ --excludes [STR [STR ...]]
|
|
56
|
-
│ List of
|
|
57
|
-
│
|
|
58
|
-
│
|
|
59
|
-
│
|
|
60
|
-
|
|
49
|
+
╭─ positional arguments ──────────────────────────────────────────────────────────────────────────────────────╮
|
|
50
|
+
│ source Source directory to back up. (required) │
|
|
51
|
+
│ destination Destination directory for the backup. (required) │
|
|
52
|
+
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
53
|
+
╭─ options ───────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
54
|
+
│ -h, --help show this help message and exit │
|
|
55
|
+
│ --excludes [STR [STR ...]] │
|
|
56
|
+
│ List of directories to exclude from backup. (default: __pycache__ .cache .temp .tmp .tox .nox) │
|
|
57
|
+
│ --verbosity {debug,info,warning,error} │
|
|
58
|
+
│ Log level for console logging. (default: warning) │
|
|
59
|
+
│ --log-file-level {debug,info,warning,error} │
|
|
60
|
+
│ Log level for the log file (default: info) │
|
|
61
|
+
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
61
62
|
```
|
|
62
63
|
[comment]: <> (✂✂✂ auto generated backup help end ✂✂✂)
|
|
63
64
|
|
|
@@ -174,7 +175,7 @@ The file contains only the relative path to the first hardlink of this file cont
|
|
|
174
175
|
|
|
175
176
|
[comment]: <> (✂✂✂ auto generated dev help start ✂✂✂)
|
|
176
177
|
```
|
|
177
|
-
usage: ./dev-cli.py [-h] {benchmark-hashes,coverage,install,lint,mypy,nox,pip-audit,publish,shell-completion,test,update,update-readme-history,update-test-snapshot-files,version}
|
|
178
|
+
usage: ./dev-cli.py [-h] {benchmark-hashes,coverage,install,lint,mypy,nox,pip-audit,publish,scan-benchmark,shell-completion,test,update,update-readme-history,update-test-snapshot-files,version}
|
|
178
179
|
|
|
179
180
|
|
|
180
181
|
|
|
@@ -192,6 +193,8 @@ usage: ./dev-cli.py [-h] {benchmark-hashes,coverage,install,lint,mypy,nox,pip-au
|
|
|
192
193
|
│ • nox Run nox │
|
|
193
194
|
│ • pip-audit Run pip-audit check against current requirements files │
|
|
194
195
|
│ • publish Build and upload this project to PyPi │
|
|
196
|
+
│ • scan-benchmark │
|
|
197
|
+
│ Benchmark our filesystem scan routine. │
|
|
195
198
|
│ • shell-completion │
|
|
196
199
|
│ Setup shell completion for this CLI (Currently only for bash shell) │
|
|
197
200
|
│ • test Run unittests │
|
|
@@ -228,6 +231,12 @@ Overview of main changes:
|
|
|
228
231
|
|
|
229
232
|
[comment]: <> (✂✂✂ auto generated history start ✂✂✂)
|
|
230
233
|
|
|
234
|
+
* [v1.4.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.4.0...v1.4.1)
|
|
235
|
+
* 2026-01-16 - Bugfix large file handling
|
|
236
|
+
* [v1.4.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.3.0...v1.4.0)
|
|
237
|
+
* 2026-01-16 - Create log file in backup and a summary.txt
|
|
238
|
+
* 2026-01-16 - Run CI tests on macos, too.
|
|
239
|
+
* 2026-01-16 - add dev cli command "scan-benchmark"
|
|
231
240
|
* [v1.3.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.2.0...v1.3.0)
|
|
232
241
|
* 2026-01-15 - Verify SHA256SUMS files in "rebuild" command, too.
|
|
233
242
|
* 2026-01-15 - Code cleanup: use more generic names for and in BackupProgress
|
|
@@ -240,14 +249,14 @@ Overview of main changes:
|
|
|
240
249
|
* 2026-01-14 - Enhance progress bars
|
|
241
250
|
* 2026-01-14 - A a note to rsync --link-dest
|
|
242
251
|
* 2026-01-14 - Use cli_base.cli_tools.test_utils.base_testcases
|
|
252
|
+
|
|
253
|
+
<details><summary>Expand older history entries ...</summary>
|
|
254
|
+
|
|
243
255
|
* [v1.1.0](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.1...v1.1.0)
|
|
244
256
|
* 2026-01-14 - Change backup timestamp directory to old schema: '%Y-%m-%d-%H%M%S'
|
|
245
257
|
* 2026-01-14 - Add "Overview of main changes" to README
|
|
246
258
|
* [v1.0.1](https://github.com/jedie/PyHardLinkBackup/compare/v1.0.0...v1.0.1)
|
|
247
259
|
* 2026-01-13 - Store SHA256SUMS files in backup directories
|
|
248
|
-
|
|
249
|
-
<details><summary>Expand older history entries ...</summary>
|
|
250
|
-
|
|
251
260
|
* [v1.0.0](https://github.com/jedie/PyHardLinkBackup/compare/v0.13.0...v1.0.0)
|
|
252
261
|
* 2026-01-13 - Change "./cli.py" to "phlb" (because it's the name installed via pipx)
|
|
253
262
|
* 2026-01-13 - Update README
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import dataclasses
|
|
2
|
+
import datetime
|
|
2
3
|
import logging
|
|
3
4
|
import os
|
|
4
5
|
import shutil
|
|
5
6
|
import sys
|
|
6
7
|
import time
|
|
7
|
-
from datetime import datetime
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
|
|
10
10
|
from rich import print # noqa
|
|
11
11
|
|
|
12
12
|
from PyHardLinkBackup.constants import CHUNK_SIZE
|
|
13
|
+
from PyHardLinkBackup.logging_setup import LoggingManager
|
|
13
14
|
from PyHardLinkBackup.utilities.file_hash_database import FileHashDatabase
|
|
14
15
|
from PyHardLinkBackup.utilities.file_size_database import FileSizeDatabase
|
|
15
16
|
from PyHardLinkBackup.utilities.filesystem import (
|
|
@@ -20,9 +21,10 @@ from PyHardLinkBackup.utilities.filesystem import (
|
|
|
20
21
|
read_and_hash_file,
|
|
21
22
|
supports_hardlinks,
|
|
22
23
|
)
|
|
23
|
-
from PyHardLinkBackup.utilities.humanize import human_filesize
|
|
24
|
+
from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager, human_filesize
|
|
24
25
|
from PyHardLinkBackup.utilities.rich_utils import DisplayFileTreeProgress
|
|
25
26
|
from PyHardLinkBackup.utilities.sha256sums import store_hash
|
|
27
|
+
from PyHardLinkBackup.utilities.tee import TeeStdoutContext
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
logger = logging.getLogger(__name__)
|
|
@@ -31,6 +33,7 @@ logger = logging.getLogger(__name__)
|
|
|
31
33
|
@dataclasses.dataclass
|
|
32
34
|
class BackupResult:
|
|
33
35
|
backup_dir: Path
|
|
36
|
+
log_file: Path
|
|
34
37
|
#
|
|
35
38
|
backup_count: int = 0
|
|
36
39
|
backup_size: int = 0
|
|
@@ -135,12 +138,13 @@ def backup_one_file(
|
|
|
135
138
|
backup_result.hardlinked_size += size
|
|
136
139
|
else:
|
|
137
140
|
logger.info('Copy unique file: %s to %s', src_path, dst_path)
|
|
141
|
+
file_hash = copy_and_hash(src_path, dst_path)
|
|
138
142
|
hash_db[file_hash] = dst_path
|
|
139
143
|
backup_result.copied_files += 1
|
|
140
144
|
backup_result.copied_size += size
|
|
141
145
|
|
|
142
146
|
# Keep original file metadata (permission bits, time stamps, and flags)
|
|
143
|
-
shutil.
|
|
147
|
+
shutil.copystat(src_path, dst_path)
|
|
144
148
|
else:
|
|
145
149
|
# A file with this size not backuped before -> Can't be duplicate -> copy and hash
|
|
146
150
|
file_hash = copy_and_hash(src_path, dst_path)
|
|
@@ -152,7 +156,13 @@ def backup_one_file(
|
|
|
152
156
|
store_hash(dst_path, file_hash)
|
|
153
157
|
|
|
154
158
|
|
|
155
|
-
def backup_tree(
|
|
159
|
+
def backup_tree(
|
|
160
|
+
*,
|
|
161
|
+
src_root: Path,
|
|
162
|
+
backup_root: Path,
|
|
163
|
+
excludes: tuple[str, ...],
|
|
164
|
+
log_manager: LoggingManager,
|
|
165
|
+
) -> BackupResult:
|
|
156
166
|
src_root = src_root.resolve()
|
|
157
167
|
if not src_root.is_dir():
|
|
158
168
|
print('Error: Source directory does not exist!')
|
|
@@ -176,15 +186,23 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
|
|
|
176
186
|
sys.exit(1)
|
|
177
187
|
|
|
178
188
|
# Step 1: Scan source directory:
|
|
179
|
-
|
|
189
|
+
excludes: set = set(excludes)
|
|
190
|
+
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
191
|
+
src_file_count, src_total_size = humanized_fs_scan(src_root, excludes=excludes)
|
|
180
192
|
|
|
181
193
|
phlb_conf_dir = backup_root / '.phlb'
|
|
182
194
|
phlb_conf_dir.mkdir(parents=False, exist_ok=True)
|
|
183
195
|
|
|
184
|
-
|
|
185
|
-
|
|
196
|
+
timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
|
|
197
|
+
backup_main_dir = backup_root / src_root.name
|
|
198
|
+
backup_dir = backup_main_dir / timestamp
|
|
186
199
|
backup_dir.mkdir(parents=True, exist_ok=False)
|
|
187
200
|
|
|
201
|
+
log_file = backup_main_dir / f'{timestamp}-backup.log'
|
|
202
|
+
log_manager.start_file_logging(log_file)
|
|
203
|
+
|
|
204
|
+
logger.info('Backup %s to %s', src_root, backup_dir)
|
|
205
|
+
|
|
188
206
|
print(f'\nBackup to {backup_dir}...\n')
|
|
189
207
|
|
|
190
208
|
with DisplayFileTreeProgress(src_file_count, src_total_size) as progress:
|
|
@@ -192,7 +210,7 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
|
|
|
192
210
|
size_db = FileSizeDatabase(phlb_conf_dir)
|
|
193
211
|
hash_db = FileHashDatabase(backup_root, phlb_conf_dir)
|
|
194
212
|
|
|
195
|
-
backup_result = BackupResult(backup_dir=backup_dir)
|
|
213
|
+
backup_result = BackupResult(backup_dir=backup_dir, log_file=log_file)
|
|
196
214
|
|
|
197
215
|
next_update = 0
|
|
198
216
|
for entry in iter_scandir_files(src_root, excludes=excludes):
|
|
@@ -219,21 +237,25 @@ def backup_tree(*, src_root: Path, backup_root: Path, excludes: set[str]) -> Bac
|
|
|
219
237
|
# Finalize progress indicator values:
|
|
220
238
|
progress.update(completed_file_count=backup_result.backup_count, completed_size=backup_result.backup_size)
|
|
221
239
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
f' *
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
f'
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
240
|
+
summary_file = backup_main_dir / f'{timestamp}-summary.txt'
|
|
241
|
+
with TeeStdoutContext(summary_file):
|
|
242
|
+
print(f'\nBackup complete: {backup_dir} (total size {human_filesize(backup_result.backup_size)})\n')
|
|
243
|
+
print(f' Total files processed: {backup_result.backup_count}')
|
|
244
|
+
print(f' * Symlinked files: {backup_result.symlink_files}')
|
|
245
|
+
print(
|
|
246
|
+
f' * Hardlinked files: {backup_result.hardlinked_files}'
|
|
247
|
+
f' (saved {human_filesize(backup_result.hardlinked_size)})'
|
|
248
|
+
)
|
|
249
|
+
print(f' * Copied files: {backup_result.copied_files} (total {human_filesize(backup_result.copied_size)})')
|
|
250
|
+
print(
|
|
251
|
+
f' of which small (<{size_db.MIN_SIZE} Bytes)'
|
|
252
|
+
f' files: {backup_result.copied_small_files}'
|
|
253
|
+
f' (total {human_filesize(backup_result.copied_small_size)})'
|
|
254
|
+
)
|
|
255
|
+
if backup_result.error_count > 0:
|
|
256
|
+
print(f' Errors during backup: {backup_result.error_count} (see log for details)')
|
|
257
|
+
print()
|
|
258
|
+
|
|
259
|
+
logger.info('Backup completed. Summary created: %s', summary_file)
|
|
238
260
|
|
|
239
261
|
return backup_result
|
|
@@ -3,13 +3,22 @@ from pathlib import Path
|
|
|
3
3
|
from typing import Annotated
|
|
4
4
|
|
|
5
5
|
import tyro
|
|
6
|
-
from cli_base.cli_tools.verbosity import setup_logging
|
|
7
|
-
from cli_base.tyro_commands import TyroVerbosityArgType
|
|
8
6
|
from rich import print # noqa
|
|
9
7
|
|
|
10
8
|
from PyHardLinkBackup import rebuild_databases
|
|
11
9
|
from PyHardLinkBackup.backup import backup_tree
|
|
12
10
|
from PyHardLinkBackup.cli_app import app
|
|
11
|
+
from PyHardLinkBackup.logging_setup import (
|
|
12
|
+
DEFAULT_CONSOLE_LOG_LEVEL,
|
|
13
|
+
DEFAULT_LOG_FILE_LEVEL,
|
|
14
|
+
LoggingManager,
|
|
15
|
+
TyroConsoleLogLevelArgType,
|
|
16
|
+
TyroLogFileLevelArgType,
|
|
17
|
+
)
|
|
18
|
+
from PyHardLinkBackup.utilities.tyro_cli_shared_args import (
|
|
19
|
+
DEFAULT_EXCLUDE_DIRECTORIES,
|
|
20
|
+
TyroExcludeDirectoriesArgType,
|
|
21
|
+
)
|
|
13
22
|
|
|
14
23
|
|
|
15
24
|
logger = logging.getLogger(__name__)
|
|
@@ -32,22 +41,22 @@ def backup(
|
|
|
32
41
|
),
|
|
33
42
|
],
|
|
34
43
|
/,
|
|
35
|
-
excludes:
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
help='List of directory or file names to exclude from backup.',
|
|
39
|
-
),
|
|
40
|
-
] = ('__pycache__', '.cache', '.temp', '.tmp', '.tox', '.nox'),
|
|
41
|
-
verbosity: TyroVerbosityArgType = 2,
|
|
44
|
+
excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
|
|
45
|
+
verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
|
|
46
|
+
log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
|
|
42
47
|
) -> None:
|
|
43
48
|
"""
|
|
44
49
|
Backup the source directory to the destination directory using hard links for deduplication.
|
|
45
50
|
"""
|
|
46
|
-
|
|
51
|
+
log_manager = LoggingManager(
|
|
52
|
+
console_level=verbosity,
|
|
53
|
+
file_level=log_file_level,
|
|
54
|
+
)
|
|
47
55
|
backup_tree(
|
|
48
56
|
src_root=src,
|
|
49
57
|
backup_root=dst,
|
|
50
|
-
excludes=
|
|
58
|
+
excludes=excludes,
|
|
59
|
+
log_manager=log_manager,
|
|
51
60
|
)
|
|
52
61
|
|
|
53
62
|
|
|
@@ -61,11 +70,18 @@ def rebuild(
|
|
|
61
70
|
),
|
|
62
71
|
],
|
|
63
72
|
/,
|
|
64
|
-
verbosity:
|
|
73
|
+
verbosity: TyroConsoleLogLevelArgType = DEFAULT_CONSOLE_LOG_LEVEL,
|
|
74
|
+
log_file_level: TyroLogFileLevelArgType = DEFAULT_LOG_FILE_LEVEL,
|
|
65
75
|
) -> None:
|
|
66
76
|
"""
|
|
67
77
|
Rebuild the file hash and size database by scanning all backup files. And also verify SHA256SUMS
|
|
68
78
|
and/or store missing hashes in SHA256SUMS files.
|
|
69
79
|
"""
|
|
70
|
-
|
|
71
|
-
|
|
80
|
+
log_manager = LoggingManager(
|
|
81
|
+
console_level=verbosity,
|
|
82
|
+
file_level=log_file_level,
|
|
83
|
+
)
|
|
84
|
+
rebuild_databases.rebuild(
|
|
85
|
+
backup_root=backup_root,
|
|
86
|
+
log_manager=log_manager,
|
|
87
|
+
)
|
|
@@ -10,16 +10,34 @@ from cli_base.tyro_commands import TyroVerbosityArgType
|
|
|
10
10
|
from rich import print # noqa
|
|
11
11
|
|
|
12
12
|
from PyHardLinkBackup.cli_dev import app
|
|
13
|
-
from PyHardLinkBackup.utilities.filesystem import iter_scandir_files
|
|
13
|
+
from PyHardLinkBackup.utilities.filesystem import humanized_fs_scan, iter_scandir_files
|
|
14
|
+
from PyHardLinkBackup.utilities.humanize import PrintTimingContextManager
|
|
15
|
+
from PyHardLinkBackup.utilities.tyro_cli_shared_args import DEFAULT_EXCLUDE_DIRECTORIES, TyroExcludeDirectoriesArgType
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
21
|
+
@app.command
|
|
22
|
+
def scan_benchmark(
|
|
23
|
+
base_path: Path,
|
|
24
|
+
/,
|
|
25
|
+
excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
|
|
26
|
+
verbosity: TyroVerbosityArgType = 1,
|
|
27
|
+
) -> None:
|
|
28
|
+
"""
|
|
29
|
+
Benchmark our filesystem scan routine.
|
|
30
|
+
"""
|
|
31
|
+
setup_logging(verbosity=verbosity)
|
|
32
|
+
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
33
|
+
humanized_fs_scan(path=base_path, excludes=set(excludes))
|
|
34
|
+
|
|
35
|
+
|
|
19
36
|
@app.command
|
|
20
37
|
def benchmark_hashes(
|
|
21
38
|
base_path: Path,
|
|
22
39
|
/,
|
|
40
|
+
excludes: TyroExcludeDirectoriesArgType = DEFAULT_EXCLUDE_DIRECTORIES,
|
|
23
41
|
max_duration: int = 30, # in seconds
|
|
24
42
|
min_file_size: int = 15 * 1024, # 15 KiB
|
|
25
43
|
max_file_size: int = 100 * 1024 * 1024, # 100 MiB
|
|
@@ -70,40 +88,41 @@ def benchmark_hashes(
|
|
|
70
88
|
stop_time = start_time + max_duration
|
|
71
89
|
next_update = start_time + 2
|
|
72
90
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
91
|
+
with PrintTimingContextManager('Filesystem scan completed in'):
|
|
92
|
+
for dir_entry in iter_scandir_files(path=base_path, excludes=set(excludes)):
|
|
93
|
+
entry_stat = dir_entry.stat()
|
|
94
|
+
file_size = entry_stat.st_size
|
|
95
|
+
if not (min_file_size <= file_size <= max_file_size):
|
|
96
|
+
continue
|
|
78
97
|
|
|
79
|
-
start_time = time.perf_counter()
|
|
80
|
-
file_content = Path(dir_entry.path).read_bytes()
|
|
81
|
-
duration = time.perf_counter() - start_time
|
|
82
|
-
total_read_time += duration
|
|
83
|
-
|
|
84
|
-
for algo in algorithms:
|
|
85
|
-
# Actual measurement:
|
|
86
98
|
start_time = time.perf_counter()
|
|
87
|
-
|
|
99
|
+
file_content = Path(dir_entry.path).read_bytes()
|
|
88
100
|
duration = time.perf_counter() - start_time
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
101
|
+
total_read_time += duration
|
|
102
|
+
|
|
103
|
+
for algo in algorithms:
|
|
104
|
+
# Actual measurement:
|
|
105
|
+
start_time = time.perf_counter()
|
|
106
|
+
hashlib.new(algo, file_content)
|
|
107
|
+
duration = time.perf_counter() - start_time
|
|
108
|
+
|
|
109
|
+
results[algo].add(duration)
|
|
110
|
+
|
|
111
|
+
file_count += 1
|
|
112
|
+
total_size += entry_stat.st_size
|
|
113
|
+
|
|
114
|
+
now = time.time()
|
|
115
|
+
if now >= stop_time:
|
|
116
|
+
print('Reached max duration limit, stopping benchmark...')
|
|
117
|
+
break
|
|
118
|
+
|
|
119
|
+
if now >= next_update:
|
|
120
|
+
percent = (now - (stop_time - max_duration)) / max_duration * 100
|
|
121
|
+
print(
|
|
122
|
+
f'{int(percent)}% Processed {file_count} files so far,'
|
|
123
|
+
f' total size: {total_size / 1024 / 1024:.1f} MiB...'
|
|
124
|
+
)
|
|
125
|
+
next_update = now + 2
|
|
107
126
|
|
|
108
127
|
print(f'\nTotal files hashed: {file_count}, total size: {total_size / 1024 / 1024:.1f} MiB')
|
|
109
128
|
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated, Literal
|
|
5
|
+
|
|
6
|
+
import tyro
|
|
7
|
+
from bx_py_utils.path import assert_is_dir
|
|
8
|
+
from rich import (
|
|
9
|
+
get_console,
|
|
10
|
+
print, # noqa
|
|
11
|
+
)
|
|
12
|
+
from rich.logging import RichHandler
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
LogLevelLiteral = Literal['debug', 'info', 'warning', 'error']
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
TyroConsoleLogLevelArgType = Annotated[
|
|
21
|
+
LogLevelLiteral,
|
|
22
|
+
tyro.conf.arg(
|
|
23
|
+
help='Log level for console logging.',
|
|
24
|
+
),
|
|
25
|
+
]
|
|
26
|
+
DEFAULT_CONSOLE_LOG_LEVEL: TyroConsoleLogLevelArgType = 'warning'
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
TyroLogFileLevelArgType = Annotated[
|
|
30
|
+
LogLevelLiteral,
|
|
31
|
+
tyro.conf.arg(
|
|
32
|
+
help='Log level for the log file',
|
|
33
|
+
),
|
|
34
|
+
]
|
|
35
|
+
DEFAULT_LOG_FILE_LEVEL: TyroLogFileLevelArgType = 'info'
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def log_level_name2int(level_name: str) -> int:
|
|
39
|
+
level_name = level_name.upper()
|
|
40
|
+
level_mapping = logging.getLevelNamesMapping()
|
|
41
|
+
try:
|
|
42
|
+
return level_mapping[level_name]
|
|
43
|
+
except KeyError as err:
|
|
44
|
+
raise ValueError(f'Invalid log level name: {level_name}') from err
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
console = get_console()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LoggingManager:
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
*,
|
|
54
|
+
console_level: TyroConsoleLogLevelArgType,
|
|
55
|
+
file_level: TyroLogFileLevelArgType,
|
|
56
|
+
):
|
|
57
|
+
self.console_level_name = console_level
|
|
58
|
+
self.console_level: int = log_level_name2int(console_level)
|
|
59
|
+
self.file_level_name = file_level
|
|
60
|
+
self.file_level: int = log_level_name2int(file_level)
|
|
61
|
+
|
|
62
|
+
self.lowest_level = min(self.console_level, self.file_level)
|
|
63
|
+
|
|
64
|
+
if console_level == logging.DEBUG:
|
|
65
|
+
log_format = '(%(name)s) %(message)s'
|
|
66
|
+
else:
|
|
67
|
+
log_format = '%(message)s'
|
|
68
|
+
|
|
69
|
+
console.print(
|
|
70
|
+
f'(Set [bold]console[bold] log level: [cyan]{self.console_level_name}[/cyan])',
|
|
71
|
+
justify='right',
|
|
72
|
+
)
|
|
73
|
+
handler = RichHandler(console=console, omit_repeated_times=False)
|
|
74
|
+
handler.setLevel(self.console_level)
|
|
75
|
+
logging.basicConfig(
|
|
76
|
+
level=self.lowest_level,
|
|
77
|
+
format=log_format,
|
|
78
|
+
datefmt='[%x %X.%f]',
|
|
79
|
+
handlers=[handler],
|
|
80
|
+
force=True,
|
|
81
|
+
)
|
|
82
|
+
sys.excepthook = self.log_unhandled_exception
|
|
83
|
+
|
|
84
|
+
def start_file_logging(self, log_file: Path):
|
|
85
|
+
console.print(
|
|
86
|
+
f'(initialize log file [bold]{log_file}[/bold] with level: [cyan]{self.file_level_name}[/cyan])',
|
|
87
|
+
justify='right',
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
assert_is_dir(log_file.parent)
|
|
91
|
+
|
|
92
|
+
root_logger = logging.getLogger()
|
|
93
|
+
|
|
94
|
+
file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
|
|
95
|
+
file_handler.setLevel(self.file_level)
|
|
96
|
+
|
|
97
|
+
formatter = logging.Formatter(
|
|
98
|
+
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
99
|
+
datefmt='%Y-%m-%d %H:%M:%S',
|
|
100
|
+
)
|
|
101
|
+
file_handler.setFormatter(formatter)
|
|
102
|
+
|
|
103
|
+
root_logger.addHandler(file_handler)
|
|
104
|
+
|
|
105
|
+
def log_unhandled_exception(self, exc_type, exc_value, exc_traceback):
|
|
106
|
+
if issubclass(exc_type, KeyboardInterrupt):
|
|
107
|
+
logger.info('Program interrupted by user (KeyboardInterrupt). Exiting...')
|
|
108
|
+
sys.__excepthook__(exc_type, exc_value, exc_traceback)
|
|
109
|
+
else:
|
|
110
|
+
logger.exception(
|
|
111
|
+
'Unhandled exception occurred:',
|
|
112
|
+
exc_info=(exc_type, exc_value, exc_traceback),
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class NoopLoggingManager(LoggingManager):
|
|
117
|
+
"""
|
|
118
|
+
Only for tests: A logging manager that does nothing.
|
|
119
|
+
"""
|
|
120
|
+
def __init__(self, *args, **kwargs):
|
|
121
|
+
pass
|
|
122
|
+
|
|
123
|
+
def start_file_logging(self, log_file: Path):
|
|
124
|
+
pass
|