FileBackup 0.4.4__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {filebackup-0.4.4 → filebackup-0.5.1}/.github/workflows/CICD_impl.yml +8 -0
  2. {filebackup-0.4.4 → filebackup-0.5.1}/BuildBinary.py +49 -6
  3. {filebackup-0.4.4 → filebackup-0.5.1}/PKG-INFO +2 -2
  4. {filebackup-0.4.4 → filebackup-0.5.1}/pyproject.toml +2 -2
  5. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/CommandLine/OffsiteEntryPoint.py +7 -0
  6. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/Offsite.py +191 -150
  7. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/__init__.py +1 -1
  8. {filebackup-0.4.4 → filebackup-0.5.1}/tests/CommandLine/EntryPoint_UnitTest.py +3 -0
  9. {filebackup-0.4.4 → filebackup-0.5.1}/uv.lock +35 -9
  10. {filebackup-0.4.4 → filebackup-0.5.1}/.copier-answers.yml +0 -0
  11. {filebackup-0.4.4 → filebackup-0.5.1}/.github/CODEOWNERS +0 -0
  12. {filebackup-0.4.4 → filebackup-0.5.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  13. {filebackup-0.4.4 → filebackup-0.5.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  14. {filebackup-0.4.4 → filebackup-0.5.1}/.github/ISSUE_TEMPLATE/custom.md +0 -0
  15. {filebackup-0.4.4 → filebackup-0.5.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  16. {filebackup-0.4.4 → filebackup-0.5.1}/.github/actions/release_impl/action.yml +0 -0
  17. {filebackup-0.4.4 → filebackup-0.5.1}/.github/pull_request_template.md +0 -0
  18. {filebackup-0.4.4 → filebackup-0.5.1}/.github/release_sources.yaml +0 -0
  19. {filebackup-0.4.4 → filebackup-0.5.1}/.github/workflows/CICD.yml +0 -0
  20. {filebackup-0.4.4 → filebackup-0.5.1}/.github/workflows/codeql.yml +0 -0
  21. {filebackup-0.4.4 → filebackup-0.5.1}/.gitignore +0 -0
  22. {filebackup-0.4.4 → filebackup-0.5.1}/.pre-commit-config.yaml +0 -0
  23. {filebackup-0.4.4 → filebackup-0.5.1}/.python-version +0 -0
  24. {filebackup-0.4.4 → filebackup-0.5.1}/CODE_OF_CONDUCT.md +0 -0
  25. {filebackup-0.4.4 → filebackup-0.5.1}/CONTRIBUTING.md +0 -0
  26. {filebackup-0.4.4 → filebackup-0.5.1}/DEVELOPMENT.md +0 -0
  27. {filebackup-0.4.4 → filebackup-0.5.1}/GOVERNANCE.md +0 -0
  28. {filebackup-0.4.4 → filebackup-0.5.1}/LICENSE +0 -0
  29. {filebackup-0.4.4 → filebackup-0.5.1}/MAINTAINERS.md +0 -0
  30. {filebackup-0.4.4 → filebackup-0.5.1}/README.md +0 -0
  31. {filebackup-0.4.4 → filebackup-0.5.1}/SECURITY.md +0 -0
  32. {filebackup-0.4.4 → filebackup-0.5.1}/minisign_key.pub +0 -0
  33. {filebackup-0.4.4 → filebackup-0.5.1}/src/AutoGitSemVer.yaml +0 -0
  34. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/CommandLine/CommandLineArguments.py +0 -0
  35. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/CommandLine/EntryPoint.py +0 -0
  36. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/CommandLine/MirrorEntryPoint.py +0 -0
  37. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/CommandLine/__init__.py +0 -0
  38. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/FastGlacierDataStore.py +0 -0
  39. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/FileSystemDataStore.py +0 -0
  40. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/Interfaces/BulkStorageDataStore.py +0 -0
  41. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/Interfaces/DataStore.py +0 -0
  42. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/Interfaces/FileBasedDataStore.py +0 -0
  43. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/Interfaces/__init__.py +0 -0
  44. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/S3BrowserDataStore.py +0 -0
  45. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/SFTPDataStore.py +0 -0
  46. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/DataStore/__init__.py +0 -0
  47. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/Impl/Common.py +0 -0
  48. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/Impl/__init__.py +0 -0
  49. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/Mirror.py +0 -0
  50. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/Snapshot.py +0 -0
  51. {filebackup-0.4.4 → filebackup-0.5.1}/src/FileBackup/py.typed +0 -0
  52. {filebackup-0.4.4 → filebackup-0.5.1}/tests/Mirror_UnitTest.py +0 -0
  53. {filebackup-0.4.4 → filebackup-0.5.1}/tests/Offsite_UnitTest.py +0 -0
  54. {filebackup-0.4.4 → filebackup-0.5.1}/tests/Snapshot_UnitTest.py +0 -0
  55. {filebackup-0.4.4 → filebackup-0.5.1}/tests/TestHelpers.py +0 -0
@@ -66,6 +66,9 @@ jobs:
66
66
  python-version: ${{ matrix.python_version }}
67
67
  enable-cache: true
68
68
 
69
+ - name: Install Dependencies
70
+ run: uv sync --frozen
71
+
69
72
  - name: Run pre-commit scripts
70
73
  run: uv run pre-commit run --verbose
71
74
 
@@ -250,6 +253,9 @@ jobs:
250
253
  python-version: ${{ inputs.python_package_version }}
251
254
  enable-cache: true
252
255
 
256
+ - name: Install Dependencies
257
+ run: uv sync --frozen
258
+
253
259
  - name: Build Binary
254
260
  run: uv run python BuildBinary.py Build --verbose
255
261
 
@@ -261,6 +267,8 @@ jobs:
261
267
  with:
262
268
  name: Binary.${{ matrix.os }}
263
269
  path: FileBackup-${{ env.PACKAGE_VERSION }}.*
270
+
271
+ # ----------------------------------------------------------------------
264
272
  validate_binary:
265
273
  needs: [python_package, build_binary]
266
274
 
@@ -1,15 +1,21 @@
1
1
  import os
2
2
  import shutil
3
+ import textwrap
4
+ import uuid
3
5
 
4
6
  from pathlib import Path
5
7
  from typing import Annotated, Optional
6
8
 
7
9
  import typer
8
10
 
11
+ from dbrownell_Common.ContextlibEx import ExitStack
9
12
  from dbrownell_Common import SubprocessEx
10
13
  from dbrownell_Common.Streams.DoneManager import DoneManager, Flags as DoneManagerFlags
14
+ import inflect
11
15
  from typer.core import TyperGroup
12
16
 
17
+ import FileBackup
18
+
13
19
 
14
20
  # ----------------------------------------------------------------------
15
21
  class NaturalOrderGrouper(TyperGroup):
@@ -46,12 +52,49 @@ def Build(
46
52
  with DoneManager.CreateCommandLine(
47
53
  flags=DoneManagerFlags.Create(verbose=verbose, debug=debug),
48
54
  ) as dm:
49
- command_line = "cxfreeze --script src/FileBackup/CommandLine/EntryPoint.py --target-name=FileBackup"
50
-
51
- dm.WriteVerbose(f"Command line: {command_line}\n\n")
52
-
53
- with dm.YieldStream() as stream:
54
- dm.result = SubprocessEx.Stream(command_line, stream)
55
+ # inflect uses the typeguard library, which relies on inspect to read the source code of
56
+ # inflect to enforce that arguments are the correct types when invoking functions. This
57
+ # works fine when running the application normally, but does not work by default with
58
+ # cx_Freeze because it does not include the source code of inflect in the built binary. This
59
+ # means that typeguard can't determine the types of the arguments, which causes it to raise
60
+ # an exception. The fix is to include the inflect source code in the built binary so that
61
+ # typeguard can find the function's types and not raise an exception.
62
+ inflect_filename = Path(inflect.__file__).relative_to(Path(__file__).parent)
63
+
64
+ configuration_filename = Path("setup{}.py".format(str(uuid.uuid4()).replace("-", "")))
65
+
66
+ configuration_filename.write_text(
67
+ textwrap.dedent(
68
+ f"""\
69
+ from cx_Freeze import setup, Executable
70
+
71
+ setup(
72
+ name = "FileBackup",
73
+ version = "{FileBackup.__version__}",
74
+ options = {{
75
+ "build_exe": {{
76
+ "include_files": [
77
+ (r"{inflect_filename}", "lib/inflect/{inflect_filename.name}"),
78
+ ],
79
+ "packages": []
80
+ }},
81
+ }},
82
+ executables = [
83
+ Executable(
84
+ "src/FileBackup/CommandLine/EntryPoint.py",
85
+ target_name="FileBackup",
86
+ base=None,
87
+ ),
88
+ ],
89
+ )
90
+ """,
91
+ ),
92
+ encoding="utf-8",
93
+ )
94
+
95
+ with ExitStack(configuration_filename.unlink):
96
+ with dm.YieldStream() as stream:
97
+ dm.result = SubprocessEx.Stream(f'python "{configuration_filename}" build_exe', stream)
55
98
 
56
99
 
57
100
  # ----------------------------------------------------------------------
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FileBackup
3
- Version: 0.4.4
3
+ Version: 0.5.1
4
4
  Summary: Tool for creating and restoring file system backups.
5
5
  Project-URL: Homepage, https://github.com/davidbrownell/FileBackup
6
6
  Project-URL: Documentation, https://github.com/davidbrownell/FileBackup
@@ -28,7 +28,7 @@ Classifier: Topic :: System :: Archiving :: Mirroring
28
28
  Classifier: Topic :: System :: Systems Administration
29
29
  Classifier: Topic :: Utilities
30
30
  Requires-Python: >=3.10
31
- Requires-Dist: dbrownell-common>=0.14.6
31
+ Requires-Dist: dbrownell-common>=0.15.0
32
32
  Requires-Dist: paramiko>=3.5.1
33
33
  Requires-Dist: typer>=0.15.3
34
34
  Description-Content-Type: text/markdown
@@ -7,7 +7,7 @@ authors = [
7
7
  ]
8
8
  requires-python = ">= 3.10"
9
9
  dependencies = [
10
- "dbrownell-common>=0.14.6",
10
+ "dbrownell-common>=0.15.0",
11
11
  "paramiko>=3.5.1",
12
12
  "typer>=0.15.3",
13
13
  ]
@@ -23,7 +23,7 @@ keywords = [
23
23
  license = "MIT"
24
24
 
25
25
  classifiers = [
26
- "Development Status :: 4 - Beta", # TODO
26
+ "Development Status :: 4 - Beta",
27
27
  "Environment :: Console",
28
28
  "Intended Audience :: Developers",
29
29
  "Intended Audience :: End Users/Desktop",
@@ -241,6 +241,12 @@ def Restore( # pylint: disable=dangerous-default-value
241
241
  help="Working directory to use when decompressing archives; provide this value during a dry run and subsequent execution to only download and extract the backup content once.",
242
242
  ),
243
243
  ] = None,
244
+ continue_on_errors: Annotated[
245
+ bool,
246
+ typer.Option(
247
+ "--continue-on-errors", help="Continue restoring files even if some files cannot be restored."
248
+ ),
249
+ ] = False,
244
250
  ) -> None:
245
251
  """Restores content from an offsite data store."""
246
252
 
@@ -263,6 +269,7 @@ def Restore( # pylint: disable=dangerous-default-value
263
269
  quiet=quiet,
264
270
  dry_run=dry_run,
265
271
  overwrite=overwrite,
272
+ continue_on_errors=continue_on_errors,
266
273
  )
267
274
 
268
275
 
@@ -30,16 +30,17 @@ import uuid
30
30
  from contextlib import contextmanager
31
31
  from dataclasses import dataclass
32
32
  from enum import auto, Enum
33
+ from io import StringIO
33
34
  from pathlib import Path
34
35
  from typing import Any, Callable, cast, Iterator, Pattern
35
36
 
36
- from dbrownell_Common.ContextlibEx import ExitStack # type: ignore[import-untyped]
37
- from dbrownell_Common import ExecuteTasks # type: ignore[import-untyped]
38
- from dbrownell_Common.InflectEx import inflect # type: ignore[import-untyped]
39
- from dbrownell_Common import PathEx # type: ignore[import-untyped]
40
- from dbrownell_Common.Streams.DoneManager import DoneManager # type: ignore[import-untyped]
41
- from dbrownell_Common import SubprocessEx # type: ignore[import-untyped]
42
- from dbrownell_Common import TextwrapEx # type: ignore[import-untyped]
37
+ from dbrownell_Common.ContextlibEx import ExitStack
38
+ from dbrownell_Common import ExecuteTasks
39
+ from dbrownell_Common.InflectEx import inflect
40
+ from dbrownell_Common import PathEx
41
+ from dbrownell_Common.Streams.DoneManager import DoneManager, Flags as DoneManagerFlags
42
+ from dbrownell_Common import SubprocessEx
43
+ from dbrownell_Common import TextwrapEx
43
44
 
44
45
  from FileBackup.DataStore.FileSystemDataStore import FileSystemDataStore
45
46
  from FileBackup.DataStore.Interfaces.BulkStorageDataStore import BulkStorageDataStore
@@ -530,6 +531,7 @@ def Restore(
530
531
  quiet: bool,
531
532
  dry_run: bool,
532
533
  overwrite: bool,
534
+ continue_on_errors: bool = False,
533
535
  ) -> None:
534
536
  with Common.YieldDataStore(
535
537
  dm,
@@ -708,7 +710,7 @@ def Restore(
708
710
  # ----------------------------------------------------------------------
709
711
  def ExecuteTask(
710
712
  status: ExecuteTasks.Status,
711
- ) -> Path:
713
+ ) -> ExecuteTasks.TransformResultComplete:
712
714
  # This function will create the following directory structure:
713
715
  #
714
716
  # <working_dir>
@@ -717,93 +719,109 @@ def Restore(
717
719
  # └── decompressed (temporary)
718
720
  # └── final
719
721
 
720
- assert working_dir is not None
721
- this_working_dir = working_dir / directory
722
- final_dir = this_working_dir / "final"
723
-
724
- if final_dir.is_dir():
725
- # The destination already exists, no need to process it further
726
- return final_dir
727
-
728
- with _YieldTransferredArchive(
729
- data_store, # type: ignore
730
- directory,
731
- this_working_dir / "transferred",
732
- lambda bytes_transferred: cast(
733
- None,
734
- status.OnProgress(
735
- ProcessDirectoryState.Transferring.value + 1,
736
- bytes_transferred,
737
- ),
738
- ),
739
- ) as (transferred_dir, transferred_dir_is_temporary):
740
- with _YieldDecompressedFiles(
741
- transferred_dir,
742
- this_working_dir / "decompressed",
722
+ sink = StringIO()
723
+
724
+ with DoneManager.Create(
725
+ sink,
726
+ "",
727
+ line_prefix="",
728
+ flags=DoneManagerFlags.Create(verbose=dm.is_verbose, debug=dm.is_debug),
729
+ ) as dm_sink:
730
+ assert working_dir is not None
731
+ this_working_dir = working_dir / directory
732
+ final_dir = this_working_dir / "final"
733
+
734
+ if final_dir.is_dir():
735
+ # The destination already exists, no need to process it further
736
+ return ExecuteTasks.TransformResultComplete(final_dir, 0)
737
+
738
+ with _YieldTransferredArchive(
739
+ data_store, # type: ignore
743
740
  directory,
744
- encryption_password,
745
- lambda message: cast(
741
+ this_working_dir / "transferred",
742
+ lambda bytes_transferred: cast(
746
743
  None,
747
744
  status.OnProgress(
748
- ProcessDirectoryState.Extracting.value + 1,
749
- message,
745
+ ProcessDirectoryState.Transferring.value + 1,
746
+ bytes_transferred,
750
747
  ),
751
748
  ),
752
- ) as (decompressed_dir, decompressed_dir_is_temporary):
753
- # Validate the contents
754
- _VerifyFiles(
749
+ ) as (transferred_dir, transferred_dir_is_temporary):
750
+ with _YieldDecompressedFiles(
751
+ dm_sink,
752
+ transferred_dir,
753
+ this_working_dir / "decompressed",
755
754
  directory,
756
- decompressed_dir,
755
+ encryption_password,
757
756
  lambda message: cast(
758
757
  None,
759
758
  status.OnProgress(
760
- ProcessDirectoryState.Verifying.value + 1,
759
+ ProcessDirectoryState.Extracting.value + 1,
761
760
  message,
762
761
  ),
763
762
  ),
764
- )
763
+ continue_on_errors=continue_on_errors,
764
+ ) as (decompressed_dir, decompressed_dir_is_temporary):
765
+ # Validate the contents
766
+ _VerifyFiles(
767
+ dm_sink,
768
+ directory,
769
+ decompressed_dir,
770
+ lambda message: cast(
771
+ None,
772
+ status.OnProgress(
773
+ ProcessDirectoryState.Verifying.value + 1,
774
+ message,
775
+ ),
776
+ ),
777
+ continue_on_errors=continue_on_errors,
778
+ )
765
779
 
766
- # Move/Copy the content. Note that the code assumes a flat
767
- # directory structure and doesn't do anything to account for
768
- # nested dirs. This assumption matches the current archive
769
- # format.
770
- if transferred_dir_is_temporary or decompressed_dir_is_temporary:
771
- func = cast(Callable[[Path, Path], None], shutil.move)
772
- else:
773
- # ----------------------------------------------------------------------
774
- def CreateSymLink(
775
- source: Path,
776
- dest: Path,
777
- ) -> None:
778
- dest /= source.name
779
- os.symlink(source, dest, target_is_directory=source.is_dir())
780
+ # Move/Copy the content. Note that the code assumes a flat
781
+ # directory structure and doesn't do anything to account for
782
+ # nested dirs. This assumption matches the current archive
783
+ # format.
784
+ if transferred_dir_is_temporary or decompressed_dir_is_temporary:
785
+ func = cast(Callable[[Path, Path], None], shutil.move)
786
+ else:
787
+ # ----------------------------------------------------------------------
788
+ def CreateSymLink(
789
+ source: Path,
790
+ dest: Path,
791
+ ) -> None:
792
+ dest /= source.name
793
+ os.symlink(source, dest, target_is_directory=source.is_dir())
780
794
 
781
- # ----------------------------------------------------------------------
795
+ # ----------------------------------------------------------------------
782
796
 
783
- func = CreateSymLink
797
+ func = CreateSymLink
784
798
 
785
- temp_dest_dir = final_dir.parent / (final_dir.name + "__temp__")
799
+ temp_dest_dir = final_dir.parent / (final_dir.name + "__temp__")
786
800
 
787
- shutil.rmtree(temp_dest_dir, ignore_errors=True)
788
- temp_dest_dir.mkdir(parents=True)
801
+ shutil.rmtree(temp_dest_dir, ignore_errors=True)
802
+ temp_dest_dir.mkdir(parents=True)
789
803
 
790
- items = [
791
- item
792
- for item in decompressed_dir.iterdir()
793
- if item.name != INDEX_HASH_FILENAME
794
- ]
804
+ items = [
805
+ item
806
+ for item in decompressed_dir.iterdir()
807
+ if item.name != INDEX_HASH_FILENAME
808
+ ]
795
809
 
796
- for item_index, item in enumerate(items):
797
- status.OnProgress(
798
- ProcessDirectoryState.Moving.value + 1,
799
- f"Moving {item_index + 1} of {len(items)}...",
800
- )
810
+ for item_index, item in enumerate(items):
811
+ status.OnProgress(
812
+ ProcessDirectoryState.Moving.value + 1,
813
+ f"Moving {item_index + 1} of {len(items)}...",
814
+ )
801
815
 
802
- func(item, temp_dest_dir)
816
+ if item.exists():
817
+ func(item, temp_dest_dir)
803
818
 
804
- shutil.move(temp_dest_dir, final_dir)
819
+ shutil.move(temp_dest_dir, final_dir)
805
820
 
806
- return final_dir
821
+ if dm_sink.result != 0:
822
+ status.Log(sink.getvalue())
823
+
824
+ return ExecuteTasks.TransformResultComplete(final_dir, dm_sink.result)
807
825
 
808
826
  # ----------------------------------------------------------------------
809
827
 
@@ -811,7 +829,7 @@ def Restore(
811
829
 
812
830
  # ----------------------------------------------------------------------
813
831
 
814
- directory_working_dirs: list[Path | None | Exception] = ExecuteTasks.TransformTasksEx(
832
+ directory_working_dirs: list[object | None | Exception] = ExecuteTasks.TransformTasksEx(
815
833
  preprocess_dm,
816
834
  "Processing",
817
835
  [ExecuteTasks.TaskData(str(directory), directory) for directory in directories],
@@ -821,7 +839,8 @@ def Restore(
821
839
  refresh_per_second=Common.EXECUTE_TASKS_REFRESH_PER_SECOND,
822
840
  )
823
841
 
824
- if preprocess_dm.result != 0:
842
+ # Allow the process to continue if warnings were encountered
843
+ if preprocess_dm.result < 0:
825
844
  return
826
845
 
827
846
  assert all(isinstance(working_dir, Path) for working_dir in directory_working_dirs), (
@@ -980,6 +999,8 @@ def Restore(
980
999
 
981
1000
  # ----------------------------------------------------------------------
982
1001
  def WriteImpl(
1002
+ dm: DoneManager,
1003
+ action: str,
983
1004
  local_filename: Path,
984
1005
  content_filename: Path | None,
985
1006
  ) -> None:
@@ -998,25 +1019,29 @@ def Restore(
998
1019
  commit_actions.append(CommitDir)
999
1020
  return
1000
1021
 
1022
+ content_filename = content_filename.resolve()
1001
1023
  temp_filename = temp_directory / str(uuid.uuid4())
1002
1024
 
1003
- with content_filename.resolve().open("rb") as source:
1004
- with temp_filename.open("wb") as dest:
1005
- dest.write(source.read())
1025
+ if not content_filename.is_file():
1026
+ dm.WriteError(f"The file could not be {action} as its archive data is missing.\n")
1027
+ else:
1028
+ with content_filename.open("rb") as source:
1029
+ with temp_filename.open("wb") as dest:
1030
+ dest.write(source.read())
1006
1031
 
1007
- # ----------------------------------------------------------------------
1008
- def CommitFile() -> None:
1009
- if local_filename.is_dir():
1010
- shutil.rmtree(local_filename)
1011
- elif local_filename.is_file():
1012
- local_filename.unlink()
1032
+ # ----------------------------------------------------------------------
1033
+ def CommitFile() -> None:
1034
+ if local_filename.is_dir():
1035
+ shutil.rmtree(local_filename)
1036
+ elif local_filename.is_file():
1037
+ local_filename.unlink()
1013
1038
 
1014
- local_filename.parent.mkdir(parents=True, exist_ok=True)
1015
- shutil.move(temp_filename, local_filename)
1039
+ local_filename.parent.mkdir(parents=True, exist_ok=True)
1040
+ shutil.move(temp_filename, local_filename)
1016
1041
 
1017
- # ----------------------------------------------------------------------
1042
+ # ----------------------------------------------------------------------
1018
1043
 
1019
- commit_actions.append(CommitFile)
1044
+ commit_actions.append(CommitFile)
1020
1045
 
1021
1046
  # ----------------------------------------------------------------------
1022
1047
  def OnAddInstruction(
@@ -1029,7 +1054,7 @@ def Restore(
1029
1054
  )
1030
1055
  return
1031
1056
 
1032
- WriteImpl(instruction.local_filename, instruction.file_content_path)
1057
+ WriteImpl(dm, "restored", instruction.local_filename, instruction.file_content_path)
1033
1058
 
1034
1059
  # ----------------------------------------------------------------------
1035
1060
  def OnModifyInstruction(
@@ -1037,7 +1062,7 @@ def Restore(
1037
1062
  instruction: Instruction,
1038
1063
  ) -> None:
1039
1064
  assert instruction.file_content_path is not None
1040
- WriteImpl(instruction.local_filename, instruction.file_content_path)
1065
+ WriteImpl(dm, "modified", instruction.local_filename, instruction.file_content_path)
1041
1066
 
1042
1067
  # ----------------------------------------------------------------------
1043
1068
  def OnRemoveInstruction(
@@ -1114,7 +1139,7 @@ def Restore(
1114
1139
  ) as execute_dm:
1115
1140
  on_instruction_func(execute_dm, instruction)
1116
1141
 
1117
- if execute_dm.result != 0:
1142
+ if execute_dm.result != 0 and not continue_on_errors:
1118
1143
  break
1119
1144
 
1120
1145
  instructions_dm.WriteLine("")
@@ -1382,11 +1407,14 @@ def _YieldTransferredArchive(
1382
1407
  # ----------------------------------------------------------------------
1383
1408
  @contextmanager
1384
1409
  def _YieldDecompressedFiles(
1410
+ dm: DoneManager,
1385
1411
  transferred_dir: Path,
1386
1412
  decompressed_dir: Path,
1387
1413
  directory_name: str,
1388
1414
  encryption_password: str | None,
1389
1415
  status_func: Callable[[str], None],
1416
+ *,
1417
+ continue_on_errors: bool,
1390
1418
  ) -> Iterator[
1391
1419
  tuple[
1392
1420
  Path,
@@ -1411,18 +1439,17 @@ def _YieldDecompressedFiles(
1411
1439
  #
1412
1440
  password = encryption_password or str(uuid.uuid4())
1413
1441
 
1414
- # Validate
1415
- status_func("Validating archive...")
1416
-
1417
1442
  archive_filename = transferred_dir / (ARCHIVE_FILENAME + ".001")
1418
1443
 
1419
1444
  if not archive_filename.is_file():
1420
1445
  raise Exception(f"The archive file '{archive_filename.name}' was not found.")
1421
1446
 
1422
- result = SubprocessEx.Run(f'{_GetZipBinary()} t "{archive_filename}" "-p{password}"')
1423
- if result.returncode != 0:
1424
- raise Exception(
1425
- textwrap.dedent(
1447
+ # Validate
1448
+ status_func("Validating archive...")
1449
+ with dm.Nested("Validating archive...") as validate_dm:
1450
+ result = SubprocessEx.Run(f'{_GetZipBinary()} t "{archive_filename}" "-p{password}"')
1451
+ if result.returncode != 0:
1452
+ message = textwrap.dedent(
1426
1453
  """\
1427
1454
  Archive validation failed for the directory '{}' ({}).
1428
1455
 
@@ -1438,29 +1465,32 @@ def _YieldDecompressedFiles(
1438
1465
  4,
1439
1466
  skip_first_line=True,
1440
1467
  ),
1441
- ),
1442
- )
1468
+ )
1443
1469
 
1444
- # Extract
1445
- status_func("Extracting archive...")
1470
+ if continue_on_errors:
1471
+ validate_dm.WriteWarning(message)
1472
+ else:
1473
+ raise Exception(message)
1446
1474
 
1475
+ # Extract
1447
1476
  with _YieldTempDirectory(decompressed_dir, "extracting the archive") as decompressed_dir:
1448
- decompressed_dir.mkdir(parents=True, exist_ok=True)
1477
+ status_func("Extracting archive...")
1478
+ with dm.Nested("Extracting archive...") as extract_dm:
1479
+ decompressed_dir.mkdir(parents=True, exist_ok=True)
1449
1480
 
1450
- result = SubprocessEx.Run(
1451
- f'{_GetZipBinary()} x "{archive_filename}" "-p{password}"',
1452
- cwd=decompressed_dir,
1453
- )
1481
+ result = SubprocessEx.Run(
1482
+ f'{_GetZipBinary()} x "{archive_filename}" "-p{password}"',
1483
+ cwd=decompressed_dir,
1484
+ )
1454
1485
 
1455
- if result.returncode != 0:
1456
- raise Exception(
1457
- textwrap.dedent(
1486
+ if result.returncode != 0:
1487
+ message = textwrap.dedent(
1458
1488
  """\
1459
- Archive extraction failed for the directory '{}' ({}).
1489
+ Archive extraction failed for the directory '{}' ({}).
1460
1490
 
1461
- {}
1491
+ {}
1462
1492
 
1463
- """,
1493
+ """,
1464
1494
  ).format(
1465
1495
  directory_name,
1466
1496
  result.returncode,
@@ -1469,67 +1499,74 @@ def _YieldDecompressedFiles(
1469
1499
  4,
1470
1500
  skip_first_line=True,
1471
1501
  ),
1472
- ),
1473
- )
1502
+ )
1503
+
1504
+ if continue_on_errors:
1505
+ extract_dm.WriteWarning(message)
1506
+ else:
1507
+ raise Exception(message)
1474
1508
 
1475
1509
  yield decompressed_dir, True
1476
1510
 
1477
1511
 
1478
1512
  # ----------------------------------------------------------------------
1479
1513
  def _VerifyFiles(
1514
+ dm: DoneManager,
1480
1515
  directory_name: str,
1481
1516
  contents_dir: Path,
1482
1517
  status_func: Callable[[str], None],
1518
+ *,
1519
+ continue_on_errors: bool,
1483
1520
  ) -> None:
1484
1521
  # Ensure that the index is present
1485
1522
  for index_filename in [INDEX_FILENAME, INDEX_HASH_FILENAME]:
1486
1523
  if not (contents_dir / index_filename).is_file():
1487
1524
  raise Exception(f"The index file '{index_filename}' does not exist.")
1488
1525
 
1489
- # Ensure that the content is valid
1490
- all_filenames: list[Path] = []
1526
+ with dm.Nested("Verifying files...") as verify_dm:
1527
+ # Ensure that the content is valid
1528
+ all_filenames: list[Path] = []
1491
1529
 
1492
- for root_str, _, filenames in os.walk(contents_dir):
1493
- root = Path(root_str)
1530
+ for root_str, _, filenames in os.walk(contents_dir):
1531
+ root = Path(root_str)
1494
1532
 
1495
- all_filenames += [root / filename for filename in filenames if filename != INDEX_HASH_FILENAME]
1533
+ all_filenames += [root / filename for filename in filenames if filename != INDEX_HASH_FILENAME]
1496
1534
 
1497
- data_store = FileSystemDataStore()
1535
+ data_store = FileSystemDataStore()
1498
1536
 
1499
- errors: list[str] = []
1537
+ errors: list[str] = []
1500
1538
 
1501
- for filename_index, filename in enumerate(all_filenames):
1502
- if filename.name == INDEX_FILENAME:
1503
- expected_hash_value = (contents_dir / INDEX_HASH_FILENAME).read_text().strip()
1504
- else:
1505
- expected_hash_value = filename.name
1539
+ for filename_index, filename in enumerate(all_filenames):
1540
+ if filename.name == INDEX_FILENAME:
1541
+ expected_hash_value = (contents_dir / INDEX_HASH_FILENAME).read_text().strip()
1542
+ else:
1543
+ expected_hash_value = filename.name
1506
1544
 
1507
- file_size = filename.stat().st_size or 1
1545
+ file_size = filename.stat().st_size or 1
1508
1546
 
1509
- status_template = f"Validating file {filename_index + 1} of {len(all_filenames)} [{PathEx.GetSizeDisplay(file_size)}] {{:.02f}}%..."
1547
+ status_template = f"Validating file {filename_index + 1} of {len(all_filenames)} [{PathEx.GetSizeDisplay(file_size)}] {{:.02f}}%..."
1510
1548
 
1511
- actual_hash_value = Common.CalculateHash(
1512
- data_store,
1513
- filename,
1514
- lambda bytes_transferred: status_func(
1515
- status_template.format((bytes_transferred / file_size) * 100)
1516
- ),
1517
- )
1518
-
1519
- if actual_hash_value != expected_hash_value:
1520
- errors.append(
1521
- textwrap.dedent(
1522
- f"""\
1523
- Filename: {filename.relative_to(contents_dir)}
1524
- Expected: {expected_hash_value}
1525
- Actual: {actual_hash_value}
1526
- """,
1549
+ actual_hash_value = Common.CalculateHash(
1550
+ data_store,
1551
+ filename,
1552
+ lambda bytes_transferred: status_func(
1553
+ status_template.format((bytes_transferred / file_size) * 100)
1527
1554
  ),
1528
1555
  )
1529
1556
 
1530
- if errors:
1531
- raise Exception(
1532
- textwrap.dedent(
1557
+ if actual_hash_value != expected_hash_value:
1558
+ errors.append(
1559
+ textwrap.dedent(
1560
+ f"""\
1561
+ Filename: {filename.relative_to(contents_dir)}
1562
+ Expected: {expected_hash_value}
1563
+ Actual: {actual_hash_value}
1564
+ """,
1565
+ ),
1566
+ )
1567
+
1568
+ if errors:
1569
+ message = textwrap.dedent(
1533
1570
  """\
1534
1571
  Corrupt files were encountered in the directory '{}'.
1535
1572
 
@@ -1543,5 +1580,9 @@ def _VerifyFiles(
1543
1580
  4,
1544
1581
  skip_first_line=True,
1545
1582
  ),
1546
- ),
1547
- )
1583
+ )
1584
+
1585
+ if continue_on_errors:
1586
+ verify_dm.WriteWarning(message)
1587
+ else:
1588
+ raise Exception(message)
@@ -9,4 +9,4 @@
9
9
  # Note that this value will be overwritten by calls to `python ../../Build.py update_version` based
10
10
  # on changes observed in the git repository. The default value below will be used until the value
11
11
  # here is explicitly updated by the Continuous Integration system as part of a commit.
12
- __version__ = "0.4.4"
12
+ __version__ = "0.5.1"
@@ -447,6 +447,7 @@ class TestOffsite:
447
447
  kwargs = restore.call_args_list[0].kwargs
448
448
 
449
449
  assert kwargs == {
450
+ "continue_on_errors": False,
450
451
  "ssd": False,
451
452
  "quiet": False,
452
453
  "dry_run": False,
@@ -483,6 +484,7 @@ class TestOffsite:
483
484
  "--overwrite",
484
485
  "--ssd",
485
486
  "--quiet",
487
+ "--continue-on-errors",
486
488
  ]
487
489
 
488
490
  result = CliRunner().invoke(app, args)
@@ -501,6 +503,7 @@ class TestOffsite:
501
503
  kwargs = restore.call_args_list[0].kwargs
502
504
 
503
505
  assert kwargs == {
506
+ "continue_on_errors": True,
504
507
  "ssd": True,
505
508
  "quiet": True,
506
509
  "dry_run": True,
@@ -387,7 +387,7 @@ wheels = [
387
387
 
388
388
  [[package]]
389
389
  name = "dbrownell-common"
390
- version = "0.14.6"
390
+ version = "0.15.0"
391
391
  source = { registry = "https://pypi.org/simple" }
392
392
  dependencies = [
393
393
  { name = "inflect" },
@@ -395,8 +395,9 @@ dependencies = [
395
395
  { name = "typer" },
396
396
  { name = "typer-config" },
397
397
  ]
398
+ sdist = { url = "https://files.pythonhosted.org/packages/75/83/785a3d7e8622f1cb805625f9cca9024c1d6985afc6626efa0946b00d2f27/dbrownell_common-0.15.0.tar.gz", hash = "sha256:6a7e2caa89e2eadef473901f35f5eab2963deb74b335c9ed8f3e05c2ba9f184f", size = 97508, upload-time = "2025-07-03T17:27:55.32Z" }
398
399
  wheels = [
399
- { url = "https://files.pythonhosted.org/packages/a5/8c/f3d2c3a6712aaf54be2f9ed37fed5515d302b9cbce0ecd589f6253efab31/dbrownell_common-0.14.6-py3-none-any.whl", hash = "sha256:6799b088c4007ab1b4cec169138aab220d86da198e383dc1a6e5c9119f3f2b35", size = 41485, upload-time = "2025-04-03T21:13:34.193Z" },
400
+ { url = "https://files.pythonhosted.org/packages/37/dc/b1663f95bf3582d0fd07498700f4708e71ef9c5e3bc7441d90353884908b/dbrownell_common-0.15.0-py3-none-any.whl", hash = "sha256:554238d000dfc8acc78fa6d9df778706bb4f84f5b70f35f75cf0ccdd8f0f09de", size = 43206, upload-time = "2025-07-03T17:27:54.115Z" },
400
401
  ]
401
402
 
402
403
  [[package]]
@@ -464,7 +465,7 @@ dev = [
464
465
 
465
466
  [package.metadata]
466
467
  requires-dist = [
467
- { name = "dbrownell-common", specifier = ">=0.14.6" },
468
+ { name = "dbrownell-common", specifier = ">=0.15.0" },
468
469
  { name = "paramiko", specifier = ">=3.5.1" },
469
470
  { name = "typer", specifier = ">=0.15.3" },
470
471
  ]
@@ -524,11 +525,15 @@ wheels = [
524
525
 
525
526
  [[package]]
526
527
  name = "inflect"
527
- version = "4.1.1"
528
+ version = "7.5.0"
528
529
  source = { registry = "https://pypi.org/simple" }
529
- sdist = { url = "https://files.pythonhosted.org/packages/c6/9f/163fbcda011f284d13261f11aa1e14daf28f68729e475c79f7d223c01a36/inflect-4.1.1.tar.gz", hash = "sha256:0527947406025991506b252620da69a5efdc72bf61aa6b4b4976e3bd71c19660", size = 71278, upload-time = "2020-11-15T01:14:15.736Z" }
530
+ dependencies = [
531
+ { name = "more-itertools" },
532
+ { name = "typeguard" },
533
+ ]
534
+ sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" }
530
535
  wheels = [
531
- { url = "https://files.pythonhosted.org/packages/4e/9d/81e86c99531ec3dda860f04656e48304cb9dc7d2b9e0ac3e7a806ae5747f/inflect-4.1.1-py3-none-any.whl", hash = "sha256:b0d13bbb6e12313b8fc4a171c86c4f4544cf09a8498e118d25473558200bd171", size = 31507, upload-time = "2020-11-15T01:14:14.173Z" },
536
+ { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197, upload-time = "2024-12-28T17:11:15.931Z" },
532
537
  ]
533
538
 
534
539
  [[package]]
@@ -615,6 +620,15 @@ wheels = [
615
620
  { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
616
621
  ]
617
622
 
623
+ [[package]]
624
+ name = "more-itertools"
625
+ version = "10.7.0"
626
+ source = { registry = "https://pypi.org/simple" }
627
+ sdist = { url = "https://files.pythonhosted.org/packages/ce/a0/834b0cebabbfc7e311f30b46c8188790a37f89fc8d756660346fe5abfd09/more_itertools-10.7.0.tar.gz", hash = "sha256:9fddd5403be01a94b204faadcff459ec3568cf110265d3c54323e1e866ad29d3", size = 127671, upload-time = "2025-04-22T14:17:41.838Z" }
628
+ wheels = [
629
+ { url = "https://files.pythonhosted.org/packages/2b/9f/7ba6f94fc1e9ac3d2b853fdff3035fb2fa5afbed898c4a72b8a020610594/more_itertools-10.7.0-py3-none-any.whl", hash = "sha256:d43980384673cb07d2f7d2d918c616b30c659c089ee23953f601d6609c67510e", size = 65278, upload-time = "2025-04-22T14:17:40.49Z" },
630
+ ]
631
+
618
632
  [[package]]
619
633
  name = "nodeenv"
620
634
  version = "1.9.1"
@@ -823,16 +837,16 @@ wheels = [
823
837
 
824
838
  [[package]]
825
839
  name = "rich"
826
- version = "13.9.4"
840
+ version = "14.0.0"
827
841
  source = { registry = "https://pypi.org/simple" }
828
842
  dependencies = [
829
843
  { name = "markdown-it-py" },
830
844
  { name = "pygments" },
831
845
  { name = "typing-extensions", marker = "python_full_version < '3.11'" },
832
846
  ]
833
- sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149, upload-time = "2024-11-01T16:43:57.873Z" }
847
+ sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078, upload-time = "2025-03-30T14:15:14.23Z" }
834
848
  wheels = [
835
- { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424, upload-time = "2024-11-01T16:43:55.817Z" },
849
+ { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" },
836
850
  ]
837
851
 
838
852
  [[package]]
@@ -1052,6 +1066,18 @@ wheels = [
1052
1066
  { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" },
1053
1067
  ]
1054
1068
 
1069
+ [[package]]
1070
+ name = "typeguard"
1071
+ version = "4.4.2"
1072
+ source = { registry = "https://pypi.org/simple" }
1073
+ dependencies = [
1074
+ { name = "typing-extensions" },
1075
+ ]
1076
+ sdist = { url = "https://files.pythonhosted.org/packages/70/60/8cd6a3d78d00ceeb2193c02b7ed08f063d5341ccdfb24df88e61f383048e/typeguard-4.4.2.tar.gz", hash = "sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49", size = 75746, upload-time = "2025-02-16T16:28:26.205Z" }
1077
+ wheels = [
1078
+ { url = "https://files.pythonhosted.org/packages/cf/4b/9a77dc721aa0b7f74440a42e4ef6f9a4fae7324e17f64f88b96f4c25cc05/typeguard-4.4.2-py3-none-any.whl", hash = "sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c", size = 35801, upload-time = "2025-02-16T16:28:24.793Z" },
1079
+ ]
1080
+
1055
1081
  [[package]]
1056
1082
  name = "typer"
1057
1083
  version = "0.15.3"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes