FileBackup 0.4.3__py3-none-any.whl → 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- FileBackup/CommandLine/OffsiteEntryPoint.py +7 -0
- FileBackup/Offsite.py +224 -161
- FileBackup/__init__.py +1 -1
- {filebackup-0.4.3.dist-info → filebackup-0.5.1.dist-info}/METADATA +2 -2
- {filebackup-0.4.3.dist-info → filebackup-0.5.1.dist-info}/RECORD +8 -8
- {filebackup-0.4.3.dist-info → filebackup-0.5.1.dist-info}/WHEEL +0 -0
- {filebackup-0.4.3.dist-info → filebackup-0.5.1.dist-info}/entry_points.txt +0 -0
- {filebackup-0.4.3.dist-info → filebackup-0.5.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -241,6 +241,12 @@ def Restore( # pylint: disable=dangerous-default-value
|
|
|
241
241
|
help="Working directory to use when decompressing archives; provide this value during a dry run and subsequent execution to only download and extract the backup content once.",
|
|
242
242
|
),
|
|
243
243
|
] = None,
|
|
244
|
+
continue_on_errors: Annotated[
|
|
245
|
+
bool,
|
|
246
|
+
typer.Option(
|
|
247
|
+
"--continue-on-errors", help="Continue restoring files even if some files cannot be restored."
|
|
248
|
+
),
|
|
249
|
+
] = False,
|
|
244
250
|
) -> None:
|
|
245
251
|
"""Restores content from an offsite data store."""
|
|
246
252
|
|
|
@@ -263,6 +269,7 @@ def Restore( # pylint: disable=dangerous-default-value
|
|
|
263
269
|
quiet=quiet,
|
|
264
270
|
dry_run=dry_run,
|
|
265
271
|
overwrite=overwrite,
|
|
272
|
+
continue_on_errors=continue_on_errors,
|
|
266
273
|
)
|
|
267
274
|
|
|
268
275
|
|
FileBackup/Offsite.py
CHANGED
|
@@ -30,16 +30,17 @@ import uuid
|
|
|
30
30
|
from contextlib import contextmanager
|
|
31
31
|
from dataclasses import dataclass
|
|
32
32
|
from enum import auto, Enum
|
|
33
|
+
from io import StringIO
|
|
33
34
|
from pathlib import Path
|
|
34
35
|
from typing import Any, Callable, cast, Iterator, Pattern
|
|
35
36
|
|
|
36
|
-
from dbrownell_Common.ContextlibEx import ExitStack
|
|
37
|
-
from dbrownell_Common import ExecuteTasks
|
|
38
|
-
from dbrownell_Common.InflectEx import inflect
|
|
39
|
-
from dbrownell_Common import PathEx
|
|
40
|
-
from dbrownell_Common.Streams.DoneManager import DoneManager
|
|
41
|
-
from dbrownell_Common import SubprocessEx
|
|
42
|
-
from dbrownell_Common import TextwrapEx
|
|
37
|
+
from dbrownell_Common.ContextlibEx import ExitStack
|
|
38
|
+
from dbrownell_Common import ExecuteTasks
|
|
39
|
+
from dbrownell_Common.InflectEx import inflect
|
|
40
|
+
from dbrownell_Common import PathEx
|
|
41
|
+
from dbrownell_Common.Streams.DoneManager import DoneManager, Flags as DoneManagerFlags
|
|
42
|
+
from dbrownell_Common import SubprocessEx
|
|
43
|
+
from dbrownell_Common import TextwrapEx
|
|
43
44
|
|
|
44
45
|
from FileBackup.DataStore.FileSystemDataStore import FileSystemDataStore
|
|
45
46
|
from FileBackup.DataStore.Interfaces.BulkStorageDataStore import BulkStorageDataStore
|
|
@@ -530,6 +531,7 @@ def Restore(
|
|
|
530
531
|
quiet: bool,
|
|
531
532
|
dry_run: bool,
|
|
532
533
|
overwrite: bool,
|
|
534
|
+
continue_on_errors: bool = False,
|
|
533
535
|
) -> None:
|
|
534
536
|
with Common.YieldDataStore(
|
|
535
537
|
dm,
|
|
@@ -555,7 +557,7 @@ def Restore(
|
|
|
555
557
|
)
|
|
556
558
|
return
|
|
557
559
|
|
|
558
|
-
with _YieldTempDirectory("staging content") as staging_directory:
|
|
560
|
+
with _YieldTempDirectory(working_dir / "staging", "staging content") as staging_directory:
|
|
559
561
|
# ----------------------------------------------------------------------
|
|
560
562
|
@dataclass(frozen=True)
|
|
561
563
|
class Instruction:
|
|
@@ -708,90 +710,118 @@ def Restore(
|
|
|
708
710
|
# ----------------------------------------------------------------------
|
|
709
711
|
def ExecuteTask(
|
|
710
712
|
status: ExecuteTasks.Status,
|
|
711
|
-
) ->
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
713
|
+
) -> ExecuteTasks.TransformResultComplete:
|
|
714
|
+
# This function will create the following directory structure:
|
|
715
|
+
#
|
|
716
|
+
# <working_dir>
|
|
717
|
+
# └── <directory>
|
|
718
|
+
# └── transferred (temporary)
|
|
719
|
+
# └── decompressed (temporary)
|
|
720
|
+
# └── final
|
|
721
|
+
|
|
722
|
+
sink = StringIO()
|
|
723
|
+
|
|
724
|
+
with DoneManager.Create(
|
|
725
|
+
sink,
|
|
726
|
+
"",
|
|
727
|
+
line_prefix="",
|
|
728
|
+
flags=DoneManagerFlags.Create(verbose=dm.is_verbose, debug=dm.is_debug),
|
|
729
|
+
) as dm_sink:
|
|
730
|
+
assert working_dir is not None
|
|
731
|
+
this_working_dir = working_dir / directory
|
|
732
|
+
final_dir = this_working_dir / "final"
|
|
733
|
+
|
|
734
|
+
if final_dir.is_dir():
|
|
735
|
+
# The destination already exists, no need to process it further
|
|
736
|
+
return ExecuteTasks.TransformResultComplete(final_dir, 0)
|
|
737
|
+
|
|
738
|
+
with _YieldTransferredArchive(
|
|
739
|
+
data_store, # type: ignore
|
|
730
740
|
directory,
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
lambda message: cast(
|
|
741
|
+
this_working_dir / "transferred",
|
|
742
|
+
lambda bytes_transferred: cast(
|
|
734
743
|
None,
|
|
735
744
|
status.OnProgress(
|
|
736
|
-
ProcessDirectoryState.
|
|
737
|
-
|
|
745
|
+
ProcessDirectoryState.Transferring.value + 1,
|
|
746
|
+
bytes_transferred,
|
|
738
747
|
),
|
|
739
748
|
),
|
|
740
|
-
) as (
|
|
741
|
-
|
|
742
|
-
|
|
749
|
+
) as (transferred_dir, transferred_dir_is_temporary):
|
|
750
|
+
with _YieldDecompressedFiles(
|
|
751
|
+
dm_sink,
|
|
752
|
+
transferred_dir,
|
|
753
|
+
this_working_dir / "decompressed",
|
|
743
754
|
directory,
|
|
744
|
-
|
|
755
|
+
encryption_password,
|
|
745
756
|
lambda message: cast(
|
|
746
757
|
None,
|
|
747
758
|
status.OnProgress(
|
|
748
|
-
ProcessDirectoryState.
|
|
759
|
+
ProcessDirectoryState.Extracting.value + 1,
|
|
749
760
|
message,
|
|
750
761
|
),
|
|
751
762
|
),
|
|
752
|
-
|
|
763
|
+
continue_on_errors=continue_on_errors,
|
|
764
|
+
) as (decompressed_dir, decompressed_dir_is_temporary):
|
|
765
|
+
# Validate the contents
|
|
766
|
+
_VerifyFiles(
|
|
767
|
+
dm_sink,
|
|
768
|
+
directory,
|
|
769
|
+
decompressed_dir,
|
|
770
|
+
lambda message: cast(
|
|
771
|
+
None,
|
|
772
|
+
status.OnProgress(
|
|
773
|
+
ProcessDirectoryState.Verifying.value + 1,
|
|
774
|
+
message,
|
|
775
|
+
),
|
|
776
|
+
),
|
|
777
|
+
continue_on_errors=continue_on_errors,
|
|
778
|
+
)
|
|
753
779
|
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
780
|
+
# Move/Copy the content. Note that the code assumes a flat
|
|
781
|
+
# directory structure and doesn't do anything to account for
|
|
782
|
+
# nested dirs. This assumption matches the current archive
|
|
783
|
+
# format.
|
|
784
|
+
if transferred_dir_is_temporary or decompressed_dir_is_temporary:
|
|
785
|
+
func = cast(Callable[[Path, Path], None], shutil.move)
|
|
786
|
+
else:
|
|
787
|
+
# ----------------------------------------------------------------------
|
|
788
|
+
def CreateSymLink(
|
|
789
|
+
source: Path,
|
|
790
|
+
dest: Path,
|
|
791
|
+
) -> None:
|
|
792
|
+
dest /= source.name
|
|
793
|
+
os.symlink(source, dest, target_is_directory=source.is_dir())
|
|
768
794
|
|
|
769
|
-
|
|
795
|
+
# ----------------------------------------------------------------------
|
|
770
796
|
|
|
771
|
-
|
|
797
|
+
func = CreateSymLink
|
|
772
798
|
|
|
773
|
-
|
|
799
|
+
temp_dest_dir = final_dir.parent / (final_dir.name + "__temp__")
|
|
774
800
|
|
|
775
|
-
|
|
776
|
-
|
|
801
|
+
shutil.rmtree(temp_dest_dir, ignore_errors=True)
|
|
802
|
+
temp_dest_dir.mkdir(parents=True)
|
|
777
803
|
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
804
|
+
items = [
|
|
805
|
+
item
|
|
806
|
+
for item in decompressed_dir.iterdir()
|
|
807
|
+
if item.name != INDEX_HASH_FILENAME
|
|
808
|
+
]
|
|
783
809
|
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
810
|
+
for item_index, item in enumerate(items):
|
|
811
|
+
status.OnProgress(
|
|
812
|
+
ProcessDirectoryState.Moving.value + 1,
|
|
813
|
+
f"Moving {item_index + 1} of {len(items)}...",
|
|
814
|
+
)
|
|
815
|
+
|
|
816
|
+
if item.exists():
|
|
817
|
+
func(item, temp_dest_dir)
|
|
789
818
|
|
|
790
|
-
|
|
819
|
+
shutil.move(temp_dest_dir, final_dir)
|
|
791
820
|
|
|
792
|
-
|
|
821
|
+
if dm_sink.result != 0:
|
|
822
|
+
status.Log(sink.getvalue())
|
|
793
823
|
|
|
794
|
-
return
|
|
824
|
+
return ExecuteTasks.TransformResultComplete(final_dir, dm_sink.result)
|
|
795
825
|
|
|
796
826
|
# ----------------------------------------------------------------------
|
|
797
827
|
|
|
@@ -799,7 +829,7 @@ def Restore(
|
|
|
799
829
|
|
|
800
830
|
# ----------------------------------------------------------------------
|
|
801
831
|
|
|
802
|
-
directory_working_dirs: list[
|
|
832
|
+
directory_working_dirs: list[object | None | Exception] = ExecuteTasks.TransformTasksEx(
|
|
803
833
|
preprocess_dm,
|
|
804
834
|
"Processing",
|
|
805
835
|
[ExecuteTasks.TaskData(str(directory), directory) for directory in directories],
|
|
@@ -809,7 +839,8 @@ def Restore(
|
|
|
809
839
|
refresh_per_second=Common.EXECUTE_TASKS_REFRESH_PER_SECOND,
|
|
810
840
|
)
|
|
811
841
|
|
|
812
|
-
if
|
|
842
|
+
# Allow the process to continue if warnings were encountered
|
|
843
|
+
if preprocess_dm.result < 0:
|
|
813
844
|
return
|
|
814
845
|
|
|
815
846
|
assert all(isinstance(working_dir, Path) for working_dir in directory_working_dirs), (
|
|
@@ -960,13 +991,16 @@ def Restore(
|
|
|
960
991
|
with dm.Nested("\nProcessing instructions...") as all_instructions_dm:
|
|
961
992
|
all_instructions_dm.WriteLine("")
|
|
962
993
|
|
|
963
|
-
temp_directory =
|
|
994
|
+
temp_directory = working_dir / "instructions"
|
|
995
|
+
temp_directory.mkdir(parents=True, exist_ok=True)
|
|
964
996
|
|
|
965
997
|
with ExitStack(lambda: shutil.rmtree(temp_directory)):
|
|
966
998
|
commit_actions: list[Callable[[], None]] = []
|
|
967
999
|
|
|
968
1000
|
# ----------------------------------------------------------------------
|
|
969
1001
|
def WriteImpl(
|
|
1002
|
+
dm: DoneManager,
|
|
1003
|
+
action: str,
|
|
970
1004
|
local_filename: Path,
|
|
971
1005
|
content_filename: Path | None,
|
|
972
1006
|
) -> None:
|
|
@@ -985,25 +1019,29 @@ def Restore(
|
|
|
985
1019
|
commit_actions.append(CommitDir)
|
|
986
1020
|
return
|
|
987
1021
|
|
|
1022
|
+
content_filename = content_filename.resolve()
|
|
988
1023
|
temp_filename = temp_directory / str(uuid.uuid4())
|
|
989
1024
|
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1025
|
+
if not content_filename.is_file():
|
|
1026
|
+
dm.WriteError(f"The file could not be {action} as its archive data is missing.\n")
|
|
1027
|
+
else:
|
|
1028
|
+
with content_filename.open("rb") as source:
|
|
1029
|
+
with temp_filename.open("wb") as dest:
|
|
1030
|
+
dest.write(source.read())
|
|
993
1031
|
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1032
|
+
# ----------------------------------------------------------------------
|
|
1033
|
+
def CommitFile() -> None:
|
|
1034
|
+
if local_filename.is_dir():
|
|
1035
|
+
shutil.rmtree(local_filename)
|
|
1036
|
+
elif local_filename.is_file():
|
|
1037
|
+
local_filename.unlink()
|
|
1000
1038
|
|
|
1001
|
-
|
|
1002
|
-
|
|
1039
|
+
local_filename.parent.mkdir(parents=True, exist_ok=True)
|
|
1040
|
+
shutil.move(temp_filename, local_filename)
|
|
1003
1041
|
|
|
1004
|
-
|
|
1042
|
+
# ----------------------------------------------------------------------
|
|
1005
1043
|
|
|
1006
|
-
|
|
1044
|
+
commit_actions.append(CommitFile)
|
|
1007
1045
|
|
|
1008
1046
|
# ----------------------------------------------------------------------
|
|
1009
1047
|
def OnAddInstruction(
|
|
@@ -1016,7 +1054,7 @@ def Restore(
|
|
|
1016
1054
|
)
|
|
1017
1055
|
return
|
|
1018
1056
|
|
|
1019
|
-
WriteImpl(instruction.local_filename, instruction.file_content_path)
|
|
1057
|
+
WriteImpl(dm, "restored", instruction.local_filename, instruction.file_content_path)
|
|
1020
1058
|
|
|
1021
1059
|
# ----------------------------------------------------------------------
|
|
1022
1060
|
def OnModifyInstruction(
|
|
@@ -1024,7 +1062,7 @@ def Restore(
|
|
|
1024
1062
|
instruction: Instruction,
|
|
1025
1063
|
) -> None:
|
|
1026
1064
|
assert instruction.file_content_path is not None
|
|
1027
|
-
WriteImpl(instruction.local_filename, instruction.file_content_path)
|
|
1065
|
+
WriteImpl(dm, "modified", instruction.local_filename, instruction.file_content_path)
|
|
1028
1066
|
|
|
1029
1067
|
# ----------------------------------------------------------------------
|
|
1030
1068
|
def OnRemoveInstruction(
|
|
@@ -1101,7 +1139,7 @@ def Restore(
|
|
|
1101
1139
|
) as execute_dm:
|
|
1102
1140
|
on_instruction_func(execute_dm, instruction)
|
|
1103
1141
|
|
|
1104
|
-
if execute_dm.result != 0:
|
|
1142
|
+
if execute_dm.result != 0 and not continue_on_errors:
|
|
1105
1143
|
break
|
|
1106
1144
|
|
|
1107
1145
|
instructions_dm.WriteLine("")
|
|
@@ -1287,9 +1325,9 @@ def _CommitFileBasedDataStore(
|
|
|
1287
1325
|
# ----------------------------------------------------------------------
|
|
1288
1326
|
@contextmanager
|
|
1289
1327
|
def _YieldTempDirectory(
|
|
1328
|
+
temp_directory: Path,
|
|
1290
1329
|
desc: str,
|
|
1291
1330
|
) -> Iterator[Path]:
|
|
1292
|
-
temp_directory = PathEx.CreateTempDirectory()
|
|
1293
1331
|
should_delete = True
|
|
1294
1332
|
|
|
1295
1333
|
try:
|
|
@@ -1299,7 +1337,8 @@ def _YieldTempDirectory(
|
|
|
1299
1337
|
raise
|
|
1300
1338
|
finally:
|
|
1301
1339
|
if should_delete:
|
|
1302
|
-
|
|
1340
|
+
if temp_directory.is_dir():
|
|
1341
|
+
shutil.rmtree(temp_directory)
|
|
1303
1342
|
else:
|
|
1304
1343
|
sys.stderr.write(
|
|
1305
1344
|
f"**** The temporary directory '{temp_directory}' was preserved due to errors while {desc}.\n",
|
|
@@ -1308,9 +1347,10 @@ def _YieldTempDirectory(
|
|
|
1308
1347
|
|
|
1309
1348
|
# ----------------------------------------------------------------------
|
|
1310
1349
|
@contextmanager
|
|
1311
|
-
def
|
|
1350
|
+
def _YieldTransferredArchive(
|
|
1312
1351
|
data_store: FileBasedDataStore,
|
|
1313
1352
|
directory: str,
|
|
1353
|
+
working_dir: Path,
|
|
1314
1354
|
status_func: Callable[[str], None],
|
|
1315
1355
|
) -> Iterator[
|
|
1316
1356
|
tuple[
|
|
@@ -1318,6 +1358,8 @@ def _YieldRestoredArchive(
|
|
|
1318
1358
|
bool, # is temporary directory
|
|
1319
1359
|
],
|
|
1320
1360
|
]:
|
|
1361
|
+
"""Transfer content from the data store to the local filesystem."""
|
|
1362
|
+
|
|
1321
1363
|
if data_store.is_local_filesystem:
|
|
1322
1364
|
working_dir = data_store.GetWorkingDir() / directory
|
|
1323
1365
|
assert working_dir.is_dir(), working_dir
|
|
@@ -1327,7 +1369,7 @@ def _YieldRestoredArchive(
|
|
|
1327
1369
|
|
|
1328
1370
|
status_func("Calculating files to transfer...")
|
|
1329
1371
|
|
|
1330
|
-
with _YieldTempDirectory("transferring archive files") as
|
|
1372
|
+
with _YieldTempDirectory(working_dir, "transferring archive files") as working_dir:
|
|
1331
1373
|
# Map the remote filenames to local filenames
|
|
1332
1374
|
filename_map: dict[Path, Path] = {}
|
|
1333
1375
|
|
|
@@ -1339,7 +1381,7 @@ def _YieldRestoredArchive(
|
|
|
1339
1381
|
relative_root = root.relative_to(data_store_dir)
|
|
1340
1382
|
|
|
1341
1383
|
for filename in filenames:
|
|
1342
|
-
filename_map[root / filename] =
|
|
1384
|
+
filename_map[root / filename] = working_dir / relative_root / filename
|
|
1343
1385
|
|
|
1344
1386
|
if not filename_map:
|
|
1345
1387
|
raise Exception(f"The directory '{directory}' does not contain any files.")
|
|
@@ -1359,24 +1401,30 @@ def _YieldRestoredArchive(
|
|
|
1359
1401
|
),
|
|
1360
1402
|
)
|
|
1361
1403
|
|
|
1362
|
-
yield
|
|
1404
|
+
yield working_dir, True
|
|
1363
1405
|
|
|
1364
1406
|
|
|
1365
1407
|
# ----------------------------------------------------------------------
|
|
1366
1408
|
@contextmanager
|
|
1367
|
-
def
|
|
1409
|
+
def _YieldDecompressedFiles(
|
|
1410
|
+
dm: DoneManager,
|
|
1411
|
+
transferred_dir: Path,
|
|
1412
|
+
decompressed_dir: Path,
|
|
1368
1413
|
directory_name: str,
|
|
1369
|
-
archive_dir: Path,
|
|
1370
1414
|
encryption_password: str | None,
|
|
1371
1415
|
status_func: Callable[[str], None],
|
|
1416
|
+
*,
|
|
1417
|
+
continue_on_errors: bool,
|
|
1372
1418
|
) -> Iterator[
|
|
1373
1419
|
tuple[
|
|
1374
1420
|
Path,
|
|
1375
1421
|
bool, # is temporary directory
|
|
1376
1422
|
],
|
|
1377
1423
|
]:
|
|
1378
|
-
|
|
1379
|
-
|
|
1424
|
+
"""Decompress the archive if necessary"""
|
|
1425
|
+
|
|
1426
|
+
if (transferred_dir / INDEX_FILENAME).is_file():
|
|
1427
|
+
yield transferred_dir, False
|
|
1380
1428
|
return
|
|
1381
1429
|
|
|
1382
1430
|
# By default, 7zip will prompt for a password with archives that were created
|
|
@@ -1391,18 +1439,17 @@ def _YieldRestoredFiles(
|
|
|
1391
1439
|
#
|
|
1392
1440
|
password = encryption_password or str(uuid.uuid4())
|
|
1393
1441
|
|
|
1394
|
-
|
|
1395
|
-
status_func("Validating archive...")
|
|
1396
|
-
|
|
1397
|
-
archive_filename = archive_dir / (ARCHIVE_FILENAME + ".001")
|
|
1442
|
+
archive_filename = transferred_dir / (ARCHIVE_FILENAME + ".001")
|
|
1398
1443
|
|
|
1399
1444
|
if not archive_filename.is_file():
|
|
1400
1445
|
raise Exception(f"The archive file '{archive_filename.name}' was not found.")
|
|
1401
1446
|
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1447
|
+
# Validate
|
|
1448
|
+
status_func("Validating archive...")
|
|
1449
|
+
with dm.Nested("Validating archive...") as validate_dm:
|
|
1450
|
+
result = SubprocessEx.Run(f'{_GetZipBinary()} t "{archive_filename}" "-p{password}"')
|
|
1451
|
+
if result.returncode != 0:
|
|
1452
|
+
message = textwrap.dedent(
|
|
1406
1453
|
"""\
|
|
1407
1454
|
Archive validation failed for the directory '{}' ({}).
|
|
1408
1455
|
|
|
@@ -1418,27 +1465,32 @@ def _YieldRestoredFiles(
|
|
|
1418
1465
|
4,
|
|
1419
1466
|
skip_first_line=True,
|
|
1420
1467
|
),
|
|
1421
|
-
)
|
|
1422
|
-
)
|
|
1468
|
+
)
|
|
1423
1469
|
|
|
1424
|
-
|
|
1425
|
-
|
|
1470
|
+
if continue_on_errors:
|
|
1471
|
+
validate_dm.WriteWarning(message)
|
|
1472
|
+
else:
|
|
1473
|
+
raise Exception(message)
|
|
1426
1474
|
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1475
|
+
# Extract
|
|
1476
|
+
with _YieldTempDirectory(decompressed_dir, "extracting the archive") as decompressed_dir:
|
|
1477
|
+
status_func("Extracting archive...")
|
|
1478
|
+
with dm.Nested("Extracting archive...") as extract_dm:
|
|
1479
|
+
decompressed_dir.mkdir(parents=True, exist_ok=True)
|
|
1480
|
+
|
|
1481
|
+
result = SubprocessEx.Run(
|
|
1482
|
+
f'{_GetZipBinary()} x "{archive_filename}" "-p{password}"',
|
|
1483
|
+
cwd=decompressed_dir,
|
|
1484
|
+
)
|
|
1432
1485
|
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
textwrap.dedent(
|
|
1486
|
+
if result.returncode != 0:
|
|
1487
|
+
message = textwrap.dedent(
|
|
1436
1488
|
"""\
|
|
1437
|
-
|
|
1489
|
+
Archive extraction failed for the directory '{}' ({}).
|
|
1438
1490
|
|
|
1439
|
-
|
|
1491
|
+
{}
|
|
1440
1492
|
|
|
1441
|
-
|
|
1493
|
+
""",
|
|
1442
1494
|
).format(
|
|
1443
1495
|
directory_name,
|
|
1444
1496
|
result.returncode,
|
|
@@ -1447,67 +1499,74 @@ def _YieldRestoredFiles(
|
|
|
1447
1499
|
4,
|
|
1448
1500
|
skip_first_line=True,
|
|
1449
1501
|
),
|
|
1450
|
-
)
|
|
1451
|
-
)
|
|
1502
|
+
)
|
|
1452
1503
|
|
|
1453
|
-
|
|
1504
|
+
if continue_on_errors:
|
|
1505
|
+
extract_dm.WriteWarning(message)
|
|
1506
|
+
else:
|
|
1507
|
+
raise Exception(message)
|
|
1508
|
+
|
|
1509
|
+
yield decompressed_dir, True
|
|
1454
1510
|
|
|
1455
1511
|
|
|
1456
1512
|
# ----------------------------------------------------------------------
|
|
1457
|
-
def
|
|
1513
|
+
def _VerifyFiles(
|
|
1514
|
+
dm: DoneManager,
|
|
1458
1515
|
directory_name: str,
|
|
1459
1516
|
contents_dir: Path,
|
|
1460
1517
|
status_func: Callable[[str], None],
|
|
1518
|
+
*,
|
|
1519
|
+
continue_on_errors: bool,
|
|
1461
1520
|
) -> None:
|
|
1462
1521
|
# Ensure that the index is present
|
|
1463
1522
|
for index_filename in [INDEX_FILENAME, INDEX_HASH_FILENAME]:
|
|
1464
1523
|
if not (contents_dir / index_filename).is_file():
|
|
1465
1524
|
raise Exception(f"The index file '{index_filename}' does not exist.")
|
|
1466
1525
|
|
|
1467
|
-
|
|
1468
|
-
|
|
1526
|
+
with dm.Nested("Verifying files...") as verify_dm:
|
|
1527
|
+
# Ensure that the content is valid
|
|
1528
|
+
all_filenames: list[Path] = []
|
|
1469
1529
|
|
|
1470
|
-
|
|
1471
|
-
|
|
1530
|
+
for root_str, _, filenames in os.walk(contents_dir):
|
|
1531
|
+
root = Path(root_str)
|
|
1472
1532
|
|
|
1473
|
-
|
|
1533
|
+
all_filenames += [root / filename for filename in filenames if filename != INDEX_HASH_FILENAME]
|
|
1474
1534
|
|
|
1475
|
-
|
|
1535
|
+
data_store = FileSystemDataStore()
|
|
1476
1536
|
|
|
1477
|
-
|
|
1537
|
+
errors: list[str] = []
|
|
1478
1538
|
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1539
|
+
for filename_index, filename in enumerate(all_filenames):
|
|
1540
|
+
if filename.name == INDEX_FILENAME:
|
|
1541
|
+
expected_hash_value = (contents_dir / INDEX_HASH_FILENAME).read_text().strip()
|
|
1542
|
+
else:
|
|
1543
|
+
expected_hash_value = filename.name
|
|
1484
1544
|
|
|
1485
|
-
|
|
1545
|
+
file_size = filename.stat().st_size or 1
|
|
1486
1546
|
|
|
1487
|
-
|
|
1547
|
+
status_template = f"Validating file {filename_index + 1} of {len(all_filenames)} [{PathEx.GetSizeDisplay(file_size)}] {{:.02f}}%..."
|
|
1488
1548
|
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
),
|
|
1495
|
-
)
|
|
1496
|
-
|
|
1497
|
-
if actual_hash_value != expected_hash_value:
|
|
1498
|
-
errors.append(
|
|
1499
|
-
textwrap.dedent(
|
|
1500
|
-
f"""\
|
|
1501
|
-
Filename: {filename.relative_to(contents_dir)}
|
|
1502
|
-
Expected: {expected_hash_value}
|
|
1503
|
-
Actual: {actual_hash_value}
|
|
1504
|
-
""",
|
|
1549
|
+
actual_hash_value = Common.CalculateHash(
|
|
1550
|
+
data_store,
|
|
1551
|
+
filename,
|
|
1552
|
+
lambda bytes_transferred: status_func(
|
|
1553
|
+
status_template.format((bytes_transferred / file_size) * 100)
|
|
1505
1554
|
),
|
|
1506
1555
|
)
|
|
1507
1556
|
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1557
|
+
if actual_hash_value != expected_hash_value:
|
|
1558
|
+
errors.append(
|
|
1559
|
+
textwrap.dedent(
|
|
1560
|
+
f"""\
|
|
1561
|
+
Filename: {filename.relative_to(contents_dir)}
|
|
1562
|
+
Expected: {expected_hash_value}
|
|
1563
|
+
Actual: {actual_hash_value}
|
|
1564
|
+
""",
|
|
1565
|
+
),
|
|
1566
|
+
)
|
|
1567
|
+
|
|
1568
|
+
if errors:
|
|
1569
|
+
message = textwrap.dedent(
|
|
1511
1570
|
"""\
|
|
1512
1571
|
Corrupt files were encountered in the directory '{}'.
|
|
1513
1572
|
|
|
@@ -1521,5 +1580,9 @@ def _VerifyRestoredFiles(
|
|
|
1521
1580
|
4,
|
|
1522
1581
|
skip_first_line=True,
|
|
1523
1582
|
),
|
|
1524
|
-
)
|
|
1525
|
-
|
|
1583
|
+
)
|
|
1584
|
+
|
|
1585
|
+
if continue_on_errors:
|
|
1586
|
+
verify_dm.WriteWarning(message)
|
|
1587
|
+
else:
|
|
1588
|
+
raise Exception(message)
|
FileBackup/__init__.py
CHANGED
|
@@ -9,4 +9,4 @@
|
|
|
9
9
|
# Note that this value will be overwritten by calls to `python ../../Build.py update_version` based
|
|
10
10
|
# on changes observed in the git repository. The default value below will be used until the value
|
|
11
11
|
# here is explicitly updated by the Continuous Integration system as part of a commit.
|
|
12
|
-
__version__ = "0.
|
|
12
|
+
__version__ = "0.5.1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FileBackup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Tool for creating and restoring file system backups.
|
|
5
5
|
Project-URL: Homepage, https://github.com/davidbrownell/FileBackup
|
|
6
6
|
Project-URL: Documentation, https://github.com/davidbrownell/FileBackup
|
|
@@ -28,7 +28,7 @@ Classifier: Topic :: System :: Archiving :: Mirroring
|
|
|
28
28
|
Classifier: Topic :: System :: Systems Administration
|
|
29
29
|
Classifier: Topic :: Utilities
|
|
30
30
|
Requires-Python: >=3.10
|
|
31
|
-
Requires-Dist: dbrownell-common>=0.
|
|
31
|
+
Requires-Dist: dbrownell-common>=0.15.0
|
|
32
32
|
Requires-Dist: paramiko>=3.5.1
|
|
33
33
|
Requires-Dist: typer>=0.15.3
|
|
34
34
|
Description-Content-Type: text/markdown
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
FileBackup/Mirror.py,sha256=GDgTWVNUYn5O7vlH5YVi0LppwQN4R99J61g1UCBwlfI,23544
|
|
2
|
-
FileBackup/Offsite.py,sha256=
|
|
2
|
+
FileBackup/Offsite.py,sha256=_QOywhHX8rmSJJ0TC3dKF4m-cFGx-l9Splng351crqw,65496
|
|
3
3
|
FileBackup/Snapshot.py,sha256=Dj-XFjAFWkqNZIlMHqTOI1Ygdm1jB3orIwDxMQSrupQ,26970
|
|
4
|
-
FileBackup/__init__.py,sha256=
|
|
4
|
+
FileBackup/__init__.py,sha256=mbDIiinNsmldCAGbrTOXoPVPYmGGiRz94d8JOF3bEyc,597
|
|
5
5
|
FileBackup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
FileBackup/CommandLine/CommandLineArguments.py,sha256=7oLUf9tODd_OjBNzIjQiiyZ9ujy-KbW53Pg0dF0MQG8,2378
|
|
7
7
|
FileBackup/CommandLine/EntryPoint.py,sha256=Fn62wtKV6uiTJ1dQpolNFUyWWeUOKKgetOFSXPgoe-w,1813
|
|
8
8
|
FileBackup/CommandLine/MirrorEntryPoint.py,sha256=WlGIz_AZ7OdE-hVL19lQExUXbEjQ8_4bbIuITWxdWO4,6704
|
|
9
|
-
FileBackup/CommandLine/OffsiteEntryPoint.py,sha256=
|
|
9
|
+
FileBackup/CommandLine/OffsiteEntryPoint.py,sha256=OscTJi57_-dYX2NsfzTONqWhxm-NvON-aY80r6-aRwk,12634
|
|
10
10
|
FileBackup/CommandLine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
11
|
FileBackup/DataStore/FastGlacierDataStore.py,sha256=5J4W9FqB92hnHoRoZLo-DfMQ6uyjaOAWWrnuc0pvnJo,3017
|
|
12
12
|
FileBackup/DataStore/FileSystemDataStore.py,sha256=XUvIQe2Yl-W6OPfvsJlPzDPsGclDOkKlozO0GJ-05bo,6791
|
|
@@ -19,8 +19,8 @@ FileBackup/DataStore/Interfaces/FileBasedDataStore.py,sha256=eMA_cQ-CED83z5uMDQZ
|
|
|
19
19
|
FileBackup/DataStore/Interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
FileBackup/Impl/Common.py,sha256=2WFMx5BdCnZmo-cuLYZlIx4Nr_ZBKmJWfMaFVUpsqKo,24935
|
|
21
21
|
FileBackup/Impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
filebackup-0.
|
|
23
|
-
filebackup-0.
|
|
24
|
-
filebackup-0.
|
|
25
|
-
filebackup-0.
|
|
26
|
-
filebackup-0.
|
|
22
|
+
filebackup-0.5.1.dist-info/METADATA,sha256=Rk1-xC4FGaXbba_ORWFeVHAvd86amimeq0yDtBf9Ft8,5762
|
|
23
|
+
filebackup-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
24
|
+
filebackup-0.5.1.dist-info/entry_points.txt,sha256=2SUVO3jHsi06MYfUEdqxaVwVvTQVjxzIWlHexRKWRgQ,121
|
|
25
|
+
filebackup-0.5.1.dist-info/licenses/LICENSE,sha256=2dPVskJuHRLwSF9fQiAfln1RVWya2DsKNPvgknRLTYw,1074
|
|
26
|
+
filebackup-0.5.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|