FileBackup 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

FileBackup/CommandLine/CommandLineArguments.py

@@ -0,0 +1,77 @@
+# ----------------------------------------------------------------------
+# |
+# |  CommandLineArguments.py
+# |
+# |  David Brownell <db@DavidBrownell.com>
+# |      2024-06-12 13:20:05
+# |
+# ----------------------------------------------------------------------
+# |
+# |  Copyright David Brownell 2024
+# |  Distributed under the MIT License.
+# |
+# ----------------------------------------------------------------------
+
+import re
+
+from typing import Pattern
+
+import typer
+
+
+# ----------------------------------------------------------------------
+def ToRegex(
+    values: list[str],
+) -> list[Pattern]:
+    expressions: list[Pattern] = []
+
+    for value in values:
+        try:
+            expressions.append(re.compile("^{}$".format(value)))
+        except re.error as ex:
+            raise typer.BadParameter(
+                "The regular expression '{}' is not valid ({}).".format(value, ex)
+            )
+
+    return expressions
+
+
+# ----------------------------------------------------------------------
+input_filename_or_dirs_argument = typer.Argument(
+    ..., exists=True, resolve_path=True, help="Input filename or directory."
+)
+
+ssd_option = typer.Option(
+    "--ssd",
+    help="Processes tasks in parallel to leverage the capabilities of solid-state-drives.",
+)
+ssd_option_default = False
+
+quiet_option = typer.Option("--quiet", help="Reduce the amount of information displayed.")
+quiet_option_default = False
+
+force_option = typer.Option(
+    "--force",
+    help="Ignore previous backup information and overwrite all data in the destination data store.",
+)
+force_option_default = False
+
+verbose_option = typer.Option("--verbose", help="Write verbose information to the terminal.")
+verbose_option_default = False
+
+debug_option = typer.Option("--debug", help="Write debug information to the terminal.")
+debug_option_default = False
+
+file_include_option = typer.Option(
+    "--file-include",
+    callback=ToRegex,
+    help="Regular expression (based on a posix path) used to include files and/or directories when preserving content.",
+)
+file_include_option_default: list[str] = []
+
+file_exclude_option = typer.Option(
+    "--file-exclude",
+    callback=ToRegex,
+    help="Regular expression (based on a posix path) used to exclude files and/or directories when preserving content.",
+)
+file_exclude_option_default: list[str] = []

FileBackup/{EntryPoint.py → CommandLine/EntryPoint.py}

@@ -12,7 +12,8 @@ import typer
 
 from typer.core import TyperGroup  # type: ignore [import-untyped]
 
-from FileBackup import Math, __version__
+from FileBackup import __version__
+from FileBackup.CommandLine import MirrorEntryPoint
 
 
 # ----------------------------------------------------------------------
@@ -33,56 +34,12 @@ app = typer.Typer(
 )
 
 
-# ----------------------------------------------------------------------
-@app.command("Add")
-def Add(
-    x: int,
-    y: int,
-) -> None:
-    """Adds 2 values."""
-
-    sys.stdout.write(str(Math.Add(x, y)))
-
-
-# ----------------------------------------------------------------------
-@app.command("Sub")
-def Sub(
-    x: int,
-    y: int,
-) -> None:
-    """Subtracts 2 values."""
-
-    sys.stdout.write(str(Math.Sub(x, y)))
-
-
-# ----------------------------------------------------------------------
-@app.command("Mult")
-def Mult(
-    x: int,
-    y: int,
-) -> None:
-    """Multiplies 2 values."""
-
-    sys.stdout.write(str(Math.Mult(x, y)))
-
-
-# ----------------------------------------------------------------------
-@app.command("Div")
-def Div(
-    x: int,
-    y: int,
-) -> None:
-    """Divides 1 value by another."""
-
-    sys.stdout.write(str(Math.Div(x, y)))
+app.add_typer(MirrorEntryPoint.app, name="mirror", help=MirrorEntryPoint.__doc__)
 
 
-# ----------------------------------------------------------------------
-@app.command("Version")
-def Version() -> None:
-    """Prints the version of the package."""
-
-    sys.stdout.write(__version__)
+@app.command("version", no_args_is_help=False)
+def Version():
+    sys.stdout.write(f"FileBackup v{__version__}\n")
 
 
 # ----------------------------------------------------------------------

FileBackup/CommandLine/MirrorEntryPoint.py

@@ -0,0 +1,194 @@
+# ----------------------------------------------------------------------
+# |
+# |  MirrorEntryPoint.py
+# |
+# |  David Brownell <db@DavidBrownell.com>
+# |      2024-06-12 13:16:52
+# |
+# ----------------------------------------------------------------------
+# |
+# |  Copyright David Brownell 2024
+# |  Distributed under the MIT License.
+# |
+# ----------------------------------------------------------------------
+"""\
+Mirrors backup content: files created locally will be added to the backup data store; files deleted
+locally will be removed from the backup data store; files modified locally will be modified at the
+backup data store.
+"""
+
+import datetime
+import textwrap
+
+from pathlib import Path
+from typing import Annotated, cast, Pattern
+
+import typer
+
+from dbrownell_Common.Streams.DoneManager import DoneManager, Flags as DoneManagerFlags  # type: ignore [import-untyped]
+from typer.core import TyperGroup  # type: ignore [import-untyped]
+
+from FileBackup.CommandLine import CommandLineArguments
+from FileBackup.Impl import Common
+from FileBackup import Mirror
+
+
+# ----------------------------------------------------------------------
+class NaturalOrderGrouper(TyperGroup):
+    # pylint: disable=missing-class-docstring
+    # ----------------------------------------------------------------------
+    def list_commands(self, *args, **kwargs):  # pylint: disable=unused-argument
+        return self.commands.keys()
+
+
+# ----------------------------------------------------------------------
+app = typer.Typer(
+    cls=NaturalOrderGrouper,
+    help=__doc__,
+    no_args_is_help=True,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_enable=False,
+)
+
+
+# ----------------------------------------------------------------------
+_destination_argument = typer.Argument(
+    ...,
+    help="Destination data store used when mirroring local content; see the comments below for information on the different data store destination formats.",
+)
+
+
+# ----------------------------------------------------------------------
+@app.command(
+    "execute",
+    epilog=Common.GetDestinationHelp(),
+    no_args_is_help=True,
+)
+def Execute(
+    destination: Annotated[str, _destination_argument],
+    input_filename_or_dirs: Annotated[
+        list[Path],
+        CommandLineArguments.input_filename_or_dirs_argument,
+    ],
+    ssd: Annotated[bool, CommandLineArguments.ssd_option] = CommandLineArguments.ssd_option_default,
+    force: Annotated[
+        bool, CommandLineArguments.force_option
+    ] = CommandLineArguments.force_option_default,
+    verbose: Annotated[
+        bool, CommandLineArguments.verbose_option
+    ] = CommandLineArguments.verbose_option_default,
+    quiet: Annotated[
+        bool, CommandLineArguments.quiet_option
+    ] = CommandLineArguments.quiet_option_default,
+    debug: Annotated[
+        bool, CommandLineArguments.debug_option
+    ] = CommandLineArguments.debug_option_default,
+    file_include_params: Annotated[
+        list[str],
+        CommandLineArguments.file_include_option,
+    ] = CommandLineArguments.file_include_option_default,
+    file_exclude_params: Annotated[
+        list[str],
+        CommandLineArguments.file_exclude_option,
+    ] = CommandLineArguments.file_exclude_option_default,
+) -> None:
+    """Mirrors content to a backup data store."""
+
+    file_includes = cast(list[Pattern], file_include_params)
+    file_excludes = cast(list[Pattern], file_exclude_params)
+
+    del file_include_params
+    del file_exclude_params
+
+    with DoneManager.CreateCommandLine(
+        flags=DoneManagerFlags.Create(verbose=verbose, debug=debug),
+    ) as dm:
+        dm.WriteVerbose(str(datetime.datetime.now()) + "\n\n")
+
+        Mirror.Backup(
+            dm,
+            destination,
+            input_filename_or_dirs,
+            ssd=ssd,
+            force=force,
+            quiet=quiet,
+            file_includes=file_includes,
+            file_excludes=file_excludes,
+        )
+
+
+# ----------------------------------------------------------------------
+@app.command(
+    "validate",
+    no_args_is_help=True,
+    epilog=textwrap.dedent(
+        """\
+        {}
+        Validation Types
+        ================
+            standard: Validates that files and directories at the destination exist and file sizes match the expected values.
+            complete: Validates that files and directories at the destination exist and file hashes match the expected values.
+        """,
+    )
+    .replace("\n", "\n\n")
+    .format(Common.GetDestinationHelp()),
+)
+def Validate(
+    destination: Annotated[str, _destination_argument],
+    validate_type: Annotated[
+        Mirror.ValidateType,
+        typer.Argument(
+            case_sensitive=False,
+            help="Specifies the type of validation to use; the the comments below for information on the different validation types.",
+        ),
+    ] = Mirror.ValidateType.standard,
+    ssd: Annotated[bool, CommandLineArguments.ssd_option] = CommandLineArguments.ssd_option_default,
+    verbose: Annotated[
+        bool, CommandLineArguments.verbose_option
+    ] = CommandLineArguments.verbose_option_default,
+    quiet: Annotated[
+        bool, CommandLineArguments.quiet_option
+    ] = CommandLineArguments.quiet_option_default,
+    debug: Annotated[
+        bool, CommandLineArguments.debug_option
+    ] = CommandLineArguments.debug_option_default,
+) -> None:
+    """Validates perviously mirrored content in the backup data store."""
+
+    with DoneManager.CreateCommandLine(
+        flags=DoneManagerFlags.Create(verbose=verbose, debug=debug),
+    ) as dm:
+        dm.WriteVerbose(str(datetime.datetime.now()) + "\n\n")
+
+        Mirror.Validate(
+            dm,
+            destination,
+            validate_type,
+            ssd=ssd,
+            quiet=quiet,
+        )
+
+
+# ----------------------------------------------------------------------
+@app.command(
+    "cleanup",
+    epilog=Common.GetDestinationHelp(),
+    no_args_is_help=True,
+)
+def Cleanup(
+    destination: Annotated[str, _destination_argument],
+    verbose: Annotated[
+        bool, CommandLineArguments.verbose_option
+    ] = CommandLineArguments.verbose_option_default,
+    debug: Annotated[
+        bool, CommandLineArguments.debug_option
+    ] = CommandLineArguments.debug_option_default,
+) -> None:
+    """Cleans a backup data store after a mirror execution that was interrupted or failed."""
+
+    with DoneManager.CreateCommandLine(
+        flags=DoneManagerFlags.Create(verbose=verbose, debug=debug),
+    ) as dm:
+        dm.WriteVerbose(str(datetime.datetime.now()) + "\n\n")
+
+        Mirror.Cleanup(dm, destination)

FileBackup/DataStore/FastGlacierDataStore.py

@@ -0,0 +1,79 @@
+# ----------------------------------------------------------------------
+# |
+# |  FastGlacierDataStore.py
+# |
+# |  David Brownell <db@DavidBrownell.com>
+# |      2024-06-10 14:32:10
+# |
+# ----------------------------------------------------------------------
+# |
+# |  Copyright David Brownell 2024
+# |  Distributed under the MIT License.
+# |
+# ----------------------------------------------------------------------
+"""Contains the FastGlacierDataStore object"""
+
+from pathlib import Path
+from typing import Optional
+
+from dbrownell_Common.Streams.DoneManager import DoneManager  # type: ignore[import-untyped]
+from dbrownell_Common import SubprocessEx  # type: ignore[import-untyped]
+from dbrownell_Common.Types import override  # type: ignore[import-untyped]
+
+from FileBackup.DataStore.Interfaces.BulkStorageDataStore import BulkStorageDataStore
+
+
+# ----------------------------------------------------------------------
+class FastGlacierDataStore(BulkStorageDataStore):
+    """Data store that uses the Fast Glacier application (https://fastglacier.com/)"""
+
+    # ----------------------------------------------------------------------
+    def __init__(
+        self,
+        account_name: str,
+        aws_region: str,
+        glacier_dir: Optional[Path],
+    ):
+        super(FastGlacierDataStore, self).__init__()
+
+        self.account_name = account_name
+        self.aws_region = aws_region
+
+        self._glacier_dir = glacier_dir or Path()
+
+        self._validated_command_line = False
+
+    # ----------------------------------------------------------------------
+    @override
+    def ExecuteInParallel(self) -> bool:
+        return False
+
+    # ----------------------------------------------------------------------
+    @override
+    def Upload(
+        self,
+        dm: DoneManager,
+        local_path: Path,
+    ) -> None:
+        if self._validated_command_line is False:
+            with dm.Nested(
+                "Validating Fast Glacier on the command line...",
+                suffix="\n",
+            ) as check_dm:
+                result = SubprocessEx.Run("glacier-con --version")
+
+                check_dm.WriteVerbose(result.output)
+
+                if result.returncode != 0 and "glacier-con.exe upload" not in result.output:
+                    check_dm.WriteError(
+                        "Fast Glacier is not available; please make sure it exists in the path and run the script again.\n"
+                    )
+                    return
+
+            with dm.Nested("Uploading to Fast Glacier...") as upload_dm:
+                command_line = f'glacier-con upload "{self.account_name}" "{local_path / "*"}" "{self.aws_region}" "{self._glacier_dir.as_posix()}"'
+
+                upload_dm.WriteVerbose(f"Command Line: {command_line}\n\n")
+
+                with upload_dm.YieldStream() as stream:
+                    upload_dm.result = SubprocessEx.Stream(command_line, stream)

FileBackup/DataStore/FileSystemDataStore.py

@@ -0,0 +1,218 @@
+# ----------------------------------------------------------------------
+# |
+# |  FileSystemDataStore.py
+# |
+# |  David Brownell <db@DavidBrownell.com>
+# |      2024-06-10 13:42:04
+# |
+# ----------------------------------------------------------------------
+# |
+# |  Copyright David Brownell 2024
+# |  Distributed under the MIT License.
+# |
+# ----------------------------------------------------------------------
+"""Contains the FileSystemDataStore object"""
+
+import itertools
+import os
+import shutil
+
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Generator, Optional
+
+from dbrownell_Common import PathEx  # type: ignore[import-untyped]
+from dbrownell_Common.Types import override  # type: ignore[import-untyped]
+
+from FileBackup.DataStore.Interfaces.FileBasedDataStore import FileBasedDataStore, ItemType
+
+
+# ----------------------------------------------------------------------
+class FileSystemDataStore(FileBasedDataStore):
+    """DataStore associated with a standard, local file system"""
+
+    # ----------------------------------------------------------------------
+    def __init__(
+        self,
+        root: Path = Path.cwd(),
+        *,
+        ssd: bool = False,
+        is_local_filesystem_override_value_for_testing: Optional[bool] = None,
+    ) -> None:
+        assert not root.exists() or root.is_dir(), root
+
+        super(FileSystemDataStore, self).__init__(
+            is_local_filesystem=(
+                is_local_filesystem_override_value_for_testing
+                if is_local_filesystem_override_value_for_testing is not None
+                else root.root == Path.cwd().root
+            ),
+        )
+
+        self._working_dir: Path = root
+        self._ssd = ssd
+
+    # ----------------------------------------------------------------------
+    @override
+    def ExecuteInParallel(self) -> bool:
+        return self._ssd
+
+    # ----------------------------------------------------------------------
+    @override
+    def ValidateBackupInputs(
+        self,
+        input_filename_or_dirs: list[Path],
+    ) -> None:
+        for input_filename_or_dir in input_filename_or_dirs:
+            if input_filename_or_dir.is_file():
+                input_dir = input_filename_or_dir.parent
+            elif input_filename_or_dir.is_dir():
+                input_dir = input_filename_or_dir
+            else:
+                raise Exception(f"'{input_filename_or_dir}' is not a supported item type.")
+
+            if PathEx.IsDescendant(self._working_dir, input_dir):
+                raise Exception(
+                    f"The directory '{input_filename_or_dir}' overlaps with the destination path '{self._working_dir}'."
+                )
+
+    # ----------------------------------------------------------------------
+    @override
+    def SnapshotFilenameToDestinationName(
+        self,
+        path: Path,
+    ) -> Path:
+        if path.parts[0] == "/":
+            path = Path(*path.parts[1:])
+        elif path.parts[0]:
+            # Probably on Windows
+            path = Path(path.parts[0].replace(":", "_").rstrip("\\")) / Path(*path.parts[1:])
+
+        return self.GetWorkingDir() / path
+
+    # ----------------------------------------------------------------------
+    @override
+    def GetBytesAvailable(self) -> Optional[int]:
+        # Find a directory that exists
+        for potential_dir in itertools.chain(
+            [self._working_dir],
+            self._working_dir.parents,
+            [Path.cwd()],
+        ):
+            if potential_dir.is_dir():
+                return shutil.disk_usage(potential_dir).free
+
+        assert False, self._working_dir
+
+    # ----------------------------------------------------------------------
+    @override
+    def GetWorkingDir(self) -> Path:
+        return self._working_dir
+
+    # ----------------------------------------------------------------------
+    @override
+    def SetWorkingDir(
+        self,
+        path: Path,
+    ) -> None:
+        self._working_dir /= path
+
+    # ----------------------------------------------------------------------
+    @override
+    def GetItemType(
+        self,
+        path: Path,
+    ) -> Optional[ItemType]:
+        path = self._working_dir / path
+
+        if not path.exists():
+            return None
+
+        if path.is_symlink():
+            return ItemType.SymLink
+
+        if path.is_file():
+            return ItemType.File
+
+        if path.is_dir():
+            return ItemType.Dir
+
+        raise Exception(f"'{path}' is not a known type.")
+
+    # ----------------------------------------------------------------------
+    @override
+    def GetFileSize(
+        self,
+        path: Path,
+    ) -> int:
+        return (self._working_dir / path).stat().st_size
+
+    # ----------------------------------------------------------------------
+    @override
+    def RemoveDir(
+        self,
+        path: Path,
+    ) -> None:
+        shutil.rmtree(self._working_dir / path)
+
+    # ----------------------------------------------------------------------
+    @override
+    def RemoveFile(
+        self,
+        path: Path,
+    ) -> None:
+        (self._working_dir / path).unlink()
+
+    # ----------------------------------------------------------------------
+    @override
+    def MakeDirs(
+        self,
+        path: Path,
+    ) -> None:
+        (self._working_dir / path).mkdir(parents=True, exist_ok=True)
+
+    # ----------------------------------------------------------------------
+    @override
+    @contextmanager
+    def Open(
+        self,
+        filename: Path,
+        *args,
+        **kwargs,
+    ):
+        with (self._working_dir / filename).open(*args, **kwargs) as f:
+            yield f
+
+    # ----------------------------------------------------------------------
+    @override
+    def Rename(
+        self,
+        old_path: Path,
+        new_path: Path,
+    ) -> None:
+        old_path = self._working_dir / old_path
+        new_path = self._working_dir / new_path
+
+        if new_path.is_file():
+            new_path.unlink()
+        elif new_path.is_dir():
+            shutil.rmtree(new_path)
+
+        shutil.move(old_path, new_path)
+
+    # ----------------------------------------------------------------------
+    @override
+    def Walk(
+        self,
+        path: Path = Path(),
+    ) -> Generator[
+        tuple[
+            Path,  # root
+            list[str],  # directories
+            list[str],  # files
+        ],
+        None,
+        None,
+    ]:
+        for root, directories, filenames in os.walk(self._working_dir / path):
+            yield Path(root), directories, filenames

FileBackup/DataStore/Interfaces/BulkStorageDataStore.py

@@ -0,0 +1,36 @@
+# ----------------------------------------------------------------------
+# |
+# |  BulkStorageDataStore.py
+# |
+# |  David Brownell <db@DavidBrownell.com>
+# |      2024-06-10 13:33:45
+# |
+# ----------------------------------------------------------------------
+# |
+# |  Copyright David Brownell 2024
+# |  Distributed under the MIT License.
+# |
+# ----------------------------------------------------------------------
+"""Contains the BulkStorageDataStore object"""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from dbrownell_Common.Streams.DoneManager import DoneManager  # type: ignore[import-untyped]
+
+from FileBackup.DataStore.Interfaces.DataStore import DataStore  # type: ignore[import-untyped]
+
+
+# ----------------------------------------------------------------------
+class BulkStorageDataStore(DataStore):
+    """Abstraction for data stores that can upload content in bulk but not easily retrieve it (such as cloud storage)"""
+
+    # ----------------------------------------------------------------------
+    @abstractmethod
+    def Upload(
+        self,
+        dm: DoneManager,
+        local_path: Path,
+    ) -> None:
+        """Uploads all content in the provided path and its children"""
+        raise Exception("Abstract method")  # pragma: no cover