prefect-managedfiletransfer 0.1.0__py3-none-any.whl

prefect_managedfiletransfer/AssetDownloadResult.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+from pathlib import Path
+
+
+class AssetDownloadResult:
+    """
+    Represents the result of an asset download operation.
+    """
+
+    def __init__(
+        self,
+        success: bool,
+        file_path: Path | None,
+        download_skipped: bool = False,
+        last_modified: datetime | None = None,
+        size: int = 0,
+        error: str | None = None,
+    ):
+        """
+        Represents the result of an asset download operation.
+        :param success: True if the download was successful, False otherwise.
+        :param file_path: The path to the downloaded file, or None if the download failed.
+        :param download_skipped: True if the download was skipped because the file was not newer than the destination file.
+        :param last_modified: The last modified time of the file, if available.
+        """
+        self.last_modified = last_modified
+        self.success = success
+        self.file_path = file_path
+        self.download_skipped = download_skipped
+        self.error = error
+        self.size = size
+
+        if not success and file_path is not None:
+            raise ValueError("If success is False, file_path must be None")
+
+    def __repr__(self):
+        if self.error:
+            return f"AssetDownloadResult(success={self.success}, error={self.error})"
+
+        return f"AssetDownloadResult(success={self.success}, file_path={self.file_path}, download_skipped={self.download_skipped}, last_modified={self.last_modified})"

prefect_managedfiletransfer/FileMatcher.py

@@ -0,0 +1,41 @@
+from datetime import timedelta
+from pydantic import BaseModel, Field
+from prefect_managedfiletransfer.SortFilesBy import SortFilesBy
+
+from pathlib import Path
+
+
+class FileMatcher(BaseModel):
+    """
+    Represents a file matcher with a source path and a pattern to match files.
+    This is used to find files in a directory that match a specific pattern.
+    """
+
+    source_folder: Path = Field(
+        default=Path("."),
+        description="Path to the source directory to look for files.",
+    )
+    pattern_to_match: str = Field(
+        default="*",
+        description="Pattern to match files in the source directory. Supports glob patterns like '*.txt' or 'file_*.csv'.",
+    )
+    minimum_age: str | int | timedelta | None = Field(
+        default=None,
+        description="Only transfer files older than this in secs (or other time with suffix s|m|h|d|w|month|year). Default off.",
+    )
+    maximum_age: str | int | timedelta | None = Field(
+        default=None,
+        description="Only transfer files newer than this in secs (or other time with suffix s|m|h|d|w|month|year). Default off.",
+    )
+    sort: SortFilesBy = Field(
+        default=SortFilesBy.PATH_ASC,
+        description="Sort files by a specific attribute. Default is PATH_ASC.",
+    )
+    skip: int = Field(
+        default=0,
+        description="Number of files to skip in the sorted list. Default is 0.",
+    )
+    take: int | None = Field(
+        default=None,
+        description="Number of files to take from the sorted list. If None, all files are taken. Default is None.",
+    )

prefect_managedfiletransfer/FileToFolderMapping.py

@@ -0,0 +1,92 @@
+from pydantic import BaseModel, Field
+
+from pathlib import Path
+
+from prefect_managedfiletransfer.RemoteAsset import RemoteAsset
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class FileToFolderMapping(BaseModel):
+    source_path_pattern_to_match: str = Field(
+        default="*",
+        description="Pattern to match files against in the source directory. Supports glob patterns. E.g. 'folder/*.txt' or '*path*/file.*'",
+    )
+
+    destination_folder: Path = Field(
+        default=Path("."),
+        description="Path of the destination folder files matching the pattern should be placed into",
+    )
+
+    def is_match(self, file_path: Path) -> bool:
+        """
+        Check if the given file path matches the source path pattern.
+        """
+
+        if not self.source_path_pattern_to_match:
+            return False
+
+        if not file_path:
+            return False
+
+        return file_path.match(self.source_path_pattern_to_match)
+
+    # define constructor to allow for easy instantiation
+    def __init__(
+        self,
+        source_path_pattern_to_match: str = "*",
+        destination_folder: str = ".",
+    ):
+        super().__init__(
+            source_path_pattern_to_match=source_path_pattern_to_match,
+            destination_folder=Path(destination_folder),
+        )
+
+    @staticmethod
+    def apply_mappings(
+        mappings: list["FileToFolderMapping"],
+        source_files: list[RemoteAsset],
+        destination_path,
+    ) -> list[tuple[RemoteAsset, Path]]:
+        source_destination_pairs = []
+        for remote_asset in source_files:
+            logger.info(f"Found file: {remote_asset}")
+
+            # calculate the destination path - target + mapping
+            target_file_path: Path | None = None
+
+            for mapping in mappings:
+                if mapping.is_match(remote_asset.path):
+                    if not destination_path:
+                        target_file_path = (
+                            mapping.destination_folder / remote_asset.path.name
+                        )
+                    elif (
+                        not mapping.destination_folder
+                        or mapping.destination_folder == Path(".")
+                    ):
+                        target_file_path = destination_path / remote_asset.path.name
+                    else:
+                        target_file_path = (
+                            destination_path
+                            / mapping.destination_folder
+                            / remote_asset.path.name
+                        )
+                    logger.info(
+                        f"File {remote_asset.path} matched {mapping.source_path_pattern_to_match} -> {mapping.destination_folder}"
+                    )
+                    break
+
+            if not target_file_path:
+                logger.info(
+                    f"No mapping found for {remote_asset.path}, using default destination path"
+                )
+                target_file_path = destination_path / remote_asset.path.name
+
+            assert target_file_path is not None
+
+            source_destination_pairs.append((remote_asset, target_file_path))
+
+        return source_destination_pairs

prefect_managedfiletransfer/PathUtil.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+from prefect_managedfiletransfer.RemoteConnectionType import RemoteConnectionType
+
+
+class PathUtil:
+    @staticmethod
+    def resolve_path(
+        path_type: RemoteConnectionType,
+        basepath: Path | str | None,
+        path: Path | str,
+    ) -> Path:
+        if path_type == RemoteConnectionType.LOCAL:
+            remote_source_path = PathUtil._local_resolve_path(
+                basepath, path, validate=True
+            )
+        else:
+            remote_source_path = PathUtil._resolve_remote_path(
+                basepath, path, validate=True
+            )
+
+        return remote_source_path
+
+    @staticmethod
+    def _local_resolve_path(
+        basepath: str | Path | None, path: str | Path, validate: bool = False
+    ) -> Path:
+        # Only resolve the base path at runtime, default to the current directory
+        resolved_basepath = (
+            Path(basepath).expanduser().resolve() if basepath else Path(".").resolve()
+        )
+
+        # Determine the path to access relative to the base path, ensuring that paths
+        # outside of the base path are off limits
+        if path is None:
+            return resolved_basepath
+
+        resolved_path: Path = Path(path).expanduser()
+
+        if not resolved_path.is_absolute():
+            resolved_path = resolved_basepath / resolved_path
+        else:
+            resolved_path = resolved_path.resolve()
+
+        if validate:
+            if resolved_basepath not in resolved_path.parents and (
+                resolved_basepath != resolved_path
+            ):
+                raise ValueError(
+                    f"Provided path {resolved_path} is outside of the base path {resolved_basepath}."
+                )
+        return resolved_path
+
+    @staticmethod
+    def _resolve_remote_path(
+        basepath: str | Path | None, path: str | Path, validate: bool = False
+    ) -> Path:
+        if path is None and basepath is None:
+            return Path(".")
+        elif path is None:
+            return basepath
+
+        path = str(path).strip()
+        basepath = str(basepath).strip() if basepath else ""
+
+        if not path.startswith("/") and len(basepath) > 0:
+            resolved_path = f"{basepath.rstrip('/')}/{path}"
+        else:
+            resolved_path = path
+
+        if validate:
+            if (
+                len(basepath) > 0
+                and Path(basepath) not in Path(resolved_path).parents
+                and basepath != resolved_path
+            ):
+                raise ValueError(
+                    f"Provided path {resolved_path} is outside of the base path {basepath}."
+                )
+            if len(basepath) > 0 and ".." in resolved_path:
+                raise ValueError(
+                    f"Provided path {resolved_path} contains '..' so cannot be validated to be within basepath. Remove basepath if .. is required"
+                )
+        return Path(resolved_path)

prefect_managedfiletransfer/RCloneCommandBuilder.py

@@ -0,0 +1,217 @@
+import importlib
+import logging
+from pathlib import Path
+import platform
+import subprocess
+import importlib.resources
+from prefect_managedfiletransfer.RCloneConfig import RCloneConfig
+
+logger = logging.getLogger(__name__)
+
+
+class RCloneCommandBuilder:
+    def __init__(
+        self,
+        rclone_config_file: Path | None = None,
+        rclone_config: RCloneConfig | None = None,
+    ):
+        self.rclone_config = rclone_config
+        self.rclone_config_file = rclone_config_file
+
+        self._rclone_executable = "rclone"
+        packaged_folder = Path(
+            str(
+                importlib.resources.files("prefect_managedfiletransfer").joinpath(
+                    "rclone"
+                )
+            )
+        )
+        packaged_executable = packaged_folder / self._rclone_executable
+
+        if platform.system() == "Windows":
+            self._rclone_executable = "rclone.exe"
+            packaged_executable = packaged_folder / self._rclone_executable
+            logger.debug(
+                f"rclone windows packaged executable is at {packaged_executable}"
+            )
+        elif platform.system() == "Darwin":
+            packaged_executable = packaged_folder / "osx" / self._rclone_executable
+            logger.debug(
+                f"rclone macOS packaged executable is at {packaged_executable}"
+            )
+        else:
+            logger.debug(
+                f"rclone linux packaged executable is at {packaged_executable}"
+            )
+
+        # Check if rclone is available on the system path
+        try:
+            subprocess.run(
+                f"{self._rclone_executable} version",
+                shell=True,
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            logger.debug(
+                f"Using rclone executable on PATH: '{self._rclone_executable}'"
+            )
+        except subprocess.CalledProcessError:
+            logger.warning(
+                "rclone is not available on PATH - using packaged version at "
+                + str(packaged_executable)
+            )
+            self._rclone_executable = str(packaged_executable)
+
+        try:
+            subprocess.run(
+                [self._rclone_executable, "version"],
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            logger.critical(
+                f"rclone executable {self._rclone_executable} is not available or not executable"
+            )
+            raise FileNotFoundError(
+                f"rclone executable {self._rclone_executable} is not available or not executable"
+            )
+
+        self.rclone_command = [self._rclone_executable]
+
+        if rclone_config_file is not None and rclone_config is not None:
+            raise ValueError(
+                "rclone_config_file and rclone_config_contents cannot be used at the same time"
+            )
+
+        if rclone_config_file is not None and not rclone_config_file.exists():
+            logger.critical(f"rclone config file {rclone_config_file} does not exist")
+            raise FileNotFoundError(
+                f"rclone config file {rclone_config_file} does not exist"
+            )
+
+        if rclone_config_file is not None and rclone_config_file.exists():
+            logger.info(f"Using rclone config file {rclone_config_file.absolute()}")
+            self.rclone_command.insert(1, "--config")
+            self.rclone_command.insert(2, str(rclone_config_file.absolute()))
+        elif rclone_config is not None:
+            logger.info(f"Using rclone config for remote {rclone_config.remote_name}")
+        else:
+            logger.info(
+                "No rclone config file or rclone config object provided, using default rclone config, probably [HOME]/.config/rclone/rclone.conf"
+            )
+
+    def uploadTo(
+        self,
+        source_file,
+        destination_file: Path,
+        update_only_if_newer_mode=False,
+    ) -> "RCloneCommandBuilder":
+        rclone_remote_pathstr = str(destination_file).lstrip()
+
+        rclone_remote_pathstr = self._apply_remote_prefix(rclone_remote_pathstr)
+
+        return self.copyTo(
+            source_file, update_only_if_newer_mode, rclone_remote_pathstr
+        )
+
+    def downloadTo(
+        self,
+        source_file,
+        destination_file,
+        update_only_if_newer_mode=False,
+    ) -> "RCloneCommandBuilder":
+        rclone_remote_pathstr = str(source_file).lstrip()
+
+        rclone_remote_pathstr = self._apply_remote_prefix(rclone_remote_pathstr)
+
+        return self.copyTo(
+            rclone_remote_pathstr, update_only_if_newer_mode, destination_file
+        )
+
+    def deleteFile(
+        self,
+        remote_file,
+    ) -> "RCloneCommandBuilder":
+        rclone_remote_pathstr = str(remote_file).lstrip()
+
+        rclone_remote_pathstr = self._apply_remote_prefix(rclone_remote_pathstr)
+        logger.info(f"Using rclone to delete {str(remote_file)}")
+
+        self.rclone_command += ["deletefile", rclone_remote_pathstr]
+
+        return self
+
+    def copyTo(self, source_file, update_only_if_newer_mode, rclone_remote_pathstr):
+        if update_only_if_newer_mode:
+            logger.info(
+                "Using rclone with --update flag to skip files that are newer on the destination"
+            )
+            self.rclone_command.append("--update")
+
+        logger.info(
+            f"Using rclone to copy {str(source_file)} to {rclone_remote_pathstr}"
+        )
+
+        self.rclone_command += [
+            "copyto",
+            str(source_file),
+            rclone_remote_pathstr,
+            "--check-first",  # try and catch issues before starting upload
+            "--checksum",  # be thorough and check checksums
+            "--max-duration",
+            "30m",  # don't let the upload take forever, this is a safety net
+        ]
+
+        return self
+
+    def _apply_remote_prefix(self, rclone_remote_pathstr):
+        if self.rclone_config is not None and not rclone_remote_pathstr.startswith(
+            f"{self.rclone_config.remote_name}:"
+        ):
+            logger.debug(
+                f"Destination {rclone_remote_pathstr} does not start with {self.rclone_config.remote_name}:, adding it"
+            )
+            rclone_remote_pathstr = (
+                f"{self.rclone_config.remote_name}:{rclone_remote_pathstr}"
+            )
+
+        return rclone_remote_pathstr
+
+    def lsf(
+        self, remote_folder: Path, pattern_to_match: str | None = None
+    ) -> "RCloneCommandBuilder":
+        rclone_remote_pathstr = str(remote_folder).rstrip(
+            "/"
+        )  # Ensure no trailing slash
+        rclone_remote_pathstr = self._apply_remote_prefix(rclone_remote_pathstr)
+
+        self.rclone_command += [
+            "lsf",
+            "--files-only",
+            "--format",
+            "tsp",  # time, size, path
+            rclone_remote_pathstr,
+        ]
+
+        if pattern_to_match:
+            logger.info(
+                f"Using rclone to list files in {rclone_remote_pathstr} matching pattern {pattern_to_match}"
+            )
+            self.rclone_command += ["--include", pattern_to_match]
+
+        return self
+
+    def build(self, custom_executable_path: str | Path = None) -> list[str]:
+        """
+        Build the rclone command as a list of strings.
+        """
+
+        if custom_executable_path:
+            logger.info(f"Using custom rclone executable at {custom_executable_path}")
+            self._rclone_executable = custom_executable_path
+
+        self.rclone_command[0] = str(self._rclone_executable)
+
+        return self.rclone_command.copy()

prefect_managedfiletransfer/RCloneConfig.py

@@ -0,0 +1,10 @@
+class RCloneConfig:
+    def __init__(self, remote_name: str):
+        self.remote_name = remote_name
+        self._config_contents: str
+
+    def get_config(self):
+        return self._config_contents
+
+    async def update_config(self, config_contents: str):
+        self._config_contents = config_contents

prefect_managedfiletransfer/RCloneConfigFileBlock.py

@@ -0,0 +1,51 @@
+from pydantic import Field
+from prefect.blocks.core import Block
+
+
+class RCloneConfigFileBlock(Block):
+    """
+    Block for storing RClone configuration file contents.
+    This block is used to store the contents of an RClone configuration file, which can be used to configure RClone for file transfers.
+    The block is updated with tokends when they are refreshed, allowing for dynamic updates to the RClone configuration.
+
+    Generate a config locally with `rclone config create my_sharepoint onedrive` like below, then save the contents in a block with remote_name=my_sharepoint,config_file_contents=
+    [my_sharepoint]
+    type = onedrive
+    token = {"access_token":"...","token_type":"Bearer","refresh_token":"...","expiry":"2000-00-00T00:00:00.000000000Z"}
+    drive_id = b!-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+    drive_type = documentLibrary
+
+    Attributes:
+        remote_name (str): The name of the remote connection.
+        config_file_contents (str): The contents of the RClone configuration file.
+
+    Example:
+        Load a stored value:
+        ```python
+        from prefect_managedfiletransfer import RCloneConfigFileBlock
+        block = RCloneConfigFileBlock.load("BLOCK_NAME")
+        ```
+
+        Creating a block:
+        ```python
+        from prefect_managedfiletransfer import RCloneConfigFileBlock
+        block = RCloneConfigFileBlock(
+            remote_name="my_sharepoint",
+            config_file_contents="..."
+        )
+        block.save("my_sharepoint_block", overwrite=True)
+        ```
+    """
+
+    _block_type_name = "RClone Remote Config File [ManagedFileTransfer]"
+    _logo_url = "https://github.com/rclone/rclone/blob/master/graphics/logo/logo_symbol/logo_symbol_color_64px.png?raw=true"
+    _documentation_url = "https://ImperialCollegeLondon.github.io/prefect-managedfiletransfer/blocks/#prefect-managedfiletransfer.blocks.RCloneConfigFileBlock"  # noqa
+
+    remote_name: str = Field(
+        title="Remote Name",
+        description="The name of the remote connection in the RClone configuration.",
+    )
+    config_file_contents: str = Field(
+        title="Config File Contents",
+        description="The contents of the RClone configuration file.",
+    )

prefect_managedfiletransfer/RCloneConfigSavedInPrefect.py

@@ -0,0 +1,19 @@
+from prefect_managedfiletransfer import RCloneConfigFileBlock
+from prefect_managedfiletransfer.RCloneConfig import RCloneConfig
+
+
+class RCloneConfigSavedInPrefect(RCloneConfig):
+    """
+    RCloneDynamicConfig that uses a Prefect block to store the rclone config file contents. Is saved after sucessful uploads to update the token when it is refreshed
+    """
+
+    def __init__(self, block: RCloneConfigFileBlock):
+        self._block = block
+        self.remote_name = block.remote_name
+
+    def get_config(self):
+        return self._block.config_file_contents
+
+    async def update_config(self, config_contents: str):
+        self._block.config_file_contents = config_contents
+        await self._block.save(overwrite=True)

prefect_managedfiletransfer/RemoteAsset.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from pathlib import Path
+
+
+class RemoteAsset:
+    """
+    Represents a remote asset with its path and last modified time.
+    """
+
+    def __init__(self, path: Path, last_modified: datetime, size: int | None = None):
+        self.path: Path = path
+        self.last_modified: datetime = last_modified
+        self.size: int | None = size
+
+    def __repr__(self):
+        return f"RemoteAsset(path={self.path}, last_modified={self.last_modified}, size={self.size})"

prefect_managedfiletransfer/RemoteConnectionType.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class RemoteConnectionType(Enum):
+    LOCAL = "local"
+    SFTP = "sftp"
+    RCLONE = "rclone"

prefect_managedfiletransfer/ServerWithBasicAuthBlock.py

@@ -0,0 +1,64 @@
+from typing import Optional
+from prefect.blocks.core import Block
+from pydantic import Field, SecretStr
+
+from prefect_managedfiletransfer.constants import CONSTANTS
+
+
+class ServerWithBasicAuthBlock(Block):
+    """
+    A connection to a remote server with basic authentication.
+
+    Attributes:
+        username (str): The username for authentication.
+        password (SecretStr): The password for authentication.
+        host (str): The host of the server.
+        port (int): The port of the server.
+
+    Example:
+        Load a stored value:
+        ```python
+        from prefect_managedfiletransfer import ServerWithBasicAuthBlock
+        block = ServerWithBasicAuthBlock.load("BLOCK_NAME")
+        ```
+    """
+
+    _block_type_name = "Server - Basic Auth [ManagedFileTransfer]"
+    _logo_url = CONSTANTS.SERVER_LOGO_URL
+    _documentation_url = "https://ImperialCollegeLondon.github.io/prefect-managedfiletransfer/blocks/#prefect-managedfiletransfer.blocks.ServerWithBasicAuthBlock"  # noqa
+
+    username: str = Field(
+        title="The username for authentication.",
+        description="The username for authentication.",
+    )
+    password: Optional[SecretStr] = Field(
+        default=None,
+        title="The password for authentication.",
+        description="The password for authentication.",
+    )
+    host: str = Field(
+        title="The host of the server.", description="The host of the server."
+    )
+    port: int = Field(default=22, description="The port of the server.")
+
+    @classmethod
+    def seed_value_for_example(cls):
+        """
+        Seeds the field, value, so the block can be loaded.
+        """
+        block = cls(
+            username="example_user",
+            password=SecretStr("example_password"),
+            host="example.com",
+            port=22,
+        )
+        block.save("sample-block", overwrite=True)
+
+    def isValid(self) -> bool:
+        """Checks if the server credentials are available and valid."""
+        return (
+            self.username
+            and self.password.get_secret_value()
+            and self.host
+            and self.port > 0
+        )