prefect-client 2.20.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

prefect/filesystems.py

@@ -1,33 +1,22 @@
 import abc
-import io
-import json
 import urllib.parse
 from pathlib import Path
-from shutil import ignore_patterns
-from tempfile import TemporaryDirectory
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, Optional
 
 import anyio
 import fsspec
-
-from prefect._internal.compatibility.deprecated import deprecated_class
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import Field, SecretStr, validator
-else:
-    from pydantic import Field, SecretStr, validator
+from pydantic import Field, SecretStr, field_validator
 
 from prefect._internal.schemas.validators import (
     stringify_path,
     validate_basepath,
-    validate_github_access_token,
 )
 from prefect.blocks.core import Block
 from prefect.utilities.asyncutils import run_sync_in_worker_thread, sync_compatible
 from prefect.utilities.compat import copytree
 from prefect.utilities.filesystem import filter_files
-from prefect.utilities.processutils import run_process
+
+from ._internal.compatibility.migration import getattr_migration
 
 
 class ReadableFileSystem(Block, abc.ABC):
@@ -55,7 +44,7 @@ class ReadableDeploymentStorage(Block, abc.ABC):
 
     @abc.abstractmethod
     async def get_directory(
-        self, from_path: str = None, local_path: str = None
+        self, from_path: Optional[str] = None, local_path: Optional[str] = None
     ) -> None:
         pass
 
@@ -65,13 +54,16 @@ class WritableDeploymentStorage(Block, abc.ABC):
 
     @abc.abstractmethod
     async def get_directory(
-        self, from_path: str = None, local_path: str = None
+        self, from_path: Optional[str] = None, local_path: Optional[str] = None
     ) -> None:
         pass
 
     @abc.abstractmethod
     async def put_directory(
-        self, local_path: str = None, to_path: str = None, ignore_file: str = None
+        self,
+        local_path: Optional[str] = None,
+        to_path: Optional[str] = None,
+        ignore_file: Optional[str] = None,
     ) -> None:
         pass
 
@@ -99,11 +91,11 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         default=None, description="Default local path for this block to write to."
     )
 
-    @validator("basepath", pre=True)
+    @field_validator("basepath", mode="before")
     def cast_pathlib(cls, value):
         return stringify_path(value)
 
-    def _resolve_path(self, path: str) -> Path:
+    def _resolve_path(self, path: str, validate: bool = False) -> Path:
         # Only resolve the base path at runtime, default to the current directory
         basepath = (
             Path(self.basepath).expanduser().resolve()
@@ -116,22 +108,23 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         if path is None:
             return basepath
 
-        path: Path = Path(path).expanduser()
+        resolved_path: Path = Path(path).expanduser()
 
-        if not path.is_absolute():
-            path = basepath / path
+        if not resolved_path.is_absolute():
+            resolved_path = basepath / resolved_path
         else:
-            path = path.resolve()
-            if basepath not in path.parents and (basepath != path):
+            resolved_path = resolved_path.resolve()
+
+        if validate:
+            if basepath not in resolved_path.parents and (basepath != resolved_path):
                 raise ValueError(
-                    f"Provided path {path} is outside of the base path {basepath}."
+                    f"Provided path {resolved_path} is outside of the base path {basepath}."
                 )
-
-        return path
+        return resolved_path
 
     @sync_compatible
     async def get_directory(
-        self, from_path: str = None, local_path: str = None
+        self, from_path: Optional[str] = None, local_path: Optional[str] = None
     ) -> None:
         """
         Copies a directory from one place to another on the local filesystem.
@@ -181,7 +174,10 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
 
     @sync_compatible
     async def put_directory(
-        self, local_path: str = None, to_path: str = None, ignore_file: str = None
+        self,
+        local_path: Optional[str] = None,
+        to_path: Optional[str] = None,
+        ignore_file: Optional[str] = None,
     ) -> None:
         """
         Copies a directory from one place to another on the local filesystem.
@@ -189,7 +185,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         Defaults to copying the entire contents of the current working directory to the block's basepath.
         An `ignore_file` path may be provided that can include gitignore style expressions for filepaths to ignore.
         """
-        destination_path = self._resolve_path(to_path)
+        destination_path = self._resolve_path(to_path, validate=True)
 
         if not local_path:
             local_path = Path(".").absolute()
@@ -280,7 +276,7 @@ class RemoteFileSystem(WritableFileSystem, WritableDeploymentStorage):
     # Cache for the configured fsspec file system used for access
     _filesystem: fsspec.AbstractFileSystem = None
 
-    @validator("basepath")
+    @field_validator("basepath")
     def check_basepath(cls, value):
         return validate_basepath(value)
 
@@ -422,368 +418,6 @@ class RemoteFileSystem(WritableFileSystem, WritableDeploymentStorage):
         return self._filesystem
 
 
-@deprecated_class(
-    start_date="Mar 2024", help="Use the `S3Bucket` block from prefect-aws instead."
-)
-class S3(WritableFileSystem, WritableDeploymentStorage):
-    """
-    DEPRECATION WARNING:
-
-    This class is deprecated as of March 2024 and will not be available after September 2024.
-    It has been replaced by `S3Bucket` from the `prefect-aws` package, which offers enhanced functionality
-    and better a better user experience.
-
-    Store data as a file on AWS S3.
-
-    Example:
-        Load stored S3 config:
-        ```python
-        from prefect.filesystems import S3
-
-        s3_block = S3.load("BLOCK_NAME")
-        ```
-    """
-
-    _block_type_name = "S3"
-    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/d74b16fe84ce626345adf235a47008fea2869a60-225x225.png"
-    _documentation_url = "https://docs.prefect.io/concepts/filesystems/#s3"
-
-    bucket_path: str = Field(
-        default=...,
-        description="An S3 bucket path.",
-        examples=["my-bucket/a-directory-within"],
-    )
-    aws_access_key_id: Optional[SecretStr] = Field(
-        default=None,
-        title="AWS Access Key ID",
-        description="Equivalent to the AWS_ACCESS_KEY_ID environment variable.",
-        examples=["AKIAIOSFODNN7EXAMPLE"],
-    )
-    aws_secret_access_key: Optional[SecretStr] = Field(
-        default=None,
-        title="AWS Secret Access Key",
-        description="Equivalent to the AWS_SECRET_ACCESS_KEY environment variable.",
-        examples=["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"],
-    )
-
-    _remote_file_system: RemoteFileSystem = None
-
-    @property
-    def basepath(self) -> str:
-        return f"s3://{self.bucket_path}"
-
-    @property
-    def filesystem(self) -> RemoteFileSystem:
-        settings = {}
-        if self.aws_access_key_id:
-            settings["key"] = self.aws_access_key_id.get_secret_value()
-        if self.aws_secret_access_key:
-            settings["secret"] = self.aws_secret_access_key.get_secret_value()
-        self._remote_file_system = RemoteFileSystem(
-            basepath=f"s3://{self.bucket_path}", settings=settings
-        )
-        return self._remote_file_system
-
-    @sync_compatible
-    async def get_directory(
-        self, from_path: Optional[str] = None, local_path: Optional[str] = None
-    ) -> bytes:
-        """
-        Downloads a directory from a given remote path to a local directory.
-
-        Defaults to downloading the entire contents of the block's basepath to the current working directory.
-        """
-        return await self.filesystem.get_directory(
-            from_path=from_path, local_path=local_path
-        )
-
-    @sync_compatible
-    async def put_directory(
-        self,
-        local_path: Optional[str] = None,
-        to_path: Optional[str] = None,
-        ignore_file: Optional[str] = None,
-    ) -> int:
-        """
-        Uploads a directory from a given local path to a remote directory.
-
-        Defaults to uploading the entire contents of the current working directory to the block's basepath.
-        """
-        return await self.filesystem.put_directory(
-            local_path=local_path, to_path=to_path, ignore_file=ignore_file
-        )
-
-    @sync_compatible
-    async def read_path(self, path: str) -> bytes:
-        return await self.filesystem.read_path(path)
-
-    @sync_compatible
-    async def write_path(self, path: str, content: bytes) -> str:
-        return await self.filesystem.write_path(path=path, content=content)
-
-
-@deprecated_class(
-    start_date="Mar 2024", help="Use the `GcsBucket` block from prefect-gcp instead."
-)
-class GCS(WritableFileSystem, WritableDeploymentStorage):
-    """
-    DEPRECATION WARNING:
-
-    This class is deprecated as of March 2024 and will not be available after September 2024.
-    It has been replaced by `GcsBucket` from the `prefect-gcp` package, which offers enhanced functionality
-    and better a better user experience.
-    Store data as a file on Google Cloud Storage.
-
-    Example:
-        Load stored GCS config:
-        ```python
-        from prefect.filesystems import GCS
-
-        gcs_block = GCS.load("BLOCK_NAME")
-        ```
-    """
-
-    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/422d13bb838cf247eb2b2cf229ce6a2e717d601b-256x256.png"
-    _documentation_url = "https://docs.prefect.io/concepts/filesystems/#gcs"
-
-    bucket_path: str = Field(
-        default=...,
-        description="A GCS bucket path.",
-        examples=["my-bucket/a-directory-within"],
-    )
-    service_account_info: Optional[SecretStr] = Field(
-        default=None,
-        description="The contents of a service account keyfile as a JSON string.",
-    )
-    project: Optional[str] = Field(
-        default=None,
-        description=(
-            "The project the GCS bucket resides in. If not provided, the project will"
-            " be inferred from the credentials or environment."
-        ),
-    )
-
-    @property
-    def basepath(self) -> str:
-        return f"gcs://{self.bucket_path}"
-
-    @property
-    def filesystem(self) -> RemoteFileSystem:
-        settings = {}
-        if self.service_account_info:
-            try:
-                settings["token"] = json.loads(
-                    self.service_account_info.get_secret_value()
-                )
-            except json.JSONDecodeError:
-                raise ValueError(
-                    "Unable to load provided service_account_info. Please make sure"
-                    " that the provided value is a valid JSON string."
-                )
-        remote_file_system = RemoteFileSystem(
-            basepath=f"gcs://{self.bucket_path}", settings=settings
-        )
-        return remote_file_system
-
-    @sync_compatible
-    async def get_directory(
-        self, from_path: Optional[str] = None, local_path: Optional[str] = None
-    ) -> bytes:
-        """
-        Downloads a directory from a given remote path to a local directory.
-
-        Defaults to downloading the entire contents of the block's basepath to the current working directory.
-        """
-        return await self.filesystem.get_directory(
-            from_path=from_path, local_path=local_path
-        )
-
-    @sync_compatible
-    async def put_directory(
-        self,
-        local_path: Optional[str] = None,
-        to_path: Optional[str] = None,
-        ignore_file: Optional[str] = None,
-    ) -> int:
-        """
-        Uploads a directory from a given local path to a remote directory.
-
-        Defaults to uploading the entire contents of the current working directory to the block's basepath.
-        """
-        return await self.filesystem.put_directory(
-            local_path=local_path, to_path=to_path, ignore_file=ignore_file
-        )
-
-    @sync_compatible
-    async def read_path(self, path: str) -> bytes:
-        return await self.filesystem.read_path(path)
-
-    @sync_compatible
-    async def write_path(self, path: str, content: bytes) -> str:
-        return await self.filesystem.write_path(path=path, content=content)
-
-
-@deprecated_class(
-    start_date="Mar 2024",
-    help="Use the `AzureBlobStorageContainer` block from prefect-azure instead.",
-)
-class Azure(WritableFileSystem, WritableDeploymentStorage):
-    """
-    DEPRECATION WARNING:
-
-    This class is deprecated as of March 2024 and will not be available after September 2024.
-    It has been replaced by `AzureBlobStorageContainer` from the `prefect-azure` package, which
-    offers enhanced functionality and better a better user experience.
-
-    Store data as a file on Azure Datalake and Azure Blob Storage.
-
-    Example:
-        Load stored Azure config:
-        ```python
-        from prefect.filesystems import Azure
-
-        az_block = Azure.load("BLOCK_NAME")
-        ```
-    """
-
-    _block_type_name = "Azure"
-    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/54e3fa7e00197a4fbd1d82ed62494cb58d08c96a-250x250.png"
-    _documentation_url = "https://docs.prefect.io/concepts/filesystems/#azure"
-
-    bucket_path: str = Field(
-        default=...,
-        description="An Azure storage bucket path.",
-        examples=["my-bucket/a-directory-within"],
-    )
-    azure_storage_connection_string: Optional[SecretStr] = Field(
-        default=None,
-        title="Azure storage connection string",
-        description=(
-            "Equivalent to the AZURE_STORAGE_CONNECTION_STRING environment variable."
-        ),
-    )
-    azure_storage_account_name: Optional[SecretStr] = Field(
-        default=None,
-        title="Azure storage account name",
-        description=(
-            "Equivalent to the AZURE_STORAGE_ACCOUNT_NAME environment variable."
-        ),
-    )
-    azure_storage_account_key: Optional[SecretStr] = Field(
-        default=None,
-        title="Azure storage account key",
-        description="Equivalent to the AZURE_STORAGE_ACCOUNT_KEY environment variable.",
-    )
-    azure_storage_tenant_id: Optional[SecretStr] = Field(
-        None,
-        title="Azure storage tenant ID",
-        description="Equivalent to the AZURE_TENANT_ID environment variable.",
-    )
-    azure_storage_client_id: Optional[SecretStr] = Field(
-        None,
-        title="Azure storage client ID",
-        description="Equivalent to the AZURE_CLIENT_ID environment variable.",
-    )
-    azure_storage_client_secret: Optional[SecretStr] = Field(
-        None,
-        title="Azure storage client secret",
-        description="Equivalent to the AZURE_CLIENT_SECRET environment variable.",
-    )
-    azure_storage_anon: bool = Field(
-        default=True,
-        title="Azure storage anonymous connection",
-        description=(
-            "Set the 'anon' flag for ADLFS. This should be False for systems that"
-            " require ADLFS to use DefaultAzureCredentials."
-        ),
-    )
-    azure_storage_container: Optional[SecretStr] = Field(
-        default=None,
-        title="Azure storage container",
-        description=(
-            "Blob Container in Azure Storage Account. If set the 'bucket_path' will"
-            " be interpreted using the following URL format:"
-            "'az://<container>@<storage_account>.dfs.core.windows.net/<bucket_path>'."
-        ),
-    )
-    _remote_file_system: RemoteFileSystem = None
-
-    @property
-    def basepath(self) -> str:
-        if self.azure_storage_container:
-            return (
-                f"az://{self.azure_storage_container.get_secret_value()}"
-                f"@{self.azure_storage_account_name.get_secret_value()}"
-                f".dfs.core.windows.net/{self.bucket_path}"
-            )
-        else:
-            return f"az://{self.bucket_path}"
-
-    @property
-    def filesystem(self) -> RemoteFileSystem:
-        settings = {}
-        if self.azure_storage_connection_string:
-            settings[
-                "connection_string"
-            ] = self.azure_storage_connection_string.get_secret_value()
-        if self.azure_storage_account_name:
-            settings[
-                "account_name"
-            ] = self.azure_storage_account_name.get_secret_value()
-        if self.azure_storage_account_key:
-            settings["account_key"] = self.azure_storage_account_key.get_secret_value()
-        if self.azure_storage_tenant_id:
-            settings["tenant_id"] = self.azure_storage_tenant_id.get_secret_value()
-        if self.azure_storage_client_id:
-            settings["client_id"] = self.azure_storage_client_id.get_secret_value()
-        if self.azure_storage_client_secret:
-            settings[
-                "client_secret"
-            ] = self.azure_storage_client_secret.get_secret_value()
-        settings["anon"] = self.azure_storage_anon
-        self._remote_file_system = RemoteFileSystem(
-            basepath=self.basepath, settings=settings
-        )
-        return self._remote_file_system
-
-    @sync_compatible
-    async def get_directory(
-        self, from_path: Optional[str] = None, local_path: Optional[str] = None
-    ) -> bytes:
-        """
-        Downloads a directory from a given remote path to a local directory.
-
-        Defaults to downloading the entire contents of the block's basepath to the current working directory.
-        """
-        return await self.filesystem.get_directory(
-            from_path=from_path, local_path=local_path
-        )
-
-    @sync_compatible
-    async def put_directory(
-        self,
-        local_path: Optional[str] = None,
-        to_path: Optional[str] = None,
-        ignore_file: Optional[str] = None,
-    ) -> int:
-        """
-        Uploads a directory from a given local path to a remote directory.
-
-        Defaults to uploading the entire contents of the current working directory to the block's basepath.
-        """
-        return await self.filesystem.put_directory(
-            local_path=local_path, to_path=to_path, ignore_file=ignore_file
-        )
-
-    @sync_compatible
-    async def read_path(self, path: str) -> bytes:
-        return await self.filesystem.read_path(path)
-
-    @sync_compatible
-    async def write_path(self, path: str, content: bytes) -> str:
-        return await self.filesystem.write_path(path=path, content=content)
-
-
 class SMB(WritableFileSystem, WritableDeploymentStorage):
     """
     Store data as a file on a SMB share.
@@ -815,7 +449,7 @@ class SMB(WritableFileSystem, WritableDeploymentStorage):
         default=None, title="SMB Password", description="Password for SMB access."
     )
     smb_host: str = Field(
-        default=..., tile="SMB server/hostname", description="SMB server/hostname."
+        default=..., title="SMB server/hostname", description="SMB server/hostname."
     )
     smb_port: Optional[int] = Field(
         default=None, title="SMB port", description="SMB port (default: 445)."
@@ -883,137 +517,4 @@ class SMB(WritableFileSystem, WritableDeploymentStorage):
         return await self.filesystem.write_path(path=path, content=content)
 
 
-@deprecated_class(
-    start_date="Mar 2024",
-    help="Use the `GitHubRepository` block from prefect-github instead.",
-)
-class GitHub(ReadableDeploymentStorage):
-    """
-        DEPRECATION WARNING:
-
-        This class is deprecated as of March 2024 and will not be available after September 2024.
-        It has been replaced by `GitHubRepository` from the `prefect-github` package, which offers
-        enhanced functionality and better a better user experience.
-    q
-        Interact with files stored on GitHub repositories.
-    """
-
-    _block_type_name = "GitHub"
-    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/41971cfecfea5f79ff334164f06ecb34d1038dd4-250x250.png"
-    _documentation_url = "https://docs.prefect.io/concepts/filesystems/#github"
-
-    repository: str = Field(
-        default=...,
-        description=(
-            "The URL of a GitHub repository to read from, in either HTTPS or SSH"
-            " format."
-        ),
-    )
-    reference: Optional[str] = Field(
-        default=None,
-        description="An optional reference to pin to; can be a branch name or tag.",
-    )
-    access_token: Optional[SecretStr] = Field(
-        name="Personal Access Token",
-        default=None,
-        description=(
-            "A GitHub Personal Access Token (PAT) with repo scope."
-            " To use a fine-grained PAT, provide '{username}:{PAT}' as the value."
-        ),
-    )
-    include_git_objects: bool = Field(
-        default=True,
-        description=(
-            "Whether to include git objects when copying the repo contents to a"
-            " directory."
-        ),
-    )
-
-    @validator("access_token")
-    def _ensure_credentials_go_with_https(cls, v: str, values: dict) -> str:
-        return validate_github_access_token(v, values)
-
-    def _create_repo_url(self) -> str:
-        """Format the URL provided to the `git clone` command.
-
-        For private repos: https://<oauth-key>@github.com/<username>/<repo>.git
-        All other repos should be the same as `self.repository`.
-        """
-        url_components = urllib.parse.urlparse(self.repository)
-        if url_components.scheme == "https" and self.access_token is not None:
-            updated_components = url_components._replace(
-                netloc=f"{self.access_token.get_secret_value()}@{url_components.netloc}"
-            )
-            full_url = urllib.parse.urlunparse(updated_components)
-        else:
-            full_url = self.repository
-
-        return full_url
-
-    @staticmethod
-    def _get_paths(
-        dst_dir: Union[str, None], src_dir: str, sub_directory: str
-    ) -> Tuple[str, str]:
-        """Returns the fully formed paths for GitHubRepository contents in the form
-        (content_source, content_destination).
-        """
-        if dst_dir is None:
-            content_destination = Path(".").absolute()
-        else:
-            content_destination = Path(dst_dir)
-
-        content_source = Path(src_dir)
-
-        if sub_directory:
-            content_destination = content_destination.joinpath(sub_directory)
-            content_source = content_source.joinpath(sub_directory)
-
-        return str(content_source), str(content_destination)
-
-    @sync_compatible
-    async def get_directory(
-        self, from_path: Optional[str] = None, local_path: Optional[str] = None
-    ) -> None:
-        """
-        Clones a GitHub project specified in `from_path` to the provided `local_path`;
-        defaults to cloning the repository reference configured on the Block to the
-        present working directory.
-
-        Args:
-            from_path: If provided, interpreted as a subdirectory of the underlying
-                repository that will be copied to the provided local path.
-            local_path: A local path to clone to; defaults to present working directory.
-        """
-        # CONSTRUCT COMMAND
-        cmd = ["git", "clone", self._create_repo_url()]
-        if self.reference:
-            cmd += ["-b", self.reference]
-
-        # Limit git history
-        cmd += ["--depth", "1"]
-
-        # Clone to a temporary directory and move the subdirectory over
-        with TemporaryDirectory(suffix="prefect") as tmp_dir:
-            cmd.append(tmp_dir)
-
-            err_stream = io.StringIO()
-            out_stream = io.StringIO()
-            process = await run_process(cmd, stream_output=(out_stream, err_stream))
-            if process.returncode != 0:
-                err_stream.seek(0)
-                raise OSError(f"Failed to pull from remote:\n {err_stream.read()}")
-
-            content_source, content_destination = self._get_paths(
-                dst_dir=local_path, src_dir=tmp_dir, sub_directory=from_path
-            )
-
-            ignore_func = None
-            if not self.include_git_objects:
-                ignore_func = ignore_patterns(".git")
-
-            copytree(
-                src=content_source,
-                dst=content_destination,
-                dirs_exist_ok=True,
-                ignore=ignore_func,
-            )
+__getattr__ = getattr_migration(__name__)