genesis-flow 1.0.0__py3-none-any.whl

mlflow/store/artifact/models_artifact_repo.py

@@ -0,0 +1,259 @@
+import logging
+import os
+import urllib.parse
+from pathlib import Path
+from typing import Optional, Union
+
+import mlflow
+from mlflow.exceptions import MlflowException
+from mlflow.store.artifact.artifact_repo import ArtifactRepository
+from mlflow.store.artifact.databricks_models_artifact_repo import DatabricksModelsArtifactRepository
+from mlflow.store.artifact.unity_catalog_models_artifact_repo import (
+    UnityCatalogModelsArtifactRepository,
+)
+from mlflow.store.artifact.unity_catalog_oss_models_artifact_repo import (
+    UnityCatalogOSSModelsArtifactRepository,
+)
+from mlflow.store.artifact.utils.models import (
+    _parse_model_uri,
+    get_model_name_and_version,
+    is_using_databricks_registry,
+)
+from mlflow.utils.uri import (
+    add_databricks_profile_info_to_artifact_uri,
+    get_databricks_profile_uri_from_artifact_uri,
+    is_databricks_unity_catalog_uri,
+    is_models_uri,
+    is_oss_unity_catalog_uri,
+)
+
+REGISTERED_MODEL_META_FILE_NAME = "registered_model_meta"
+
+_logger = logging.getLogger(__name__)
+
+
+class ModelsArtifactRepository(ArtifactRepository):
+    """
+    Handles artifacts associated with a model version in the model registry via URIs of the form:
+      - `models:/<model_name>/<model_version>`
+      - `models:/<model_name>/<stage>`  (refers to the latest model version in the given stage)
+      - `models:/<model_name>/latest` (refers to the latest of all model versions)
+    It is a light wrapper that resolves the artifact path to an absolute URI then instantiates
+    and uses the artifact repository for that URI.
+    """
+
+    def __init__(self, artifact_uri: str, tracking_uri: Optional[str] = None) -> None:
+        from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
+
+        super().__init__(artifact_uri, tracking_uri)
+        registry_uri = mlflow.get_registry_uri()
+        self.is_logged_model_uri = self._is_logged_model_uri(artifact_uri)
+        if is_databricks_unity_catalog_uri(uri=registry_uri) and not self.is_logged_model_uri:
+            self.repo = UnityCatalogModelsArtifactRepository(
+                artifact_uri=artifact_uri, registry_uri=registry_uri
+            )
+            self.model_name = self.repo.model_name
+            self.model_version = self.repo.model_version
+        elif is_oss_unity_catalog_uri(uri=registry_uri) and not self.is_logged_model_uri:
+            self.repo = UnityCatalogOSSModelsArtifactRepository(
+                artifact_uri=artifact_uri, registry_uri=registry_uri
+            )
+            self.model_name = self.repo.model_name
+            self.model_version = self.repo.model_version
+        elif is_using_databricks_registry(artifact_uri) and not self.is_logged_model_uri:
+            # Use the DatabricksModelsArtifactRepository if a databricks profile is being used.
+            self.repo = DatabricksModelsArtifactRepository(artifact_uri)
+            self.model_name = self.repo.model_name
+            self.model_version = self.repo.model_version
+        else:
+            (
+                self.model_name,
+                self.model_version,
+                underlying_uri,
+            ) = ModelsArtifactRepository._get_model_uri_infos(artifact_uri)
+            self.repo = get_artifact_repository(underlying_uri)
+            # TODO: it may be nice to fall back to the source URI explicitly here if for some reason
+            #  we don't get a download URI here, or fail during the download itself.
+
+    @staticmethod
+    def is_models_uri(uri):
+        return urllib.parse.urlparse(uri).scheme == "models"
+
+    @staticmethod
+    def split_models_uri(uri):
+        """
+        Split 'models:/<name>/<version>/path/to/model' into
+        ('models:/<name>/<version>', 'path/to/model').
+        Split 'models://<scope>:<prefix>@databricks/<name>/<version>/path/to/model' into
+        ('models://<scope>:<prefix>@databricks/<name>/<version>', 'path/to/model').
+        Split 'models:/<name>@alias/path/to/model' into
+        ('models:/<name>@alias', 'path/to/model').
+        """
+        uri = uri.rstrip("/")
+        parsed_url = urllib.parse.urlparse(uri)
+        path = parsed_url.path
+        netloc = parsed_url.netloc
+        if path.count("/") >= 2 and not path.endswith("/"):
+            splits = path.split("/", 3)
+            cut_index = 2 if "@" in splits[1] else 3
+            model_name_and_version = splits[:cut_index]
+            artifact_path = "/".join(splits[cut_index:])
+            base_part = f"models://{netloc}" if netloc else "models:"
+            return base_part + "/".join(model_name_and_version), artifact_path
+        return uri, ""
+
+    @staticmethod
+    def _is_logged_model_uri(uri: Union[str, Path]) -> bool:
+        """
+        Returns True if the URI is a logged model URI (e.g. 'models:/<model_id>'), False otherwise.
+        """
+        uri = str(uri)
+        return is_models_uri(uri) and _parse_model_uri(uri).model_id is not None
+
+    @staticmethod
+    def _get_model_uri_infos(uri):
+        # Note: to support a registry URI that is different from the tracking URI here,
+        # we'll need to add setting of registry URIs via environment variables.
+
+        from mlflow import MlflowClient
+
+        databricks_profile_uri = (
+            get_databricks_profile_uri_from_artifact_uri(uri) or mlflow.get_registry_uri()
+        )
+        client = MlflowClient(registry_uri=databricks_profile_uri)
+        name_and_version_or_id = get_model_name_and_version(client, uri)
+        if len(name_and_version_or_id) == 1:
+            name = None
+            version = None
+            model_id = name_and_version_or_id[0]
+            download_uri = client.get_logged_model(model_id).artifact_location
+        else:
+            name, version = name_and_version_or_id
+            download_uri = client.get_model_version_download_uri(name, version)
+
+        return (
+            name,
+            version,
+            add_databricks_profile_info_to_artifact_uri(download_uri, databricks_profile_uri),
+        )
+
+    @staticmethod
+    def get_underlying_uri(uri):
+        _, _, underlying_uri = ModelsArtifactRepository._get_model_uri_infos(uri)
+
+        return underlying_uri
+
+    def log_artifact(self, local_file, artifact_path=None):
+        """
+        Log a local file as an artifact, optionally taking an ``artifact_path`` to place it in
+        within the run's artifacts. Run artifacts can be organized into directories, so you can
+        place the artifact in a directory this way.
+
+        Args:
+            local_file: Path to artifact to log.
+            artifact_path: Directory within the run's artifact directory in which to log the
+                artifact.
+        """
+        if self.is_logged_model_uri:
+            return self.repo.log_artifact(local_file, artifact_path)
+        raise ValueError(
+            "log_artifact is not supported for models:/<name>/<version> URIs. "
+            "Use register_model instead."
+        )
+
+    def log_artifacts(self, local_dir, artifact_path=None):
+        """
+        Log the files in the specified local directory as artifacts, optionally taking
+        an ``artifact_path`` to place them in within the run's artifacts.
+
+        Args:
+            local_dir: Directory of local artifacts to log.
+            artifact_path: Directory within the run's artifact directory in which to log the
+                artifacts.
+        """
+        if self.is_logged_model_uri:
+            return self.repo.log_artifacts(local_dir, artifact_path)
+        raise ValueError(
+            "log_artifacts is not supported for models:/<name>/<version> URIs. "
+            "Use register_model instead."
+        )
+
+    def list_artifacts(self, path):
+        """
+        Return all the artifacts for this run_id directly under path. If path is a file, returns
+        an empty list. Will error if path is neither a file nor directory.
+
+        Args:
+            path: Relative source path that contain desired artifacts.
+
+        Returns:
+            List of artifacts as FileInfo listed directly under path.
+        """
+        return self.repo.list_artifacts(path)
+
+    def _add_registered_model_meta_file(self, model_path):
+        from mlflow.utils.yaml_utils import write_yaml
+
+        write_yaml(
+            model_path,
+            REGISTERED_MODEL_META_FILE_NAME,
+            {
+                "model_name": self.model_name,
+                "model_version": self.model_version,
+            },
+            overwrite=True,
+            ensure_yaml_extension=False,
+        )
+
+    def download_artifacts(self, artifact_path, dst_path=None, lineage_header_info=None):
+        """
+        Download an artifact file or directory to a local directory if applicable, and return a
+        local path for it.
+        For registered models, when the artifact is downloaded, the model name and version
+        are saved in the "registered_model_meta" file on the caller's side.
+        The caller is responsible for managing the lifecycle of the downloaded artifacts.
+
+        Args:
+            artifact_path: Relative source path to the desired artifacts.
+            dst_path: Absolute path of the local filesystem destination directory to which to
+                download the specified artifacts. This directory must already exist.
+                If unspecified, the artifacts will either be downloaded to a new
+                uniquely-named directory on the local filesystem or will be returned
+                directly in the case of the LocalArtifactRepository.
+            lineage_header_info: Linear header information.
+
+        Returns:
+            Absolute path of the local filesystem location containing the desired artifacts.
+        """
+
+        from mlflow.models.model import MLMODEL_FILE_NAME
+
+        # Pass lineage header info if model is registered in UC
+        if isinstance(self.repo, UnityCatalogModelsArtifactRepository):
+            model_path = self.repo.download_artifacts(
+                artifact_path, dst_path, lineage_header_info=lineage_header_info
+            )
+        else:
+            model_path = self.repo.download_artifacts(artifact_path, dst_path)
+        # NB: only add the registered model metadata iff the artifact path is at the root model
+        # directory. For individual files or subdirectories within the model directory, do not
+        # create the metadata file.
+        if os.path.isdir(model_path) and MLMODEL_FILE_NAME in os.listdir(model_path):
+            self._add_registered_model_meta_file(model_path)
+
+        return model_path
+
+    def _download_file(self, remote_file_path, local_path):
+        """
+        Download the file at the specified relative remote path and saves
+        it at the specified local path.
+
+        Args:
+            remote_file_path: Source path to the remote file, relative to the root
+                directory of the artifact repository.
+            local_path: The path to which to save the downloaded file.
+        """
+        self.repo._download_file(remote_file_path, local_path)
+
+    def delete_artifacts(self, artifact_path=None):
+        raise MlflowException("Not implemented yet")

mlflow/store/artifact/optimized_s3_artifact_repo.py

@@ -0,0 +1,356 @@
+import json
+import logging
+import os
+import posixpath
+import urllib.parse
+from mimetypes import guess_type
+
+from mlflow.entities import FileInfo
+from mlflow.environment_variables import (
+    MLFLOW_ENABLE_MULTIPART_UPLOAD,
+    MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE,
+    MLFLOW_S3_UPLOAD_EXTRA_ARGS,
+)
+from mlflow.exceptions import MlflowException
+from mlflow.protos.databricks_artifacts_pb2 import ArtifactCredentialInfo
+from mlflow.store.artifact.artifact_repo import _retry_with_new_creds
+from mlflow.store.artifact.cloud_artifact_repo import (
+    CloudArtifactRepository,
+    _complete_futures,
+    _compute_num_chunks,
+    _validate_chunk_size_aws,
+)
+from mlflow.store.artifact.s3_artifact_repo import _get_s3_client
+from mlflow.utils.file_utils import read_chunk
+from mlflow.utils.request_utils import cloud_storage_http_request
+from mlflow.utils.rest_utils import augmented_raise_for_status
+
+_logger = logging.getLogger(__name__)
+_BUCKET_REGION = "BucketRegion"
+_RESPONSE_METADATA = "ResponseMetadata"
+_HTTP_HEADERS = "HTTPHeaders"
+_HTTP_HEADER_BUCKET_REGION = "x-amz-bucket-region"
+_BUCKET_LOCATION_NAME = "BucketLocationName"
+
+
+class OptimizedS3ArtifactRepository(CloudArtifactRepository):
+    """
+    An optimized version of the S3 Artifact Repository.
+
+    This class is used for uploading and downloading S3 artifacts for UC models. While it largely
+    copies the behavior of the S3ArtifactRepository, the `log_artifact`, `log_artifacts`, and
+    `_download_file` methods are optimized by replacing boto3 client operations with the use of
+    presigned URLs for both uploads and downloads.
+    """
+
+    def __init__(
+        self,
+        artifact_uri,
+        access_key_id=None,
+        secret_access_key=None,
+        session_token=None,
+        credential_refresh_def=None,
+        addressing_style=None,
+        s3_endpoint_url=None,
+        s3_upload_extra_args=None,
+    ):
+        super().__init__(artifact_uri)
+        self._access_key_id = access_key_id
+        self._secret_access_key = secret_access_key
+        self._session_token = session_token
+        self._credential_refresh_def = credential_refresh_def
+        self._addressing_style = addressing_style
+        self._s3_endpoint_url = s3_endpoint_url
+        self.bucket, self.bucket_path = self.parse_s3_compliant_uri(self.artifact_uri)
+        self._region_name = self._get_region_name()
+        self._s3_upload_extra_args = s3_upload_extra_args if s3_upload_extra_args else {}
+
+    def _refresh_credentials(self):
+        if not self._credential_refresh_def:
+            return self._get_s3_client()
+        new_creds = self._credential_refresh_def()
+        self._access_key_id = new_creds["access_key_id"]
+        self._secret_access_key = new_creds["secret_access_key"]
+        self._session_token = new_creds["session_token"]
+        self._s3_upload_extra_args = new_creds["s3_upload_extra_args"]
+        return self._get_s3_client()
+
+    def _get_region_name(self):
+        from botocore.exceptions import ClientError
+
+        temp_client = _get_s3_client(
+            addressing_style=self._addressing_style,
+            access_key_id=self._access_key_id,
+            secret_access_key=self._secret_access_key,
+            session_token=self._session_token,
+            s3_endpoint_url=self._s3_endpoint_url,
+        )
+        try:
+            head_bucket_resp = temp_client.head_bucket(Bucket=self.bucket)
+            # A normal response will have the region in the Bucket_Region field of the response
+            if _BUCKET_REGION in head_bucket_resp:
+                return head_bucket_resp[_BUCKET_REGION]
+            # If the bucket exists but the caller does not have permissions, the http headers
+            # are passed back as part of the metadata of a normal, non-throwing response.  In
+            # this case we use the x-amz-bucket-region field of the HTTP headers which should
+            # always be populated with the region.
+            if (
+                _RESPONSE_METADATA in head_bucket_resp
+                and _HTTP_HEADERS in head_bucket_resp[_RESPONSE_METADATA]
+                and _HTTP_HEADER_BUCKET_REGION
+                in head_bucket_resp[_RESPONSE_METADATA][_HTTP_HEADERS]
+            ):
+                return head_bucket_resp[_RESPONSE_METADATA][_HTTP_HEADERS][
+                    _HTTP_HEADER_BUCKET_REGION
+                ]
+            # Directory buckets do not have a Bucket_Region and instead have a
+            # Bucket_Location_Name.  This name cannot be used as the region name
+            # however, so we warn that this has happened and allow the exception
+            # at the end to be raised.
+            if _BUCKET_LOCATION_NAME in head_bucket_resp:
+                _logger.warning(
+                    f"Directory bucket {self.bucket} found with BucketLocationName "
+                    f"{head_bucket_resp[_BUCKET_LOCATION_NAME]}."
+                )
+            raise Exception(f"Unable to get the region name for bucket {self.bucket}.")
+        except ClientError as error:
+            # If a client error occurs, we check to see if the x-amz-bucket-region field is set
+            # in the response and return that.  If it is not present, this will raise due to the
+            # key not being present.
+            return error.response[_RESPONSE_METADATA][_HTTP_HEADERS][_HTTP_HEADER_BUCKET_REGION]
+
+    def _get_s3_client(self):
+        return _get_s3_client(
+            addressing_style=self._addressing_style,
+            access_key_id=self._access_key_id,
+            secret_access_key=self._secret_access_key,
+            session_token=self._session_token,
+            region_name=self._region_name,
+            s3_endpoint_url=self._s3_endpoint_url,
+        )
+
+    def parse_s3_compliant_uri(self, uri):
+        """Parse an S3 URI, returning (bucket, path)"""
+        parsed = urllib.parse.urlparse(uri)
+        if parsed.scheme != "s3":
+            raise Exception(f"Not an S3 URI: {uri}")
+        path = parsed.path
+        if path.startswith("/"):
+            path = path[1:]
+        return parsed.netloc, path
+
+    @staticmethod
+    def get_s3_file_upload_extra_args():
+        s3_file_upload_extra_args = MLFLOW_S3_UPLOAD_EXTRA_ARGS.get()
+        if s3_file_upload_extra_args:
+            return json.loads(s3_file_upload_extra_args)
+        else:
+            return None
+
+    def _upload_file(self, s3_client, local_file, bucket, key):
+        extra_args = {}
+        extra_args.update(self._s3_upload_extra_args)
+        guessed_type, guessed_encoding = guess_type(local_file)
+        if guessed_type is not None:
+            extra_args["ContentType"] = guessed_type
+        if guessed_encoding is not None:
+            extra_args["ContentEncoding"] = guessed_encoding
+        environ_extra_args = self.get_s3_file_upload_extra_args()
+        if environ_extra_args is not None:
+            extra_args.update(environ_extra_args)
+
+        def try_func(creds):
+            creds.upload_file(Filename=local_file, Bucket=bucket, Key=key, ExtraArgs=extra_args)
+
+        _retry_with_new_creds(
+            try_func=try_func, creds_func=self._refresh_credentials, orig_creds=s3_client
+        )
+
+    def log_artifact(self, local_file, artifact_path=None):
+        artifact_file_path = os.path.basename(local_file)
+        if artifact_path:
+            artifact_file_path = posixpath.join(artifact_path, artifact_file_path)
+        self._upload_to_cloud(
+            cloud_credential_info=self._get_s3_client(),
+            src_file_path=local_file,
+            artifact_file_path=artifact_file_path,
+        )
+
+    def _get_write_credential_infos(self, remote_file_paths):
+        """
+        Instead of returning ArtifactCredentialInfo objects, we instead return a list of initialized
+        S3 client. We do so because S3 clients cannot be instantiated within each thread.
+        """
+        return [self._get_s3_client() for _ in remote_file_paths]
+
+    def _upload_to_cloud(self, cloud_credential_info, src_file_path, artifact_file_path):
+        dest_path = posixpath.join(self.bucket_path, artifact_file_path)
+        key = posixpath.normpath(dest_path)
+        if (
+            MLFLOW_ENABLE_MULTIPART_UPLOAD.get()
+            and os.path.getsize(src_file_path) > MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE.get()
+        ):
+            self._multipart_upload(cloud_credential_info, src_file_path, self.bucket, key)
+        else:
+            self._upload_file(cloud_credential_info, src_file_path, self.bucket, key)
+
+    def _multipart_upload(self, cloud_credential_info, local_file, bucket, key):
+        # Create multipart upload
+        s3_client = cloud_credential_info
+        response = s3_client.create_multipart_upload(Bucket=bucket, Key=key)
+        upload_id = response["UploadId"]
+
+        num_parts = _compute_num_chunks(local_file, MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE.get())
+        _validate_chunk_size_aws(MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE.get())
+
+        # define helper functions for uploading data
+        def _upload_part(part_number, local_file, start_byte, size):
+            data = read_chunk(local_file, size, start_byte)
+
+            def try_func(creds):
+                # Create presigned URL for each part
+                presigned_url = creds.generate_presigned_url(
+                    "upload_part",
+                    Params={
+                        "Bucket": bucket,
+                        "Key": key,
+                        "UploadId": upload_id,
+                        "PartNumber": part_number,
+                    },
+                )
+                with cloud_storage_http_request("put", presigned_url, data=data) as response:
+                    augmented_raise_for_status(response)
+                    return response.headers["ETag"]
+
+            return _retry_with_new_creds(
+                try_func=try_func, creds_func=self._refresh_credentials, orig_creds=s3_client
+            )
+
+        try:
+            # Upload each part with retries
+            futures = {}
+            for index in range(num_parts):
+                part_number = index + 1
+                start_byte = index * MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE.get()
+                future = self.chunk_thread_pool.submit(
+                    _upload_part,
+                    part_number=part_number,
+                    local_file=local_file,
+                    start_byte=start_byte,
+                    size=MLFLOW_MULTIPART_UPLOAD_CHUNK_SIZE.get(),
+                )
+                futures[future] = part_number
+
+            results, errors = _complete_futures(futures, local_file)
+            if errors:
+                raise MlflowException(
+                    f"Failed to upload at least one part of {local_file}. Errors: {errors}"
+                )
+            parts = [
+                {"PartNumber": part_number, "ETag": results[part_number]}
+                for part_number in sorted(results)
+            ]
+
+            # Complete multipart upload
+            s3_client.complete_multipart_upload(
+                Bucket=bucket,
+                Key=key,
+                UploadId=upload_id,
+                MultipartUpload={"Parts": parts},
+            )
+        except Exception as e:
+            _logger.warning(
+                "Encountered an unexpected error during multipart upload: %s, aborting", e
+            )
+            s3_client.abort_multipart_upload(
+                Bucket=bucket,
+                Key=key,
+                UploadId=upload_id,
+            )
+            raise e
+
+    def list_artifacts(self, path=None):
+        artifact_path = self.bucket_path
+        dest_path = self.bucket_path
+        if path:
+            dest_path = posixpath.join(dest_path, path)
+        infos = []
+        dest_path = dest_path.rstrip("/") if dest_path else ""
+        prefix = dest_path + "/" if dest_path else ""
+        s3_client = self._get_s3_client()
+        paginator = s3_client.get_paginator("list_objects_v2")
+        results = paginator.paginate(Bucket=self.bucket, Prefix=prefix, Delimiter="/")
+        for result in results:
+            # Subdirectories will be listed as "common prefixes" due to the way we made the request
+            for obj in result.get("CommonPrefixes", []):
+                subdir_path = obj.get("Prefix")
+                self._verify_listed_object_contains_artifact_path_prefix(
+                    listed_object_path=subdir_path, artifact_path=artifact_path
+                )
+                subdir_rel_path = posixpath.relpath(path=subdir_path, start=artifact_path)
+                if subdir_rel_path.endswith("/"):
+                    subdir_rel_path = subdir_rel_path[:-1]
+                infos.append(FileInfo(subdir_rel_path, True, None))
+            # Objects listed directly will be files
+            for obj in result.get("Contents", []):
+                file_path = obj.get("Key")
+                self._verify_listed_object_contains_artifact_path_prefix(
+                    listed_object_path=file_path, artifact_path=artifact_path
+                )
+                file_rel_path = posixpath.relpath(path=file_path, start=artifact_path)
+                file_size = int(obj.get("Size"))
+                infos.append(FileInfo(file_rel_path, False, file_size))
+        return sorted(infos, key=lambda f: f.path)
+
+    @staticmethod
+    def _verify_listed_object_contains_artifact_path_prefix(listed_object_path, artifact_path):
+        if not listed_object_path.startswith(artifact_path):
+            raise MlflowException(
+                "The path of the listed S3 object does not begin with the specified"
+                f" artifact path. Artifact path: {artifact_path}. Object path:"
+                f" {listed_object_path}."
+            )
+
+    def _get_presigned_uri(self, remote_file_path):
+        s3_client = self._get_s3_client()
+        s3_full_path = posixpath.join(self.bucket_path, remote_file_path)
+        return s3_client.generate_presigned_url(
+            "get_object", Params={"Bucket": self.bucket, "Key": s3_full_path}
+        )
+
+    def _get_read_credential_infos(self, remote_file_paths):
+        return [
+            ArtifactCredentialInfo(signed_uri=self._get_presigned_uri(path))
+            for path in remote_file_paths
+        ]
+
+    def _download_from_cloud(self, remote_file_path, local_path):
+        s3_client = self._get_s3_client()
+        s3_full_path = posixpath.join(self.bucket_path, remote_file_path)
+
+        def try_func(creds):
+            creds.download_file(self.bucket, s3_full_path, local_path)
+
+        _retry_with_new_creds(
+            try_func=try_func, creds_func=self._refresh_credentials, orig_creds=s3_client
+        )
+
+    def delete_artifacts(self, artifact_path=None):
+        dest_path = self.bucket_path
+        if artifact_path:
+            dest_path = posixpath.join(dest_path, artifact_path)
+
+        dest_path = dest_path.rstrip("/") if dest_path else ""
+        s3_client = self._get_s3_client()
+        paginator = s3_client.get_paginator("list_objects_v2")
+        results = paginator.paginate(Bucket=self.bucket, Prefix=dest_path)
+        for result in results:
+            keys = []
+            for to_delete_obj in result.get("Contents", []):
+                file_path = to_delete_obj.get("Key")
+                self._verify_listed_object_contains_artifact_path_prefix(
+                    listed_object_path=file_path, artifact_path=dest_path
+                )
+                keys.append({"Key": file_path})
+            if keys:
+                s3_client.delete_objects(Bucket=self.bucket, Delete={"Objects": keys})