amapy-plugin-s3 1.0.0__tar.gz

amapy-plugin-s3-1.0.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: amapy-plugin-s3
+Version: 1.0.0
+Summary: The S3 plugin for the asset manager project. It provides functionalities to manage and integrate Amazon S3 storage within the asset management system.
+Author-email: Swarup Mahanti <swarup.mahanti@roche.com>
+Maintainer-email: Swarup Mahanti <swarup.mahanti@roche.com>
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aioboto3<=10.2.0
+Requires-Dist: amapy-contents==1.*
+Requires-Dist: amapy-pluggy==1.*
+Requires-Dist: amapy-utils==1.*
+Requires-Dist: backoff==2.2.*
+
+asset-plugin-s3

amapy-plugin-s3-1.0.0/README.md

@@ -0,0 +1 @@
+asset-plugin-s3

amapy-plugin-s3-1.0.0/amapy_plugin_s3.egg-info/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: amapy-plugin-s3
+Version: 1.0.0
+Summary: The S3 plugin for the asset manager project. It provides functionalities to manage and integrate Amazon S3 storage within the asset management system.
+Author-email: Swarup Mahanti <swarup.mahanti@roche.com>
+Maintainer-email: Swarup Mahanti <swarup.mahanti@roche.com>
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aioboto3<=10.2.0
+Requires-Dist: amapy-contents==1.*
+Requires-Dist: amapy-pluggy==1.*
+Requires-Dist: amapy-utils==1.*
+Requires-Dist: backoff==2.2.*
+
+asset-plugin-s3

amapy-plugin-s3-1.0.0/amapy_plugin_s3.egg-info/SOURCES.txt

@@ -0,0 +1,42 @@
+README.md
+pyproject.toml
+amapy_plugin_s3.egg-info/PKG-INFO
+amapy_plugin_s3.egg-info/SOURCES.txt
+amapy_plugin_s3.egg-info/dependency_links.txt
+amapy_plugin_s3.egg-info/requires.txt
+amapy_plugin_s3.egg-info/top_level.txt
+asset_plugin_s3/__init__.py
+asset_plugin_s3/aws_auth.py
+asset_plugin_s3/aws_blob.py
+asset_plugin_s3/aws_http_handler.py
+asset_plugin_s3/aws_mount_handler.py
+asset_plugin_s3/aws_storage.py
+asset_plugin_s3/aws_storage_mixin.py
+asset_plugin_s3/bucket_cors.py
+asset_plugin_s3/s3_proxy.py
+asset_plugin_s3/signed_url.py
+asset_plugin_s3/mounted_bucket/__init__.py
+asset_plugin_s3/mounted_bucket/async_mount_copy.py
+asset_plugin_s3/mounted_bucket/async_mount_transporter.py
+asset_plugin_s3/mounted_bucket/mounted_resource.py
+asset_plugin_s3/mounted_bucket/mounted_url.py
+asset_plugin_s3/mounted_bucket/tests/__init__.py
+asset_plugin_s3/mounted_bucket/tests/test_mounted_url.py
+asset_plugin_s3/tests/__init__.py
+asset_plugin_s3/tests/test_aws_storage.py
+asset_plugin_s3/transporter/__init__.py
+asset_plugin_s3/transporter/aws_transport_resource.py
+asset_plugin_s3/transporter/async_aws/__init__.py
+asset_plugin_s3/transporter/async_aws/async_aws_transporter.py
+asset_plugin_s3/transporter/async_aws/async_copy.py
+asset_plugin_s3/transporter/async_aws/async_download.py
+asset_plugin_s3/transporter/async_aws/async_update_blob.py
+asset_plugin_s3/transporter/async_aws/async_upload.py
+asset_plugin_s3/transporter/async_aws/tests/__init__.py
+asset_plugin_s3/transporter/async_aws/tests/test_async_aws_transporter.py
+asset_plugin_s3/transporter/legacy_aws/__init__.py
+asset_plugin_s3/transporter/legacy_aws/async_download.py
+asset_plugin_s3/transporter/legacy_aws/async_upload.py
+asset_plugin_s3/transporter/legacy_aws/legacy_aws_transporter.py
+asset_plugin_s3/transporter/legacy_aws/tests/__init__.py
+asset_plugin_s3/transporter/legacy_aws/tests/test_legacy_aws_transporter.py

amapy-plugin-s3-1.0.0/amapy_plugin_s3.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

amapy-plugin-s3-1.0.0/amapy_plugin_s3.egg-info/requires.txt

@@ -0,0 +1,5 @@
+aioboto3<=10.2.0
+amapy-contents==1.*
+amapy-pluggy==1.*
+amapy-utils==1.*
+backoff==2.2.*

amapy-plugin-s3-1.0.0/amapy_plugin_s3.egg-info/top_level.txt

@@ -0,0 +1 @@
+asset_plugin_s3

amapy-plugin-s3-1.0.0/asset_plugin_s3/__init__.py

@@ -0,0 +1 @@
+from .aws_storage import AwsStoragePlugin

amapy-plugin-s3-1.0.0/asset_plugin_s3/aws_auth.py

@@ -0,0 +1,79 @@
+import hmac
+import os
+from datetime import datetime
+from hashlib import sha256
+
+from botocore.auth import S3SigV4Auth
+
+from asset_pluggy.storage.storage_credentials import StorageCredentials
+from asset_utils.common import exceptions
+
+
+def get_aws_id_k_date() -> dict:
+    """
+    TODO: will move it into the server, added for testing purpose only
+    Returns
+    -------
+    str
+        the k_date hex string
+    """
+    secret_key = StorageCredentials.shared().credentials.get("aws_secret_access_key")
+    today = datetime.utcnow().strftime('%Y%m%d')  # always use the UTC time for k_date
+    hex_k_date = hmac.new(f"AWS4{secret_key}".encode("utf-8"), today.encode("utf-8"), sha256).hexdigest()
+    return {"aws_access_key_id": StorageCredentials.shared().credentials.get("aws_access_key_id"),
+            "k_date": hex_k_date}
+
+
+class AwsAuth(S3SigV4Auth):
+    """
+    Custom AWS authentication class for S3 that uses K_DATE from the environment if the secret key is not provided
+    """
+
+    def signature(self, string_to_sign, request):
+        """
+        Returns the signature for the given string and request
+
+        Parameters
+        ----------
+        string_to_sign : str
+            The string to sign
+        request : obj
+            The request object
+
+        Returns
+        -------
+        str
+            The signature for the given string and request
+        """
+        key = self.credentials.secret_key
+        if key:
+            return super().signature(string_to_sign, request)
+        else:
+            return self._sign_with_k_date(string_to_sign)
+
+    def _sign_with_k_date(self, string_to_sign):
+        """Returns the signature for the given string using K_DATE from the environment
+
+        Parameters
+        ----------
+        string_to_sign : str
+            The string to sign
+
+        Returns
+        -------
+        str
+            The signature for the given string using K_DATE
+
+        Raises
+        ------
+        AssetException
+            If K_DATE is not found in the environment
+        """
+        k_date_str = os.environ.get("ASSET_K_DATE")
+        if not k_date_str:
+            raise exceptions.AssetException("ASSET_K_DATE not found in environment")
+        k_date = bytes.fromhex(k_date_str)
+        k_region = self._sign(k_date, self._region_name)
+        k_service = self._sign(k_region, self._service_name)
+        k_signing = self._sign(k_service, msg="aws4_request")
+        return self._sign(k_signing, string_to_sign, hex=True)

amapy-plugin-s3-1.0.0/asset_plugin_s3/aws_blob.py

@@ -0,0 +1,231 @@
+from typing import Any
+
+from asset_pluggy.storage import BlobData
+from asset_pluggy.storage import BlobStoreURL
+from asset_utils.common.exceptions import AssetException
+from asset_utils.utils.file_utils import FileUtils
+
+
+class AwsBlob(BlobData):
+    """Class to handle AWS S3 blob data."""
+    _aws_obj = None
+    _multipart_size = None
+
+    def initialize(self, data: Any, url_object: BlobStoreURL):
+        """Initializes the AwsBlob object with data from an S3 object or S3 object summary.
+
+        Parameters
+        ----------
+        data : Any
+            A dict or s3.ObjectSummary or s3.object.
+        url_object : BlobStoreURL
+            The URL of the blob.
+        """
+        self._aws_obj = data
+        try:
+            if type(self._aws_obj) is dict:  # json data from S3Proxy
+                self._initialize_from_dict(data=self._aws_obj)
+                return
+            elif hasattr(self._aws_obj, 'content_type'):  # s3.Object
+                self._initialize_from_s3_object(s3_obj=self._aws_obj)
+            else:
+                self._initialize_from_s3_summary_object(s3_summary_obj=self._aws_obj)
+        except Exception as e:
+            raise AssetException(f"Error initializing blob from {url_object.url}") from e
+        self.host = url_object.host
+        self.url = url_object.url_for_blob(host=self.host, bucket=self.bucket, name=self.name)
+
+    def _initialize_from_s3_object(self, s3_obj: Any):
+        """Initializes the AwsBlob object with data from an S3 object.
+
+        Parameters
+        ----------
+        s3_obj : Any
+            The S3.object.
+        """
+        self.bucket = s3_obj.bucket_name
+        self.name = s3_obj.key
+        self.size = s3_obj.content_length
+        # s3 mostly returns content-type as binary/octet-stream so we try to guess it
+        # self.content_type = FileUtils.mime_type(self.name) or s3_obj.content_type
+        # aws returns files + directory unlike gs which returns only files
+        # self.is_file = bool(self.content_type != 'application/x-directory')
+        # check all possible checksums, the structure here is different thant s3.ObjectSummary
+        # note: aws_blob returns inconsistent check_sum_algorithm keys, sometimes the keys are present
+        # if the value is null - so we add an extra protection to ensure hash can not be null
+        if hasattr(s3_obj, "checksum_crc32") and getattr(s3_obj, "checksum_crc32"):
+            self.hashes["crc32"] = getattr(s3_obj, "checksum_crc32")
+        if hasattr(s3_obj, "checksum_crc32_c") and getattr(s3_obj, "checksum_crc32_c"):
+            self.hashes["crc32_c"] = getattr(s3_obj, "checksum_crc32_c")
+        if hasattr(s3_obj, "checksum_sha1") and getattr(s3_obj, "checksum_sha1"):
+            self.hashes["sha1"] = getattr(s3_obj, "checksum_sha1")
+        if hasattr(s3_obj, "checksum_sha256") and getattr(s3_obj, "checksum_sha256"):
+            self.hashes["sha256"] = getattr(s3_obj, "checksum_sha256")
+
+        self.hashes["etag"] = s3_obj.e_tag
+        if '-' in s3_obj.e_tag and self._parse_etag(s3_obj.e_tag)[1] == '1':
+            # if number of parts is 1, then self.size is the multipart size
+            self._multipart_size = self.size
+
+    def _initialize_from_s3_summary_object(self, s3_summary_obj: Any):
+        """Initializes the AwsBlob object with data from an S3 object summary.
+
+        Parameters
+        ----------
+        s3_summary_obj : Any
+            The s3.ObjectSummary.
+        """
+        self.bucket = s3_summary_obj.bucket_name
+        self.name = s3_summary_obj.key
+        self.size = s3_summary_obj.size
+        # if hasattr(s3_summary_obj, "checksum_algorithm"):
+        #     # todo: verify whats the checksum value when this attribute is present in the object.
+        #     hash_type = getattr(s3_summary_obj, "checksum_algorithm") or "md5"
+        # e-tag has extra quotes, so we strip quotes
+        # etag has 2 parts separated by "-", first part is the md5,
+        # second part is the number of parts in which the file was uploaded
+        self.hashes["etag"] = s3_summary_obj.e_tag
+        if '-' in s3_summary_obj.e_tag and self._parse_etag(s3_summary_obj.e_tag)[1] == '1':
+            # if number of parts is 1, then self.size is the multipart size
+            self._multipart_size = self.size
+
+    def _initialize_from_dict(self, data: dict):
+        """Initialize the AwsBlob object from a dictionary.
+
+        Parameters
+        ----------
+        data : dict
+            The dictionary containing the blob data.
+        """
+        self.bucket = data.get("bucket")
+        self.hashes = data.get("hashes")
+        self.host = data.get("host")
+        self.name = data.get("name")
+        self.path_in_asset = data.get("path_in_asset")
+        self.size = data.get("size")
+        self.url = data.get("url")
+
+    @property
+    def content_type(self):
+        """Get the content type of the blob.
+
+        Refactored into a property to avoid extra network calls involved in getting the content-type of s3
+
+        Returns the content type of the blob. If the blob is an S3 object, it returns the content type of
+        the S3 object. If the blob is an S3 object summary, it returns the MIME type of the blob based on
+        its name or the content type of the S3 object summary.
+
+        Returns
+        -------
+        str
+            The content type of the blob.
+        """
+        if hasattr(self._aws_obj, 'content_type'):  # s3.Object
+            return getattr(self._aws_obj, 'content_type')
+        else:
+            # note: this is refactored into a separate property for better list-blobs performance,
+            # self._parse_content_type(obj=s3_summary_obj)
+            # makes this slower by 0.6 seconds per call, so if you are checking 500 objects,
+            # it would take about 4 minutes which is painfully slow
+            return FileUtils.mime_type(self.name) or self._parse_content_type(obj=self._aws_obj)
+
+    @property
+    def is_file(self):
+        """Check if the blob is a file.
+
+        Refactored into property because of content_type refactoring above.
+
+        Returns
+        -------
+        bool
+            True if the blob is a file, False otherwise.
+
+        Notes
+        -----
+        aws returns files + directory unlike gs which returns only files.
+        """
+        # self.is_file = bool(self.content_type != 'application/x-directory')
+        return bool(self.content_type != 'application/x-directory')
+
+    @property
+    def multipart_size(self) -> int:
+        """Get the part size of the multipart upload.
+
+        Returns
+        -------
+        int
+            The part size of the multipart upload.
+
+        Raises
+        ------
+        AssetException
+            If the blob is a multipart upload but the multipart_size is not initialized.
+        """
+        if self.is_multipart and not self._multipart_size:
+            raise AssetException("Multipart size is not initialized. Please set the multipart size.")
+        return self._multipart_size
+
+    @multipart_size.setter
+    def multipart_size(self, size: int):
+        """Set the part size of the multipart upload.
+
+        Parameters
+        ----------
+        size : int
+            The part size of the multipart upload.
+        """
+        self._multipart_size = size
+
+    @property
+    def is_multipart(self) -> bool:
+        """Check if the blob is a multipart upload.
+
+        Returns True if the ETag of the blob indicates a multipart upload, False otherwise.
+
+        Returns
+        -------
+        bool
+            True if the blob is a multipart upload, False otherwise.
+        """
+        return bool('-' in self.hashes.get("etag", ""))
+
+    def _parse_etag(self, etag: str) -> list:
+        """Parse the ETag of the blob if it is multipart.
+
+        Parameters
+        ----------
+        etag : str
+            The ETag of the blob.
+
+        Returns
+        -------
+        list
+            A list containing the ETag and the number of parts in the upload.
+        """
+        # remove extra quotes
+        return etag[1:-1].split("-")
+
+    def _parse_content_type(self, obj: Any):
+        """Get the content-type of an S3 object summary.
+
+        Parameters
+        ----------
+        obj : Any
+            The S3 object summary.
+
+        Returns
+        -------
+        str
+            The content-type of the S3 object summary.
+        """
+        try:
+            return obj.get()['ContentType']
+        except Exception as e:
+            self.log.info(str(e))
+            return None
+
+    def compute_hash(self) -> tuple:
+        raise NotImplementedError
+
+    def get_hash_preferences(self) -> list:
+        return [*super().get_hash_preferences(), "etag"]

amapy-plugin-s3-1.0.0/asset_plugin_s3/aws_http_handler.py

@@ -0,0 +1,222 @@
+import os
+from functools import cached_property
+from typing import Union
+
+import boto3
+import botocore
+from botocore.errorfactory import ClientError
+
+from asset_pluggy.storage import StorageData, StorageURL, BlobStoreURL
+from asset_pluggy.storage import storage_utils
+from asset_plugin_s3.aws_auth import AwsAuth, get_aws_id_k_date
+from asset_plugin_s3.aws_blob import AwsBlob
+from asset_plugin_s3.transporter import AsyncAwsTransporter
+from asset_utils.utils import utils
+
+
+class AwsHttpHandler:
+    """AWS Http Handler class for handling AWS S3 operations using boto3 http requests."""
+
+    def __init__(self, credentials: dict = None):
+        self.cred = credentials
+
+    @cached_property
+    def credentials(self) -> dict:
+        """Fetches and returns AWS credentials.
+
+        This method first checks if the AWS credentials are provided by the user. If not, it fetches the
+        aws_access_key_id and k_date from the server. It also updates the auth type map to redirect to the
+        custom AwsAuth class.
+
+        Returns
+        -------
+        dict
+            A dictionary containing AWS credentials.
+        """
+        cred = self.cred
+        # cred = None  # for testing purposes
+        # update the auth type map to redirect to the custom AwsAuth class
+        botocore.auth.AUTH_TYPE_MAPS.update({"s3v4": AwsAuth})
+        # if the aws_access_key_id and aws_secret_access_key are there it means the user has provided the credentials.
+        # is not then we fetch the aws_access_key_id and k_date from the server
+        if not cred or "aws_access_key_id" not in cred and "aws_secret_access_key" not in cred:
+            server_creds = get_aws_id_k_date()  # TODO: fetch it from the server instead of using a test function
+            cred = {} if cred is None else cred
+            cred["aws_access_key_id"] = server_creds["aws_access_key_id"]
+            cred["aws_secret_access_key"] = ""
+            os.environ["ASSET_K_DATE"] = server_creds["k_date"]  # set the k_date in the environment variable
+        return cred
+
+    def allows_object_add(self):
+        """Checks if object addition is allowed."""
+        return True
+
+    def allows_proxy(self):
+        """Checks if proxy is allowed."""
+        return True
+
+    def get_transporter(self) -> AsyncAwsTransporter:
+        """Returns an instance of AsyncAwsTransporter.
+
+        The AsyncAwsTransporter is initialized with the AWS credentials.
+
+        Returns
+        -------
+        Transporter
+            An instance of AsyncAwsTransporter.
+        """
+        return AsyncAwsTransporter.shared(credentials=self.credentials)
+
+    @property
+    def s3_client(self):
+        return boto3.client('s3', **self.credentials)
+
+    @property
+    def s3_resource(self):
+        return boto3.resource('s3', **self.credentials)
+
+    def get_storage_url(self, url_string: str, ignore: str = None) -> StorageURL:
+        return BlobStoreURL(url=url_string, ignore=ignore)
+
+    def get_blob(self, url_string: str) -> AwsBlob:
+        """Get the blob instance from the given URL."""
+        aws_url = BlobStoreURL(url=url_string)
+        blob_data = self.fetch_blob_data(url=aws_url)
+        return AwsBlob(data=blob_data, url_object=aws_url)
+
+    def fetch_blob_data(self, url: BlobStoreURL):
+        return self.fetch_data_from_bucket(bucket_name=url.bucket, blob_name=url.path)
+
+    def fetch_data_from_bucket(self, bucket_name, blob_name):
+        return self.s3_resource.Object(bucket_name, blob_name)
+
+    def url_is_file(self, url: Union[StorageURL, str]) -> bool:
+        """Checks if the URL is a file.
+
+        Blobs are files, so if a blob exists then it's a file
+        else either the url doesn't exist or it's a directory
+        """
+        if type(url) is str:
+            url = BlobStoreURL(url=url)
+        return self.check_if_blob_exists(url)
+
+    def blob_exists(self, url_string: str) -> bool:
+        """Checks if a blob exists at the given URL."""
+        return self.check_if_blob_exists(url=BlobStoreURL(url=url_string))
+
+    def check_if_blob_exists(self, url: BlobStoreURL) -> bool:
+        # no blob path means the url is a bucket
+        if not url.path:
+            return False
+        try:
+            exists = self.s3_client.head_object(Bucket=url.bucket, Key=url.path)
+            return bool(exists)
+        except ClientError:
+            return False
+
+    def list_blobs(self, url: Union[str, StorageURL], ignore: str = None) -> [StorageData]:
+        """Returns a list of AwsBlobs located at the url."""
+        if type(url) is str:
+            url = BlobStoreURL(url=url, ignore=ignore)
+        blobs_list = self.fetch_blobs_list(url=url)
+        return list(map(lambda x: AwsBlob(data=x, url_object=url), blobs_list))
+
+    def fetch_blobs_list(self, url: BlobStoreURL):
+        aws_bucket = self.s3_resource.Bucket(url.bucket)
+        # fetch blobs from the bucket filtered by prefix
+        blobs = list(aws_bucket.objects.filter(Prefix=url.path))
+        # filter blobs based on pattern and ignore
+        return storage_utils.filter_blobs(blobs=blobs,
+                                          name_key="key",
+                                          pattern=url.pattern,
+                                          ignore=url.ignore)
+
+    def delete_blobs(self, url_strings: [str]) -> None:
+        """Deletes blobs at the given URLs."""
+        self.delete_blob_urls(urls=list(map(lambda x: BlobStoreURL(url=x), url_strings)))
+
+    def delete_blob_urls(self, urls: [BlobStoreURL]):
+        # group by bucket
+        groups = {}
+        for url in urls:
+            keys_list = groups.get(url.bucket, [])
+            keys_list.append(url.path)
+            groups[url.bucket] = keys_list
+        # delete batch by batch
+        for bucket in groups:
+            self.batch_delete_s3(s3_client=self.s3_client, bucket=bucket, keys_list=groups.get(bucket))
+
+    def batch_delete_s3(self, s3_client, bucket: str, keys_list: list):
+        for batch in utils.batch(keys_list, batch_size=100):
+            batch_to_delete = list(map(lambda x: {"Key": x}, batch))
+            s3_client.delete_objects(Bucket=bucket, Delete={'Objects': batch_to_delete, 'Quiet': True})
+
+    def filter_duplicate_blobs(self, src_blobs: [StorageData], dst_blobs: [StorageData]) -> (list, list):
+        """Filters the source blobs to determine which blobs are new and which need to be replaced in the destination.
+
+        If a blob in `src_blobs` has the same path_in_asset as a blob in `dst_blobs`, it compares their hashes.
+        If the hashes are different, the blob is added to the replace_blobs list. If the path_in_asset is not
+        found in `dst_blobs`, the blob is considered new and is added to the new_blobs list. For upload and download
+        operations, the multipart sizes of the blobs are updated before hash comparison.
+
+        Parameters
+        ----------
+        src_blobs : list
+            A list of source blobs.
+        dst_blobs : list
+            A list of destination blobs.
+
+        Returns
+        -------
+        tuple
+            A tuple containing two lists: new_blobs and replace_blobs. new_blobs is a list of blobs that are new and
+            replace_blobs is a list of blobs that need to be replaced in the destination.
+        """
+        # TODO: improve the overall implementation
+        if not dst_blobs:  # nothing to filter against
+            return src_blobs, []
+        if all(isinstance(blob, AwsBlob) for blob in src_blobs) and all(
+                isinstance(blob, AwsBlob) for blob in dst_blobs):  # asset cp remote copy
+            new_blobs, replace_blobs = [], []
+            # compare the path_in_asset and hash of the blobs
+            dst_blob_map = {obj.path_in_asset: obj for obj in dst_blobs}
+            for src_blob in src_blobs:
+                if src_blob.path_in_asset in dst_blob_map:
+                    # no need to update the multipart sizes before hash comparison
+                    if not src_blob.compare_hash(dst_blob_map[src_blob.path_in_asset]):
+                        replace_blobs.append(src_blob)
+                else:
+                    # new path_in_asset new object
+                    new_blobs.append(src_blob)
+            return new_blobs, replace_blobs
+
+        # src_blobs or dst_blobs must be PosixBlob objects
+        new_blobs, replace_blobs = [], []
+        # compare the path_in_asset and hash of the blobs
+        dst_blob_map = {obj.path_in_asset: obj for obj in dst_blobs}
+        need_hash_compare = []
+        for src_blob in src_blobs:
+            if src_blob.path_in_asset in dst_blob_map:
+                # need to compare hash
+                need_hash_compare.append(src_blob)
+            else:
+                # new path_in_asset new object
+                new_blobs.append(src_blob)
+
+        if all(isinstance(blob, AwsBlob) for blob in need_hash_compare):  # asset cp download
+            # update the multipart sizes of the blobs that need hash comparison
+            self.get_transporter().update_multipart_blobs(blobs=need_hash_compare)
+            for src_blob in need_hash_compare:
+                posix_blob = dst_blob_map[src_blob.path_in_asset]
+                if not posix_blob.compare_hash(src_blob):
+                    replace_blobs.append(src_blob)
+        else:  # asset cp upload
+            # dst_blobs must be AwsBlob objects, update the multipart sizes
+            self.get_transporter().update_multipart_blobs(
+                blobs=[dst_blob_map[obj.path_in_asset] for obj in need_hash_compare])
+            for posix_blob in need_hash_compare:
+                dst_blob = dst_blob_map[posix_blob.path_in_asset]
+                if not posix_blob.compare_hash(dst_blob):
+                    replace_blobs.append(posix_blob)
+
+        return new_blobs, replace_blobs