flytekitplugins-async-fsspec 1.10.3b5__tar.gz

flytekitplugins-async-fsspec-1.10.3b5/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: flytekitplugins-async-fsspec
+Version: 1.10.3b5
+Summary: This package holds the data persistence plugins for flytekit
+Author: flyteorg
+Author-email: admin@flyte.org
+License: apache2
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Requires-Dist: flytekit

flytekitplugins-async-fsspec-1.10.3b5/README.md

@@ -0,0 +1,14 @@
+# Flytekit Async fsspec Plugin
+
+The Flyte async fsspec plugin is a powerful addition to the Flyte ecosystem designed to optimize the performance of object transmission. This plugin focuses on overriding key methods of the file systems in fsspec to introduce efficiency improvements, resulting in accelerated data transfers between Flyte workflows and object storage.
+
+Currently, the async fsspec plugin improves the following file systems:
+1. s3fs
+
+To install the plugin, run the following command:
+
+```bash
+pip install flytekitplugins-async-fsspec
+```
+
+Once installed, the plugin will automatically override the original file system and register optimized ones, seamlessly integrating with your Flyte workflows.

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins/async_fsspec/__init__.py

@@ -0,0 +1,16 @@
+"""
+.. currentmodule:: flytekitplugins.async_fsspec
+
+This package contains things that are useful when extending Flytekit.
+
+.. autosummary::
+   :template: custom.rst
+   :toctree: generated/
+
+   AsyncS3FileSystem
+"""
+import fsspec
+
+from .s3fs.s3fs import AsyncS3FileSystem
+
+fsspec.register_implementation("s3", AsyncS3FileSystem)

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins/async_fsspec/s3fs/constants.py

@@ -0,0 +1,6 @@
+DEFAULT_UPLOAD_CHUNK_SIZE = 50 * 2**20  # 50MB
+DEFAULT_CONCURRENT_UPLOAD = 4
+SINGLE_OBJECT_UPLOAD_LIMIT = 5 * 2**30  # 5GB
+DEFAULT_DOWNLOAD_CHUNK_SIZE = 50 * 2**20  # 50MB
+DEFAULT_CONCURRENT_DOWNLOAD = 4
+DEFAULT_DOWNLOAD_BODY_READ_SIZE = 2**16  # from s3fs

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins/async_fsspec/s3fs/s3fs.py

@@ -0,0 +1,240 @@
+import asyncio
+import mimetypes
+import os
+
+from fsspec.callbacks import _DEFAULT_CALLBACK
+from s3fs import S3FileSystem
+from s3fs.core import S3_RETRYABLE_ERRORS, version_id_kw
+
+from .constants import (
+    DEFAULT_CONCURRENT_DOWNLOAD,
+    DEFAULT_CONCURRENT_UPLOAD,
+    DEFAULT_DOWNLOAD_BODY_READ_SIZE,
+    DEFAULT_DOWNLOAD_CHUNK_SIZE,
+    DEFAULT_UPLOAD_CHUNK_SIZE,
+    SINGLE_OBJECT_UPLOAD_LIMIT,
+)
+
+
+class AsyncS3FileSystem(S3FileSystem):
+    def __init__(self, **s3kwargs):
+        super().__init__(**s3kwargs)
+
+    async def _put_file(
+        self,
+        lpath,
+        rpath,
+        callback=_DEFAULT_CALLBACK,
+        chunksize=DEFAULT_UPLOAD_CHUNK_SIZE,
+        concurrent_upload=DEFAULT_CONCURRENT_UPLOAD,
+        **kwargs,
+    ):
+        """
+        Put a file from lpath to rpath.
+        Args:
+            lpath (str): The local path of the file to be uploaded.
+            rpath (str): The remote path which the file should be uploaded to.
+            callback (function, optional): The callback function.
+            chunksize (int, optional): Upload chunksize. Defaults to 50 * 2**20 (50MB).
+            concurrent_upload (int, optional): The number of concurrent upload when using multipart upload. Defaults to 4.
+        """
+        bucket, key, _ = self.split_path(rpath)
+        if os.path.isdir(lpath):
+            if key:
+                # don't make remote "directory"
+                return
+            else:
+                await self._mkdir(lpath)
+        size = os.path.getsize(lpath)
+        callback.set_size(size)
+
+        if "ContentType" not in kwargs:
+            content_type, _ = mimetypes.guess_type(lpath)
+            if content_type is not None:
+                kwargs["ContentType"] = content_type
+
+        with open(lpath, "rb") as f0:
+            if size < min(SINGLE_OBJECT_UPLOAD_LIMIT, 2 * chunksize):
+                chunk = f0.read()
+                await self._call_s3("put_object", Bucket=bucket, Key=key, Body=chunk, **kwargs)
+                callback.relative_update(size)
+            else:
+                mpu = await self._call_s3("create_multipart_upload", Bucket=bucket, Key=key, **kwargs)
+
+                # async function to upload a single chunk
+                async def upload_chunk(chunk, part_number):
+                    result = await self._call_s3(
+                        "upload_part",
+                        Bucket=bucket,
+                        PartNumber=part_number,
+                        UploadId=mpu["UploadId"],
+                        Body=chunk,
+                        Key=key,
+                    )
+                    callback.relative_update(len(chunk))
+                    return {"PartNumber": part_number, "ETag": result["ETag"]}
+
+                tasks = set()
+                part_number = 1
+                parts = []
+                read_end = False
+                while True:
+                    while len(tasks) < concurrent_upload:
+                        chunk = f0.read(chunksize)
+                        if not chunk:
+                            read_end = True
+                            break
+                        tasks.add(asyncio.create_task(upload_chunk(chunk, part_number)))
+                        part_number += 1
+                    if read_end:
+                        break
+                    done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+                    parts.extend(map(lambda x: x.result(), done))
+                    tasks = pending
+
+                parts.extend(await asyncio.gather(*tasks))
+                parts.sort(key=lambda part: part["PartNumber"])
+                await self._call_s3(
+                    "complete_multipart_upload",
+                    Bucket=bucket,
+                    Key=key,
+                    UploadId=mpu["UploadId"],
+                    MultipartUpload={"Parts": parts},
+                )
+        while rpath:
+            self.invalidate_cache(rpath)
+            rpath = self._parent(rpath)
+
+    async def _get_file(
+        self,
+        rpath,
+        lpath,
+        callback=_DEFAULT_CALLBACK,
+        version_id=None,
+        chunksize=DEFAULT_DOWNLOAD_CHUNK_SIZE,
+        concurrent_download=DEFAULT_CONCURRENT_DOWNLOAD,
+    ):
+        """
+        Get a file from rpath to lpath.
+        Args:
+            rpath (str): The remote path of the file to be downloaded.
+            lpath (str): The local path which the file should be downloaded to.
+            callback (function, optional): The callback function.
+            chunksize (int, optional): Download chunksize. Defaults to 50 * 2**20 (50MB).
+            version_id (str, optional): The version id of the file. Defaults to None.
+        """
+        if os.path.isdir(lpath):
+            return
+
+        # get the file size
+        file_info = await self._info(path=rpath, version_id=version_id)
+        file_size = file_info["size"]
+
+        bucket, key, vers = self.split_path(rpath)
+
+        # the async function to get a range of the remote file
+        async def _open_file(start_byte: int, end_byte: int = None):
+            kw = self.req_kw.copy()
+            if end_byte:
+                kw["Range"] = f"bytes={start_byte}-{end_byte}"
+            else:
+                kw["Range"] = f"bytes={start_byte}"
+            resp = await self._call_s3(
+                "get_object",
+                Bucket=bucket,
+                Key=key,
+                **version_id_kw(version_id or vers),
+                **kw,
+            )
+            return resp["Body"], resp.get("ContentLength", None)
+
+        # Refer to s3fs's implementation
+        async def handle_read_error(body, failed_reads, restart_byte, end_byte=None):
+            if failed_reads >= self.retries:
+                raise
+            try:
+                body.close()
+            except Exception:
+                pass
+
+            await asyncio.sleep(min(1.7**failed_reads * 0.1, 15))
+            body, _ = await _open_file(restart_byte, end_byte)
+            return body
+
+        # According to s3fs documentation, some file systems might not be able to measure the file’s size,
+        # in which case, the returned dict will include 'size': None. When we cannot get the file size
+        # in advance, we keep using the original implementation of s3fs.
+        if file_size is None:
+            # From s3fs
+            body, content_length = await _open_file(start_byte=0)
+            callback.set_size(content_length)
+
+            failed_reads = 0
+            bytes_read = 0
+
+            try:
+                with open(lpath, "wb") as f0:
+                    while True:
+                        try:
+                            chunk = await body.read(DEFAULT_DOWNLOAD_BODY_READ_SIZE)
+                        except S3_RETRYABLE_ERRORS:
+                            failed_reads += 1
+                            body = await handle_read_error(body, failed_reads, bytes_read)
+                            continue
+
+                        if not chunk:
+                            break
+
+                        f0.write(chunk)
+                        bytes_read += len(chunk)
+                        callback.relative_update(len(chunk))
+            finally:
+                try:
+                    body.close()
+                except Exception:
+                    pass
+        else:
+            callback.set_size(file_size)
+            with open(lpath, "wb") as f0:
+                # async function to download a single chunk
+                async def download_chunk(chunk_index: int):
+                    start_byte = chunk_index * chunksize
+                    end_byte = min(start_byte + chunksize, file_size) - 1
+                    body, _ = await _open_file(start_byte, end_byte)
+                    failed_reads = 0
+                    bytes_read = 0
+                    try:
+                        while True:
+                            try:
+                                chunk = await body.read(DEFAULT_DOWNLOAD_BODY_READ_SIZE)
+                            except S3_RETRYABLE_ERRORS:
+                                failed_reads += 1
+                                body = await handle_read_error(body, failed_reads, start_byte + bytes_read, end_byte)
+                                continue
+
+                            if not chunk:
+                                break
+
+                            f0.seek(start_byte + bytes_read)
+                            f0.write(chunk)
+                            bytes_read += len(chunk)
+                            callback.relative_update(len(chunk))
+                    finally:
+                        try:
+                            body.close()
+                        except Exception:
+                            pass
+
+                chunk_count = file_size // chunksize
+                if file_size % chunksize > 0:
+                    chunk_count += 1
+
+                tasks = set()
+                current_chunk = 0
+                while current_chunk < chunk_count:
+                    while current_chunk < chunk_count and len(tasks) < concurrent_download:
+                        tasks.add(asyncio.create_task(download_chunk(current_chunk)))
+                        current_chunk += 1
+                    _, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+                    tasks = pending
+                await asyncio.gather(*tasks)

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: flytekitplugins-async-fsspec
+Version: 1.10.3b5
+Summary: This package holds the data persistence plugins for flytekit
+Author: flyteorg
+Author-email: admin@flyte.org
+License: apache2
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Requires-Dist: flytekit

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+setup.py
+flytekitplugins/async_fsspec/__init__.py
+flytekitplugins/async_fsspec/s3fs/__init__.py
+flytekitplugins/async_fsspec/s3fs/constants.py
+flytekitplugins/async_fsspec/s3fs/s3fs.py
+flytekitplugins_async_fsspec.egg-info/PKG-INFO
+flytekitplugins_async_fsspec.egg-info/SOURCES.txt
+flytekitplugins_async_fsspec.egg-info/dependency_links.txt
+flytekitplugins_async_fsspec.egg-info/entry_points.txt
+flytekitplugins_async_fsspec.egg-info/namespace_packages.txt
+flytekitplugins_async_fsspec.egg-info/requires.txt
+flytekitplugins_async_fsspec.egg-info/top_level.txt
+tests/test_s3fs.py

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[flytekit.plugins]
+async_fsspec = flytekitplugins.async_fsspec

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/namespace_packages.txt

@@ -0,0 +1 @@
+flytekitplugins

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/requires.txt

@@ -0,0 +1 @@
+flytekit

flytekitplugins-async-fsspec-1.10.3b5/flytekitplugins_async_fsspec.egg-info/top_level.txt

@@ -0,0 +1 @@
+flytekitplugins

flytekitplugins-async-fsspec-1.10.3b5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

flytekitplugins-async-fsspec-1.10.3b5/setup.py

@@ -0,0 +1,37 @@
+from setuptools import setup
+
+PLUGIN_NAME = "async_fsspec"
+
+microlib_name = "flytekitplugins-async-fsspec"
+
+plugin_requires = ["flytekit"]
+
+__version__ = "1.10.3b5"
+
+setup(
+    name=microlib_name,
+    version=__version__,
+    author="flyteorg",
+    author_email="admin@flyte.org",
+    description="This package holds the data persistence plugins for flytekit",
+    namespace_packages=["flytekitplugins"],
+    packages=[f"flytekitplugins.{PLUGIN_NAME}", f"flytekitplugins.{PLUGIN_NAME}.s3fs"],
+    install_requires=plugin_requires,
+    license="apache2",
+    python_requires=">=3.8",
+    classifiers=[
+        "Intended Audience :: Science/Research",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]},
+)

flytekitplugins-async-fsspec-1.10.3b5/tests/test_s3fs.py

@@ -0,0 +1,201 @@
+import os
+from unittest import mock
+from unittest.mock import MagicMock, mock_open
+
+import pytest
+from flytekitplugins.async_fsspec import AsyncS3FileSystem
+from flytekitplugins.async_fsspec.s3fs.constants import DEFAULT_DOWNLOAD_CHUNK_SIZE, DEFAULT_UPLOAD_CHUNK_SIZE
+
+
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._parent")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem.invalidate_cache")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._call_s3")
+@mock.patch("mimetypes.guess_type")
+@mock.patch("os.path.getsize")
+@pytest.mark.asyncio
+async def test_put_file_single_object_upload(
+    mock_getsize, mock_guess_type, mock_call_s3, mock_invalidate_cache, mock_parent
+):
+    mock_bucket = "mock-bucket"
+    mock_file_name = "mock_file_name"
+    mock_file_size = 32 * 2**20  # 32MB
+    mock_getsize.return_value = mock_file_size
+    mock_guess_type.return_value = (None, None)
+    mock_parent.return_value = None
+    mock_body = os.urandom(mock_file_size)
+    m = mock_open(read_data=mock_body)
+
+    with mock.patch("builtins.open", m):
+        asyncs3fs = AsyncS3FileSystem()
+        await asyncs3fs._put_file(lpath=f"/{mock_file_name}", rpath=f"s3://{mock_bucket}/{mock_file_name}")
+
+    mock_call_s3.assert_called_once_with("put_object", Bucket=mock_bucket, Key=mock_file_name, Body=mock_body)
+
+
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._parent")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem.invalidate_cache")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._call_s3")
+@mock.patch("mimetypes.guess_type")
+@mock.patch("os.path.getsize")
+@pytest.mark.asyncio
+async def test_put_file_multipart_upload(
+    mock_getsize, mock_guess_type, mock_call_s3, mock_invalidate_cache, mock_parent
+):
+    mock_bucket = "mock-bucket"
+    mock_file_name = "mock_file_name"
+    mock_upload_id = "mock_upload_id"
+    mock_ETag = "mock_ETag"
+    mock_file_size = 256 * 2**20  # 256MB
+    mock_getsize.return_value = mock_file_size
+    mock_guess_type.return_value = (None, None)
+
+    def call_s3_side_effect(*args, **kwargs):
+        if args[0] == "create_multipart_upload":
+            return {"UploadId": mock_upload_id}
+        elif args[0] == "upload_part":
+            part_number = kwargs["PartNumber"]
+            return {"ETag": f"{mock_ETag}{part_number}"}
+        elif args[0] == "complete_multipart_upload":
+            return None
+
+    mock_call_s3.side_effect = call_s3_side_effect
+
+    mock_parent.return_value = None
+
+    mock_body = os.urandom(mock_file_size)
+    m = mock_open(read_data=mock_body)
+
+    with mock.patch("builtins.open", m):
+        asyncs3fs = AsyncS3FileSystem()
+        await asyncs3fs._put_file(lpath=f"/{mock_file_name}", rpath=f"s3://{mock_bucket}/{mock_file_name}")
+
+    mock_chunk_count = mock_file_size // DEFAULT_UPLOAD_CHUNK_SIZE
+    if mock_file_size % DEFAULT_UPLOAD_CHUNK_SIZE > 0:
+        mock_chunk_count += 1
+    put_object_calls = []
+    for i in range(mock_chunk_count):
+        part_number = i + 1
+        start_byte = i * DEFAULT_UPLOAD_CHUNK_SIZE
+        end_byte = min(start_byte + DEFAULT_UPLOAD_CHUNK_SIZE, mock_file_size)
+        body = mock_body[start_byte:end_byte]
+        put_object_calls.append(
+            mock.call(
+                "upload_part",
+                Bucket=mock_bucket,
+                Key=mock_file_name,
+                PartNumber=part_number,
+                UploadId=mock_upload_id,
+                Body=body,
+            ),
+        )
+
+    mock_call_s3.assert_has_calls(
+        put_object_calls
+        + [
+            mock.call("create_multipart_upload", Bucket=mock_bucket, Key=mock_file_name),
+            mock.call(
+                "complete_multipart_upload",
+                Bucket=mock_bucket,
+                Key=mock_file_name,
+                UploadId=mock_upload_id,
+                MultipartUpload={
+                    "Parts": [{"PartNumber": i, "ETag": f"{mock_ETag}{i}"} for i in range(1, mock_chunk_count + 1)]
+                },
+            ),
+        ],
+        any_order=True,
+    )
+    assert mock_call_s3.call_count == 2 + mock_chunk_count
+
+
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._call_s3")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._info")
+@mock.patch("os.path.isdir")
+@pytest.mark.asyncio
+async def test_get_file_file_size_is_none(mock_isdir, mock_info, mock_call_s3):
+    mock_bucket = "mock-bucket"
+    mock_file_name = "mock_file_name"
+    mock_file_size = 32 * 2**20  # 32MB
+    mock_isdir.return_value = False
+    mock_info.return_value = {"size": None}
+
+    file_been_read = 0
+
+    async def read_side_effect(*args, **kwargs):
+        read_size = args[0]
+        nonlocal file_been_read
+        real_read_size = min(read_size, mock_file_size - file_been_read)
+        if real_read_size == 0:
+            return None
+        file_been_read += real_read_size
+        return os.urandom(real_read_size)
+
+    mock_chunk = MagicMock()
+    mock_chunk.read.side_effect = read_side_effect
+    mock_call_s3.return_value = {"Body": mock_chunk, "ContentLength": mock_file_size}
+
+    m = mock_open()
+
+    with mock.patch("builtins.open", m):
+        asyncs3fs = AsyncS3FileSystem()
+        await asyncs3fs._get_file(lpath=f"/{mock_file_name}", rpath=f"s3://{mock_bucket}/{mock_file_name}")
+
+    assert file_been_read == mock_file_size
+    mock_call_s3.assert_called_once_with("get_object", Bucket=mock_bucket, Key=mock_file_name, Range="bytes=0")
+
+
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._call_s3")
+@mock.patch("flytekitplugins.async_fsspec.AsyncS3FileSystem._info")
+@mock.patch("os.path.isdir")
+@pytest.mark.asyncio
+async def test_get_file_file_size_is_not_none(mock_isdir, mock_info, mock_call_s3):
+    mock_bucket = "mock-bucket"
+    mock_file_name = "mock_file_name"
+    mock_file_size = 256 * 2**20  # 256MB
+    mock_isdir.return_value = False
+    mock_info.return_value = {"size": mock_file_size}
+
+    file_been_read = 0
+
+    def call_s3_side_effect(*args, **kwargs):
+        start_byte, end_byte = kwargs["Range"][6:].split("-")
+        start_byte, end_byte = int(start_byte), int(end_byte)
+        chunk_size = end_byte - start_byte + 1
+        chunk_been_read = 0
+
+        async def read_side_effect(*args, **kwargs):
+            nonlocal chunk_been_read
+            nonlocal file_been_read
+            read_size = args[0]
+            real_read_size = min(read_size, chunk_size - chunk_been_read)
+            if real_read_size == 0:
+                return None
+            chunk_been_read += real_read_size
+            file_been_read += real_read_size
+            return os.urandom(real_read_size)
+
+        mock_chunk = MagicMock()
+        mock_chunk.read.side_effect = read_side_effect
+        return {"Body": mock_chunk, "ContentLength": chunk_size}
+
+    mock_call_s3.side_effect = call_s3_side_effect
+
+    m = mock_open()
+    with mock.patch("builtins.open", m):
+        asyncs3fs = AsyncS3FileSystem()
+        await asyncs3fs._get_file(lpath=f"/{mock_file_name}", rpath=f"s3://{mock_bucket}/{mock_file_name}")
+
+    assert file_been_read == mock_file_size
+
+    mock_chunk_count = mock_file_size // DEFAULT_DOWNLOAD_CHUNK_SIZE
+    if mock_file_size % DEFAULT_DOWNLOAD_CHUNK_SIZE > 0:
+        mock_chunk_count += 1
+    get_object_calls = []
+    for i in range(mock_chunk_count):
+        start_byte = i * DEFAULT_DOWNLOAD_CHUNK_SIZE
+        end_byte = min(start_byte + DEFAULT_DOWNLOAD_CHUNK_SIZE, mock_file_size) - 1
+        get_object_calls.append(
+            mock.call("get_object", Bucket=mock_bucket, Key=mock_file_name, Range=f"bytes={start_byte}-{end_byte}")
+        )
+    mock_call_s3.assert_has_calls(get_object_calls)
+    assert mock_call_s3.call_count == len(get_object_calls)