modal 0.74.58__tar.gz → 0.74.59__tar.gz

{modal-0.74.58 → modal-0.74.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 0.74.58
+Version: 0.74.59
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-0.74.58 → modal-0.74.59}/modal/cli/_download.py

@@ -1,5 +1,6 @@
 # Copyright Modal Labs 2023
 import asyncio
+import functools
 import os
 import shutil
 import sys
@@ -70,21 +71,30 @@ async def _volume_download(
                 if is_pipe:
                     if entry.type == FileEntryType.FILE:
                         progress_task_id = progress_cb(name=entry.path, size=entry.size)
+                        file_progress_cb = functools.partial(progress_cb, task_id=progress_task_id)
+
                         async for chunk in volume.read_file(entry.path):
                             sys.stdout.buffer.write(chunk)
-                            progress_cb(task_id=progress_task_id, advance=len(chunk))
-                        progress_cb(task_id=progress_task_id, complete=True)
+                            file_progress_cb(advance=len(chunk))
+
+                        file_progress_cb(complete=True)
                 else:
                     if entry.type == FileEntryType.FILE:
                         progress_task_id = progress_cb(name=entry.path, size=entry.size)
                         output_path.parent.mkdir(parents=True, exist_ok=True)
+                        file_progress_cb = functools.partial(progress_cb, task_id=progress_task_id)
+
                         with output_path.open("wb") as fp:
-                            b = 0
-                            async for chunk in volume.read_file(entry.path):
-                                b += fp.write(chunk)
-                                progress_cb(task_id=progress_task_id, advance=len(chunk))
+                            if isinstance(volume, _Volume):
+                                b = await volume.read_file_into_fileobj(entry.path, fp, file_progress_cb)
+                            else:
+                                b = 0
+                                async for chunk in volume.read_file(entry.path):
+                                    b += fp.write(chunk)
+                                    file_progress_cb(advance=len(chunk))
+
                         logger.debug(f"Wrote {b} bytes to {output_path}")
-                        progress_cb(task_id=progress_task_id, complete=True)
+                        file_progress_cb(complete=True)
                     elif entry.type == FileEntryType.DIRECTORY:
                         output_path.mkdir(parents=True, exist_ok=True)
             finally:

{modal-0.74.58 → modal-0.74.59}/modal/client.pyi

@@ -27,7 +27,7 @@ class _Client:
     _snapshotted: bool
 
     def __init__(
-        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.74.58"
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.74.59"
     ): ...
     def is_closed(self) -> bool: ...
     @property
@@ -86,7 +86,7 @@ class Client:
     _snapshotted: bool
 
     def __init__(
-        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.74.58"
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.74.59"
     ): ...
     def is_closed(self) -> bool: ...
     @property

{modal-0.74.58 → modal-0.74.59}/modal/volume.py

@@ -463,6 +463,96 @@ class _Volume(_Object, type_prefix="vo"):
                 yield value
 
 
+    @live_method
+    async def read_file_into_fileobj(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        progress_cb: Optional[Callable[..., Any]] = None
+    ) -> int:
+        """mdmd:hidden
+        Read volume file into file-like IO object.
+        """
+        if progress_cb is None:
+            def progress_cb(*_, **__):
+                pass
+
+        if self._is_v1:
+            return await self._read_file_into_fileobj1(path, fileobj, progress_cb)
+        else:
+            return await self._read_file_into_fileobj2(path, fileobj, progress_cb)
+
+
+    async def _read_file_into_fileobj1(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        progress_cb: Callable[..., Any]
+    ) -> int:
+        num_bytes_written = 0
+
+        async for chunk in self._read_file1(path):
+            num_chunk_bytes_written = 0
+
+            while num_chunk_bytes_written < len(chunk):
+                # TODO(dflemstr): this is a small write, but nonetheless might block the event loop for some time:
+                n = fileobj.write(chunk)
+                num_chunk_bytes_written += n
+                progress_cb(advance=n)
+
+            num_bytes_written += len(chunk)
+
+        return num_bytes_written
+
+
+    async def _read_file_into_fileobj2(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        progress_cb: Callable[..., Any]
+    ) -> int:
+        req = api_pb2.VolumeGetFile2Request(volume_id=self.object_id, path=path)
+
+        try:
+            response = await retry_transient_errors(self._client.stub.VolumeGetFile2, req)
+        except GRPCError as exc:
+            raise FileNotFoundError(exc.message) if exc.status == Status.NOT_FOUND else exc
+
+        # TODO(dflemstr): Sane default limit? Make configurable?
+        download_semaphore = asyncio.Semaphore(multiprocessing.cpu_count())
+        write_lock = asyncio.Lock()
+        start_pos = fileobj.tell()
+
+        async def download_block(idx, url) -> int:
+            block_start_pos = start_pos + idx * BLOCK_SIZE
+            num_bytes_written = 0
+
+            async with download_semaphore, ClientSessionRegistry.get_session().get(url) as get_response:
+                async for chunk in get_response.content:
+                    num_chunk_bytes_written = 0
+
+                    while num_chunk_bytes_written < len(chunk):
+                        async with write_lock:
+                            fileobj.seek(block_start_pos + num_bytes_written + num_chunk_bytes_written)
+                            # TODO(dflemstr): this is a small write, but nonetheless might block the event loop for some
+                            #  time:
+                            n = fileobj.write(chunk)
+
+                        num_chunk_bytes_written += n
+                        progress_cb(advance=n)
+
+                    num_bytes_written += len(chunk)
+
+            return num_bytes_written
+
+        coros = [download_block(idx, url) for idx, url in enumerate(response.get_urls)]
+
+        total_size = sum(await asyncio.gather(*coros))
+        fileobj.seek(start_pos + total_size)
+
+        return total_size
+
+
     @live_method
     async def remove_file(self, path: str, recursive: bool = False) -> None:
         """Remove a file or directory from a volume."""

{modal-0.74.58 → modal-0.74.59}/modal/volume.pyi

@@ -87,6 +87,18 @@ class _Volume(modal._object._Object):
     def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
     def _read_file1(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
     def _read_file2(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
+    async def read_file_into_fileobj(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+    ) -> int: ...
+    async def _read_file_into_fileobj1(
+        self, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+    ) -> int: ...
+    async def _read_file_into_fileobj2(
+        self, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+    ) -> int: ...
     async def remove_file(self, path: str, recursive: bool = False) -> None: ...
     async def copy_files(self, src_paths: collections.abc.Sequence[str], dst_path: str) -> None: ...
     async def batch_upload(self, force: bool = False) -> _AbstractVolumeUploadContextManager: ...
@@ -234,6 +246,44 @@ class Volume(modal.object.Object):
 
     _read_file2: ___read_file2_spec[typing_extensions.Self]
 
+    class __read_file_into_fileobj_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(
+            self,
+            /,
+            path: str,
+            fileobj: typing.IO[bytes],
+            progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+        ) -> int: ...
+        async def aio(
+            self,
+            /,
+            path: str,
+            fileobj: typing.IO[bytes],
+            progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+        ) -> int: ...
+
+    read_file_into_fileobj: __read_file_into_fileobj_spec[typing_extensions.Self]
+
+    class ___read_file_into_fileobj1_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(
+            self, /, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+        ) -> int: ...
+        async def aio(
+            self, /, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+        ) -> int: ...
+
+    _read_file_into_fileobj1: ___read_file_into_fileobj1_spec[typing_extensions.Self]
+
+    class ___read_file_into_fileobj2_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(
+            self, /, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+        ) -> int: ...
+        async def aio(
+            self, /, path: str, fileobj: typing.IO[bytes], progress_cb: collections.abc.Callable[..., typing.Any]
+        ) -> int: ...
+
+    _read_file_into_fileobj2: ___read_file_into_fileobj2_spec[typing_extensions.Self]
+
     class __remove_file_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, path: str, recursive: bool = False) -> None: ...
         async def aio(self, /, path: str, recursive: bool = False) -> None: ...

{modal-0.74.58 → modal-0.74.59}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 0.74.58
+Version: 0.74.59
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-0.74.58 → modal-0.74.59}/modal_version/_version_generated.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 
 # Note: Reset this value to -1 whenever you make a minor `0.X` release of the client.
-build_number = 58  # git: ea8f826
+build_number = 59  # git: 57679cb