modal 1.2.1.dev5__tar.gz → 1.2.1.dev6__tar.gz

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.2.1.dev5
+Version: 1.2.1.dev6
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal/cli/_download.py

@@ -1,6 +1,7 @@
 # Copyright Modal Labs 2023
 import asyncio
 import functools
+import multiprocessing
 import os
 import shutil
 import sys
@@ -23,12 +24,22 @@ async def _volume_download(
     remote_path: str,
     local_destination: Path,
     overwrite: bool,
-    progress_cb: Callable,
+    concurrency: Optional[int] = None,
+    progress_cb: Optional[Callable] = None,
 ):
+    if progress_cb is None:
+
+        def progress_cb(*_, **__):
+            pass
+
+    if concurrency is None:
+        concurrency = max(128, 2 * multiprocessing.cpu_count())
+
     is_pipe = local_destination == PIPE_PATH
 
     q: asyncio.Queue[tuple[Optional[Path], Optional[FileEntry]]] = asyncio.Queue()
-    num_consumers = 1 if is_pipe else 10  # concurrency limit for downloading files
+    num_consumers = 1 if is_pipe else concurrency  # concurrency limit for downloading files
+    download_semaphore = asyncio.Semaphore(concurrency)
 
     async def producer():
         iterator: AsyncIterator[FileEntry]
@@ -86,7 +97,12 @@ async def _volume_download(
 
                         with output_path.open("wb") as fp:
                             if isinstance(volume, _Volume):
-                                b = await volume.read_file_into_fileobj(entry.path, fp, file_progress_cb)
+                                b = await volume._read_file_into_fileobj(
+                                    path=entry.path,
+                                    fileobj=fp,
+                                    download_semaphore=download_semaphore,
+                                    progress_cb=file_progress_cb,
+                                )
                             else:
                                 b = 0
                                 async for chunk in volume.read_file(entry.path):

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal/cli/volume.py

@@ -96,7 +96,13 @@ async def get(
     console = make_console()
     progress_handler = ProgressHandler(type="download", console=console)
     with progress_handler.live:
-        await _volume_download(volume, remote_path, destination, force, progress_cb=progress_handler.progress)
+        await _volume_download(
+            volume=volume,
+            remote_path=remote_path,
+            local_destination=destination,
+            overwrite=force,
+            progress_cb=progress_handler.progress,
+        )
     console.print(OutputManager.step_completed("Finished downloading files to local!"))
 
 

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.1.dev5",
+        version: str = "1.2.1.dev6",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.1.dev5",
+        version: str = "1.2.1.dev6",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal/volume.py

@@ -670,16 +670,33 @@ class _Volume(_Object, type_prefix="vo"):
 
     @live_method
     async def read_file_into_fileobj(
-        self, path: str, fileobj: typing.IO[bytes], progress_cb: Optional[Callable[..., Any]] = None
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        progress_cb: Optional[Callable[..., Any]] = None,
     ) -> int:
         """mdmd:hidden
         Read volume file into file-like IO object.
         """
+        return await self._read_file_into_fileobj(path, fileobj, progress_cb=progress_cb)
+
+    @live_method
+    async def _read_file_into_fileobj(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        concurrency: Optional[int] = None,
+        download_semaphore: Optional[asyncio.Semaphore] = None,
+        progress_cb: Optional[Callable[..., Any]] = None,
+    ) -> int:
         if progress_cb is None:
 
             def progress_cb(*_, **__):
                 pass
 
+        if concurrency is None:
+            concurrency = multiprocessing.cpu_count()
+
         req = api_pb2.VolumeGetFile2Request(volume_id=self.object_id, path=path)
 
         try:
@@ -687,8 +704,9 @@ class _Volume(_Object, type_prefix="vo"):
         except modal.exception.NotFoundError as exc:
             raise FileNotFoundError(exc.args[0])
 
-        # TODO(dflemstr): Sane default limit? Make configurable?
-        download_semaphore = asyncio.Semaphore(multiprocessing.cpu_count())
+        if download_semaphore is None:
+            download_semaphore = asyncio.Semaphore(concurrency)
+
         write_lock = asyncio.Lock()
         start_pos = fileobj.tell()
 

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal/volume.pyi

@@ -626,6 +626,14 @@ class _Volume(modal._object._Object):
         """
         ...
 
+    async def _read_file_into_fileobj(
+        self,
+        path: str,
+        fileobj: typing.IO[bytes],
+        concurrency: typing.Optional[int] = None,
+        download_semaphore: typing.Optional[asyncio.locks.Semaphore] = None,
+        progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+    ) -> int: ...
     async def remove_file(self, path: str, recursive: bool = False) -> None:
         """Remove a file or directory from a volume."""
         ...
@@ -1065,6 +1073,28 @@ class Volume(modal.object.Object):
 
     read_file_into_fileobj: __read_file_into_fileobj_spec[typing_extensions.Self]
 
+    class ___read_file_into_fileobj_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(
+            self,
+            /,
+            path: str,
+            fileobj: typing.IO[bytes],
+            concurrency: typing.Optional[int] = None,
+            download_semaphore: typing.Optional[asyncio.locks.Semaphore] = None,
+            progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+        ) -> int: ...
+        async def aio(
+            self,
+            /,
+            path: str,
+            fileobj: typing.IO[bytes],
+            concurrency: typing.Optional[int] = None,
+            download_semaphore: typing.Optional[asyncio.locks.Semaphore] = None,
+            progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
+        ) -> int: ...
+
+    _read_file_into_fileobj: ___read_file_into_fileobj_spec[typing_extensions.Self]
+
     class __remove_file_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, path: str, recursive: bool = False) -> None:
             """Remove a file or directory from a volume."""

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.2.1.dev5
+Version: 1.2.1.dev6
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.2.1.dev5 → modal-1.2.1.dev6}/modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.2.1.dev5"
+__version__ = "1.2.1.dev6"