modal 0.62.115__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/token_flow.pyi

@@ -5,46 +5,51 @@ import typing
 import typing_extensions
 
 class _TokenFlow:
-    def __init__(self, client: modal.client._Client):
-        ...
-
-    def start(self, utm_source: typing.Union[str, None] = None, next_url: typing.Union[str, None] = None) -> typing.AsyncContextManager[typing.Tuple[str, str, str]]:
-        ...
-
-    async def finish(self, timeout: float = 40.0, grpc_extra_timeout: float = 5.0) -> typing.Union[modal_proto.api_pb2.TokenFlowWaitResponse, None]:
-        ...
-
+    def __init__(self, client: modal.client._Client): ...
+    def start(
+        self, utm_source: typing.Optional[str] = None, next_url: typing.Optional[str] = None
+    ) -> typing.AsyncContextManager[tuple[str, str, str]]: ...
+    async def finish(
+        self, timeout: float = 40.0, grpc_extra_timeout: float = 5.0
+    ) -> typing.Optional[modal_proto.api_pb2.TokenFlowWaitResponse]: ...
 
 class TokenFlow:
-    def __init__(self, client: modal.client.Client):
-        ...
+    def __init__(self, client: modal.client.Client): ...
 
     class __start_spec(typing_extensions.Protocol):
-        def __call__(self, utm_source: typing.Union[str, None] = None, next_url: typing.Union[str, None] = None) -> synchronicity.combined_types.AsyncAndBlockingContextManager[typing.Tuple[str, str, str]]:
-            ...
-
-        def aio(self, utm_source: typing.Union[str, None] = None, next_url: typing.Union[str, None] = None) -> typing.AsyncContextManager[typing.Tuple[str, str, str]]:
-            ...
+        def __call__(
+            self, utm_source: typing.Optional[str] = None, next_url: typing.Optional[str] = None
+        ) -> synchronicity.combined_types.AsyncAndBlockingContextManager[tuple[str, str, str]]: ...
+        def aio(
+            self, utm_source: typing.Optional[str] = None, next_url: typing.Optional[str] = None
+        ) -> typing.AsyncContextManager[tuple[str, str, str]]: ...
 
     start: __start_spec
 
     class __finish_spec(typing_extensions.Protocol):
-        def __call__(self, timeout: float = 40.0, grpc_extra_timeout: float = 5.0) -> typing.Union[modal_proto.api_pb2.TokenFlowWaitResponse, None]:
-            ...
-
-        async def aio(self, *args, **kwargs) -> typing.Union[modal_proto.api_pb2.TokenFlowWaitResponse, None]:
-            ...
+        def __call__(
+            self, timeout: float = 40.0, grpc_extra_timeout: float = 5.0
+        ) -> typing.Optional[modal_proto.api_pb2.TokenFlowWaitResponse]: ...
+        async def aio(
+            self, timeout: float = 40.0, grpc_extra_timeout: float = 5.0
+        ) -> typing.Optional[modal_proto.api_pb2.TokenFlowWaitResponse]: ...
 
     finish: __finish_spec
 
-
-async def _new_token(*, profile: typing.Union[str, None] = None, activate: bool = True, verify: bool = True, source: typing.Union[str, None] = None, next_url: typing.Union[str, None] = None):
-    ...
-
-
-async def _set_token(token_id: str, token_secret: str, *, profile: typing.Union[str, None] = None, activate: bool = True, verify: bool = True):
-    ...
-
-
-def _open_url(url: str) -> bool:
-    ...
+async def _new_token(
+    *,
+    profile: typing.Optional[str] = None,
+    activate: bool = True,
+    verify: bool = True,
+    source: typing.Optional[str] = None,
+    next_url: typing.Optional[str] = None,
+): ...
+async def _set_token(
+    token_id: str,
+    token_secret: str,
+    *,
+    profile: typing.Optional[str] = None,
+    activate: bool = True,
+    verify: bool = True,
+): ...
+def _open_url(url: str) -> bool: ...

modal/volume.py

@@ -2,35 +2,33 @@
 import asyncio
 import concurrent.futures
 import enum
+import functools
 import os
 import platform
 import re
 import time
+import typing
+from collections.abc import AsyncGenerator, AsyncIterator, Generator, Sequence
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 from typing import (
     IO,
-    AsyncGenerator,
-    AsyncIterator,
+    Any,
     BinaryIO,
     Callable,
-    Generator,
-    List,
     Optional,
-    Sequence,
-    Type,
     Union,
 )
 
-import aiostream
 from grpclib import GRPCError, Status
 from synchronicity.async_wrap import asynccontextmanager
 
-from modal.exception import InvalidError, VolumeUploadTimeoutError, deprecation_warning
+import modal_proto.api_pb2
+from modal.exception import VolumeUploadTimeoutError
 from modal_proto import api_pb2
 
 from ._resolver import Resolver
-from ._utils.async_utils import TaskContext, asyncnullcontext, synchronize_api
+from ._utils.async_utils import TaskContext, aclosing, async_map, asyncnullcontext, synchronize_api
 from ._utils.blob_utils import (
     FileUploadSpec,
     blob_iter,
@@ -38,15 +36,16 @@ from ._utils.blob_utils import (
     get_file_upload_spec_from_fileobj,
     get_file_upload_spec_from_path,
 )
-from ._utils.grpc_utils import retry_transient_errors, unary_stream
+from ._utils.deprecation import deprecation_error, deprecation_warning, renamed_parameter
+from ._utils.grpc_utils import retry_transient_errors
+from ._utils.name_utils import check_object_name
 from .client import _Client
 from .config import logger
-from .exception import deprecation_error
 from .object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
 
 # Max duration for uploading to volumes files
 # As a guide, files >40GiB will take >10 minutes to upload.
-VOLUME_PUT_FILE_CLIENT_TIMEOUT = 30 * 60
+VOLUME_PUT_FILE_CLIENT_TIMEOUT = 60 * 60
 
 
 class FileEntryType(enum.IntEnum):
@@ -76,12 +75,6 @@ class FileEntry:
             size=proto.size,
         )
 
-    def __getattr__(self, name: str):
-        deprecation_error(
-            (2024, 4, 15),
-            f"The FileEntry dataclass was introduced to replace a private Protobuf message. This dataclass does not have the {name} attribute.",
-        )
-
 
 class _Volume(_Object, type_prefix="vo"):
     """A writeable volume that can be used to share files between one or more Modal functions.
@@ -108,7 +101,7 @@ class _Volume(_Object, type_prefix="vo"):
     ```python
     import modal
 
-    app = modal.App()  # Note: "app" was called "stub" up until April 2024
+    app = modal.App()
     volume = modal.Volume.from_name("my-persisted-volume", create_if_missing=True)
 
     @app.function(volumes={"/root/foo": volume})
@@ -125,69 +118,57 @@ class _Volume(_Object, type_prefix="vo"):
     ```
     """
 
-    _lock: asyncio.Lock
+    _lock: Optional[asyncio.Lock] = None
 
-    def _initialize_from_empty(self):
+    async def _get_lock(self):
         # To (mostly*) prevent multiple concurrent operations on the same volume, which can cause problems under
         # some unlikely circumstances.
         # *: You can bypass this by creating multiple handles to the same volume, e.g. via lookup. But this
         # covers the typical case = good enough.
-        self._lock = asyncio.Lock()
-
-    @staticmethod
-    def new() -> "_Volume":
-        """`Volume.new` is deprecated.
 
-        Please use `Volume.from_name` (for persisted) or `Volume.ephemeral` (for ephemeral) volumes.
-        """
-        deprecation_warning((2024, 3, 20), Volume.new.__doc__)
-
-        async def _load(self: _Volume, resolver: Resolver, existing_object_id: Optional[str]):
-            status_row = resolver.add_status_row()
-            if existing_object_id:
-                # Volume already exists; do nothing.
-                self._hydrate(existing_object_id, resolver.client, None)
-                return
-
-            status_row.message("Creating volume...")
-            req = api_pb2.VolumeCreateRequest(app_id=resolver.app_id)
-            resp = await retry_transient_errors(resolver.client.stub.VolumeCreate, req)
-            status_row.finish("Created volume.")
-            self._hydrate(resp.volume_id, resolver.client, None)
-
-        return _Volume._from_loader(_load, "Volume()")
+        # Note: this function runs no async code but is marked as async to ensure it's
+        # being run inside the synchronicity event loop and binds the lock to the
+        # correct event loop on Python 3.9 which eagerly assigns event loops on
+        # constructions of locks
+        if self._lock is None:
+            self._lock = asyncio.Lock()
+        return self._lock
 
     @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     def from_name(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> "_Volume":
-        """Create a reference to a persisted volume. Optionally create it lazily.
+        """Reference a Volume by name, creating if necessary.
 
-        **Example Usage**
+        In contrast to `modal.Volume.lookup`, this is a lazy method
+        that defers hydrating the local object with metadata from
+        Modal servers until the first time is is actually used.
 
         ```python
-        import modal
-
-        volume = modal.Volume.from_name("my-volume", create_if_missing=True)
+        vol = modal.Volume.from_name("my-volume", create_if_missing=True)
 
-        app = modal.App()  # Note: "app" was called "stub" up until April 2024
+        app = modal.App()
 
         # Volume refers to the same object, even across instances of `app`.
-        @app.function(volumes={"/vol": volume})
+        @app.function(volumes={"/data": vol})
         def f():
             pass
         ```
         """
+        check_object_name(name, "Volume")
 
         async def _load(self: _Volume, resolver: Resolver, existing_object_id: Optional[str]):
             req = api_pb2.VolumeGetOrCreateRequest(
-                deployment_name=label,
+                deployment_name=name,
                 namespace=namespace,
                 environment_name=_get_environment_name(environment_name, resolver),
                 object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
+                version=version,
             )
             response = await resolver.client.stub.VolumeGetOrCreate(req)
             self._hydrate(response.volume_id, resolver.client, None)
@@ -197,20 +178,24 @@ class _Volume(_Object, type_prefix="vo"):
     @classmethod
     @asynccontextmanager
     async def ephemeral(
-        cls: Type["_Volume"],
+        cls: type["_Volume"],
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
         _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
-    ) -> AsyncIterator["_Volume"]:
+    ) -> AsyncGenerator["_Volume", None]:
         """Creates a new ephemeral volume within a context manager:
 
         Usage:
         ```python
-        with Volume.ephemeral() as vol:
-            assert vol.listdir() == []
+        import modal
+        with modal.Volume.ephemeral() as vol:
+            assert vol.listdir("/") == []
+        ```
 
-        async with Volume.ephemeral() as vol:
-            assert await vol.listdir() == []
+        ```python notest
+        async with modal.Volume.ephemeral() as vol:
+            assert await vol.listdir("/") == []
         ```
         """
         if client is None:
@@ -218,6 +203,7 @@ class _Volume(_Object, type_prefix="vo"):
         request = api_pb2.VolumeGetOrCreateRequest(
             object_creation_type=api_pb2.OBJECT_CREATION_TYPE_EPHEMERAL,
             environment_name=_get_environment_name(environment_name),
+            version=version,
         )
         response = await client.stub.VolumeGetOrCreate(request)
         async with TaskContext() as tc:
@@ -226,33 +212,31 @@ class _Volume(_Object, type_prefix="vo"):
             yield cls._new_hydrated(response.volume_id, client, None, is_another_app=True)
 
     @staticmethod
-    def persisted(
-        label: str,
-        namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
-        environment_name: Optional[str] = None,
-        cloud: Optional[str] = None,
-    ) -> "_Volume":
-        """Deprecated! Use `Volume.from_name(name, create_if_missing=True)`."""
-        deprecation_warning((2024, 3, 1), _Volume.persisted.__doc__)
-        return _Volume.from_name(label, namespace, environment_name, create_if_missing=True)
-
-    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     async def lookup(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> "_Volume":
-        """Lookup a volume with a given name
+        """Lookup a named Volume.
 
-        ```python
-        n = modal.Volume.lookup("my-volume")
-        print(n.listdir("/"))
+        In contrast to `modal.Volume.from_name`, this is an eager method
+        that will hydrate the local object with metadata from Modal servers.
+
+        ```python notest
+        vol = modal.Volume.lookup("my-volume")
+        print(vol.listdir("/"))
         ```
         """
         obj = _Volume.from_name(
-            label, namespace=namespace, environment_name=environment_name, create_if_missing=create_if_missing
+            name,
+            namespace=namespace,
+            environment_name=environment_name,
+            create_if_missing=create_if_missing,
+            version=version,
         )
         if client is None:
             client = await _Client.from_env()
@@ -266,8 +250,10 @@ class _Volume(_Object, type_prefix="vo"):
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> str:
         """mdmd:hidden"""
+        check_object_name(deployment_name, "Volume")
         if client is None:
             client = await _Client.from_env()
         request = api_pb2.VolumeGetOrCreateRequest(
@@ -275,13 +261,14 @@ class _Volume(_Object, type_prefix="vo"):
             namespace=namespace,
             environment_name=_get_environment_name(environment_name),
             object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
+            version=version,
         )
         resp = await retry_transient_errors(client.stub.VolumeGetOrCreate, request)
         return resp.volume_id
 
     @live_method
     async def _do_reload(self, lock=True):
-        async with self._lock if lock else asyncnullcontext():
+        async with (await self._get_lock()) if lock else asyncnullcontext():
             req = api_pb2.VolumeReloadRequest(volume_id=self.object_id)
             _ = await retry_transient_errors(self._client.stub.VolumeReload, req)
 
@@ -292,7 +279,7 @@ class _Volume(_Object, type_prefix="vo"):
         If successful, the changes made are now persisted in durable storage and available to other containers accessing
         the volume.
         """
-        async with self._lock:
+        async with await self._get_lock():
             req = api_pb2.VolumeCommitRequest(volume_id=self.object_id)
             try:
                 # TODO(gongy): only apply indefinite retries on 504 status.
@@ -343,23 +330,30 @@ class _Volume(_Object, type_prefix="vo"):
         # This allows us to remove the server shim after 0.62 is no longer supported.
         deprecation = deprecation_warning if (major_number, minor_number) <= (0, 62) else deprecation_error
         if path.endswith("**"):
+            msg = (
+                "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. "
+                "Please pass recursive=True instead. For the CLI, just remove the glob suffix."
+            )
             deprecation(
                 (2024, 4, 23),
-                "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. Please pass recursive=True instead. For the CLI, just remove the glob suffix.",
+                msg,
             )
         elif path.endswith("*"):
             deprecation(
                 (2024, 4, 23),
-                "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. Please remove the glob `*` suffix.",
+                (
+                    "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. "
+                    "Please remove the glob `*` suffix."
+                ),
             )
 
         req = api_pb2.VolumeListFilesRequest(volume_id=self.object_id, path=path, recursive=recursive)
-        async for batch in unary_stream(self._client.stub.VolumeListFiles, req):
+        async for batch in self._client.stub.VolumeListFiles.unary_stream(req):
             for entry in batch.entries:
                 yield FileEntry._from_proto(entry)
 
     @live_method
-    async def listdir(self, path: str, *, recursive: bool = False) -> List[FileEntry]:
+    async def listdir(self, path: str, *, recursive: bool = False) -> list[FileEntry]:
         """List all files under a path prefix in the modal.Volume.
 
         Passing a directory path lists all files in the directory. For a file path, return only that
@@ -369,7 +363,7 @@ class _Volume(_Object, type_prefix="vo"):
         return [entry async for entry in self.iterdir(path, recursive=recursive)]
 
     @live_method_gen
-    async def read_file(self, path: Union[str, bytes]) -> AsyncIterator[bytes]:
+    async def read_file(self, path: str) -> AsyncIterator[bytes]:
         """
         Read a file from the modal.Volume.
 
@@ -383,8 +377,6 @@ class _Volume(_Object, type_prefix="vo"):
         print(len(data))  # == 1024 * 1024
         ```
         """
-        if isinstance(path, str):
-            path = path.encode("utf-8")
         req = api_pb2.VolumeGetFileRequest(volume_id=self.object_id, path=path)
         try:
             response = await retry_transient_errors(self._client.stub.VolumeGetFile, req)
@@ -399,14 +391,12 @@ class _Volume(_Object, type_prefix="vo"):
                 yield data
 
     @live_method
-    async def read_file_into_fileobj(self, path: Union[str, bytes], fileobj: IO[bytes]) -> int:
+    async def read_file_into_fileobj(self, path: str, fileobj: IO[bytes]) -> int:
         """mdmd:hidden
 
         Read volume file into file-like IO object.
         In the future, this will replace the current generator implementation of the `read_file` method.
         """
-        if isinstance(path, str):
-            path = path.encode("utf-8")
 
         chunk_size_bytes = 8 * 1024 * 1024
         start = 0
@@ -420,7 +410,7 @@ class _Volume(_Object, type_prefix="vo"):
 
         n = fileobj.write(response.data)
         if n != len(response.data):
-            raise IOError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
+            raise OSError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
         elif n == response.size:
             return response.size
         elif n > response.size:
@@ -441,7 +431,7 @@ class _Volume(_Object, type_prefix="vo"):
 
             n = fileobj.write(response.data)
             if n != len(response.data):
-                raise IOError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
+                raise OSError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
             written += n
             if written == file_size:
                 break
@@ -449,15 +439,13 @@ class _Volume(_Object, type_prefix="vo"):
         return written
 
     @live_method
-    async def remove_file(self, path: Union[str, bytes], recursive: bool = False) -> None:
+    async def remove_file(self, path: str, recursive: bool = False) -> None:
         """Remove a file or directory from a volume."""
-        if isinstance(path, str):
-            path = path.encode("utf-8")
         req = api_pb2.VolumeRemoveFileRequest(volume_id=self.object_id, path=path, recursive=recursive)
         await retry_transient_errors(self._client.stub.VolumeRemoveFile, req)
 
     @live_method
-    async def copy_files(self, src_paths: Sequence[Union[str, bytes]], dst_path: Union[str, bytes]) -> None:
+    async def copy_files(self, src_paths: Sequence[str], dst_path: str) -> None:
         """
         Copy files within the volume from src_paths to dst_path.
         The semantics of the copy operation follow those of the UNIX cp command.
@@ -481,10 +469,6 @@ class _Volume(_Object, type_prefix="vo"):
         like `os.rename()` and then `commit()` the volume. The `copy_files()` method is useful when you don't have
         the volume mounted as a filesystem, e.g. when running a script on your local computer.
         """
-        src_paths = [path.encode("utf-8") for path in src_paths if isinstance(path, str)]
-        if isinstance(dst_path, str):
-            dst_path = dst_path.encode("utf-8")
-
         request = api_pb2.VolumeCopyFilesRequest(volume_id=self.object_id, src_paths=src_paths, dst_path=dst_path)
         await retry_transient_errors(self._client.stub.VolumeCopyFiles, request, base_delay=1)
 
@@ -515,32 +499,25 @@ class _Volume(_Object, type_prefix="vo"):
             self._client.stub.VolumeDelete, api_pb2.VolumeDeleteRequest(volume_id=self.object_id)
         )
 
-    # @staticmethod  # TODO uncomment when enforcing deprecation of instance method invocation
-    async def delete(*args, label: str = "", client: Optional[_Client] = None, environment_name: Optional[str] = None):
-        # -- Backwards-compatibility section
-        # TODO(michael) Upon enforcement of this deprecation, remove *args and the default argument for label=.
-        if isinstance(self := args[0], _Volume):
-            msg = (
-                "Calling Volume.delete as an instance method is deprecated."
-                " Please update your code to call it as a static method, passing"
-                " the name of the volume to delete, e.g. `modal.Volume.delete('my-volume')`."
-            )
-            deprecation_warning((2024, 4, 23), msg)
-            await self._instance_delete()
-            return
-        elif isinstance(args[0], type):
-            args = args[1:]
-
-        if args and isinstance(args[0], str):
-            if label:
-                raise InvalidError("`label` specified as both positional and keyword argument")
-            label = args[0]
-        # -- Backwards-compatibility code ends here
-
-        obj = await _Volume.lookup(label, client=client, environment_name=environment_name)
+    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
+    async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
+        obj = await _Volume.lookup(name, client=client, environment_name=environment_name)
         req = api_pb2.VolumeDeleteRequest(volume_id=obj.object_id)
         await retry_transient_errors(obj._client.stub.VolumeDelete, req)
 
+    @staticmethod
+    async def rename(
+        old_name: str,
+        new_name: str,
+        *,
+        client: Optional[_Client] = None,
+        environment_name: Optional[str] = None,
+    ):
+        obj = await _Volume.lookup(old_name, client=client, environment_name=environment_name)
+        req = api_pb2.VolumeRenameRequest(volume_id=obj.object_id, name=new_name)
+        await retry_transient_errors(obj._client.stub.VolumeRename, req)
+
 
 class _VolumeUploadContextManager:
     """Context manager for batch-uploading files to a Volume."""
@@ -548,13 +525,17 @@ class _VolumeUploadContextManager:
     _volume_id: str
     _client: _Client
     _force: bool
-    _upload_generators: List[Generator[Callable[[], FileUploadSpec], None, None]]
+    progress_cb: Callable[..., Any]
+    _upload_generators: list[Generator[Callable[[], FileUploadSpec], None, None]]
 
-    def __init__(self, volume_id: str, client: _Client, force: bool = False):
+    def __init__(
+        self, volume_id: str, client: _Client, progress_cb: Optional[Callable[..., Any]] = None, force: bool = False
+    ):
         """mdmd:hidden"""
         self._volume_id = volume_id
         self._client = client
         self._upload_generators = []
+        self._progress_cb = progress_cb or (lambda *_, **__: None)
         self._force = force
 
     async def __aenter__(self):
@@ -576,11 +557,13 @@ class _VolumeUploadContextManager:
                     for fut in asyncio.as_completed(futs):
                         yield await fut
 
-            # Compute checksums
-            files_stream = aiostream.stream.iterate(gen_file_upload_specs())
-            # Upload files
-            uploads_stream = aiostream.stream.map(files_stream, self._upload_file, task_limit=20)
-            files: List[api_pb2.MountFile] = await aiostream.stream.list(uploads_stream)
+            # Compute checksums & Upload files
+            files: list[api_pb2.MountFile] = []
+            async with aclosing(async_map(gen_file_upload_specs(), self._upload_file, concurrency=20)) as stream:
+                async for item in stream:
+                    files.append(item)
+
+            self._progress_cb(complete=True)
 
             request = api_pb2.VolumePutFilesRequest(
                 volume_id=self._volume_id,
@@ -646,7 +629,7 @@ class _VolumeUploadContextManager:
 
     async def _upload_file(self, file_spec: FileUploadSpec) -> api_pb2.MountFile:
         remote_filename = file_spec.mount_filename
-
+        progress_task_id = self._progress_cb(name=remote_filename, size=file_spec.size)
         request = api_pb2.MountPutFileRequest(sha256_hex=file_spec.sha256_hex)
         response = await retry_transient_errors(self._client.stub.MountPutFile, request, base_delay=1)
 
@@ -655,7 +638,13 @@ class _VolumeUploadContextManager:
             if file_spec.use_blob:
                 logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
                 with file_spec.source() as fp:
-                    blob_id = await blob_upload_file(fp, self._client.stub)
+                    blob_id = await blob_upload_file(
+                        fp,
+                        self._client.stub,
+                        functools.partial(self._progress_cb, progress_task_id),
+                        sha256_hex=file_spec.sha256_hex,
+                        md5_hex=file_spec.md5_hex,
+                    )
                 logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
                 request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
             else:
@@ -663,6 +652,7 @@ class _VolumeUploadContextManager:
                     f"Uploading file {file_spec.source_description} to {remote_filename} ({file_spec.size} bytes)"
                 )
                 request2 = api_pb2.MountPutFileRequest(data=file_spec.content, sha256_hex=file_spec.sha256_hex)
+                self._progress_cb(task_id=progress_task_id, complete=True)
 
             while (time.monotonic() - start_time) < VOLUME_PUT_FILE_CLIENT_TIMEOUT:
                 response = await retry_transient_errors(self._client.stub.MountPutFile, request2, base_delay=1)
@@ -671,7 +661,8 @@ class _VolumeUploadContextManager:
 
             if not response.exists:
                 raise VolumeUploadTimeoutError(f"Uploading of {file_spec.source_description} timed out")
-
+        else:
+            self._progress_cb(task_id=progress_task_id, complete=True)
         return api_pb2.MountFile(
             filename=remote_filename,
             sha256_hex=file_spec.sha256_hex,
@@ -697,16 +688,11 @@ def _open_files_error_annotation(mount_path: str) -> Optional[str]:
             cmdline = " ".join([part.decode() for part in parts]).rstrip(" ")
 
         cwd = PurePosixPath(os.readlink(f"/proc/{pid}/cwd"))
-        # NOTE(staffan): Python 3.8 doesn't have is_relative_to(), so we're stuck with catching ValueError until
-        # we drop Python 3.8 support.
-        try:
-            _rel_cwd = cwd.relative_to(mount_path)
+        if cwd.is_relative_to(mount_path):
             if pid == self_pid:
                 return "cwd is inside volume"
             else:
                 return f"cwd of '{cmdline}' is inside volume"
-        except ValueError:
-            pass
 
         for fd in os.listdir(f"/proc/{pid}/fd"):
             try: