modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/volume.py

@@ -1,32 +1,34 @@
 # Copyright Modal Labs 2023
 import asyncio
 import concurrent.futures
+import enum
+import functools
+import os
+import platform
+import re
 import time
-from contextlib import nullcontext
+import typing
+from collections.abc import AsyncGenerator, AsyncIterator, Generator, Sequence
+from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 from typing import (
     IO,
-    AsyncGenerator,
-    AsyncIterator,
+    Any,
     BinaryIO,
     Callable,
-    Generator,
-    List,
     Optional,
-    Sequence,
-    Type,
     Union,
 )
 
-import aiostream
 from grpclib import GRPCError, Status
 from synchronicity.async_wrap import asynccontextmanager
 
-from modal.exception import VolumeUploadTimeoutError, deprecation_warning
+import modal_proto.api_pb2
+from modal.exception import VolumeUploadTimeoutError
 from modal_proto import api_pb2
 
 from ._resolver import Resolver
-from ._utils.async_utils import TaskContext, asyncnullcontext, synchronize_api
+from ._utils.async_utils import TaskContext, aclosing, async_map, asyncnullcontext, synchronize_api
 from ._utils.blob_utils import (
     FileUploadSpec,
     blob_iter,
@@ -34,14 +36,44 @@ from ._utils.blob_utils import (
     get_file_upload_spec_from_fileobj,
     get_file_upload_spec_from_path,
 )
-from ._utils.grpc_utils import retry_transient_errors, unary_stream
+from ._utils.deprecation import deprecation_error, deprecation_warning, renamed_parameter
+from ._utils.grpc_utils import retry_transient_errors
+from ._utils.name_utils import check_object_name
 from .client import _Client
 from .config import logger
 from .object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
 
 # Max duration for uploading to volumes files
 # As a guide, files >40GiB will take >10 minutes to upload.
-VOLUME_PUT_FILE_CLIENT_TIMEOUT = 30 * 60
+VOLUME_PUT_FILE_CLIENT_TIMEOUT = 60 * 60
+
+
+class FileEntryType(enum.IntEnum):
+    """Type of a file entry listed from a Modal volume."""
+
+    UNSPECIFIED = 0
+    FILE = 1
+    DIRECTORY = 2
+    SYMLINK = 3
+
+
+@dataclass(frozen=True)
+class FileEntry:
+    """A file or directory entry listed from a Modal volume."""
+
+    path: str
+    type: FileEntryType
+    mtime: int
+    size: int
+
+    @classmethod
+    def _from_proto(cls, proto: api_pb2.FileEntry) -> "FileEntry":
+        return cls(
+            path=proto.path,
+            type=FileEntryType(proto.type),
+            mtime=proto.mtime,
+            size=proto.size,
+        )
 
 
 class _Volume(_Object, type_prefix="vo"):
@@ -69,16 +101,16 @@ class _Volume(_Object, type_prefix="vo"):
     ```python
     import modal
 
-    stub = modal.Stub()
+    app = modal.App()
     volume = modal.Volume.from_name("my-persisted-volume", create_if_missing=True)
 
-    @stub.function(volumes={"/root/foo": volume})
+    @app.function(volumes={"/root/foo": volume})
     def f():
         with open("/root/foo/bar.txt", "w") as f:
             f.write("hello")
         volume.commit()  # Persist changes
 
-    @stub.function(volumes={"/root/foo": volume})
+    @app.function(volumes={"/root/foo": volume})
     def g():
         volume.reload()  # Fetch latest changes
         with open("/root/foo/bar.txt", "r") as f:
@@ -86,92 +118,84 @@ class _Volume(_Object, type_prefix="vo"):
     ```
     """
 
-    _lock: asyncio.Lock
+    _lock: Optional[asyncio.Lock] = None
 
-    def _initialize_from_empty(self):
+    async def _get_lock(self):
         # To (mostly*) prevent multiple concurrent operations on the same volume, which can cause problems under
         # some unlikely circumstances.
         # *: You can bypass this by creating multiple handles to the same volume, e.g. via lookup. But this
         # covers the typical case = good enough.
-        self._lock = asyncio.Lock()
 
-    @staticmethod
-    def new() -> "_Volume":
-        """`Volume.new` is deprecated.
-
-        Please use `Volume.from_name` (for persisted) or `Volume.ephemeral` (for ephemeral) volumes.
-        """
-        deprecation_warning((2024, 3, 20), Volume.new.__doc__)
-
-        async def _load(self: _Volume, resolver: Resolver, existing_object_id: Optional[str]):
-            status_row = resolver.add_status_row()
-            if existing_object_id:
-                # Volume already exists; do nothing.
-                self._hydrate(existing_object_id, resolver.client, None)
-                return
-
-            status_row.message("Creating volume...")
-            req = api_pb2.VolumeCreateRequest(app_id=resolver.app_id)
-            resp = await retry_transient_errors(resolver.client.stub.VolumeCreate, req)
-            status_row.finish("Created volume.")
-            self._hydrate(resp.volume_id, resolver.client, None)
-
-        return _Volume._from_loader(_load, "Volume()")
+        # Note: this function runs no async code but is marked as async to ensure it's
+        # being run inside the synchronicity event loop and binds the lock to the
+        # correct event loop on Python 3.9 which eagerly assigns event loops on
+        # constructions of locks
+        if self._lock is None:
+            self._lock = asyncio.Lock()
+        return self._lock
 
     @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     def from_name(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> "_Volume":
-        """Create a reference to a persisted volume. Optionally create it lazily.
+        """Reference a Volume by name, creating if necessary.
 
-        **Example Usage**
+        In contrast to `modal.Volume.lookup`, this is a lazy method
+        that defers hydrating the local object with metadata from
+        Modal servers until the first time is is actually used.
 
         ```python
-        import modal
+        vol = modal.Volume.from_name("my-volume", create_if_missing=True)
 
-        volume = modal.Volume.from_name("my-volume", create_if_missing=True)
+        app = modal.App()
 
-        stub = modal.Stub()
-
-        # Volume refers to the same object, even across instances of `stub`.
-        @stub.function(volumes={"/vol": volume})
+        # Volume refers to the same object, even across instances of `app`.
+        @app.function(volumes={"/data": vol})
         def f():
             pass
         ```
         """
+        check_object_name(name, "Volume")
 
         async def _load(self: _Volume, resolver: Resolver, existing_object_id: Optional[str]):
             req = api_pb2.VolumeGetOrCreateRequest(
-                deployment_name=label,
+                deployment_name=name,
                 namespace=namespace,
                 environment_name=_get_environment_name(environment_name, resolver),
                 object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
+                version=version,
             )
             response = await resolver.client.stub.VolumeGetOrCreate(req)
             self._hydrate(response.volume_id, resolver.client, None)
 
-        return _Volume._from_loader(_load, "Volume()")
+        return _Volume._from_loader(_load, "Volume()", hydrate_lazily=True)
 
     @classmethod
     @asynccontextmanager
     async def ephemeral(
-        cls: Type["_Volume"],
+        cls: type["_Volume"],
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
         _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
-    ) -> AsyncIterator["_Volume"]:
+    ) -> AsyncGenerator["_Volume", None]:
         """Creates a new ephemeral volume within a context manager:
 
         Usage:
         ```python
-        with Volume.ephemeral() as vol:
-            assert vol.listdir() == []
+        import modal
+        with modal.Volume.ephemeral() as vol:
+            assert vol.listdir("/") == []
+        ```
 
-        async with Volume.ephemeral() as vol:
-            assert await vol.listdir() == []
+        ```python notest
+        async with modal.Volume.ephemeral() as vol:
+            assert await vol.listdir("/") == []
         ```
         """
         if client is None:
@@ -179,6 +203,7 @@ class _Volume(_Object, type_prefix="vo"):
         request = api_pb2.VolumeGetOrCreateRequest(
             object_creation_type=api_pb2.OBJECT_CREATION_TYPE_EPHEMERAL,
             environment_name=_get_environment_name(environment_name),
+            version=version,
         )
         response = await client.stub.VolumeGetOrCreate(request)
         async with TaskContext() as tc:
@@ -187,33 +212,31 @@ class _Volume(_Object, type_prefix="vo"):
             yield cls._new_hydrated(response.volume_id, client, None, is_another_app=True)
 
     @staticmethod
-    def persisted(
-        label: str,
-        namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
-        environment_name: Optional[str] = None,
-        cloud: Optional[str] = None,
-    ) -> "_Volume":
-        """Deprecated! Use `Volume.from_name(name, create_if_missing=True)`."""
-        deprecation_warning((2024, 3, 1), _Volume.persisted.__doc__)
-        return _Volume.from_name(label, namespace, environment_name, create_if_missing=True)
-
-    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     async def lookup(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> "_Volume":
-        """Lookup a volume with a given name
+        """Lookup a named Volume.
 
-        ```python
-        n = modal.Volume.lookup("my-volume")
-        print(n.listdir("/"))
+        In contrast to `modal.Volume.from_name`, this is an eager method
+        that will hydrate the local object with metadata from Modal servers.
+
+        ```python notest
+        vol = modal.Volume.lookup("my-volume")
+        print(vol.listdir("/"))
         ```
         """
         obj = _Volume.from_name(
-            label, namespace=namespace, environment_name=environment_name, create_if_missing=create_if_missing
+            name,
+            namespace=namespace,
+            environment_name=environment_name,
+            create_if_missing=create_if_missing,
+            version=version,
         )
         if client is None:
             client = await _Client.from_env()
@@ -227,8 +250,10 @@ class _Volume(_Object, type_prefix="vo"):
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
+        version: "typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]" = None,
     ) -> str:
         """mdmd:hidden"""
+        check_object_name(deployment_name, "Volume")
         if client is None:
             client = await _Client.from_env()
         request = api_pb2.VolumeGetOrCreateRequest(
@@ -236,13 +261,14 @@ class _Volume(_Object, type_prefix="vo"):
             namespace=namespace,
             environment_name=_get_environment_name(environment_name),
             object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
+            version=version,
         )
         resp = await retry_transient_errors(client.stub.VolumeGetOrCreate, request)
         return resp.volume_id
 
     @live_method
     async def _do_reload(self, lock=True):
-        async with self._lock if lock else asyncnullcontext():
+        async with (await self._get_lock()) if lock else asyncnullcontext():
             req = api_pb2.VolumeReloadRequest(volume_id=self.object_id)
             _ = await retry_transient_errors(self._client.stub.VolumeReload, req)
 
@@ -253,7 +279,7 @@ class _Volume(_Object, type_prefix="vo"):
         If successful, the changes made are now persisted in durable storage and available to other containers accessing
         the volume.
         """
-        async with self._lock:
+        async with await self._get_lock():
             req = api_pb2.VolumeCommitRequest(volume_id=self.object_id)
             try:
                 # TODO(gongy): only apply indefinite retries on 504 status.
@@ -276,33 +302,68 @@ class _Volume(_Object, type_prefix="vo"):
         try:
             await self._do_reload()
         except GRPCError as exc:
+            # TODO(staffan): This is brittle and janky, as it relies on specific paths and error messages which can
+            #  change server-side at any time. Consider returning the open files directly in the error emitted from the
+            #  server.
+            if exc.message == "there are open files preventing the operation":
+                # Attempt to identify what open files are problematic and include information about the first (to avoid
+                # really verbose errors) open file in the error message to help troubleshooting.
+                # This is best-effort and not necessarily bulletproof, as the view of open files inside the container
+                # might differ from that outside - but it will at least catch common errors.
+                vol_path = f"/__modal/volumes/{self.object_id}"
+                annotation = _open_files_error_annotation(vol_path)
+                if annotation:
+                    raise RuntimeError(f"{exc.message}: {annotation}")
+
             raise RuntimeError(exc.message) if exc.status in (Status.FAILED_PRECONDITION, Status.NOT_FOUND) else exc
 
     @live_method_gen
-    async def iterdir(self, path: str) -> AsyncIterator[api_pb2.VolumeListFilesEntry]:
+    async def iterdir(self, path: str, *, recursive: bool = True) -> AsyncIterator[FileEntry]:
         """Iterate over all files in a directory in the volume.
 
-        * Passing a directory path lists all files in the directory (names are relative to the directory)
-        * Passing a file path returns a list containing only that file's listing description
-        * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
+        Passing a directory path lists all files in the directory. For a file path, return only that
+        file's description. If `recursive` is set to True, list all files and folders under the path
+        recursively.
         """
-        req = api_pb2.VolumeListFilesRequest(volume_id=self.object_id, path=path)
-        async for batch in unary_stream(self._client.stub.VolumeListFiles, req):
+        from modal_version import major_number, minor_number
+
+        # This allows us to remove the server shim after 0.62 is no longer supported.
+        deprecation = deprecation_warning if (major_number, minor_number) <= (0, 62) else deprecation_error
+        if path.endswith("**"):
+            msg = (
+                "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. "
+                "Please pass recursive=True instead. For the CLI, just remove the glob suffix."
+            )
+            deprecation(
+                (2024, 4, 23),
+                msg,
+            )
+        elif path.endswith("*"):
+            deprecation(
+                (2024, 4, 23),
+                (
+                    "Glob patterns in `volume get` and `Volume.listdir()` are deprecated. "
+                    "Please remove the glob `*` suffix."
+                ),
+            )
+
+        req = api_pb2.VolumeListFilesRequest(volume_id=self.object_id, path=path, recursive=recursive)
+        async for batch in self._client.stub.VolumeListFiles.unary_stream(req):
             for entry in batch.entries:
-                yield entry
+                yield FileEntry._from_proto(entry)
 
     @live_method
-    async def listdir(self, path: str) -> List[api_pb2.VolumeListFilesEntry]:
+    async def listdir(self, path: str, *, recursive: bool = False) -> list[FileEntry]:
         """List all files under a path prefix in the modal.Volume.
 
-        * Passing a directory path lists all files in the directory
-        * Passing a file path returns a list containing only that file's listing description
-        * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
+        Passing a directory path lists all files in the directory. For a file path, return only that
+        file's description. If `recursive` is set to True, list all files and folders under the path
+        recursively.
         """
-        return [entry async for entry in self.iterdir(path)]
+        return [entry async for entry in self.iterdir(path, recursive=recursive)]
 
     @live_method_gen
-    async def read_file(self, path: Union[str, bytes]) -> AsyncIterator[bytes]:
+    async def read_file(self, path: str) -> AsyncIterator[bytes]:
         """
         Read a file from the modal.Volume.
 
@@ -316,8 +377,6 @@ class _Volume(_Object, type_prefix="vo"):
         print(len(data))  # == 1024 * 1024
         ```
         """
-        if isinstance(path, str):
-            path = path.encode("utf-8")
         req = api_pb2.VolumeGetFileRequest(volume_id=self.object_id, path=path)
         try:
             response = await retry_transient_errors(self._client.stub.VolumeGetFile, req)
@@ -332,24 +391,12 @@ class _Volume(_Object, type_prefix="vo"):
                 yield data
 
     @live_method
-    async def read_file_into_fileobj(self, path: Union[str, bytes], fileobj: IO[bytes], progress: bool = False) -> int:
+    async def read_file_into_fileobj(self, path: str, fileobj: IO[bytes]) -> int:
         """mdmd:hidden
 
-        Read volume file into file-like IO object, with support for progress display.
-        Used by modal CLI. In future will replace current generator implementation of `read_file` method.
+        Read volume file into file-like IO object.
+        In the future, this will replace the current generator implementation of the `read_file` method.
         """
-        if isinstance(path, str):
-            path = path.encode("utf-8")
-
-        if progress:
-            from ._output import download_progress_bar
-
-            progress_bar = download_progress_bar()
-            task_id = progress_bar.add_task("download", path=path.decode(), start=False)
-            progress_bar.console.log(f"Requesting {path.decode()}")
-        else:
-            progress_bar = nullcontext()
-            task_id = None
 
         chunk_size_bytes = 8 * 1024 * 1024
         start = 0
@@ -363,67 +410,65 @@ class _Volume(_Object, type_prefix="vo"):
 
         n = fileobj.write(response.data)
         if n != len(response.data):
-            raise IOError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
+            raise OSError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
         elif n == response.size:
-            if progress:
-                progress_bar.console.log(f"Wrote {n} bytes to '{path.decode()}'")
             return response.size
         elif n > response.size:
             raise RuntimeError(f"length of returned data exceeds reported filesize: {n} > {response.size}")
         # else: there's more data to read. continue reading with further ranged GET requests.
-        start = n
         file_size = response.size
         written = n
 
-        if progress:
-            progress_bar.update(task_id, total=int(file_size))
-            progress_bar.start_task(task_id)
-
-        with progress_bar:
-            while True:
-                req = api_pb2.VolumeGetFileRequest(
-                    volume_id=self.object_id, path=path, start=start, len=chunk_size_bytes
-                )
-                response = await retry_transient_errors(self._client.stub.VolumeGetFile, req)
-                if response.WhichOneof("data_oneof") != "data":
-                    raise RuntimeError("expected to receive 'data' in response")
-                if len(response.data) > chunk_size_bytes:
-                    raise RuntimeError(f"received more data than requested: {len(response.data)} > {chunk_size_bytes}")
-                elif (written + len(response.data)) > file_size:
-                    raise RuntimeError(f"received data exceeds filesize of {chunk_size_bytes}")
-
-                n = fileobj.write(response.data)
-                if n != len(response.data):
-                    raise IOError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
-                start += n
-                written += n
-                if progress:
-                    progress_bar.update(task_id, advance=n)
-                if written == file_size:
-                    break
+        while True:
+            req = api_pb2.VolumeGetFileRequest(volume_id=self.object_id, path=path, start=written, len=chunk_size_bytes)
+            response = await retry_transient_errors(self._client.stub.VolumeGetFile, req)
+            if response.WhichOneof("data_oneof") != "data":
+                raise RuntimeError("expected to receive 'data' in response")
+            if len(response.data) > chunk_size_bytes:
+                raise RuntimeError(f"received more data than requested: {len(response.data)} > {chunk_size_bytes}")
+            elif (written + len(response.data)) > file_size:
+                raise RuntimeError(f"received data exceeds filesize of {chunk_size_bytes}")
+
+            n = fileobj.write(response.data)
+            if n != len(response.data):
+                raise OSError(f"failed to write {len(response.data)} bytes to output. Wrote {n}.")
+            written += n
+            if written == file_size:
+                break
 
-        if progress:
-            progress_bar.console.log(f"Wrote {written} bytes to '{path.decode()}'")
         return written
 
     @live_method
-    async def remove_file(self, path: Union[str, bytes], recursive: bool = False) -> None:
+    async def remove_file(self, path: str, recursive: bool = False) -> None:
         """Remove a file or directory from a volume."""
-        if isinstance(path, str):
-            path = path.encode("utf-8")
         req = api_pb2.VolumeRemoveFileRequest(volume_id=self.object_id, path=path, recursive=recursive)
         await retry_transient_errors(self._client.stub.VolumeRemoveFile, req)
 
     @live_method
-    async def copy_files(self, src_paths: Sequence[Union[str, bytes]], dst_path: Union[str, bytes]) -> None:
+    async def copy_files(self, src_paths: Sequence[str], dst_path: str) -> None:
         """
         Copy files within the volume from src_paths to dst_path.
         The semantics of the copy operation follow those of the UNIX cp command.
-        """
-        src_paths = [path.encode("utf-8") for path in src_paths if isinstance(path, str)]
-        if isinstance(dst_path, str):
-            dst_path = dst_path.encode("utf-8")
 
+        The `src_paths` parameter is a list. If you want to copy a single file, you should pass a list with a
+        single element.
+
+        `src_paths` and `dst_path` should refer to the desired location *inside* the volume. You do not need to prepend
+        the volume mount path.
+
+        **Usage**
+
+        ```python notest
+        vol = modal.Volume.lookup("my-modal-volume")
+
+        vol.copy_files(["bar/example.txt"], "bar2")  # Copy files to another directory
+        vol.copy_files(["bar/example.txt"], "bar/example2.txt")  # Rename a file by copying
+        ```
+
+        Note that if the volume is already mounted on the Modal function, you should use normal filesystem operations
+        like `os.rename()` and then `commit()` the volume. The `copy_files()` method is useful when you don't have
+        the volume mounted as a filesystem, e.g. when running a script on your local computer.
+        """
         request = api_pb2.VolumeCopyFilesRequest(volume_id=self.object_id, src_paths=src_paths, dst_path=dst_path)
         await retry_transient_errors(self._client.stub.VolumeCopyFiles, request, base_delay=1)
 
@@ -449,11 +494,30 @@ class _Volume(_Object, type_prefix="vo"):
         return _VolumeUploadContextManager(self.object_id, self._client, force=force)
 
     @live_method
-    async def delete(self):
+    async def _instance_delete(self):
         await retry_transient_errors(
             self._client.stub.VolumeDelete, api_pb2.VolumeDeleteRequest(volume_id=self.object_id)
         )
 
+    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
+    async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
+        obj = await _Volume.lookup(name, client=client, environment_name=environment_name)
+        req = api_pb2.VolumeDeleteRequest(volume_id=obj.object_id)
+        await retry_transient_errors(obj._client.stub.VolumeDelete, req)
+
+    @staticmethod
+    async def rename(
+        old_name: str,
+        new_name: str,
+        *,
+        client: Optional[_Client] = None,
+        environment_name: Optional[str] = None,
+    ):
+        obj = await _Volume.lookup(old_name, client=client, environment_name=environment_name)
+        req = api_pb2.VolumeRenameRequest(volume_id=obj.object_id, name=new_name)
+        await retry_transient_errors(obj._client.stub.VolumeRename, req)
+
 
 class _VolumeUploadContextManager:
     """Context manager for batch-uploading files to a Volume."""
@@ -461,13 +525,17 @@ class _VolumeUploadContextManager:
     _volume_id: str
     _client: _Client
     _force: bool
-    _upload_generators: List[Generator[Callable[[], FileUploadSpec], None, None]]
+    progress_cb: Callable[..., Any]
+    _upload_generators: list[Generator[Callable[[], FileUploadSpec], None, None]]
 
-    def __init__(self, volume_id: str, client: _Client, force: bool = False):
+    def __init__(
+        self, volume_id: str, client: _Client, progress_cb: Optional[Callable[..., Any]] = None, force: bool = False
+    ):
         """mdmd:hidden"""
         self._volume_id = volume_id
         self._client = client
         self._upload_generators = []
+        self._progress_cb = progress_cb or (lambda *_, **__: None)
         self._force = force
 
     async def __aenter__(self):
@@ -489,11 +557,13 @@ class _VolumeUploadContextManager:
                     for fut in asyncio.as_completed(futs):
                         yield await fut
 
-            # Compute checksums
-            files_stream = aiostream.stream.iterate(gen_file_upload_specs())
-            # Upload files
-            uploads_stream = aiostream.stream.map(files_stream, self._upload_file, task_limit=20)
-            files: List[api_pb2.MountFile] = await aiostream.stream.list(uploads_stream)
+            # Compute checksums & Upload files
+            files: list[api_pb2.MountFile] = []
+            async with aclosing(async_map(gen_file_upload_specs(), self._upload_file, concurrency=20)) as stream:
+                async for item in stream:
+                    files.append(item)
+
+            self._progress_cb(complete=True)
 
             request = api_pb2.VolumePutFilesRequest(
                 volume_id=self._volume_id,
@@ -559,7 +629,7 @@ class _VolumeUploadContextManager:
 
     async def _upload_file(self, file_spec: FileUploadSpec) -> api_pb2.MountFile:
         remote_filename = file_spec.mount_filename
-
+        progress_task_id = self._progress_cb(name=remote_filename, size=file_spec.size)
         request = api_pb2.MountPutFileRequest(sha256_hex=file_spec.sha256_hex)
         response = await retry_transient_errors(self._client.stub.MountPutFile, request, base_delay=1)
 
@@ -568,7 +638,13 @@ class _VolumeUploadContextManager:
             if file_spec.use_blob:
                 logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
                 with file_spec.source() as fp:
-                    blob_id = await blob_upload_file(fp, self._client.stub)
+                    blob_id = await blob_upload_file(
+                        fp,
+                        self._client.stub,
+                        functools.partial(self._progress_cb, progress_task_id),
+                        sha256_hex=file_spec.sha256_hex,
+                        md5_hex=file_spec.md5_hex,
+                    )
                 logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
                 request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
             else:
@@ -576,6 +652,7 @@ class _VolumeUploadContextManager:
                     f"Uploading file {file_spec.source_description} to {remote_filename} ({file_spec.size} bytes)"
                 )
                 request2 = api_pb2.MountPutFileRequest(data=file_spec.content, sha256_hex=file_spec.sha256_hex)
+                self._progress_cb(task_id=progress_task_id, complete=True)
 
             while (time.monotonic() - start_time) < VOLUME_PUT_FILE_CLIENT_TIMEOUT:
                 response = await retry_transient_errors(self._client.stub.MountPutFile, request2, base_delay=1)
@@ -584,7 +661,8 @@ class _VolumeUploadContextManager:
 
             if not response.exists:
                 raise VolumeUploadTimeoutError(f"Uploading of {file_spec.source_description} timed out")
-
+        else:
+            self._progress_cb(task_id=progress_task_id, complete=True)
         return api_pb2.MountFile(
             filename=remote_filename,
             sha256_hex=file_spec.sha256_hex,
@@ -594,3 +672,53 @@ class _VolumeUploadContextManager:
 
 Volume = synchronize_api(_Volume)
 VolumeUploadContextManager = synchronize_api(_VolumeUploadContextManager)
+
+
+def _open_files_error_annotation(mount_path: str) -> Optional[str]:
+    if platform.system() != "Linux":
+        return None
+
+    self_pid = os.readlink("/proc/self")
+
+    def find_open_file_for_pid(pid: str) -> Optional[str]:
+        # /proc/{pid}/cmdline is null separated
+        with open(f"/proc/{pid}/cmdline", "rb") as f:
+            raw = f.read()
+            parts = raw.split(b"\0")
+            cmdline = " ".join([part.decode() for part in parts]).rstrip(" ")
+
+        cwd = PurePosixPath(os.readlink(f"/proc/{pid}/cwd"))
+        if cwd.is_relative_to(mount_path):
+            if pid == self_pid:
+                return "cwd is inside volume"
+            else:
+                return f"cwd of '{cmdline}' is inside volume"
+
+        for fd in os.listdir(f"/proc/{pid}/fd"):
+            try:
+                path = PurePosixPath(os.readlink(f"/proc/{pid}/fd/{fd}"))
+                try:
+                    rel_path = path.relative_to(mount_path)
+                    if pid == self_pid:
+                        return f"path {rel_path} is open"
+                    else:
+                        return f"path {rel_path} is open from '{cmdline}'"
+                except ValueError:
+                    pass
+
+            except FileNotFoundError:
+                # File was closed
+                pass
+        return None
+
+    pid_re = re.compile("^[1-9][0-9]*$")
+    for dirent in os.listdir("/proc/"):
+        if pid_re.match(dirent):
+            try:
+                annotation = find_open_file_for_pid(dirent)
+                if annotation:
+                    return annotation
+            except (FileNotFoundError, PermissionError):
+                pass
+
+    return None