modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/network_file_system.py

@@ -1,8 +1,10 @@
 # Copyright Modal Labs 2023
+import functools
 import os
 import time
+from collections.abc import AsyncIterator
 from pathlib import Path, PurePosixPath
-from typing import AsyncIterator, BinaryIO, List, Optional, Tuple, Type, Union
+from typing import Any, BinaryIO, Callable, Optional, Union
 
 from grpclib import GRPCError, Status
 from synchronicity.async_wrap import asynccontextmanager
@@ -11,12 +13,14 @@ import modal
 from modal_proto import api_pb2
 
 from ._resolver import Resolver
-from ._utils.async_utils import ConcurrencyPool, TaskContext, synchronize_api
+from ._utils.async_utils import TaskContext, aclosing, async_map, sync_or_async_iter, synchronize_api
 from ._utils.blob_utils import LARGE_FILE_LIMIT, blob_iter, blob_upload_file
-from ._utils.grpc_utils import retry_transient_errors, unary_stream
+from ._utils.deprecation import renamed_parameter
+from ._utils.grpc_utils import retry_transient_errors
 from ._utils.hash_utils import get_sha256_hex
+from ._utils.name_utils import check_object_name
 from .client import _Client
-from .exception import deprecation_warning
+from .exception import InvalidError
 from .object import (
     EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
     _get_environment_name,
@@ -24,6 +28,7 @@ from .object import (
     live_method,
     live_method_gen,
 )
+from .volume import FileEntry
 
 NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT = (
     10 * 60
@@ -31,9 +36,9 @@ NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT = (
 
 
 def network_file_system_mount_protos(
-    validated_network_file_systems: List[Tuple[str, "_NetworkFileSystem"]],
+    validated_network_file_systems: list[tuple[str, "_NetworkFileSystem"]],
     allow_cross_region_volumes: bool,
-) -> List[api_pb2.SharedVolumeMount]:
+) -> list[api_pb2.SharedVolumeMount]:
     network_file_system_mounts = []
     # Relies on dicts being ordered (true as of Python 3.6).
     for path, volume in validated_network_file_systems:
@@ -59,20 +64,20 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
     import modal
 
     nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
-    stub = modal.Stub()
+    app = modal.App()
 
-    @stub.function(network_file_systems={"/root/foo": nfs})
+    @app.function(network_file_systems={"/root/foo": nfs})
     def f():
         pass
 
-    @stub.function(network_file_systems={"/root/goo": nfs})
+    @app.function(network_file_systems={"/root/goo": nfs})
     def g():
         pass
     ```
 
     Also see the CLI methods for accessing network file systems:
 
-    ```bash
+    ```
     modal nfs --help
     ```
 
@@ -86,68 +91,52 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
     """
 
     @staticmethod
-    def new(cloud: Optional[str] = None) -> "_NetworkFileSystem":
-        """`NetworkFileSystem.new` is deprecated.
-
-        Please use `NetworkFileSystem.from_name` (for persisted) or `NetworkFileSystem.ephemeral`
-        (for ephemeral) network filesystems.
-        """
-        deprecation_warning((2024, 3, 20), NetworkFileSystem.new.__doc__)
-
-        async def _load(self: _NetworkFileSystem, resolver: Resolver, existing_object_id: Optional[str]):
-            status_row = resolver.add_status_row()
-            if existing_object_id:
-                # Volume already exists; do nothing.
-                self._hydrate(existing_object_id, resolver.client, None)
-                return
-
-            if cloud:
-                deprecation_warning((2024, 1, 17), "Argument `cloud` is deprecated (has no effect).")
-
-            status_row.message("Creating network file system...")
-            req = api_pb2.SharedVolumeCreateRequest(app_id=resolver.app_id)
-            resp = await retry_transient_errors(resolver.client.stub.SharedVolumeCreate, req)
-            status_row.finish("Created network file system.")
-            self._hydrate(resp.shared_volume_id, resolver.client, None)
-
-        return _NetworkFileSystem._from_loader(_load, "NetworkFileSystem()")
-
-    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     def from_name(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
     ) -> "_NetworkFileSystem":
-        """Create a reference to a persisted network filesystem, optionally creating it lazily.
+        """Reference a NetworkFileSystem by its name, creating if necessary.
 
-        **Examples**
+        In contrast to `modal.NetworkFileSystem.lookup`, this is a lazy method
+        that defers hydrating the local object with metadata from Modal servers
+        until the first time it is actually used.
 
         ```python notest
-        volume = NetworkFileSystem.from_name("my-volume", create_if_missing=True)
+        nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
 
-        @stub.function(network_file_systems={"/vol": volume})
+        @app.function(network_file_systems={"/data": nfs})
         def f():
             pass
         ```
         """
+        check_object_name(name, "NetworkFileSystem")
 
         async def _load(self: _NetworkFileSystem, resolver: Resolver, existing_object_id: Optional[str]):
             req = api_pb2.SharedVolumeGetOrCreateRequest(
-                deployment_name=label,
+                deployment_name=name,
                 namespace=namespace,
                 environment_name=_get_environment_name(environment_name, resolver),
                 object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
             )
-            response = await resolver.client.stub.SharedVolumeGetOrCreate(req)
-            self._hydrate(response.shared_volume_id, resolver.client, None)
-
-        return _NetworkFileSystem._from_loader(_load, "NetworkFileSystem()")
+            try:
+                response = await resolver.client.stub.SharedVolumeGetOrCreate(req)
+                self._hydrate(response.shared_volume_id, resolver.client, None)
+            except GRPCError as exc:
+                if exc.status == Status.NOT_FOUND and exc.message == "App has wrong entity vo":
+                    raise InvalidError(
+                        f"Attempted to mount: `{name}` as a NetworkFileSystem " + "which already exists as a Volume"
+                    )
+                raise
+
+        return _NetworkFileSystem._from_loader(_load, "NetworkFileSystem()", hydrate_lazily=True)
 
     @classmethod
     @asynccontextmanager
     async def ephemeral(
-        cls: Type["_NetworkFileSystem"],
+        cls: type["_NetworkFileSystem"],
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
         _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
@@ -156,11 +145,13 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
 
         Usage:
         ```python
-        with NetworkFileSystem.ephemeral() as nfs:
-            assert nfs.listdir() == []
+        with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert nfs.listdir("/") == []
+        ```
 
-        async with NetworkFileSystem.ephemeral() as nfs:
-            assert await nfs.listdir() == []
+        ```python notest
+        async with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert await nfs.listdir("/") == []
         ```
         """
         if client is None:
@@ -176,44 +167,26 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
             yield cls._new_hydrated(response.shared_volume_id, client, None, is_another_app=True)
 
     @staticmethod
-    def persisted(
-        label: str,
-        namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
-        environment_name: Optional[str] = None,
-        cloud: Optional[str] = None,
-    ) -> "_NetworkFileSystem":
-        """Deprecated! Use `NetworkFileSystem.from_name(name, create_if_missing=True)`."""
-        deprecation_warning((2024, 3, 1), _NetworkFileSystem.persisted.__doc__)
-        return _NetworkFileSystem.from_name(label, namespace, environment_name, create_if_missing=True)
-
-    def persist(
-        self,
-        label: str,
-        namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
-        environment_name: Optional[str] = None,
-        cloud: Optional[str] = None,
-    ):
-        """`NetworkFileSystem().persist("my-volume")` is deprecated. Use `NetworkFileSystem.from_name("my-volume", create_if_missing=True)` instead."""
-        deprecation_warning((2024, 2, 29), _NetworkFileSystem.persist.__doc__)
-        return self.persisted(label, namespace, environment_name, cloud)
-
-    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
     async def lookup(
-        label: str,
+        name: str,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
     ) -> "_NetworkFileSystem":
-        """Lookup a network file system with a given name
+        """Lookup a named NetworkFileSystem.
 
-        ```python
-        n = modal.NetworkFileSystem.lookup("my-nfs")
-        print(n.listdir("/"))
+        In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
+        that will hydrate the local object with metadata from Modal servers.
+
+        ```python notest
+        nfs = modal.NetworkFileSystem.lookup("my-nfs")
+        print(nfs.listdir("/"))
         ```
         """
         obj = _NetworkFileSystem.from_name(
-            label, namespace=namespace, environment_name=environment_name, create_if_missing=create_if_missing
+            name, namespace=namespace, environment_name=environment_name, create_if_missing=create_if_missing
         )
         if client is None:
             client = await _Client.from_env()
@@ -229,6 +202,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
         environment_name: Optional[str] = None,
     ) -> str:
         """mdmd:hidden"""
+        check_object_name(deployment_name, "NetworkFileSystem")
         if client is None:
             client = await _Client.from_env()
         request = api_pb2.SharedVolumeGetOrCreateRequest(
@@ -240,8 +214,15 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
         resp = await retry_transient_errors(client.stub.SharedVolumeGetOrCreate, request)
         return resp.shared_volume_id
 
+    @staticmethod
+    @renamed_parameter((2024, 12, 18), "label", "name")
+    async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
+        obj = await _NetworkFileSystem.lookup(name, client=client, environment_name=environment_name)
+        req = api_pb2.SharedVolumeDeleteRequest(shared_volume_id=obj.object_id)
+        await retry_transient_errors(obj._client.stub.SharedVolumeDelete, req)
+
     @live_method
-    async def write_file(self, remote_path: str, fp: BinaryIO) -> int:
+    async def write_file(self, remote_path: str, fp: BinaryIO, progress_cb: Optional[Callable[..., Any]] = None) -> int:
         """Write from a file object to a path on the network file system, atomically.
 
         Will create any needed parent directories automatically.
@@ -249,12 +230,20 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
         If remote_path ends with `/` it's assumed to be a directory and the
         file will be uploaded with its current name to that directory.
         """
+        progress_cb = progress_cb or (lambda *_, **__: None)
+
         sha_hash = get_sha256_hex(fp)
         fp.seek(0, os.SEEK_END)
         data_size = fp.tell()
         fp.seek(0)
         if data_size > LARGE_FILE_LIMIT:
-            blob_id = await blob_upload_file(fp, self._client.stub)
+            progress_task_id = progress_cb(name=remote_path, size=data_size)
+            blob_id = await blob_upload_file(
+                fp,
+                self._client.stub,
+                progress_report_cb=functools.partial(progress_cb, progress_task_id),
+                sha256_hex=sha_hash,
+            )
             req = api_pb2.SharedVolumePutFileRequest(
                 shared_volume_id=self.object_id,
                 path=remote_path,
@@ -293,21 +282,25 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
                 yield data
 
     @live_method_gen
-    async def iterdir(self, path: str) -> AsyncIterator[api_pb2.SharedVolumeListFilesEntry]:
+    async def iterdir(self, path: str) -> AsyncIterator[FileEntry]:
         """Iterate over all files in a directory in the network file system.
 
         * Passing a directory path lists all files in the directory (names are relative to the directory)
         * Passing a file path returns a list containing only that file's listing description
-        * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
+        * Passing a glob path (including at least one * or ** sequence) returns all files matching
+        that glob path (using absolute paths)
         """
         req = api_pb2.SharedVolumeListFilesRequest(shared_volume_id=self.object_id, path=path)
-        async for batch in unary_stream(self._client.stub.SharedVolumeListFilesStream, req):
+        async for batch in self._client.stub.SharedVolumeListFilesStream.unary_stream(req):
             for entry in batch.entries:
-                yield entry
+                yield FileEntry._from_proto(entry)
 
     @live_method
     async def add_local_file(
-        self, local_path: Union[Path, str], remote_path: Optional[Union[str, PurePosixPath, None]] = None
+        self,
+        local_path: Union[Path, str],
+        remote_path: Optional[Union[str, PurePosixPath, None]] = None,
+        progress_cb: Optional[Callable[..., Any]] = None,
     ):
         local_path = Path(local_path)
         if remote_path is None:
@@ -316,13 +309,14 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
             remote_path = PurePosixPath(remote_path).as_posix()
 
         with local_path.open("rb") as local_file:
-            return await self.write_file(remote_path, local_file)
+            return await self.write_file(remote_path, local_file, progress_cb=progress_cb)
 
     @live_method
     async def add_local_dir(
         self,
         local_path: Union[Path, str],
         remote_path: Optional[Union[str, PurePosixPath, None]] = None,
+        progress_cb: Optional[Callable[..., Any]] = None,
     ):
         _local_path = Path(local_path)
         if remote_path is None:
@@ -337,17 +331,23 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
                 if subpath.is_dir():
                     continue
                 relpath_str = subpath.relative_to(_local_path).as_posix()
-                yield self.add_local_file(subpath, PurePosixPath(remote_path, relpath_str))
+                yield subpath, PurePosixPath(remote_path, relpath_str)
+
+        async def _add_local_file(paths: tuple[Path, PurePosixPath]) -> int:
+            return await self.add_local_file(paths[0], paths[1], progress_cb)
 
-        await ConcurrencyPool(20).run_coros(gen_transfers(), return_exceptions=True)
+        async with aclosing(async_map(sync_or_async_iter(gen_transfers()), _add_local_file, concurrency=20)) as stream:
+            async for _ in stream:  # consume/execute the map
+                pass
 
     @live_method
-    async def listdir(self, path: str) -> List[api_pb2.SharedVolumeListFilesEntry]:
+    async def listdir(self, path: str) -> list[FileEntry]:
         """List all files in a directory in the network file system.
 
         * Passing a directory path lists all files in the directory (names are relative to the directory)
         * Passing a file path returns a list containing only that file's listing description
-        * Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
+        * Passing a glob path (including at least one * or ** sequence) returns all files matching
+        that glob path (using absolute paths)
         """
         return [entry async for entry in self.iterdir(path)]