modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/network_file_system.pyi

@@ -1,165 +1,207 @@
+import collections.abc
 import modal.client
 import modal.object
+import modal.volume
 import modal_proto.api_pb2
 import pathlib
 import synchronicity.combined_types
 import typing
 import typing_extensions
 
-def network_file_system_mount_protos(validated_network_file_systems: typing.List[typing.Tuple[str, _NetworkFileSystem]], allow_cross_region_volumes: bool) -> typing.List[modal_proto.api_pb2.SharedVolumeMount]:
-    ...
-
+def network_file_system_mount_protos(
+    validated_network_file_systems: list[tuple[str, _NetworkFileSystem]], allow_cross_region_volumes: bool
+) -> list[modal_proto.api_pb2.SharedVolumeMount]: ...
 
 class _NetworkFileSystem(modal.object._Object):
     @staticmethod
-    def new(cloud: typing.Union[str, None] = None) -> _NetworkFileSystem:
-        ...
-
-    @staticmethod
-    def from_name(label: str, namespace=1, environment_name: typing.Union[str, None] = None, create_if_missing: bool = False) -> _NetworkFileSystem:
-        ...
-
+    def from_name(
+        name: str, namespace=1, environment_name: typing.Optional[str] = None, create_if_missing: bool = False
+    ) -> _NetworkFileSystem: ...
     @classmethod
-    def ephemeral(cls: typing.Type[_NetworkFileSystem], client: typing.Union[modal.client._Client, None] = None, environment_name: typing.Union[str, None] = None, _heartbeat_sleep: float = 300) -> typing.AsyncContextManager[_NetworkFileSystem]:
-        ...
-
+    def ephemeral(
+        cls: type[_NetworkFileSystem],
+        client: typing.Optional[modal.client._Client] = None,
+        environment_name: typing.Optional[str] = None,
+        _heartbeat_sleep: float = 300,
+    ) -> typing.AsyncContextManager[_NetworkFileSystem]: ...
     @staticmethod
-    def persisted(label: str, namespace=1, environment_name: typing.Union[str, None] = None, cloud: typing.Union[str, None] = None) -> _NetworkFileSystem:
-        ...
-
-    def persist(self, label: str, namespace=1, environment_name: typing.Union[str, None] = None, cloud: typing.Union[str, None] = None):
-        ...
-
+    async def lookup(
+        name: str,
+        namespace=1,
+        client: typing.Optional[modal.client._Client] = None,
+        environment_name: typing.Optional[str] = None,
+        create_if_missing: bool = False,
+    ) -> _NetworkFileSystem: ...
     @staticmethod
-    async def lookup(label: str, namespace=1, client: typing.Union[modal.client._Client, None] = None, environment_name: typing.Union[str, None] = None, create_if_missing: bool = False) -> _NetworkFileSystem:
-        ...
-
+    async def create_deployed(
+        deployment_name: str,
+        namespace=1,
+        client: typing.Optional[modal.client._Client] = None,
+        environment_name: typing.Optional[str] = None,
+    ) -> str: ...
     @staticmethod
-    async def create_deployed(deployment_name: str, namespace=1, client: typing.Union[modal.client._Client, None] = None, environment_name: typing.Union[str, None] = None) -> str:
-        ...
-
-    async def write_file(self, remote_path: str, fp: typing.BinaryIO) -> int:
-        ...
-
-    def read_file(self, path: str) -> typing.AsyncIterator[bytes]:
-        ...
-
-    def iterdir(self, path: str) -> typing.AsyncIterator[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-        ...
-
-    async def add_local_file(self, local_path: typing.Union[pathlib.Path, str], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None):
-        ...
-
-    async def add_local_dir(self, local_path: typing.Union[pathlib.Path, str], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None):
-        ...
-
-    async def listdir(self, path: str) -> typing.List[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-        ...
-
-    async def remove_file(self, path: str, recursive=False):
-        ...
-
+    async def delete(
+        name: str, client: typing.Optional[modal.client._Client] = None, environment_name: typing.Optional[str] = None
+    ): ...
+    async def write_file(
+        self,
+        remote_path: str,
+        fp: typing.BinaryIO,
+        progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+    ) -> int: ...
+    def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
+    def iterdir(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
+    async def add_local_file(
+        self,
+        local_path: typing.Union[pathlib.Path, str],
+        remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+        progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+    ): ...
+    async def add_local_dir(
+        self,
+        local_path: typing.Union[pathlib.Path, str],
+        remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+        progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+    ): ...
+    async def listdir(self, path: str) -> list[modal.volume.FileEntry]: ...
+    async def remove_file(self, path: str, recursive=False): ...
 
 class NetworkFileSystem(modal.object.Object):
-    def __init__(self, *args, **kwargs):
-        ...
-
+    def __init__(self, *args, **kwargs): ...
     @staticmethod
-    def new(cloud: typing.Union[str, None] = None) -> NetworkFileSystem:
-        ...
-
-    @staticmethod
-    def from_name(label: str, namespace=1, environment_name: typing.Union[str, None] = None, create_if_missing: bool = False) -> NetworkFileSystem:
-        ...
-
+    def from_name(
+        name: str, namespace=1, environment_name: typing.Optional[str] = None, create_if_missing: bool = False
+    ) -> NetworkFileSystem: ...
     @classmethod
-    def ephemeral(cls: typing.Type[NetworkFileSystem], client: typing.Union[modal.client.Client, None] = None, environment_name: typing.Union[str, None] = None, _heartbeat_sleep: float = 300) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]:
-        ...
-
-    @staticmethod
-    def persisted(label: str, namespace=1, environment_name: typing.Union[str, None] = None, cloud: typing.Union[str, None] = None) -> NetworkFileSystem:
-        ...
-
-    def persist(self, label: str, namespace=1, environment_name: typing.Union[str, None] = None, cloud: typing.Union[str, None] = None):
-        ...
+    def ephemeral(
+        cls: type[NetworkFileSystem],
+        client: typing.Optional[modal.client.Client] = None,
+        environment_name: typing.Optional[str] = None,
+        _heartbeat_sleep: float = 300,
+    ) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]: ...
 
     class __lookup_spec(typing_extensions.Protocol):
-        def __call__(self, label: str, namespace=1, client: typing.Union[modal.client.Client, None] = None, environment_name: typing.Union[str, None] = None, create_if_missing: bool = False) -> NetworkFileSystem:
-            ...
-
-        async def aio(self, *args, **kwargs) -> NetworkFileSystem:
-            ...
+        def __call__(
+            self,
+            name: str,
+            namespace=1,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+            create_if_missing: bool = False,
+        ) -> NetworkFileSystem: ...
+        async def aio(
+            self,
+            name: str,
+            namespace=1,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+            create_if_missing: bool = False,
+        ) -> NetworkFileSystem: ...
 
     lookup: __lookup_spec
 
     class __create_deployed_spec(typing_extensions.Protocol):
-        def __call__(self, deployment_name: str, namespace=1, client: typing.Union[modal.client.Client, None] = None, environment_name: typing.Union[str, None] = None) -> str:
-            ...
-
-        async def aio(self, *args, **kwargs) -> str:
-            ...
+        def __call__(
+            self,
+            deployment_name: str,
+            namespace=1,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+        ) -> str: ...
+        async def aio(
+            self,
+            deployment_name: str,
+            namespace=1,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+        ) -> str: ...
 
     create_deployed: __create_deployed_spec
 
-    class __write_file_spec(typing_extensions.Protocol):
-        def __call__(self, remote_path: str, fp: typing.BinaryIO) -> int:
-            ...
+    class __delete_spec(typing_extensions.Protocol):
+        def __call__(
+            self,
+            name: str,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+        ): ...
+        async def aio(
+            self,
+            name: str,
+            client: typing.Optional[modal.client.Client] = None,
+            environment_name: typing.Optional[str] = None,
+        ): ...
+
+    delete: __delete_spec
 
-        async def aio(self, *args, **kwargs) -> int:
-            ...
+    class __write_file_spec(typing_extensions.Protocol):
+        def __call__(
+            self,
+            remote_path: str,
+            fp: typing.BinaryIO,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ) -> int: ...
+        async def aio(
+            self,
+            remote_path: str,
+            fp: typing.BinaryIO,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ) -> int: ...
 
     write_file: __write_file_spec
 
     class __read_file_spec(typing_extensions.Protocol):
-        def __call__(self, path: str) -> typing.Iterator[bytes]:
-            ...
-
-        def aio(self, path: str) -> typing.AsyncIterator[bytes]:
-            ...
+        def __call__(self, path: str) -> typing.Iterator[bytes]: ...
+        def aio(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
 
     read_file: __read_file_spec
 
     class __iterdir_spec(typing_extensions.Protocol):
-        def __call__(self, path: str) -> typing.Iterator[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-            ...
-
-        def aio(self, path: str) -> typing.AsyncIterator[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-            ...
+        def __call__(self, path: str) -> typing.Iterator[modal.volume.FileEntry]: ...
+        def aio(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
 
     iterdir: __iterdir_spec
 
     class __add_local_file_spec(typing_extensions.Protocol):
-        def __call__(self, local_path: typing.Union[pathlib.Path, str], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
+        def __call__(
+            self,
+            local_path: typing.Union[pathlib.Path, str],
+            remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ): ...
+        async def aio(
+            self,
+            local_path: typing.Union[pathlib.Path, str],
+            remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ): ...
 
     add_local_file: __add_local_file_spec
 
     class __add_local_dir_spec(typing_extensions.Protocol):
-        def __call__(self, local_path: typing.Union[pathlib.Path, str], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
+        def __call__(
+            self,
+            local_path: typing.Union[pathlib.Path, str],
+            remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ): ...
+        async def aio(
+            self,
+            local_path: typing.Union[pathlib.Path, str],
+            remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
+            progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
+        ): ...
 
     add_local_dir: __add_local_dir_spec
 
     class __listdir_spec(typing_extensions.Protocol):
-        def __call__(self, path: str) -> typing.List[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-            ...
-
-        async def aio(self, *args, **kwargs) -> typing.List[modal_proto.api_pb2.SharedVolumeListFilesEntry]:
-            ...
+        def __call__(self, path: str) -> list[modal.volume.FileEntry]: ...
+        async def aio(self, path: str) -> list[modal.volume.FileEntry]: ...
 
     listdir: __listdir_spec
 
     class __remove_file_spec(typing_extensions.Protocol):
-        def __call__(self, path: str, recursive=False):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
+        def __call__(self, path: str, recursive=False): ...
+        async def aio(self, path: str, recursive=False): ...
 
     remove_file: __remove_file_spec

modal/object.py

@@ -1,24 +1,27 @@
 # Copyright Modal Labs 2022
 import uuid
+from collections.abc import Awaitable, Hashable, Sequence
 from functools import wraps
-from typing import Awaitable, Callable, ClassVar, Dict, Hashable, List, Optional, Type, TypeVar
+from typing import Callable, ClassVar, Optional, TypeVar
 
 from google.protobuf.message import Message
 
+from modal._utils.async_utils import aclosing
+
 from ._resolver import Resolver
 from ._utils.async_utils import synchronize_api
 from .client import _Client
-from .config import config
+from .config import config, logger
 from .exception import ExecutionError, InvalidError
 
 O = TypeVar("O", bound="_Object")
 
 _BLOCKING_O = synchronize_api(O)
 
-EPHEMERAL_OBJECT_HEARTBEAT_SLEEP = 300
+EPHEMERAL_OBJECT_HEARTBEAT_SLEEP: int = 300
 
 
-def _get_environment_name(environment_name: Optional[str], resolver: Optional[Resolver] = None) -> Optional[str]:
+def _get_environment_name(environment_name: Optional[str] = None, resolver: Optional[Resolver] = None) -> Optional[str]:
     if environment_name:
         return environment_name
     elif resolver and resolver.environment_name:
@@ -29,7 +32,7 @@ def _get_environment_name(environment_name: Optional[str], resolver: Optional[Re
 
 class _Object:
     _type_prefix: ClassVar[Optional[str]] = None
-    _prefix_to_type: ClassVar[Dict[str, type]] = {}
+    _prefix_to_type: ClassVar[dict[str, type]] = {}
 
     # For constructors
     _load: Optional[Callable[[O, Resolver, Optional[str]], Awaitable[None]]]
@@ -37,13 +40,14 @@ class _Object:
     _rep: str
     _is_another_app: bool
     _hydrate_lazily: bool
-    _deps: Optional[Callable[..., List["_Object"]]]
+    _deps: Optional[Callable[..., list["_Object"]]]
     _deduplication_key: Optional[Callable[[], Awaitable[Hashable]]] = None
 
     # For hydrated objects
     _object_id: str
     _client: _Client
     _is_hydrated: bool
+    _is_rehydrated: bool
 
     @classmethod
     def __init_subclass__(cls, type_prefix: Optional[str] = None):
@@ -62,7 +66,7 @@ class _Object:
         is_another_app: bool = False,
         preload: Optional[Callable[[O, Resolver, Optional[str]], Awaitable[None]]] = None,
         hydrate_lazily: bool = False,
-        deps: Optional[Callable[..., List["_Object"]]] = None,
+        deps: Optional[Callable[..., list["_Object"]]] = None,
         deduplication_key: Optional[Callable[[], Awaitable[Hashable]]] = None,
     ):
         self._local_uuid = str(uuid.uuid4())
@@ -77,6 +81,7 @@ class _Object:
         self._object_id = None
         self._client = None
         self._is_hydrated = False
+        self._is_rehydrated = False
 
         self._initialize_from_empty()
 
@@ -91,7 +96,9 @@ class _Object:
 
     def _initialize_from_other(self, other):
         # default implementation, can be overriden in subclasses
-        pass
+        self._object_id = other._object_id
+        self._is_hydrated = other._is_hydrated
+        self._client = other._client
 
     def _hydrate(self, object_id: str, client: _Client, metadata: Optional[Message]):
         assert isinstance(object_id, str)
@@ -116,17 +123,25 @@ class _Object:
         # the object_id is already provided by other means
         return
 
-    def _init_from_other(self, other: O):
-        # Transient use case, see Dict, Queue, and SharedVolume
-        self._init(other._rep, other._load, other._is_another_app, other._preload)
+    def _validate_is_hydrated(self: O):
+        if not self._is_hydrated:
+            object_type = self.__class__.__name__.strip("_")
+            if hasattr(self, "_app") and getattr(self._app, "_running_app", "") is None:
+                # The most common cause of this error: e.g., user called a Function without using App.run()
+                reason = ", because the App it is defined on is not running"
+            else:
+                # Technically possible, but with an ambiguous cause.
+                reason = ""
+            raise ExecutionError(
+                f"{object_type} has not been hydrated with the metadata it needs to run on Modal{reason}."
+            )
 
     def clone(self: O) -> O:
         """mdmd:hidden Clone a given hydrated object."""
 
         # Object to clone must already be hydrated, otherwise from_loader is more suitable.
-        assert self._is_hydrated
-
-        obj = _Object.__new__(type(self))
+        self._validate_is_hydrated()
+        obj = type(self).__new__(type(self))
         obj._initialize_from_other(self)
         return obj
 
@@ -138,7 +153,7 @@ class _Object:
         is_another_app: bool = False,
         preload: Optional[Callable[[O, Resolver, Optional[str]], Awaitable[None]]] = None,
         hydrate_lazily: bool = False,
-        deps: Optional[Callable[..., List["_Object"]]] = None,
+        deps: Optional[Callable[..., Sequence["_Object"]]] = None,
         deduplication_key: Optional[Callable[[], Awaitable[Hashable]]] = None,
     ):
         # TODO(erikbern): flip the order of the two first arguments
@@ -146,26 +161,34 @@ class _Object:
         obj._init(rep, load, is_another_app, preload, hydrate_lazily, deps, deduplication_key)
         return obj
 
+    @classmethod
+    def _get_type_from_id(cls: type[O], object_id: str) -> type[O]:
+        parts = object_id.split("-")
+        if len(parts) != 2:
+            raise InvalidError(f"Object id {object_id} has no dash in it")
+        prefix = parts[0]
+        if prefix not in cls._prefix_to_type:
+            raise InvalidError(f"Object prefix {prefix} does not correspond to a type")
+        return cls._prefix_to_type[prefix]
+
+    @classmethod
+    def _is_id_type(cls: type[O], object_id) -> bool:
+        return cls._get_type_from_id(object_id) == cls
+
     @classmethod
     def _new_hydrated(
-        cls: Type[O], object_id: str, client: _Client, handle_metadata: Optional[Message], is_another_app: bool = False
+        cls: type[O], object_id: str, client: _Client, handle_metadata: Optional[Message], is_another_app: bool = False
     ) -> O:
         if cls._type_prefix is not None:
             # This is called directly on a subclass, e.g. Secret.from_id
             if not object_id.startswith(cls._type_prefix + "-"):
                 raise InvalidError(f"Object {object_id} does not start with {cls._type_prefix}")
-            prefix = cls._type_prefix
+            obj_cls = cls
         else:
             # This is called on the base class, e.g. Handle.from_id
-            parts = object_id.split("-")
-            if len(parts) != 2:
-                raise InvalidError(f"Object id {object_id} has no dash in it")
-            prefix = parts[0]
-            if prefix not in cls._prefix_to_type:
-                raise InvalidError(f"Object prefix {prefix} does not correspond to a type")
+            obj_cls = cls._get_type_from_id(object_id)
 
         # Instantiate provider
-        obj_cls = cls._prefix_to_type[prefix]
         obj = _Object.__new__(obj_cls)
         rep = f"Object({object_id})"  # TODO(erikbern): dumb
         obj._init(rep, is_another_app=is_another_app)
@@ -185,7 +208,7 @@ class _Object:
         return self._local_uuid
 
     @property
-    def object_id(self):
+    def object_id(self) -> str:
         """mdmd:hidden"""
         return self._object_id
 
@@ -195,23 +218,30 @@ class _Object:
         return self._is_hydrated
 
     @property
-    def deps(self) -> Callable[..., List["_Object"]]:
+    def deps(self) -> Callable[..., list["_Object"]]:
         """mdmd:hidden"""
         return self._deps if self._deps is not None else lambda: []
 
-    async def resolve(self):
+    async def resolve(self, client: Optional[_Client] = None):
         """mdmd:hidden"""
         if self._is_hydrated:
+            # memory snapshots capture references which must be rehydrated
+            # on restore to handle staleness.
+            if self._client._snapshotted and not self._is_rehydrated:
+                logger.debug(f"rehydrating {self} after snapshot")
+                self._is_hydrated = False  # un-hydrate and re-resolve
+                c = client if client is not None else await _Client.from_env()
+                resolver = Resolver(c)
+                await resolver.load(self)
+                self._is_rehydrated = True
+                logger.debug(f"rehydrated {self} with client {id(c)}")
             return
         elif not self._hydrate_lazily:
-            raise ExecutionError(
-                "Object has not been hydrated and doesn't support lazy hydration."
-                " This might happen if an object is defined on a different stub,"
-                " or if it's on the same stub but it didn't get created because it"
-                " wasn't defined in global scope."
-            )
+            self._validate_is_hydrated()
         else:
-            resolver = Resolver()  # TODO: this resolver has no attached Client!
+            # TODO: this client and/or resolver can't be changed by a caller to X.from_name()
+            c = client if client is not None else await _Client.from_env()
+            resolver = Resolver(c)
             await resolver.load(self)
 
 
@@ -231,7 +261,8 @@ def live_method_gen(method):
     @wraps(method)
     async def wrapped(self, *args, **kwargs):
         await self.resolve()
-        async for item in method(self, *args, **kwargs):
-            yield item
+        async with aclosing(method(self, *args, **kwargs)) as stream:
+            async for item in stream:
+                yield item
 
     return wrapped