modal 0.62.115__py3-none-any.whl → 0.72.13__py3-none-any.whl

modal/client.pyi

@@ -1,164 +1,196 @@
+import asyncio.events
 import asyncio.locks
-import grpclib.exceptions
+import collections.abc
+import google.protobuf.message
+import grpclib.client
+import modal._utils.async_utils
 import modal_proto.api_grpc
+import modal_proto.modal_api_grpc
 import synchronicity.combined_types
 import typing
 import typing_extensions
 
-def _get_metadata(client_type: int, credentials: typing.Union[typing.Tuple[str, str], None], version: str) -> typing.Dict[str, str]:
-    ...
+def _get_metadata(client_type: int, credentials: typing.Optional[tuple[str, str]], version: str) -> dict[str, str]: ...
 
+ReturnType = typing.TypeVar("ReturnType")
 
-async def _http_check(url: str, timeout: float) -> str:
-    ...
-
-
-async def _grpc_exc_string(exc: grpclib.exceptions.GRPCError, method_name: str, server_url: str, timeout: float) -> str:
-    ...
+RequestType = typing.TypeVar("RequestType", bound="google.protobuf.message.Message")
 
+ResponseType = typing.TypeVar("ResponseType", bound="google.protobuf.message.Message")
 
 class _Client:
-    _client_from_env: typing.ClassVar[typing.Union[_Client, None]]
-    _client_from_env_lock: typing.ClassVar[typing.Union[asyncio.locks.Lock, None]]
-
-    def __init__(self, server_url: str, client_type: int, credentials: typing.Union[typing.Tuple[str, str], None], version: str = '0.62.115'):
-        ...
-
-    @property
-    def stub(self) -> typing.Union[modal_proto.api_grpc.ModalClientStub, None]:
-        ...
-
+    _client_from_env: typing.ClassVar[typing.Optional[_Client]]
+    _client_from_env_lock: typing.ClassVar[typing.Optional[asyncio.locks.Lock]]
+    _cancellation_context: modal._utils.async_utils.TaskContext
+    _cancellation_context_event_loop: asyncio.events.AbstractEventLoop
+    _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
+
+    def __init__(
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.72.13"
+    ): ...
+    def is_closed(self) -> bool: ...
     @property
-    def authenticated(self) -> bool:
-        ...
-
-    async def _open(self):
-        ...
-
-    async def _close(self):
-        ...
-
-    def set_pre_stop(self, pre_stop: typing.Callable[[], typing.Awaitable[None]]):
-        ...
-
-    async def _init(self):
-        ...
-
-    async def __aenter__(self):
-        ...
-
-    async def __aexit__(self, exc_type, exc, tb):
-        ...
-
+    def stub(self) -> modal_proto.modal_api_grpc.ModalClientModal: ...
+    async def _open(self): ...
+    async def _close(self, prep_for_restore: bool = False): ...
+    async def hello(self): ...
+    async def __aenter__(self): ...
+    async def __aexit__(self, exc_type, exc, tb): ...
     @classmethod
-    def anonymous(cls, server_url: str) -> typing.AsyncContextManager[_Client]:
-        ...
-
+    def anonymous(cls, server_url: str) -> typing.AsyncContextManager[_Client]: ...
     @classmethod
-    async def from_env(cls, _override_config=None) -> _Client:
-        ...
-
+    async def from_env(cls, _override_config=None) -> _Client: ...
     @classmethod
-    async def from_credentials(cls, token_id: str, token_secret: str) -> _Client:
-        ...
-
+    async def from_credentials(cls, token_id: str, token_secret: str) -> _Client: ...
     @classmethod
-    async def verify(cls, server_url: str, credentials: typing.Tuple[str, str]) -> None:
-        ...
-
+    async def verify(cls, server_url: str, credentials: tuple[str, str]) -> None: ...
     @classmethod
-    def set_env_client(cls, client: typing.Union[_Client, None]):
-        ...
-
+    def set_env_client(cls, client: typing.Optional[_Client]): ...
+    async def _call_safely(self, coro, readable_method: str): ...
+    async def _reset_on_pid_change(self): ...
+    async def _get_grpclib_method(self, method_name: str) -> typing.Any: ...
+    async def _call_unary(
+        self,
+        method_name: str,
+        request: typing.Any,
+        *,
+        timeout: typing.Optional[float] = None,
+        metadata: typing.Union[
+            collections.abc.Mapping[str, typing.Union[str, bytes]],
+            collections.abc.Collection[tuple[str, typing.Union[str, bytes]]],
+            None,
+        ] = None,
+    ) -> typing.Any: ...
+    def _call_stream(
+        self,
+        method_name: str,
+        request: typing.Any,
+        *,
+        metadata: typing.Union[
+            collections.abc.Mapping[str, typing.Union[str, bytes]],
+            collections.abc.Collection[tuple[str, typing.Union[str, bytes]]],
+            None,
+        ],
+    ) -> collections.abc.AsyncGenerator[typing.Any, None]: ...
 
 class Client:
-    _client_from_env: typing.ClassVar[typing.Union[Client, None]]
-    _client_from_env_lock: typing.ClassVar[typing.Union[asyncio.locks.Lock, None]]
-
-    def __init__(self, server_url: str, client_type: int, credentials: typing.Union[typing.Tuple[str, str], None], version: str = '0.62.115'):
-        ...
-
+    _client_from_env: typing.ClassVar[typing.Optional[Client]]
+    _client_from_env_lock: typing.ClassVar[typing.Optional[asyncio.locks.Lock]]
+    _cancellation_context: modal._utils.async_utils.TaskContext
+    _cancellation_context_event_loop: asyncio.events.AbstractEventLoop
+    _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
+
+    def __init__(
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.72.13"
+    ): ...
+    def is_closed(self) -> bool: ...
     @property
-    def stub(self) -> typing.Union[modal_proto.api_grpc.ModalClientStub, None]:
-        ...
-
-    @property
-    def authenticated(self) -> bool:
-        ...
+    def stub(self) -> modal_proto.modal_api_grpc.ModalClientModal: ...
 
     class ___open_spec(typing_extensions.Protocol):
-        def __call__(self):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
+        def __call__(self): ...
+        async def aio(self): ...
 
     _open: ___open_spec
 
     class ___close_spec(typing_extensions.Protocol):
-        def __call__(self):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
+        def __call__(self, prep_for_restore: bool = False): ...
+        async def aio(self, prep_for_restore: bool = False): ...
 
     _close: ___close_spec
 
-    class __set_pre_stop_spec(typing_extensions.Protocol):
-        def __call__(self, pre_stop: typing.Callable[[], None]):
-            ...
-
-        def aio(self, pre_stop: typing.Callable[[], typing.Awaitable[None]]):
-            ...
-
-    set_pre_stop: __set_pre_stop_spec
-
-    class ___init_spec(typing_extensions.Protocol):
-        def __call__(self):
-            ...
-
-        async def aio(self, *args, **kwargs):
-            ...
-
-    _init: ___init_spec
-
-    def __enter__(self):
-        ...
-
-    async def __aenter__(self, *args, **kwargs):
-        ...
+    class __hello_spec(typing_extensions.Protocol):
+        def __call__(self): ...
+        async def aio(self): ...
 
-    def __exit__(self, exc_type, exc, tb):
-        ...
-
-    async def __aexit__(self, *args, **kwargs):
-        ...
+    hello: __hello_spec
 
+    def __enter__(self): ...
+    async def __aenter__(self): ...
+    def __exit__(self, exc_type, exc, tb): ...
+    async def __aexit__(self, exc_type, exc, tb): ...
     @classmethod
-    def anonymous(cls, server_url: str) -> synchronicity.combined_types.AsyncAndBlockingContextManager[Client]:
-        ...
-
+    def anonymous(cls, server_url: str) -> synchronicity.combined_types.AsyncAndBlockingContextManager[Client]: ...
     @classmethod
-    def from_env(cls, _override_config=None) -> Client:
-        ...
-
+    def from_env(cls, _override_config=None) -> Client: ...
     @classmethod
-    def from_credentials(cls, token_id: str, token_secret: str) -> Client:
-        ...
-
+    def from_credentials(cls, token_id: str, token_secret: str) -> Client: ...
     @classmethod
-    def verify(cls, server_url: str, credentials: typing.Tuple[str, str]) -> None:
-        ...
-
+    def verify(cls, server_url: str, credentials: tuple[str, str]) -> None: ...
     @classmethod
-    def set_env_client(cls, client: typing.Union[Client, None]):
-        ...
-
+    def set_env_client(cls, client: typing.Optional[Client]): ...
+
+    class ___call_safely_spec(typing_extensions.Protocol):
+        def __call__(self, coro, readable_method: str): ...
+        async def aio(self, coro, readable_method: str): ...
+
+    _call_safely: ___call_safely_spec
+
+    class ___reset_on_pid_change_spec(typing_extensions.Protocol):
+        def __call__(self): ...
+        async def aio(self): ...
+
+    _reset_on_pid_change: ___reset_on_pid_change_spec
+
+    class ___get_grpclib_method_spec(typing_extensions.Protocol):
+        def __call__(self, method_name: str) -> typing.Any: ...
+        async def aio(self, method_name: str) -> typing.Any: ...
+
+    _get_grpclib_method: ___get_grpclib_method_spec
+
+    async def _call_unary(
+        self,
+        method_name: str,
+        request: typing.Any,
+        *,
+        timeout: typing.Optional[float] = None,
+        metadata: typing.Union[
+            collections.abc.Mapping[str, typing.Union[str, bytes]],
+            collections.abc.Collection[tuple[str, typing.Union[str, bytes]]],
+            None,
+        ] = None,
+    ) -> typing.Any: ...
+    def _call_stream(
+        self,
+        method_name: str,
+        request: typing.Any,
+        *,
+        metadata: typing.Union[
+            collections.abc.Mapping[str, typing.Union[str, bytes]],
+            collections.abc.Collection[tuple[str, typing.Union[str, bytes]]],
+            None,
+        ],
+    ) -> collections.abc.AsyncGenerator[typing.Any, None]: ...
+
+class UnaryUnaryWrapper(typing.Generic[RequestType, ResponseType]):
+    wrapped_method: grpclib.client.UnaryUnaryMethod[RequestType, ResponseType]
+    client: _Client
+
+    def __init__(self, wrapped_method: grpclib.client.UnaryUnaryMethod[RequestType, ResponseType], client: _Client): ...
+    @property
+    def name(self) -> str: ...
+    async def __call__(
+        self,
+        req: RequestType,
+        *,
+        timeout: typing.Optional[float] = None,
+        metadata: typing.Union[
+            collections.abc.Mapping[str, typing.Union[str, bytes]],
+            collections.abc.Collection[tuple[str, typing.Union[str, bytes]]],
+            None,
+        ] = None,
+    ) -> ResponseType: ...
+
+class UnaryStreamWrapper(typing.Generic[RequestType, ResponseType]):
+    wrapped_method: grpclib.client.UnaryStreamMethod[RequestType, ResponseType]
+
+    def __init__(
+        self, wrapped_method: grpclib.client.UnaryStreamMethod[RequestType, ResponseType], client: _Client
+    ): ...
+    @property
+    def name(self) -> str: ...
+    def unary_stream(self, request, metadata: typing.Optional[typing.Any] = None): ...
 
 HEARTBEAT_INTERVAL: float
 
 HEARTBEAT_TIMEOUT: float
-
-CLIENT_CREATE_ATTEMPT_TIMEOUT: float
-
-CLIENT_CREATE_TOTAL_TIMEOUT: float

modal/cloud_bucket_mount.py

@@ -1,11 +1,12 @@
 # Copyright Modal Labs 2022
 from dataclasses import dataclass
-from typing import List, Optional, Tuple
+from typing import Optional
 from urllib.parse import urlparse
 
 from modal_proto import api_pb2
 
 from ._utils.async_utils import synchronize_api
+from .config import logger
 from .secret import _Secret
 
 
@@ -15,18 +16,21 @@ class _CloudBucketMount:
 
     S3 buckets are mounted using [AWS S3 Mountpoint](https://github.com/awslabs/mountpoint-s3).
     S3 mounts are optimized for reading large files sequentially. It does not support every file operation; consult
-    [the AWS S3 Mountpoint documentation](https://github.com/awslabs/mountpoint-s3/blob/main/doc/SEMANTICS.md) for more information.
+    [the AWS S3 Mountpoint documentation](https://github.com/awslabs/mountpoint-s3/blob/main/doc/SEMANTICS.md)
+    for more information.
 
     **AWS S3 Usage**
 
     ```python
     import subprocess
 
-    app = modal.App()  # Note: "app" was called "stub" up until April 2024
-    secret = modal.Secret.from_dict({
-        "AWS_ACCESS_KEY_ID": "...",
-        "AWS_SECRET_ACCESS_KEY": "...",
-    })
+    app = modal.App()
+    secret = modal.Secret.from_name(
+        "aws-secret",
+        required_keys=["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]
+        # Note: providing AWS_REGION can help when automatic detection of the bucket region fails.
+    )
+
     @app.function(
         volumes={
             "/my-mount": modal.CloudBucketMount(
@@ -42,17 +46,18 @@ class _CloudBucketMount:
 
     **Cloudflare R2 Usage**
 
-    Cloudflare R2 is [S3-compatible](https://developers.cloudflare.com/r2/api/s3/api/) so its setup looks very similar to S3.
-    But additionally the `bucket_endpoint_url` argument must be passed.
+    Cloudflare R2 is [S3-compatible](https://developers.cloudflare.com/r2/api/s3/api/) so its setup looks
+    very similar to S3. But additionally the `bucket_endpoint_url` argument must be passed.
 
     ```python
     import subprocess
 
-    app = modal.App()  # Note: "app" was called "stub" up until April 2024
-    secret = modal.Secret.from_dict({
-        "AWS_ACCESS_KEY_ID": "...",
-        "AWS_SECRET_ACCESS_KEY": "...",
-    })
+    app = modal.App()
+    secret = modal.Secret.from_name(
+        "r2-secret",
+        required_keys=["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]
+    )
+
     @app.function(
         volumes={
             "/my-mount": modal.CloudBucketMount(
@@ -69,25 +74,25 @@ class _CloudBucketMount:
 
     **Google GCS Usage**
 
-    Google Cloud Storage (GCS) is partially [S3-compatible](https://cloud.google.com/storage/docs/interoperability).
-    Currently **only `read_only=True`** is supported for GCS buckets. GCS Buckets also require a secret with Google-specific
-    key names (see below) populated with a [HMAC key](https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create).
+    Google Cloud Storage (GCS) is [S3-compatible](https://cloud.google.com/storage/docs/interoperability).
+    GCS Buckets also require a secret with Google-specific key names (see below) populated with
+    a [HMAC key](https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create).
 
     ```python
     import subprocess
 
-    app = modal.App()  # Note: "app" was called "stub" up until April 2024
-    gcp_hmac_secret = modal.Secret.from_dict({
-        "GOOGLE_ACCESS_KEY_ID": "GOOG1ERM12345...",
-        "GOOGLE_ACCESS_KEY_SECRET": "HTJ123abcdef...",
-    })
+    app = modal.App()
+    gcp_hmac_secret = modal.Secret.from_name(
+        "gcp-secret",
+        required_keys=["GOOGLE_ACCESS_KEY_ID", "GOOGLE_ACCESS_KEY_SECRET"]
+    )
+
     @app.function(
         volumes={
             "/my-mount": modal.CloudBucketMount(
                 bucket_name="my-gcs-bucket",
                 bucket_endpoint_url="https://storage.googleapis.com",
                 secret=gcp_hmac_secret,
-                read_only=True,  # writing to bucket currently unsupported
             )
         }
     )
@@ -100,6 +105,8 @@ class _CloudBucketMount:
     # Endpoint URL is used to support Cloudflare R2 and Google Cloud Platform GCS.
     bucket_endpoint_url: Optional[str] = None
 
+    key_prefix: Optional[str] = None
+
     # Credentials used to access a cloud bucket.
     # If the bucket is private, the secret **must** contain AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
     # If the bucket is publicly accessible, the secret is unnecessary and can be omitted.
@@ -109,9 +116,9 @@ class _CloudBucketMount:
     requester_pays: bool = False
 
 
-def cloud_bucket_mounts_to_proto(mounts: List[Tuple[str, _CloudBucketMount]]) -> List[api_pb2.CloudBucketMount]:
+def cloud_bucket_mounts_to_proto(mounts: list[tuple[str, _CloudBucketMount]]) -> list[api_pb2.CloudBucketMount]:
     """Helper function to convert `CloudBucketMount` to a list of protobufs that can be passed to the server."""
-    cloud_bucket_mounts: List[api_pb2.CloudBucketMount] = []
+    cloud_bucket_mounts: list[api_pb2.CloudBucketMount] = []
 
     for path, mount in mounts:
         # crude mapping from mount arguments to type.
@@ -121,12 +128,12 @@ def cloud_bucket_mounts_to_proto(mounts: List[Tuple[str, _CloudBucketMount]]) ->
                 bucket_type = api_pb2.CloudBucketMount.BucketType.R2
             elif parse_result.hostname.endswith("storage.googleapis.com"):
                 bucket_type = api_pb2.CloudBucketMount.BucketType.GCP
-                if not mount.read_only:
-                    raise ValueError(
-                        f"CloudBucketMount of '{mount.bucket_name}' is invalid. Writing to GCP buckets with modal.CloudBucketMount in currently unsupported."
-                    )
             else:
-                raise ValueError(f"Unsupported bucket endpoint hostname '{parse_result.hostname}'")
+                logger.warning(
+                    "CloudBucketMount received unrecognized bucket endpoint URL. "
+                    "Assuming AWS S3 configuration as fallback."
+                )
+                bucket_type = api_pb2.CloudBucketMount.BucketType.S3
         else:
             # just assume S3; this is backwards and forwards compatible.
             bucket_type = api_pb2.CloudBucketMount.BucketType.S3
@@ -134,6 +141,11 @@ def cloud_bucket_mounts_to_proto(mounts: List[Tuple[str, _CloudBucketMount]]) ->
         if mount.requester_pays and not mount.secret:
             raise ValueError("Credentials required in order to use Requester Pays.")
 
+        if mount.key_prefix and not mount.key_prefix.endswith("/"):
+            raise ValueError("key_prefix will be prefixed to all object paths, so it must end in a '/'")
+        else:
+            key_prefix = mount.key_prefix
+
         cloud_bucket_mount = api_pb2.CloudBucketMount(
             bucket_name=mount.bucket_name,
             bucket_endpoint_url=mount.bucket_endpoint_url,
@@ -142,6 +154,7 @@ def cloud_bucket_mounts_to_proto(mounts: List[Tuple[str, _CloudBucketMount]]) ->
             read_only=mount.read_only,
             bucket_type=bucket_type,
             requester_pays=mount.requester_pays,
+            key_prefix=key_prefix,
         )
         cloud_bucket_mounts.append(cloud_bucket_mount)
 

modal/cloud_bucket_mount.pyi

@@ -4,37 +4,44 @@ import typing
 
 class _CloudBucketMount:
     bucket_name: str
-    bucket_endpoint_url: typing.Union[str, None]
-    secret: typing.Union[modal.secret._Secret, None]
+    bucket_endpoint_url: typing.Optional[str]
+    key_prefix: typing.Optional[str]
+    secret: typing.Optional[modal.secret._Secret]
     read_only: bool
     requester_pays: bool
 
-    def __init__(self, bucket_name: str, bucket_endpoint_url: typing.Union[str, None] = None, secret: typing.Union[modal.secret._Secret, None] = None, read_only: bool = False, requester_pays: bool = False) -> None:
-        ...
-
-    def __repr__(self):
-        ...
-
-    def __eq__(self, other):
-        ...
-
-
-def cloud_bucket_mounts_to_proto(mounts: typing.List[typing.Tuple[str, _CloudBucketMount]]) -> typing.List[modal_proto.api_pb2.CloudBucketMount]:
-    ...
-
+    def __init__(
+        self,
+        bucket_name: str,
+        bucket_endpoint_url: typing.Optional[str] = None,
+        key_prefix: typing.Optional[str] = None,
+        secret: typing.Optional[modal.secret._Secret] = None,
+        read_only: bool = False,
+        requester_pays: bool = False,
+    ) -> None: ...
+    def __repr__(self): ...
+    def __eq__(self, other): ...
+
+def cloud_bucket_mounts_to_proto(
+    mounts: list[tuple[str, _CloudBucketMount]],
+) -> list[modal_proto.api_pb2.CloudBucketMount]: ...
 
 class CloudBucketMount:
     bucket_name: str
-    bucket_endpoint_url: typing.Union[str, None]
-    secret: typing.Union[modal.secret.Secret, None]
+    bucket_endpoint_url: typing.Optional[str]
+    key_prefix: typing.Optional[str]
+    secret: typing.Optional[modal.secret.Secret]
     read_only: bool
     requester_pays: bool
 
-    def __init__(self, bucket_name: str, bucket_endpoint_url: typing.Union[str, None] = None, secret: typing.Union[modal.secret.Secret, None] = None, read_only: bool = False, requester_pays: bool = False) -> None:
-        ...
-
-    def __repr__(self):
-        ...
-
-    def __eq__(self, other):
-        ...
+    def __init__(
+        self,
+        bucket_name: str,
+        bucket_endpoint_url: typing.Optional[str] = None,
+        key_prefix: typing.Optional[str] = None,
+        secret: typing.Optional[modal.secret.Secret] = None,
+        read_only: bool = False,
+        requester_pays: bool = False,
+    ) -> None: ...
+    def __repr__(self): ...
+    def __eq__(self, other): ...