modal 1.2.3.dev13__tar.gz → 1.2.3.dev15__tar.gz

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.2.3.dev13
+Version: 1.2.3.dev15
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal/_utils/grpc_utils.py

@@ -29,6 +29,7 @@ from modal_proto import api_pb2
 from modal_version import __version__
 
 from .._traceback import suppress_tb_frames
+from ..config import config
 from .async_utils import retry
 from .logger import logger
 
@@ -72,6 +73,7 @@ RETRYABLE_GRPC_STATUS_CODES = [
     Status.INTERNAL,
     Status.UNKNOWN,
 ]
+SERVER_RETRY_WARNING_TIME_INTERVAL = 30.0
 
 
 @dataclass
@@ -251,6 +253,52 @@ async def retry_transient_errors(
     return await _retry_transient_errors(fn, req, retry=Retry(max_retries=max_retries))
 
 
+def get_server_retry_policy(exc: Exception) -> Optional[api_pb2.RPCRetryPolicy]:
+    """Get server retry policy."""
+    if not isinstance(exc, GRPCError) or not exc.details:
+        return None
+
+    # Server should not set multiple retry instructions, but if there is more than one, pick the first one
+    for entry in exc.details:
+        if isinstance(entry, api_pb2.RPCRetryPolicy):
+            return entry
+    return None
+
+
+def process_exception_before_retry(
+    exc: Exception,
+    final_attempt: bool,
+    fn_name: str,
+    n_retries: int,
+    delay: float,
+    total_deadline: Optional[float],
+    idempotency_key: str,
+):
+    """Process exception before retry, used by `_retry_transient_errors`."""
+    with suppress_tb_frames(1):
+        if final_attempt:
+            logger.debug(
+                f"Final attempt failed with {repr(exc)} {n_retries=} {delay=} "
+                f"{total_deadline=} for {fn_name} ({idempotency_key[:8]})"
+            )
+            if isinstance(exc, OSError):
+                raise ConnectionError(str(exc))
+            elif isinstance(exc, asyncio.TimeoutError):
+                raise ConnectionError(str(exc))
+            else:
+                raise exc
+
+        if isinstance(exc, AttributeError) and "_write_appdata" not in str(exc):
+            # StreamTerminatedError are not properly raised in grpclib<=0.4.7
+            # fixed in https://github.com/vmagamedov/grpclib/issues/185
+            # TODO: update to newer version (>=0.4.8) once stable
+            # Also be sure to remove the AttributeError from the set of exceptions
+            # we handle in the retry logic once we drop this check!
+            raise exc
+
+    logger.debug(f"Retryable failure {repr(exc)} {n_retries=} {delay=} for {fn_name} ({idempotency_key[:8]})")
+
+
 async def _retry_transient_errors(
     fn: typing.Union[
         "modal._grpc_client.UnaryUnaryWrapper[RequestType, ResponseType]",
@@ -273,12 +321,15 @@ async def _retry_transient_errors(
 
     delay = retry.base_delay
     n_retries = 0
+    n_throttled_retries = 0
 
     status_codes = [*RETRYABLE_GRPC_STATUS_CODES, *retry.additional_status_codes]
 
     idempotency_key = str(uuid.uuid4())
 
     t0 = time.time()
+    last_server_retry_warning_time = None
+
     if retry.total_timeout is not None:
         total_deadline = t0 + retry.total_timeout
     else:
@@ -290,14 +341,18 @@ async def _retry_transient_errors(
         attempt_metadata = [
             ("x-idempotency-key", idempotency_key),
             ("x-retry-attempt", str(n_retries)),
+            ("x-throttle-retry-attempt", str(n_throttled_retries)),
             *metadata,
         ]
         if n_retries > 0:
             attempt_metadata.append(("x-retry-delay", str(time.time() - t0)))
+        if n_throttled_retries > 0:
+            attempt_metadata.append(("x-throttle-retry-delay", str(time.time() - t0)))
+
         timeouts = []
         if retry.attempt_timeout is not None:
             timeouts.append(retry.attempt_timeout)
-        if retry.total_timeout is not None and total_deadline is not None:
+        if total_deadline is not None:
             timeouts.append(max(total_deadline - time.time(), retry.attempt_timeout_floor))
         if timeouts:
             timeout = min(timeouts)  # In case the function provided both types of timeouts
@@ -307,6 +362,42 @@ async def _retry_transient_errors(
             with suppress_tb_frames(1):
                 return await fn_callable(req, metadata=attempt_metadata, timeout=timeout)
         except (StreamTerminatedError, GRPCError, OSError, asyncio.TimeoutError, AttributeError) as exc:
+            # Note that we only catch AttributeError to handle a specific case that works around a bug
+            # in grpclib<=0.4.7. See above (search for `write_appdata`).
+
+            # Server side instruction for retries
+            if isinstance(exc, GRPCError) and (server_retry_policy := get_server_retry_policy(exc)):
+                server_delay = server_retry_policy.retry_after_secs
+
+                # When `total_deadline` is not defined and server gives instruction for retries, then use
+                # `max_throttle_wait`.
+                if total_deadline is None and (max_throttle_wait := config.get("max_throttle_wait")):
+                    total_deadline = t0 + max_throttle_wait
+
+                final_attempt = (
+                    total_deadline is not None
+                    and time.time() + server_delay + retry.attempt_timeout_floor >= total_deadline
+                )
+                with suppress_tb_frames(1):
+                    process_exception_before_retry(
+                        exc, final_attempt, fn.name, n_retries, server_delay, total_deadline, idempotency_key
+                    )
+
+                now = time.time()
+                if last_server_retry_warning_time is None or (
+                    now - last_server_retry_warning_time >= SERVER_RETRY_WARNING_TIME_INTERVAL
+                ):
+                    last_server_retry_warning_time = now
+                    logger.warning(
+                        f"Warning: Received {exc.status} status: {exc.message}. "
+                        f"Will retry in {server_delay:0.2f} seconds."
+                    )
+
+                n_throttled_retries += 1
+                await asyncio.sleep(server_delay)
+                continue
+
+            # Client handles retry
             if isinstance(exc, GRPCError) and exc.status not in status_codes:
                 if exc.status == Status.UNAUTHENTICATED:
                     raise AuthError(exc.message)
@@ -320,25 +411,9 @@ async def _retry_transient_errors(
                 final_attempt = False
 
             with suppress_tb_frames(1):
-                if final_attempt:
-                    logger.debug(
-                        f"Final attempt failed with {repr(exc)} {n_retries=} {delay=} "
-                        f"{total_deadline=} for {fn.name} ({idempotency_key[:8]})"
-                    )
-                    if isinstance(exc, OSError):
-                        raise ConnectionError(str(exc))
-                    elif isinstance(exc, asyncio.TimeoutError):
-                        raise ConnectionError(str(exc))
-                    else:
-                        raise exc
-
-                if isinstance(exc, AttributeError) and "_write_appdata" not in str(exc):
-                    # StreamTerminatedError are not properly raised in grpclib<=0.4.7
-                    # fixed in https://github.com/vmagamedov/grpclib/issues/185
-                    # TODO: update to newer version (>=0.4.8) once stable
-                    raise exc
-
-            logger.debug(f"Retryable failure {repr(exc)} {n_retries=} {delay=} for {fn.name} ({idempotency_key[:8]})")
+                process_exception_before_retry(
+                    exc, final_attempt, fn.name, n_retries, delay, total_deadline, idempotency_key
+                )
 
             n_retries += 1
 

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal/client.pyi

@@ -32,7 +32,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.3.dev13",
+        version: str = "1.2.3.dev15",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -163,7 +163,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.3.dev13",
+        version: str = "1.2.3.dev15",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal/config.py

@@ -51,6 +51,10 @@ Other possible configuration options are:
   Defaults to 10.
   Number of seconds to wait for logs to drain when closing the session,
   before giving up.
+* `max_throttle_wait` (in the .toml file) / `MODAL_MAX_THROTTLE_WAIT` (as an env var).
+  Defaults to None (no limit).
+  Maximum number of seconds to wait when requests are being throttled (i.e., due
+  to rate limiting or other cases that can normally be resolved through backoff).
 * `force_build` (in the .toml file) / `MODAL_FORCE_BUILD` (as an env var).
   Defaults to False.
   When set, ignores the Image cache and builds all Image layers. Note that this
@@ -217,6 +221,13 @@ def _enforce_suffix_rules(x: str) -> str:
     return x
 
 
+def _int_or_none(x: str) -> Optional[int]:
+    if not x:
+        return None
+    x_int = int(x)
+    return x_int if x_int > 0 else None
+
+
 class _Setting(typing.NamedTuple):
     default: typing.Any = None
     transform: typing.Callable[[str], typing.Any] = lambda x: x  # noqa: E731
@@ -258,6 +269,7 @@ _SETTINGS = {
         transform=lambda s: _check_value(["pickle", "cbor"])(s.lower()),
     ),
     "dev_suffix": _Setting("", transform=_enforce_suffix_rules),
+    "max_throttle_wait": _Setting(None, transform=_int_or_none),
 }
 
 

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal/functions.pyi

@@ -408,7 +408,7 @@ class Function(
 
     _call_generator: ___call_generator_spec[typing_extensions.Self]
 
-    class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER:
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
@@ -417,7 +417,7 @@ class Function(
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
 
-    remote: __remote_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    remote: __remote_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     class __remote_gen_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, *args, **kwargs) -> typing.Generator[typing.Any, None, None]:
@@ -444,7 +444,7 @@ class Function(
         """
         ...
 
-    class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """[Experimental] Calls the function with the given arguments, without waiting for the results.
 
@@ -468,7 +468,7 @@ class Function(
             ...
 
     _experimental_spawn: ___experimental_spawn_spec[
-        modal._functions.ReturnType, modal._functions.P, typing_extensions.Self
+        modal._functions.P, modal._functions.ReturnType, typing_extensions.Self
     ]
 
     class ___spawn_map_inner_spec(typing_extensions.Protocol[P_INNER, SUPERSELF]):
@@ -477,7 +477,7 @@ class Function(
 
     _spawn_map_inner: ___spawn_map_inner_spec[modal._functions.P, typing_extensions.Self]
 
-    class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """Calls the function with the given arguments, without waiting for the results.
 
@@ -498,7 +498,7 @@ class Function(
             """
             ...
 
-    spawn: __spawn_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    spawn: __spawn_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     def get_raw_f(self) -> collections.abc.Callable[..., typing.Any]:
         """Return the inner Python object wrapped by this Modal Function."""

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.2.3.dev13
+Version: 1.2.3.dev15
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.2.3.dev13 → modal-1.2.3.dev15}/modal_proto/api.proto

@@ -2642,6 +2642,12 @@ message QueuePutRequest {
   int32 partition_ttl_seconds = 6;
 }
 
+// Retry repolicy used by GRPCError.details for the server to give instructions
+// for the client to retry.
+message RPCRetryPolicy {
+  float retry_after_secs = 1;
+}
+
 // A copy google.rpc.Status for GRPCError.details:
 // https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto
 // RPCStatus is compatible with google.rpc.Status, so one can encode messages using