modAL 1.2.3.dev13__py3-none-any.whl → 1.2.3.dev14__py3-none-any.whl

modal/_utils/grpc_utils.py

@@ -29,6 +29,7 @@ from modal_proto import api_pb2
 from modal_version import __version__
 
 from .._traceback import suppress_tb_frames
+from ..config import config
 from .async_utils import retry
 from .logger import logger
 
@@ -72,6 +73,7 @@ RETRYABLE_GRPC_STATUS_CODES = [
     Status.INTERNAL,
     Status.UNKNOWN,
 ]
+SERVER_RETRY_WARNING_TIME_INTERVAL = 30.0
 
 
 @dataclass
@@ -251,6 +253,50 @@ async def retry_transient_errors(
     return await _retry_transient_errors(fn, req, retry=Retry(max_retries=max_retries))
 
 
+def get_server_retry_policy(exc: Exception) -> Optional[api_pb2.RPCRetryPolicy]:
+    """Get server retry policy."""
+    if not isinstance(exc, GRPCError) or not exc.details:
+        return None
+
+    # Server should not set multiple retry instructions, but if there is more than one, pick the first one
+    for entry in exc.details:
+        if isinstance(entry, api_pb2.RPCRetryPolicy):
+            return entry
+    return None
+
+
+def process_exception_before_retry(
+    exc: Exception,
+    final_attempt: bool,
+    fn_name: str,
+    n_retries: int,
+    delay: float,
+    total_deadline: Optional[float],
+    idempotency_key: str,
+):
+    """Process exception before retry, used by `_retry_transient_errors`."""
+    with suppress_tb_frames(1):
+        if final_attempt:
+            logger.debug(
+                f"Final attempt failed with {repr(exc)} {n_retries=} {delay=} "
+                f"{total_deadline=} for {fn_name} ({idempotency_key[:8]})"
+            )
+            if isinstance(exc, OSError):
+                raise ConnectionError(str(exc))
+            elif isinstance(exc, asyncio.TimeoutError):
+                raise ConnectionError(str(exc))
+            else:
+                raise exc
+
+        if isinstance(exc, AttributeError) and "_write_appdata" not in str(exc):
+            # StreamTerminatedError are not properly raised in grpclib<=0.4.7
+            # fixed in https://github.com/vmagamedov/grpclib/issues/185
+            # TODO: update to newer version (>=0.4.8) once stable
+            raise exc
+
+    logger.debug(f"Retryable failure {repr(exc)} {n_retries=} {delay=} for {fn_name} ({idempotency_key[:8]})")
+
+
 async def _retry_transient_errors(
     fn: typing.Union[
         "modal._grpc_client.UnaryUnaryWrapper[RequestType, ResponseType]",
@@ -273,12 +319,15 @@ async def _retry_transient_errors(
 
     delay = retry.base_delay
     n_retries = 0
+    n_throttled_retries = 0
 
     status_codes = [*RETRYABLE_GRPC_STATUS_CODES, *retry.additional_status_codes]
 
     idempotency_key = str(uuid.uuid4())
 
     t0 = time.time()
+    last_server_retry_warning_time = None
+
     if retry.total_timeout is not None:
         total_deadline = t0 + retry.total_timeout
     else:
@@ -290,14 +339,18 @@ async def _retry_transient_errors(
         attempt_metadata = [
             ("x-idempotency-key", idempotency_key),
             ("x-retry-attempt", str(n_retries)),
+            ("x-throttle-retry-attempt", str(n_throttled_retries)),
             *metadata,
         ]
         if n_retries > 0:
             attempt_metadata.append(("x-retry-delay", str(time.time() - t0)))
+        if n_throttled_retries > 0:
+            attempt_metadata.append(("x-throttle-retry-delay", str(time.time() - t0)))
+
         timeouts = []
         if retry.attempt_timeout is not None:
             timeouts.append(retry.attempt_timeout)
-        if retry.total_timeout is not None and total_deadline is not None:
+        if total_deadline is not None:
             timeouts.append(max(total_deadline - time.time(), retry.attempt_timeout_floor))
         if timeouts:
             timeout = min(timeouts)  # In case the function provided both types of timeouts
@@ -307,6 +360,39 @@ async def _retry_transient_errors(
             with suppress_tb_frames(1):
                 return await fn_callable(req, metadata=attempt_metadata, timeout=timeout)
         except (StreamTerminatedError, GRPCError, OSError, asyncio.TimeoutError, AttributeError) as exc:
+            # Server side instruction for retries
+            if isinstance(exc, GRPCError) and (server_retry_policy := get_server_retry_policy(exc)):
+                server_delay = server_retry_policy.retry_after_secs
+
+                # When `total_deadline` is not defined and server gives instruction for retries, then use
+                # `max_throttle_wait`.
+                if total_deadline is None and (max_throttle_wait := config.get("max_throttle_wait")):
+                    total_deadline = t0 + max_throttle_wait
+
+                final_attempt = (
+                    total_deadline is not None
+                    and time.time() + server_delay + retry.attempt_timeout_floor >= total_deadline
+                )
+                with suppress_tb_frames(1):
+                    process_exception_before_retry(
+                        exc, final_attempt, fn.name, n_retries, server_delay, total_deadline, idempotency_key
+                    )
+
+                now = time.time()
+                if last_server_retry_warning_time is None or (
+                    now - last_server_retry_warning_time >= SERVER_RETRY_WARNING_TIME_INTERVAL
+                ):
+                    last_server_retry_warning_time = now
+                    logger.warning(
+                        f"Warning: Received failure {exc.status}: {exc.message}. "
+                        f"Will retry in {server_delay:0.2f} seconds."
+                    )
+
+                n_throttled_retries += 1
+                await asyncio.sleep(server_delay)
+                continue
+
+            # Client handles retry
             if isinstance(exc, GRPCError) and exc.status not in status_codes:
                 if exc.status == Status.UNAUTHENTICATED:
                     raise AuthError(exc.message)
@@ -320,25 +406,9 @@ async def _retry_transient_errors(
                 final_attempt = False
 
             with suppress_tb_frames(1):
-                if final_attempt:
-                    logger.debug(
-                        f"Final attempt failed with {repr(exc)} {n_retries=} {delay=} "
-                        f"{total_deadline=} for {fn.name} ({idempotency_key[:8]})"
-                    )
-                    if isinstance(exc, OSError):
-                        raise ConnectionError(str(exc))
-                    elif isinstance(exc, asyncio.TimeoutError):
-                        raise ConnectionError(str(exc))
-                    else:
-                        raise exc
-
-                if isinstance(exc, AttributeError) and "_write_appdata" not in str(exc):
-                    # StreamTerminatedError are not properly raised in grpclib<=0.4.7
-                    # fixed in https://github.com/vmagamedov/grpclib/issues/185
-                    # TODO: update to newer version (>=0.4.8) once stable
-                    raise exc
-
-            logger.debug(f"Retryable failure {repr(exc)} {n_retries=} {delay=} for {fn.name} ({idempotency_key[:8]})")
+                process_exception_before_retry(
+                    exc, final_attempt, fn.name, n_retries, delay, total_deadline, idempotency_key
+                )
 
             n_retries += 1
 

modal/client.pyi

@@ -32,7 +32,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.3.dev13",
+        version: str = "1.2.3.dev14",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -163,7 +163,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.2.3.dev13",
+        version: str = "1.2.3.dev14",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/config.py

@@ -217,6 +217,13 @@ def _enforce_suffix_rules(x: str) -> str:
     return x
 
 
+def _int_or_none(x: str) -> Optional[int]:
+    if not x:
+        return None
+    x_int = int(x)
+    return x_int if x_int > 0 else None
+
+
 class _Setting(typing.NamedTuple):
     default: typing.Any = None
     transform: typing.Callable[[str], typing.Any] = lambda x: x  # noqa: E731
@@ -258,6 +265,7 @@ _SETTINGS = {
         transform=lambda s: _check_value(["pickle", "cbor"])(s.lower()),
     ),
     "dev_suffix": _Setting("", transform=_enforce_suffix_rules),
+    "max_throttle_wait": _Setting(None, transform=_int_or_none),
 }
 
 

{modal-1.2.3.dev13.dist-info → modal-1.2.3.dev14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.2.3.dev13
+Version: 1.2.3.dev14
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.2.3.dev13.dist-info → modal-1.2.3.dev14.dist-info}/RECORD

@@ -26,12 +26,12 @@ modal/app.pyi,sha256=nI_Tibj-rI5I7dJFJnczbHvcmVzCvvCih0tMzoQlr_Y,50835
 modal/billing.py,sha256=zmQ3bcCJlwa4KD1IA_QgdWpm1pn13c-7qfy79iEauYI,195
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=k1CqQ_0t4-vHylkqYZgYCnClg-FT0QrTPAKivL01A_k,14354
-modal/client.pyi,sha256=JRrhM5jL-pRUDESJdtpjOEHBO7VpbH0oUGv3MCqXhn8,13148
+modal/client.pyi,sha256=x5RdbYVkp8jG9PmbnHJ_Lu8kTm9g2UmtGZLCKWipErM,13148
 modal/cloud_bucket_mount.py,sha256=cvVj5Rt4nDHoQbTneoJPTaAcnl_Ab_mrsjnnzd0v9OI,6013
 modal/cloud_bucket_mount.pyi,sha256=vXusGMOUk9HjCs4lhxteuCK4xLQclPfugGWKo7LECEA,7539
 modal/cls.py,sha256=4T__mtYlQUnkatif1Wnr8Ld9DN_ySydHj2UwhRg89ok,41984
 modal/cls.pyi,sha256=p8Unbfr0yUdAzYYnD7erSRO7dH-87HkAg4oAn6srBzA,27424
-modal/config.py,sha256=xUbw_ETdR7S3guj4dyzqdd9EYwRRht3aGYQRogZbi1o,13050
+modal/config.py,sha256=xF2OQztc67ZcpppkL1pQyzIVrvmyEqhouRaRp-iwZJY,13253
 modal/container_process.py,sha256=KG2ZkFOWe2KN1b8dFSBLBoW5pWoRxBzJ20MLhDTicJs,16749
 modal/container_process.pyi,sha256=xMKr-VbQsydS8AbhAys9UTpHHnH2QRyINpPtPG7NwmI,8373
 modal/dict.py,sha256=Omfmj3fUyg7ijTI33Z97geyK7MW8T31LRm-N7pZqSwU,21582
@@ -104,7 +104,7 @@ modal/_utils/docker_utils.py,sha256=h1uETghR40mp_y3fSWuZAfbIASH1HMzuphJHghAL6DU,
 modal/_utils/function_utils.py,sha256=wL2AA8aEftxCBATPStHgKDRPxrFjPAv-rHt0Wb7y9Ts,28629
 modal/_utils/git_utils.py,sha256=qtUU6JAttF55ZxYq51y55OR58B0tDPZsZWK5dJe6W5g,3182
 modal/_utils/grpc_testing.py,sha256=H1zHqthv19eGPJz2HKXDyWXWGSqO4BRsxah3L5Xaa8A,8619
-modal/_utils/grpc_utils.py,sha256=xAdatGkwJq6c_49wFYZ2tg3xGc1fIEDi9ItgwMuypOs,13280
+modal/_utils/grpc_utils.py,sha256=z4JlF7_uTaDUwD14YEtK6ClvlMGa8oQwx1PGRbvSRtk,15936
 modal/_utils/hash_utils.py,sha256=zg3J6OGxTFGSFri1qQ12giDz90lWk8bzaxCTUCRtiX4,3034
 modal/_utils/http_utils.py,sha256=yeTFsXYr0rYMEhB7vBP7audG9Uc7OLhzKBANFDZWVt0,2451
 modal/_utils/jwt_utils.py,sha256=fxH9plyrbAemTbjSsQtzIdDXE9QXxvMC4DiUZ16G0aA,1360
@@ -158,7 +158,7 @@ modal/experimental/__init__.py,sha256=sCwNbBLcR2t-jhrpwtMAPGKt2WNqXBg0xkNZdyB-6C
 modal/experimental/flash.py,sha256=RKS3X3h4tY_mOLcMzbRs5tNCLPFL01dBAhCZSqCeBgA,26675
 modal/experimental/flash.pyi,sha256=uwinKAYxpunNNfBj58FP88DXb535Qik4F6tnJKPAIwQ,14696
 modal/experimental/ipython.py,sha256=TrCfmol9LGsRZMeDoeMPx3Hv3BFqQhYnmD_iH0pqdhk,2904
-modal-1.2.3.dev13.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-1.2.3.dev14.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
 modal_docs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal_docs/gen_cli_docs.py,sha256=c1yfBS_x--gL5bs0N4ihMwqwX8l3IBWSkBAKNNIi6bQ,3801
 modal_docs/gen_reference_docs.py,sha256=d_CQUGQ0rfw28u75I2mov9AlS773z9rG40-yq5o7g2U,6359
@@ -166,10 +166,10 @@ modal_docs/mdmd/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,2
 modal_docs/mdmd/mdmd.py,sha256=tUTImNd4UMFk1opkaw8J672gX8AkBO5gbY2S_NMxsxs,7140
 modal_docs/mdmd/signatures.py,sha256=XJaZrK7Mdepk5fdX51A8uENiLFNil85Ud0d4MH8H5f0,3218
 modal_proto/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
-modal_proto/api.proto,sha256=lJ7RboW0F29Ke4c8SkDUV0MZ81NMbfZjdk-ALJ_ffac,109898
+modal_proto/api.proto,sha256=iZoGou5hhEQjUHHCCWZxrkBtF_Bpqc_f4rDyWqsO3_k,110064
 modal_proto/api_grpc.py,sha256=-8SHvPtalTAjQtRv7zzU7Z1VoDupoJs2ifG4gl_mF0s,134442
-modal_proto/api_pb2.py,sha256=66GekGGTUMSPYvGGP-K2AJq7biNQOu6pMkdr0ID7b44,383248
-modal_proto/api_pb2.pyi,sha256=bFFZymkxUh679BZlYhLdVEinvqZ6Y6wX8VeG0nM9mn4,540395
+modal_proto/api_pb2.py,sha256=N34H92QmBNhW_5P_dmhKwQgqdnYjvFSXqSsMMDYyi6o,383758
+modal_proto/api_pb2.pyi,sha256=1ioycGxYnlNt_WKxVK8uyjzczdMWtzndwZPpMly-Gss,540983
 modal_proto/api_pb2_grpc.py,sha256=Hqw9jcbhpr-W6jsfog_tGU55ouZjITxGvA-DGNBqOLA,289714
 modal_proto/api_pb2_grpc.pyi,sha256=QLJ58ANCx147HeGJva58h0MTCLIDs9JmVjrx8bDdwlg,67776
 modal_proto/modal_api_grpc.py,sha256=MqaBZB2ZqYj6XTIgI_p5dOPjt9gKT4pAJNd_WXR3W84,21295
@@ -186,10 +186,10 @@ modal_proto/task_command_router_pb2.py,sha256=_pD2ZpU0bNzhwBdzmLoLyLtAtftI_Agxwn
 modal_proto/task_command_router_pb2.pyi,sha256=EyDgXPLr7alqjXYERV8w_MPuO404x0uCppmSkrfE9IE,14589
 modal_proto/task_command_router_pb2_grpc.py,sha256=uEQ0HdrCp8v-9bB5yIic9muA8spCShLHY6Bz9cCgOUE,10114
 modal_proto/task_command_router_pb2_grpc.pyi,sha256=s3Yxsrawdj4nr8vqQqsAxyX6ilWaGbdECy425KKbLIA,3301
-modal_version/__init__.py,sha256=HeIZKLKt1lG5JikR44WzNsyYbellHhVvSzuBFd0J1Qg,121
+modal_version/__init__.py,sha256=ya-4fgDQ6rszSVtsSMXCAD0_zM8EcLYkOuQqxnt0wXg,121
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal-1.2.3.dev13.dist-info/METADATA,sha256=XSnUFwjlqFCvBAZe_5_77lJdQcK0gOMHqd6cgTbzvDk,2490
-modal-1.2.3.dev13.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-modal-1.2.3.dev13.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-1.2.3.dev13.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
-modal-1.2.3.dev13.dist-info/RECORD,,
+modal-1.2.3.dev14.dist-info/METADATA,sha256=5EXf_2CjGPFKquGh4Au4xW8ihsebd1VO4XzNvuVcBi4,2490
+modal-1.2.3.dev14.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+modal-1.2.3.dev14.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-1.2.3.dev14.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
+modal-1.2.3.dev14.dist-info/RECORD,,

modal_proto/api.proto

@@ -2642,6 +2642,12 @@ message QueuePutRequest {
   int32 partition_ttl_seconds = 6;
 }
 
+// Retry repolicy used by GRPCError.details for the server to give instructions
+// for the client to retry.
+message RPCRetryPolicy {
+  float retry_after_secs = 1;
+}
+
 // A copy google.rpc.Status for GRPCError.details:
 // https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto
 // RPCStatus is compatible with google.rpc.Status, so one can encode messages using