modAL 1.3.6.dev13__tar.gz → 1.4.0__tar.gz

{modal-1.3.6.dev13 → modal-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.3.6.dev13
+Version: 1.4.0
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License-Expression: Apache-2.0

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_functions.py

@@ -2,7 +2,6 @@
 import asyncio
 import dataclasses
 import inspect
-import textwrap
 import time
 import typing
 import warnings
@@ -44,7 +43,6 @@ from ._utils.async_utils import (
     warn_if_generator_is_not_consumed,
 )
 from ._utils.blob_utils import MAX_OBJECT_SIZE_BYTES
-from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.function_utils import (
     ATTEMPT_TIMEOUT_GRACE_PERIOD,
     OUTPUTS_TIMEOUT,
@@ -54,6 +52,7 @@ from ._utils.function_utils import (
     _stream_function_call_data,
     get_function_type,
     is_async,
+    parse_gpu_config,
 )
 from ._utils.grpc_utils import Retry, RetryWarningMessage
 from ._utils.mount_utils import validate_network_file_systems, validate_volumes, validate_volumes_by_object_id
@@ -67,7 +66,6 @@ from .exception import (
     NotFoundError,
     OutputExpiredError,
 )
-from .gpu import GPU_T, parse_gpu_config
 from .image import _Image
 from .mount import _get_client_mount, _Mount
 from .network_file_system import _NetworkFileSystem, network_file_system_mount_protos
@@ -587,7 +585,7 @@ class _FunctionSpec:
     network_file_systems: dict[Union[str, PurePosixPath], _NetworkFileSystem]
     volumes: dict[Union[str, PurePosixPath], Union[_Volume, _CloudBucketMount]]
     # TODO(irfansharif): Somehow assert that it's the first kind, in sandboxes
-    gpus: Union[GPU_T, list[GPU_T]]
+    gpus: Optional[str | list[str]]
     cloud: Optional[str]
     cpu: Optional[Union[float, tuple[float, float]]]
     memory: Optional[Union[int, tuple[int, int]]]
@@ -666,7 +664,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         secrets: Optional[Collection[_Secret]] = None,
         schedule: Optional[Schedule] = None,
         is_generator: bool = False,
-        gpu: Union[GPU_T, list[GPU_T]] = None,
+        gpu: Optional[str | list[str]] = None,
         # TODO: maybe break this out into a separate decorator for notebooks.
         network_file_systems: dict[Union[str, PurePosixPath], _NetworkFileSystem] = {},
         volumes: dict[Union[str, PurePosixPath], Union[_Volume, _CloudBucketMount]] = {},
@@ -703,7 +701,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         include_source: bool = True,
         experimental_options: Optional[dict[str, str]] = None,
         _experimental_proxy_ip: Optional[str] = None,
-        _experimental_custom_scaling_factor: Optional[float] = None,
         restrict_output: bool = False,
         http_config: Optional[api_pb2.HTTPConfig] = None,
     ) -> "_Function":
@@ -799,11 +796,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
                         f"for clustered Functions"
                     )
 
-        if _experimental_custom_scaling_factor is not None and (
-            _experimental_custom_scaling_factor < 0 or _experimental_custom_scaling_factor > 1
-        ):
-            raise InvalidError("`_experimental_custom_scaling_factor` must be between 0.0 and 1.0 inclusive.")
-
         if not cloud and not is_builder_function:
             cloud = config.get("default_cloud")
 
@@ -1050,7 +1042,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
                     _experimental_group_size=cluster_size or 0,  # Experimental: Clustered functions
                     _experimental_concurrent_cancellations=True,
                     _experimental_proxy_ip=_experimental_proxy_ip,
-                    _experimental_custom_scaling=_experimental_custom_scaling_factor is not None,
                     # --- These are deprecated in favor of autoscaler_settings
                     warm_pool_size=min_containers or 0,
                     concurrency_limit=max_containers or 0,
@@ -1122,8 +1113,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
                     function_data.ranked_functions.extend(ranked_functions)
                     function_definition = None  # function_definition is not used in this case
                 else:
-                    # TODO(irfansharif): Assert on this specific type once we get rid of python 3.9.
-                    # assert isinstance(gpu, GPU_T)  # includes the case where gpu==None case
                     function_definition.resources.CopyFrom(
                         convert_fn_config_to_resources_config(
                             cpu=cpu, memory=memory, gpu=gpu, ephemeral_disk=ephemeral_disk, rdma=rdma
@@ -1170,7 +1159,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         obj._webhook_config = webhook_config  # only set locally
 
         # Used to check whether we should rebuild a modal.Image which uses `run_function`.
-        gpus: list[GPU_T] = gpu if isinstance(gpu, list) else [gpu]
+        gpus: list[str] = gpu if isinstance(gpu, list) else [gpu] if gpu else []
         obj._build_args = dict(  # See get_build_def
             secrets=repr(secrets),
             gpu_config=repr([parse_gpu_config(_gpu) for _gpu in gpus]),
@@ -1358,43 +1347,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         # One idea would be for FunctionUpdateScheduleParams to return the current (coalesced) settings
         # and then we could return them here (would need some ad hoc dataclass, which I don't love)
 
-    @live_method
-    async def keep_warm(self, warm_pool_size: int) -> None:
-        """mdmd:hidden
-        Set the warm pool size for the Function.
-
-        DEPRECATED: Please adapt your code to use the more general `update_autoscaler` method instead:
-
-        ```python notest
-        f = modal.Function.from_name("my-app", "function")
-
-        # Old pattern (deprecated)
-        f.keep_warm(2)
-
-        # New pattern
-        f.update_autoscaler(min_containers=2)
-        ```
-        """
-        if self._is_method:
-            raise InvalidError(
-                textwrap.dedent(
-                    """
-                The `.keep_warm()` method can not be used on Modal class *methods*.
-
-                Call `.keep_warm()` on the class *instance* instead. All methods of a class are run by the same
-                container pool, and this method applies to the size of that container pool.
-            """
-                )
-            )
-
-        deprecation_warning(
-            (2025, 5, 5),
-            "The .keep_warm() method has been deprecated in favor of the more general "
-            ".update_autoscaler(min_containers=...) method.",
-            show_source=True,
-        )
-        await self.update_autoscaler(min_containers=warm_pool_size)
-
     @classmethod
     def _from_name(
         cls,
@@ -1444,7 +1396,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         app_name: str,
         name: str,
         *,
-        namespace=None,  # mdmd:line-hidden
         environment_name: Optional[str] = None,
         client: Optional[_Client] = None,
     ) -> "_Function":
@@ -1459,17 +1410,11 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         ```
         """
         if "." in name:
-            class_name, method_name = name.split(".", 1)
-            deprecation_warning(
-                (2025, 2, 11),
-                "Looking up class methods using Function.from_name will be deprecated"
-                " in a future version of Modal.\nUse modal.Cls.from_name instead, e.g.\n\n"
-                f'{class_name} = modal.Cls.from_name("{app_name}", "{class_name}")\n'
-                f"instance = {class_name}(...)\n"
-                f"instance.{method_name}.remote(...)\n",
+            raise InvalidError(
+                f"Invalid Function name: {name!r}. "
+                "If you are trying to look up a class method, use `modal.Cls.from_name` instead."
             )
 
-        warn_if_passing_namespace(namespace, "modal.Function.from_name")
         return cls._from_name(
             app_name, name, load_context_overrides=LoadContext(environment_name=environment_name, client=client)
         )
@@ -1587,21 +1532,6 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
                 + f"or call it locally: {self._function_name}.local()"
             )
 
-    # TODO (live_method on properties is not great, since it could be blocking the event loop from async contexts)
-    @property
-    @live_method
-    async def web_url(self) -> Optional[str]:
-        """mdmd:hidden
-        Deprecated. Use the `Function.get_web_url()` method instead.
-
-        URL of a Function running as a web endpoint.
-        """
-        deprecation_msg = """The Function.web_url property will be removed in a future version of Modal.
-Use the `Function.get_web_url()` method instead.
-"""
-        deprecation_warning((2025, 5, 6), deprecation_msg, pending=True)
-        return self._web_url
-
     @live_method
     async def get_web_url(self) -> Optional[str]:
         """URL of a Function running as a web endpoint."""
@@ -1632,7 +1562,6 @@ Use the `Function.get_web_url()` method instead.
         input_queue: _SynchronizedQueue,
         order_outputs: bool,
         return_exceptions: bool,
-        wrap_returned_exceptions: bool,
     ) -> AsyncGenerator[Any, None]:
         """mdmd:hidden
 
@@ -1658,7 +1587,6 @@ Use the `Function.get_web_url()` method instead.
                     self.client,
                     order_outputs,
                     return_exceptions,
-                    wrap_returned_exceptions,
                     count_update_callback,
                 )
             ) as stream:
@@ -1672,7 +1600,6 @@ Use the `Function.get_web_url()` method instead.
                     self.client,
                     order_outputs,
                     return_exceptions,
-                    wrap_returned_exceptions,
                     count_update_callback,
                     api_pb2.FUNCTION_CALL_INVOCATION_TYPE_SYNC,
                 )
@@ -2110,13 +2037,3 @@ class _FunctionCall(typing.Generic[ReturnType], _Object, type_prefix="fc"):
         except Exception as exc:
             # TODO: kill all running function calls
             raise exc
-
-
-async def _gather(*function_calls: _FunctionCall[T]) -> typing.Sequence[T]:
-    """mdmd:hidden
-    Deprecated: Please use `modal.FunctionCall.gather()` instead."""
-    deprecation_warning(
-        (2025, 2, 24),
-        "`modal.functions.gather()` is deprecated; please use `modal.FunctionCall.gather()` instead.",
-    )
-    return await _FunctionCall.gather(*function_calls)

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_output/pty.py

@@ -147,6 +147,11 @@ async def get_app_logs_loop(
             # doesn't support server-side search)
             if search_text and search_text not in log.data:
                 return
+            # Search results may lack trailing newlines; ensure each
+            # match is a complete line so the LineBufferedOutput buffer
+            # doesn't merge it with the next match.
+            if search_text and not log.data.endswith("\n"):
+                log.data += "\n"
             if pty_shell_finish_event:
                 await output_mgr.put_pty_content(log)
             else:

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_partial_function.py

@@ -17,7 +17,7 @@ from modal_proto import api_pb2
 
 from ._functions import _Function
 from ._utils.async_utils import synchronizer
-from ._utils.deprecation import deprecation_warning
+from ._utils.deprecation import deprecation_error
 from ._utils.function_utils import callable_has_non_self_params
 from .config import config
 from .exception import InvalidError
@@ -310,7 +310,7 @@ def _method(
     def wrapper(obj: Union[Callable[..., Any], _PartialFunction]) -> _PartialFunction:
         flags = _PartialFunctionFlags.CALLABLE_INTERFACE
 
-        nonlocal is_generator  # TODO(michael): we are likely to deprecate the explicit is_generator param
+        nonlocal is_generator
         if is_generator is None:
             callable = obj.raw_f if isinstance(obj, _PartialFunction) else obj
             is_generator = inspect.isgeneratorfunction(callable) or inspect.isasyncgenfunction(callable)
@@ -403,72 +403,16 @@ def _fastapi_endpoint(
 
 def _web_endpoint(
     _warn_parentheses_missing=None,  # mdmd:line-hidden
-    *,
-    method: str = "GET",  # REST method for the created endpoint.
-    label: Optional[str] = None,  # Label for created endpoint. Final subdomain will be <workspace>--<label>.modal.run.
-    docs: bool = False,  # Whether to enable interactive documentation for this endpoint at /docs.
-    custom_domains: Optional[
-        Iterable[str]
-    ] = None,  # Create an endpoint using a custom domain fully-qualified domain name (FQDN).
-    requires_proxy_auth: bool = False,  # Require Modal-Key and Modal-Secret HTTP Headers on requests.
-) -> Callable[
-    [Union[_PartialFunction[P, ReturnType, ReturnType], Callable[P, ReturnType]]],
-    _PartialFunction[P, ReturnType, ReturnType],
-]:
-    """Register a basic web endpoint with this application.
-
+    *args,
+    **kwargs,
+) -> None:
+    """mdmd:hidden
     DEPRECATED: This decorator has been renamed to `@modal.fastapi_endpoint`.
-
-    This is the simple way to create a web endpoint on Modal. The function
-    behaves as a [FastAPI](https://fastapi.tiangolo.com/) handler and should
-    return a response object to the caller.
-
-    Endpoints created with `@modal.web_endpoint` are meant to be simple, single
-    request handlers and automatically have
-    [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) enabled.
-    For more flexibility, use `@modal.asgi_app`.
-
-    To learn how to use Modal with popular web frameworks, see the
-    [guide on web endpoints](https://modal.com/docs/guide/webhooks).
     """
-    if isinstance(_warn_parentheses_missing, str):
-        # Probably passing the method string as a positional argument.
-        raise InvalidError('Positional arguments are not allowed. Suggestion: `@modal.web_endpoint(method="GET")`.')
-    elif _warn_parentheses_missing is not None:
-        raise InvalidError(
-            "Positional arguments are not allowed. Did you forget parentheses? Suggestion: `@modal.web_endpoint()`."
-        )
-
-    deprecation_warning(
+    deprecation_error(
         (2025, 3, 5), "The `@modal.web_endpoint` decorator has been renamed to `@modal.fastapi_endpoint`."
     )
 
-    webhook_config = api_pb2.WebhookConfig(
-        type=api_pb2.WEBHOOK_TYPE_FUNCTION,
-        method=method,
-        web_endpoint_docs=docs,
-        requested_suffix=label or "",
-        ephemeral_suffix=config.get("dev_suffix"),
-        async_mode=api_pb2.WEBHOOK_ASYNC_MODE_AUTO,
-        custom_domains=_parse_custom_domains(custom_domains),
-        requires_proxy_auth=requires_proxy_auth,
-    )
-
-    flags = _PartialFunctionFlags.WEB_INTERFACE
-    params = _PartialFunctionParams(webhook_config=webhook_config)
-
-    def wrapper(
-        obj: Union[_PartialFunction[P, ReturnType, ReturnType], Callable[P, ReturnType]],
-    ) -> _PartialFunction[P, ReturnType, ReturnType]:
-        if isinstance(obj, _PartialFunction):
-            pf = obj.stack(flags, params)
-        else:
-            pf = _PartialFunction(obj, flags, params)
-        pf.validate_obj_compatibility("web_endpoint")
-        return pf
-
-    return wrapper
-
 
 def _asgi_app(
     _warn_parentheses_missing=None,  # mdmd:line-hidden

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_resources.py

@@ -3,19 +3,18 @@ from typing import Optional, Union
 
 from modal_proto import api_pb2
 
+from ._utils.function_utils import parse_gpu_config
 from .exception import InvalidError
-from .gpu import GPU_T, parse_gpu_config
 
 
 def convert_fn_config_to_resources_config(
     *,
     cpu: Optional[Union[float, tuple[float, float]]],
     memory: Optional[Union[int, tuple[int, int]]],
-    gpu: GPU_T,
+    gpu: Optional[str],
     ephemeral_disk: Optional[int],
     rdma: Optional[bool] = None,
 ) -> api_pb2.Resources:
-    gpu_config = parse_gpu_config(gpu)
     if cpu and isinstance(cpu, tuple):
         if not cpu[0]:
             raise InvalidError("CPU request must be a positive number")
@@ -45,7 +44,7 @@ def convert_fn_config_to_resources_config(
     return api_pb2.Resources(
         milli_cpu=milli_cpu,
         milli_cpu_max=milli_cpu_max,
-        gpu_config=gpu_config,
+        gpu_config=parse_gpu_config(gpu),
         memory_mb=memory_mb,
         memory_mb_max=memory_mb_max,
         ephemeral_disk_mb=ephemeral_disk,

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_utils/async_utils.py

@@ -349,19 +349,18 @@ def retry(
     direct_fn=None,
     *,
     n_attempts=3,
-    base_delay=0,
+    base_delay=0.0,
     delay_factor=2,
+    max_delay: Optional[float] = None,
     attempt_timeout: Optional[float] = 90,
-    attempt_timeout_factor=1,
+    total_timeout: Optional[float] = None,
 ):
     """Decorator that calls an async function multiple times, with a given timeout.
 
     If a `base_delay` is provided, the function is given an exponentially
     increasing delay on each run, up until the maximum number of attempts.
 
-    If `attempt_timeout_factor` is set to a value > 1, the per-attempt timeout
-    increases exponentially (starting from `attempt_timeout`, multiplied by
-    `attempt_timeout_factor` after each failed attempt).
+    If `max_delay` is set, the delay between attempts is capped at this value.
 
     Usage:
 
@@ -377,6 +376,7 @@ def retry(
         pass
     ```
     """
+    ATTEMPT_TIMEOUT_FLOOR = 2.0
 
     def decorator(fn):
         @functools.wraps(fn)
@@ -387,17 +387,35 @@ def retry(
                 local_n_attempts = n_attempts
 
             delay = base_delay
-            current_attempt_timeout = attempt_timeout
+            if total_timeout is not None:
+                total_deadline = time.time() + total_timeout
+            else:
+                total_deadline = None
+
             for i in range(local_n_attempts):
                 t0 = time.time()
+
+                timeouts = []
+                if attempt_timeout is not None:
+                    timeouts.append(attempt_timeout)
+                if total_deadline is not None:
+                    timeouts.append(max(total_deadline - time.time(), ATTEMPT_TIMEOUT_FLOOR))
+                timeout = min(timeouts) if timeouts else None
+
                 try:
-                    return await asyncio.wait_for(fn(*args, **kwargs), timeout=current_attempt_timeout)
+                    return await asyncio.wait_for(fn(*args, **kwargs), timeout=timeout)
                 except asyncio.CancelledError:
                     logger.debug(f"Function {fn} was cancelled")
                     raise
                 except Exception as e:
                     if i >= local_n_attempts - 1:
                         raise
+
+                    # We check if the timeout will be reached **after** the sleep, so we can raise an error early
+                    # without needing to actually sleep.
+                    if total_deadline is not None and time.time() + delay + ATTEMPT_TIMEOUT_FLOOR >= total_deadline:
+                        raise
+
                     logger.debug(
                         f"Failed invoking function {fn}: {e}"
                         f" (took {time.time() - t0}s, sleeping {delay}s"
@@ -405,8 +423,8 @@ def retry(
                     )
                 await asyncio.sleep(delay)
                 delay *= delay_factor
-                if current_attempt_timeout is not None:
-                    current_attempt_timeout *= attempt_timeout_factor
+                if max_delay is not None:
+                    delay = min(delay, max_delay)
 
         return f_wrapped
 

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_utils/deprecation.py

@@ -89,55 +89,34 @@ def renamed_parameter(
     return decorator
 
 
-def warn_on_renamed_autoscaler_settings(func: Callable[P, R]) -> Callable[P, R]:
-    name_map = {
-        "keep_warm": "min_containers",
-        "concurrency_limit": "max_containers",
-        "_experimental_buffer_containers": "buffer_containers",
-        "container_idle_timeout": "scaledown_window",
-    }
+def handle_deprecated_parameters(func: Callable[P, R]) -> Callable[P, R]:
+    """Meta-decorator for modal.Function decorators that errors when deprecated parameters are passed.
+
+    We use this for a select parameters that might still be suggested by LLMs.
+    """
 
     @functools.wraps(func)
     def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
-        mut_kwargs: dict[str, Any] = locals()["kwargs"]  # Avoid referencing kwargs directly due to bug in sigtools
-
-        substitutions = []
-        old_params_used = name_map.keys() & mut_kwargs.keys()
-        for old_param, new_param in name_map.items():
-            if old_param in old_params_used:
-                new_param = name_map[old_param]
-                mut_kwargs[new_param] = mut_kwargs.pop(old_param)
-                substitutions.append(f"- {old_param} -> {new_param}")
-
-        if substitutions:
-            substitution_string = "\n".join(substitutions)
-            message = (
-                "We have renamed several parameters related to autoscaling."
-                " Please update your code to use the following new names:"
-                f"\n\n{substitution_string}"
-                "\n\nSee https://modal.com/docs/guide/modal-1-0-migration for more details."
+        _kwargs: dict[str, Any] = locals()["kwargs"]  # Avoid referencing kwargs directly due to bug in sigtools
+
+        # Basic autoscaler parameters
+        autoscaler_settings = {
+            "keep_warm": "min_containers",
+            "concurrency_limit": "max_containers",
+            "container_idle_timeout": "scaledown_window",
+        }
+
+        for old_name, new_name in autoscaler_settings.items():
+            if old_name in _kwargs:
+                deprecation_error((2025, 2, 24), f"The '{old_name}' parameter has been renamed to '{new_name}'.")
+
+        # Input concurrency
+        if "allow_concurrent_inputs" in _kwargs:
+            deprecation_error(
+                (2025, 4, 9),
+                "The 'allow_concurrent_inputs' parameter is deprecated. Use the '@modal.concurrent' decorator instead.",
             )
-            deprecation_warning((2025, 2, 24), message, show_source=True)
 
         return func(*args, **kwargs)
 
     return wrapper
-
-
-def warn_if_passing_namespace(
-    namespace: Any,
-    resource_name: str,
-) -> None:
-    """Issue deprecation warning for namespace parameter if non-None value is passed.
-
-    Args:
-        namespace: The namespace parameter value (may be None or actual value)
-        resource_name: Name of the resource type for the warning message
-    """
-    if namespace is not None:
-        deprecation_warning(
-            (2025, 6, 30),
-            f"The `namespace` parameter for `{resource_name}` is deprecated and will be"
-            " removed in a future release. It is no longer needed, so can be removed"
-            " from your code.",
-        )

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_utils/function_utils.py

@@ -602,3 +602,20 @@ async def _create_input(
             ),
             idx=idx,
         )
+
+
+def parse_gpu_config(value: Optional[str]) -> api_pb2.GPUConfig:
+    if value is None:
+        return api_pb2.GPUConfig()
+    elif isinstance(value, str):
+        count = 1
+        if ":" in value:
+            value, count_str = value.split(":", 1)
+            try:
+                count = int(count_str)
+            except ValueError:
+                raise InvalidError(f"Invalid GPU count: {count_str}. Value must be an integer.")
+        gpu_type = value.upper()
+        return api_pb2.GPUConfig(gpu_type=gpu_type, count=count)
+    else:
+        raise InvalidError(f"Invalid GPU config: {value!r}. Value must be a string or `None`")

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_utils/grpc_utils.py

@@ -90,8 +90,6 @@ RETRYABLE_GRPC_STATUS_CODES = [
 ]
 SERVER_RETRY_WARNING_TIME_INTERVAL = 30.0
 # Initial per-attempt timeout for TCP/TLS handshake in connect_channel.
-# Doubles each attempt (attempt_timeout_factor=2): 0.5 + 1 + 2 + 4 + 8 = 15.5s + ~1.5s inter-attempt delays.
-CONNECT_TIMEOUT = 0.5
 DEFAULT_MAX_RETRIES = 3
 
 
@@ -225,7 +223,8 @@ def create_channel(
     return channel
 
 
-@retry(n_attempts=5, base_delay=0.1, attempt_timeout=CONNECT_TIMEOUT, attempt_timeout_factor=2)
+# With 18 attempts, the max delays between calls is: 0.1 + 0.2 + 0.4 + 0.8 + 1.6 + 3.2 + 5*11 ~ 62
+@retry(n_attempts=18, base_delay=0.1, attempt_timeout=10.0, max_delay=5.0, total_timeout=63.0)
 async def connect_channel(channel: grpclib.client.Channel):
     """Connect to socket and raise exceptions when there is a connection issue."""
     await channel.__connect__()

{modal-1.3.6.dev13 → modal-1.4.0}/modal/_utils/task_command_router_client.py

@@ -11,7 +11,6 @@ from typing import AsyncGenerator, Optional
 
 import grpclib.client
 import grpclib.config
-import grpclib.events
 from grpclib import GRPCError, Status
 from grpclib.exceptions import StreamTerminatedError
 
@@ -22,8 +21,21 @@ from modal_proto.task_command_router_grpc import TaskCommandRouterStub
 
 from .._grpc_client import grpc_error_converter
 from .._utils.grpc_utils import PermanentCloseableChannel
-from .async_utils import aclosing
-from .grpc_utils import RETRYABLE_GRPC_STATUS_CODES, connect_channel
+from .async_utils import aclosing, retry
+from .grpc_utils import RETRYABLE_GRPC_STATUS_CODES
+
+
+@retry(n_attempts=34, base_delay=1, max_delay=10, attempt_timeout=10, total_timeout=310)
+async def _connect_channel(channel: grpclib.client.Channel):
+    """Connect to the command router channel.
+
+    Uses a longer retry budget than grpc_utils.connect_channel. In rare cases the sandbox
+    may take a long time to start on the worker after scheduling.
+
+    Retries with exponential backoff (1, 2, 4, 8, 10, 10, ...) capped at 10s per delay.
+    Total sleep between attempts: 1 + 2 + 4 + 8 + 10*29 = 305s (~5 min).
+    """
+    await channel.__connect__()
 
 
 def _b64url_decode(data: str) -> bytes:
@@ -169,7 +181,7 @@ class TaskCommandRouterClient:
             closed_error_message="Unable to perform operation on a detached sandbox",
         )
 
-        await connect_channel(channel)
+        await _connect_channel(channel)
         loop = asyncio.get_running_loop()
         jwt_refresh_lock = asyncio.Lock()