modal 0.73.126__tar.gz → 0.73.128__tar.gz

{modal-0.73.126 → modal-0.73.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: modal
-Version: 0.73.126
+Version: 0.73.128
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-0.73.126 → modal-0.73.128}/modal/__init__.py

@@ -27,6 +27,7 @@ try:
         asgi_app,
         batched,
         build,
+        concurrent,
         enter,
         exit,
         fastapi_endpoint,
@@ -82,6 +83,7 @@ __all__ = [
     "asgi_app",
     "batched",
     "build",
+    "concurrent",
     "current_function_call_id",
     "current_input_id",
     "enable_output",

{modal-0.73.126 → modal-0.73.128}/modal/_functions.py

@@ -435,7 +435,8 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
         max_containers: Optional[int] = None,
         buffer_containers: Optional[int] = None,
         scaledown_window: Optional[int] = None,
-        allow_concurrent_inputs: Optional[int] = None,
+        max_concurrent_inputs: Optional[int] = None,
+        target_concurrent_inputs: Optional[int] = None,
         batch_max_size: Optional[int] = None,
         batch_wait_ms: Optional[int] = None,
         cloud: Optional[str] = None,
@@ -786,7 +787,8 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
                     runtime_perf_record=config.get("runtime_perf_record"),
                     app_name=app_name,
                     is_builder_function=is_builder_function,
-                    target_concurrent_inputs=allow_concurrent_inputs or 0,
+                    max_concurrent_inputs=max_concurrent_inputs or 0,
+                    target_concurrent_inputs=target_concurrent_inputs or 0,
                     batch_max_size=batch_max_size or 0,
                     batch_linger_ms=batch_wait_ms or 0,
                     worker_id=config.get("worker_id"),

{modal-0.73.126 → modal-0.73.128}/modal/_partial_function.py

@@ -59,6 +59,8 @@ class _PartialFunction(typing.Generic[P, ReturnType, OriginalReturnType]):
     force_build: bool
     cluster_size: Optional[int]  # Experimental: Clustered functions
     build_timeout: Optional[int]
+    max_concurrent_inputs: Optional[int]
+    target_concurrent_inputs: Optional[int]
 
     def __init__(
         self,
@@ -72,6 +74,8 @@ class _PartialFunction(typing.Generic[P, ReturnType, OriginalReturnType]):
         cluster_size: Optional[int] = None,  # Experimental: Clustered functions
         force_build: bool = False,
         build_timeout: Optional[int] = None,
+        max_concurrent_inputs: Optional[int] = None,
+        target_concurrent_inputs: Optional[int] = None,
     ):
         self.raw_f = raw_f
         self.flags = flags
@@ -89,6 +93,8 @@ class _PartialFunction(typing.Generic[P, ReturnType, OriginalReturnType]):
         self.cluster_size = cluster_size  # Experimental: Clustered functions
         self.force_build = force_build
         self.build_timeout = build_timeout
+        self.max_concurrent_inputs = max_concurrent_inputs
+        self.target_concurrent_inputs = target_concurrent_inputs
 
     def _get_raw_f(self) -> Callable[P, ReturnType]:
         return self.raw_f
@@ -143,6 +149,8 @@ class _PartialFunction(typing.Generic[P, ReturnType, OriginalReturnType]):
             batch_wait_ms=self.batch_wait_ms,
             force_build=self.force_build,
             build_timeout=self.build_timeout,
+            max_concurrent_inputs=self.max_concurrent_inputs,
+            target_concurrent_inputs=self.target_concurrent_inputs,
         )
 
 
@@ -722,3 +730,49 @@ def _batched(
         )
 
     return wrapper
+
+
+def _concurrent(
+    _warn_parentheses_missing=None,
+    *,
+    max_inputs: int,  # Hard limit on each container's input concurrency
+    target_inputs: Optional[int] = None,  # Input concurrency that Modal's autoscaler should target
+) -> Callable[[Union[Callable[..., Any], _PartialFunction]], _PartialFunction]:
+    """Decorator that allows individual containers to handle multiple inputs concurrently.
+
+    The concurrency mechanism depends on whether the function is async or not:
+    - Async functions will run inputs on a single thread as asyncio tasks.
+    - Synchronous functions will use multi-threading. The code must be thread-safe.
+
+    Input concurrency will be most useful for workflows that are IO-bound
+    (e.g., making network requests) or when running an inference server that supports
+    dynamic batching.
+
+    When `target_inputs` is set, Modal's autoscaler will try to provision resources such
+    that each container is running that many inputs concurrently. Containers may burst up to
+    up to `max_inputs` if resources are insufficient to remain at the target concurrency.
+    """
+    if _warn_parentheses_missing is not None:
+        raise InvalidError(
+            "Positional arguments are not allowed. Did you forget parentheses? Suggestion: `@modal.concurrent()`."
+        )
+
+    if target_inputs and target_inputs > max_inputs:
+        raise InvalidError("`target_inputs` parameter cannot be greater than `max_inputs`.")
+
+    def wrapper(obj: Union[Callable[..., Any], _PartialFunction]) -> _PartialFunction:
+        if isinstance(obj, _PartialFunction):
+            # Risky that we need to mutate the parameters here; should make this safer
+            obj.max_concurrent_inputs = max_inputs
+            obj.target_concurrent_inputs = target_inputs
+            obj.add_flags(_PartialFunctionFlags.FUNCTION)
+            return obj
+
+        return _PartialFunction(
+            obj,
+            _PartialFunctionFlags.FUNCTION,
+            max_concurrent_inputs=max_inputs,
+            target_concurrent_inputs=target_inputs,
+        )
+
+    return wrapper

{modal-0.73.126 → modal-0.73.128}/modal/app.py

@@ -678,6 +678,12 @@ class _App:
                 is_generator = f.is_generator
                 batch_max_size = f.batch_max_size
                 batch_wait_ms = f.batch_wait_ms
+                if f.max_concurrent_inputs:  # Using @modal.concurrent()
+                    max_concurrent_inputs = f.max_concurrent_inputs
+                    target_concurrent_inputs = f.target_concurrent_inputs
+                else:
+                    max_concurrent_inputs = allow_concurrent_inputs
+                    target_concurrent_inputs = None
             else:
                 if not is_global_object(f.__qualname__) and not serialized:
                     raise InvalidError(
@@ -709,10 +715,12 @@ class _App:
                     )
 
                 info = FunctionInfo(f, serialized=serialized, name_override=name)
+                raw_f = f
                 webhook_config = None
                 batch_max_size = None
                 batch_wait_ms = None
-                raw_f = f
+                max_concurrent_inputs = allow_concurrent_inputs
+                target_concurrent_inputs = None
 
                 cluster_size = None  # Experimental: Clustered functions
                 i6pn_enabled = i6pn
@@ -753,7 +761,8 @@ class _App:
                 max_containers=max_containers,
                 buffer_containers=buffer_containers,
                 scaledown_window=scaledown_window,
-                allow_concurrent_inputs=allow_concurrent_inputs,
+                max_concurrent_inputs=max_concurrent_inputs,
+                target_concurrent_inputs=target_concurrent_inputs,
                 batch_max_size=batch_max_size,
                 batch_wait_ms=batch_wait_ms,
                 timeout=timeout,
@@ -832,7 +841,7 @@ class _App:
         concurrency_limit: Optional[int] = None,  # Replaced with `max_containers`
         container_idle_timeout: Optional[int] = None,  # Replaced with `scaledown_window`
         _experimental_buffer_containers: Optional[int] = None,  # Now stable API with `buffer_containers`
-    ) -> Callable[[CLS_T], CLS_T]:
+    ) -> Callable[[Union[CLS_T, _PartialFunction]], CLS_T]:
         """
         Decorator to register a new Modal [Cls](/docs/reference/modal.Cls) with this App.
         """
@@ -845,8 +854,21 @@ class _App:
                 raise InvalidError("`region` and `_experimental_scheduler_placement` cannot be used together")
             scheduler_placement = SchedulerPlacement(region=region)
 
-        def wrapper(user_cls: CLS_T) -> CLS_T:
+        def wrapper(wrapped_cls: Union[CLS_T, _PartialFunction]) -> CLS_T:
             # Check if the decorated object is a class
+            if isinstance(wrapped_cls, _PartialFunction):
+                wrapped_cls.wrapped = True
+                user_cls = wrapped_cls.raw_f
+                if wrapped_cls.max_concurrent_inputs:  # Using @modal.concurrent()
+                    max_concurrent_inputs = wrapped_cls.max_concurrent_inputs
+                    target_concurrent_inputs = wrapped_cls.target_concurrent_inputs
+                else:
+                    max_concurrent_inputs = allow_concurrent_inputs
+                    target_concurrent_inputs = None
+            else:
+                user_cls = wrapped_cls
+                max_concurrent_inputs = allow_concurrent_inputs
+                target_concurrent_inputs = None
             if not inspect.isclass(user_cls):
                 raise TypeError("The @app.cls decorator must be used on a class.")
 
@@ -871,6 +893,12 @@ class _App:
             ):
                 raise InvalidError("A class must have `enable_memory_snapshot=True` to use `snap=True` on its methods.")
 
+            for method in _find_partial_methods_for_user_cls(user_cls, _PartialFunctionFlags.FUNCTION).values():
+                if method.max_concurrent_inputs:
+                    raise InvalidError(
+                        "The `@modal.concurrent` decorator cannot be used on methods; decorate the class instead."
+                    )
+
             info = FunctionInfo(None, serialized=serialized, user_cls=user_cls)
 
             cls_func = _Function.from_local(
@@ -892,7 +920,8 @@ class _App:
                 scaledown_window=scaledown_window,
                 proxy=proxy,
                 retries=retries,
-                allow_concurrent_inputs=allow_concurrent_inputs,
+                max_concurrent_inputs=max_concurrent_inputs,
+                target_concurrent_inputs=target_concurrent_inputs,
                 batch_max_size=batch_max_size,
                 batch_wait_ms=batch_wait_ms,
                 timeout=timeout,

{modal-0.73.126 → modal-0.73.128}/modal/app.pyi

@@ -1,6 +1,7 @@
 import collections.abc
 import modal._functions
 import modal._object
+import modal._partial_function
 import modal._utils.function_utils
 import modal.client
 import modal.cloud_bucket_mount
@@ -247,7 +248,7 @@ class _App:
         concurrency_limit: typing.Optional[int] = None,
         container_idle_timeout: typing.Optional[int] = None,
         _experimental_buffer_containers: typing.Optional[int] = None,
-    ) -> collections.abc.Callable[[CLS_T], CLS_T]: ...
+    ) -> collections.abc.Callable[[typing.Union[CLS_T, modal._partial_function._PartialFunction]], CLS_T]: ...
     async def spawn_sandbox(
         self,
         *entrypoint_args: str,
@@ -487,7 +488,7 @@ class App:
         concurrency_limit: typing.Optional[int] = None,
         container_idle_timeout: typing.Optional[int] = None,
         _experimental_buffer_containers: typing.Optional[int] = None,
-    ) -> collections.abc.Callable[[CLS_T], CLS_T]: ...
+    ) -> collections.abc.Callable[[typing.Union[CLS_T, modal.partial_function.PartialFunction]], CLS_T]: ...
 
     class __spawn_sandbox_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(

{modal-0.73.126 → modal-0.73.128}/modal/client.pyi

@@ -31,7 +31,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "0.73.126",
+        version: str = "0.73.128",
     ): ...
     def is_closed(self) -> bool: ...
     @property
@@ -93,7 +93,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "0.73.126",
+        version: str = "0.73.128",
     ): ...
     def is_closed(self) -> bool: ...
     @property

{modal-0.73.126 → modal-0.73.128}/modal/functions.pyi

@@ -82,7 +82,8 @@ class Function(
         max_containers: typing.Optional[int] = None,
         buffer_containers: typing.Optional[int] = None,
         scaledown_window: typing.Optional[int] = None,
-        allow_concurrent_inputs: typing.Optional[int] = None,
+        max_concurrent_inputs: typing.Optional[int] = None,
+        target_concurrent_inputs: typing.Optional[int] = None,
         batch_max_size: typing.Optional[int] = None,
         batch_wait_ms: typing.Optional[int] = None,
         cloud: typing.Optional[str] = None,

{modal-0.73.126 → modal-0.73.128}/modal/partial_function.py

@@ -5,6 +5,7 @@ from ._partial_function import (
     _asgi_app,
     _batched,
     _build,
+    _concurrent,
     _enter,
     _exit,
     _fastapi_endpoint,
@@ -28,3 +29,4 @@ build = synchronize_api(_build, target_module=__name__)
 enter = synchronize_api(_enter, target_module=__name__)
 exit = synchronize_api(_exit, target_module=__name__)
 batched = synchronize_api(_batched, target_module=__name__)
+concurrent = synchronize_api(_concurrent, target_module=__name__)

{modal-0.73.126 → modal-0.73.128}/modal/partial_function.pyi

@@ -18,6 +18,8 @@ class PartialFunction(
     force_build: bool
     cluster_size: typing.Optional[int]
     build_timeout: typing.Optional[int]
+    max_concurrent_inputs: typing.Optional[int]
+    target_concurrent_inputs: typing.Optional[int]
 
     def __init__(
         self,
@@ -31,6 +33,8 @@ class PartialFunction(
         cluster_size: typing.Optional[int] = None,
         force_build: bool = False,
         build_timeout: typing.Optional[int] = None,
+        max_concurrent_inputs: typing.Optional[int] = None,
+        target_concurrent_inputs: typing.Optional[int] = None,
     ): ...
     def _get_raw_f(self) -> collections.abc.Callable[modal._partial_function.P, modal._partial_function.ReturnType]: ...
     def _is_web_endpoint(self) -> bool: ...
@@ -118,3 +122,8 @@ def exit(
 def batched(
     _warn_parentheses_missing=None, *, max_batch_size: int, wait_ms: int
 ) -> collections.abc.Callable[[collections.abc.Callable[..., typing.Any]], PartialFunction]: ...
+def concurrent(
+    _warn_parentheses_missing=None, *, max_inputs: int, target_inputs: typing.Optional[int] = None
+) -> collections.abc.Callable[
+    [typing.Union[collections.abc.Callable[..., typing.Any], PartialFunction]], PartialFunction
+]: ...

{modal-0.73.126 → modal-0.73.128}/modal/sandbox.py

@@ -44,7 +44,11 @@ _default_image: _Image = _Image.debian_slim()
 # The maximum number of bytes that can be passed to an exec on Linux.
 # Though this is technically a 'server side' limit, it is unlikely to change.
 # getconf ARG_MAX will show this value on a host.
-ARG_MAX_BYTES = 2_097_152  # 2MiB
+#
+# By probing in production, the limit is 131072 bytes (2**17).
+# We need some bytes of overhead for the rest of the command line besides the args,
+# e.g. 'runsc exec ...'. So we use 2**16 as the limit.
+ARG_MAX_BYTES = 2**16
 
 if TYPE_CHECKING:
     import modal.app

{modal-0.73.126 → modal-0.73.128}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: modal
-Version: 0.73.126
+Version: 0.73.128
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-0.73.126 → modal-0.73.128}/modal_proto/api.proto

@@ -1216,7 +1216,7 @@ message Function {
 
   repeated VolumeMount volume_mounts = 33;
 
-  uint32 target_concurrent_inputs = 34;
+  uint32 max_concurrent_inputs = 34;
 
   repeated CustomDomainInfo custom_domain_info = 35;
 
@@ -1266,7 +1266,7 @@ message Function {
   uint64 batch_linger_ms = 61; // Miliseconds to block before a response is needed
   bool i6pn_enabled = 62;
   bool _experimental_concurrent_cancellations = 63;
-  uint32 max_concurrent_inputs = 64;
+  uint32 target_concurrent_inputs = 64;
 
   // TODO(irfansharif): Remove, once https://github.com/modal-labs/modal/pull/15645 lands.
   bool _experimental_task_templates_enabled = 65;  // forces going through the new gpu-fallbacks integration path, even if no fallback options are specified