modal 1.1.5.dev3__py3-none-any.whl → 1.1.5.dev5__py3-none-any.whl

modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.5.dev3",
+        version: str = "1.1.5.dev5",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.5.dev3",
+        version: str = "1.1.5.dev5",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/experimental/flash.py

@@ -21,7 +21,7 @@ from ..client import _Client
 from ..config import logger
 from ..exception import InvalidError
 
-_MAX_FAILURES = 3
+_MAX_FAILURES = 10
 
 
 class _FlashManager:
@@ -43,7 +43,7 @@ class _FlashManager:
         self.task_id = os.environ["MODAL_TASK_ID"]
 
     async def is_port_connection_healthy(
-        self, process: Optional[subprocess.Popen], timeout: int = 5
+        self, process: Optional[subprocess.Popen], timeout: float = 0.5
     ) -> tuple[bool, Optional[Exception]]:
         import socket
 
@@ -53,7 +53,7 @@ class _FlashManager:
             try:
                 if process is not None and process.poll() is not None:
                     return False, Exception(f"Process {process.pid} exited with code {process.returncode}")
-                with socket.create_connection(("localhost", self.port), timeout=1):
+                with socket.create_connection(("localhost", self.port), timeout=0.5):
                     return True, None
             except (ConnectionRefusedError, OSError):
                 await asyncio.sleep(0.1)
@@ -201,6 +201,7 @@ class _FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: Optional[int],
         max_containers: Optional[int],
+        buffer_containers: Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -228,6 +229,7 @@ class _FlashPrometheusAutoscaler:
         self.target_metric_value = target_metric_value
         self.min_containers = min_containers
         self.max_containers = max_containers
+        self.buffer_containers = buffer_containers
         self.scale_up_tolerance = scale_up_tolerance
         self.scale_down_tolerance = scale_down_tolerance
         self.scale_up_stabilization_window_seconds = scale_up_stabilization_window_seconds
@@ -293,6 +295,7 @@ class _FlashPrometheusAutoscaler:
                     scale_down_stabilization_window_seconds=self.scale_down_stabilization_window_seconds,
                     min_containers=self.min_containers,
                     max_containers=self.max_containers,
+                    buffer_containers=self.buffer_containers,
                 )
 
                 logger.warning(
@@ -402,6 +405,7 @@ class _FlashPrometheusAutoscaler:
         # Gets metrics from prometheus
         sum_metric = 0
         containers_with_metrics = 0
+        buffer_containers = self.buffer_containers or 0
         container_metrics_list = await asyncio.gather(
             *[
                 self._get_metrics(f"https://{container.host}:{container.port}/{self.metrics_endpoint}")
@@ -418,29 +422,36 @@ class _FlashPrometheusAutoscaler:
             sum_metric += container_metrics[target_metric][0].value
             containers_with_metrics += 1
 
-        # n_containers_missing_metric is the number of unhealthy containers + number of cold starting containers
+        # n_containers_missing = number of unhealthy containers + number of containers not registered in flash dns
         n_containers_missing_metric = current_replicas - containers_with_metrics
-        # n_containers_unhealthy is the number of live containers that are not emitting metrics i.e. unhealthy
+        # n_containers_unhealthy = number of dns registered containers that are not emitting metrics
         n_containers_unhealthy = len(containers) - containers_with_metrics
 
-        # Scale up assuming that every unhealthy container is at 2x the target metric value.
-        scale_up_target_metric_value = (sum_metric + n_containers_unhealthy * target_metric_value) / (
-            (containers_with_metrics + n_containers_unhealthy) or 1
-        )
+        # number of total containers - buffer containers
+        # This is used in 1) scale ratio denominators 2) provisioning base.
+        # Max is used to handle case when buffer_containers are first initialized.
+        num_provisioned_containers = max(current_replicas - buffer_containers, 1)
+
+        # Scale up assuming that every unhealthy container is at (1 + scale_up_tolerance)x the target metric value.
+        # This way if all containers are unhealthy, we will increase our number of containers.
+        scale_up_target_metric_value = (
+            sum_metric + (1 + self.scale_up_tolerance) * n_containers_unhealthy * target_metric_value
+        ) / (num_provisioned_containers)
 
         # Scale down assuming that every container (including cold starting containers) are at the target metric value.
+        # The denominator is just num_provisioned_containers because we don't want to account for the buffer containers.
         scale_down_target_metric_value = (sum_metric + n_containers_missing_metric * target_metric_value) / (
-            current_replicas or 1
+            num_provisioned_containers
         )
 
         scale_up_ratio = scale_up_target_metric_value / target_metric_value
         scale_down_ratio = scale_down_target_metric_value / target_metric_value
 
-        desired_replicas = current_replicas
+        desired_replicas = num_provisioned_containers
         if scale_up_ratio > 1 + self.scale_up_tolerance:
-            desired_replicas = math.ceil(current_replicas * scale_up_ratio)
+            desired_replicas = math.ceil(desired_replicas * scale_up_ratio)
         elif scale_down_ratio < 1 - self.scale_down_tolerance:
-            desired_replicas = math.ceil(current_replicas * scale_down_ratio)
+            desired_replicas = math.ceil(desired_replicas * scale_down_ratio)
 
         logger.warning(
             f"[Modal Flash] Current replicas: {current_replicas}, "
@@ -449,6 +460,7 @@ class _FlashPrometheusAutoscaler:
             f"number of containers with metrics: {containers_with_metrics}, "
             f"number of containers unhealthy: {n_containers_unhealthy}, "
             f"number of containers missing metric (includes unhealthy): {n_containers_missing_metric}, "
+            f"number of provisioned containers: {num_provisioned_containers}, "
             f"scale up ratio: {scale_up_ratio}, "
             f"scale down ratio: {scale_down_ratio}, "
             f"desired replicas: {desired_replicas}"
@@ -510,6 +522,7 @@ class _FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 60 * 5,
         min_containers: Optional[int] = None,
         max_containers: Optional[int] = None,
+        buffer_containers: Optional[int] = None,
     ) -> int:
         """
         Return the target number of containers following (simplified) Kubernetes HPA
@@ -560,6 +573,10 @@ class _FlashPrometheusAutoscaler:
             new_replicas = max(min_containers, new_replicas)
         if max_containers is not None:
             new_replicas = min(max_containers, new_replicas)
+
+        if buffer_containers is not None:
+            new_replicas += buffer_containers
+
         return new_replicas
 
     async def stop(self):
@@ -597,6 +614,8 @@ async def flash_prometheus_autoscaler(
     # How often to make autoscaling decisions.
     # Corresponds to --horizontal-pod-autoscaler-sync-period in Kubernetes.
     autoscaling_interval_seconds: int = 15,
+    # Whether to include overprovisioned containers in the scale up calculation.
+    buffer_containers: Optional[int] = None,
 ) -> _FlashPrometheusAutoscaler:
     """
     Autoscale a Flash service based on containers' Prometheus metrics.
@@ -614,19 +633,20 @@ async def flash_prometheus_autoscaler(
 
     client = await _Client.from_env()
     autoscaler = _FlashPrometheusAutoscaler(
-        client,
-        app_name,
-        cls_name,
-        metrics_endpoint,
-        target_metric,
-        target_metric_value,
-        min_containers,
-        max_containers,
-        scale_up_tolerance,
-        scale_down_tolerance,
-        scale_up_stabilization_window_seconds,
-        scale_down_stabilization_window_seconds,
-        autoscaling_interval_seconds,
+        client=client,
+        app_name=app_name,
+        cls_name=cls_name,
+        metrics_endpoint=metrics_endpoint,
+        target_metric=target_metric,
+        target_metric_value=target_metric_value,
+        min_containers=min_containers,
+        max_containers=max_containers,
+        buffer_containers=buffer_containers,
+        scale_up_tolerance=scale_up_tolerance,
+        scale_down_tolerance=scale_down_tolerance,
+        scale_up_stabilization_window_seconds=scale_up_stabilization_window_seconds,
+        scale_down_stabilization_window_seconds=scale_down_stabilization_window_seconds,
+        autoscaling_interval_seconds=autoscaling_interval_seconds,
     )
     await autoscaler.start()
     return autoscaler

modal/experimental/flash.pyi

@@ -16,7 +16,7 @@ class _FlashManager:
         ...
 
     async def is_port_connection_healthy(
-        self, process: typing.Optional[subprocess.Popen], timeout: int = 5
+        self, process: typing.Optional[subprocess.Popen], timeout: float = 0.5
     ) -> tuple[bool, typing.Optional[Exception]]: ...
     async def _start(self): ...
     async def _drain_container(self):
@@ -41,10 +41,10 @@ class FlashManager:
 
     class __is_port_connection_healthy_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(
-            self, /, process: typing.Optional[subprocess.Popen], timeout: int = 5
+            self, /, process: typing.Optional[subprocess.Popen], timeout: float = 0.5
         ) -> tuple[bool, typing.Optional[Exception]]: ...
         async def aio(
-            self, /, process: typing.Optional[subprocess.Popen], timeout: int = 5
+            self, /, process: typing.Optional[subprocess.Popen], timeout: float = 0.5
         ) -> tuple[bool, typing.Optional[Exception]]: ...
 
     is_port_connection_healthy: __is_port_connection_healthy_spec[typing_extensions.Self]
@@ -126,6 +126,7 @@ class _FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: typing.Optional[int],
         max_containers: typing.Optional[int],
+        buffer_containers: typing.Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -155,6 +156,7 @@ class _FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 300,
         min_containers: typing.Optional[int] = None,
         max_containers: typing.Optional[int] = None,
+        buffer_containers: typing.Optional[int] = None,
     ) -> int:
         """Return the target number of containers following (simplified) Kubernetes HPA
         stabilization-window semantics.
@@ -187,6 +189,7 @@ class FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: typing.Optional[int],
         max_containers: typing.Optional[int],
+        buffer_containers: typing.Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -253,6 +256,7 @@ class FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 300,
         min_containers: typing.Optional[int] = None,
         max_containers: typing.Optional[int] = None,
+        buffer_containers: typing.Optional[int] = None,
     ) -> int:
         """Return the target number of containers following (simplified) Kubernetes HPA
         stabilization-window semantics.
@@ -294,6 +298,7 @@ class __flash_prometheus_autoscaler_spec(typing_extensions.Protocol):
         scale_up_stabilization_window_seconds: int = 0,
         scale_down_stabilization_window_seconds: int = 300,
         autoscaling_interval_seconds: int = 15,
+        buffer_containers: typing.Optional[int] = None,
     ) -> FlashPrometheusAutoscaler:
         """Autoscale a Flash service based on containers' Prometheus metrics.
 
@@ -319,6 +324,7 @@ class __flash_prometheus_autoscaler_spec(typing_extensions.Protocol):
         scale_up_stabilization_window_seconds: int = 0,
         scale_down_stabilization_window_seconds: int = 300,
         autoscaling_interval_seconds: int = 15,
+        buffer_containers: typing.Optional[int] = None,
     ) -> FlashPrometheusAutoscaler:
         """Autoscale a Flash service based on containers' Prometheus metrics.
 

modal/functions.pyi

@@ -445,7 +445,7 @@ class Function(
 
     _call_generator: ___call_generator_spec[typing_extensions.Self]
 
-    class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER:
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
@@ -454,7 +454,7 @@ class Function(
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
 
-    remote: __remote_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    remote: __remote_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     class __remote_gen_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, *args, **kwargs) -> typing.Generator[typing.Any, None, None]:
@@ -481,7 +481,7 @@ class Function(
         """
         ...
 
-    class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """[Experimental] Calls the function with the given arguments, without waiting for the results.
 
@@ -505,7 +505,7 @@ class Function(
             ...
 
     _experimental_spawn: ___experimental_spawn_spec[
-        modal._functions.ReturnType, modal._functions.P, typing_extensions.Self
+        modal._functions.P, modal._functions.ReturnType, typing_extensions.Self
     ]
 
     class ___spawn_map_inner_spec(typing_extensions.Protocol[P_INNER, SUPERSELF]):
@@ -514,7 +514,7 @@ class Function(
 
     _spawn_map_inner: ___spawn_map_inner_spec[modal._functions.P, typing_extensions.Self]
 
-    class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """Calls the function with the given arguments, without waiting for the results.
 
@@ -535,7 +535,7 @@ class Function(
             """
             ...
 
-    spawn: __spawn_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    spawn: __spawn_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     def get_raw_f(self) -> collections.abc.Callable[..., typing.Any]:
         """Return the inner Python object wrapped by this Modal Function."""

{modal-1.1.5.dev3.dist-info → modal-1.1.5.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.1.5.dev3
+Version: 1.1.5.dev5
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.1.5.dev3.dist-info → modal-1.1.5.dev5.dist-info}/RECORD

@@ -22,7 +22,7 @@ modal/app.py,sha256=F4baVULljFq0CwC_7U-EKNRNx7CYeWBKudjjYUuWc4U,48416
 modal/app.pyi,sha256=AbXJCBkyt2rI_-M3VbTBYb32at0P6iRZuoC87xY_JrQ,43591
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=kyAIVB3Ay-XKJizQ_1ufUFB__EagV0MLmHJpyYyJ7J0,18636
-modal/client.pyi,sha256=c9p8XWAlM_rGXLwTQmPudc2C_CW7Melsbw20mhAj8jE,15829
+modal/client.pyi,sha256=Mfb3RbnQ9teBNGXTLG4ccodqB4Xd4MUh_3FT290X5XA,15829
 modal/cloud_bucket_mount.py,sha256=YOe9nnvSr4ZbeCn587d7_VhE9IioZYRvF9VYQTQux08,5914
 modal/cloud_bucket_mount.pyi,sha256=-qSfYAQvIoO_l2wsCCGTG5ZUwQieNKXdAO00yP1-LYU,7394
 modal/cls.py,sha256=pTEO7pHjlO7taMbIqs4oI9ZZgKDJpVKyGkO5ZT0w6tQ,40934
@@ -39,7 +39,7 @@ modal/file_io.py,sha256=OSKr77TujcXGJW1iikzYiHckLSmv07QBgBHcxxYEkoI,21456
 modal/file_io.pyi,sha256=xtO6Glf_BFwDE7QiQQo24QqcMf_Vv-iz7WojcGVlLBU,15932
 modal/file_pattern_matcher.py,sha256=A_Kdkej6q7YQyhM_2-BvpFmPqJ0oHb54B6yf9VqvPVE,8116
 modal/functions.py,sha256=kcNHvqeGBxPI7Cgd57NIBBghkfbeFJzXO44WW0jSmao,325
-modal/functions.pyi,sha256=cDqhpIM5caoCR18_8krpAmPOd4QvEbm1ypYUZ6Ze9Wo,39404
+modal/functions.pyi,sha256=F3Dll-9c3d8c8NuJuEYICUEB-sa67uWU75L111npwbA,39404
 modal/gpu.py,sha256=Fe5ORvVPDIstSq1xjmM6OoNgLYFWvogP9r5BgmD3hYg,6769
 modal/image.py,sha256=nXN9k_6gApHFy8-Bk_XT2Zu3jsDsGVrC3QcuiDC4yRY,103543
 modal/image.pyi,sha256=vKdb5PpYM8wcpq9PQegeVMjrPLzAipuV4q994NZiL84,69325
@@ -150,10 +150,10 @@ modal/cli/programs/run_jupyter.py,sha256=44Lpvqk2l3hH-uOkmAOzw60NEsfB5uaRDWDKVsh
 modal/cli/programs/run_marimo.py,sha256=HyZ2za0NYqg31-mGxFQxUIAJ3Q-jRaMocEwWwI5-cdw,2887
 modal/cli/programs/vscode.py,sha256=KbTAaIXyQBVCDXxXjmBHmKpgXkUw0q4R4KkJvUjCYgk,3380
 modal/experimental/__init__.py,sha256=fCqzo_f3vcY750vHtd7CtLs5dvdM_C0ZLLGb3zXuK9w,14913
-modal/experimental/flash.py,sha256=amsEPtzD2OX5w4YcTPKj9MAUhANEgQni1VHnYjLshrc,28647
-modal/experimental/flash.pyi,sha256=Tu9n25ZnW4dO1YjNRHIQpZb4VWSfNW5IENrY0HJW-ME,13936
+modal/experimental/flash.py,sha256=DhX0HG-K55FSK_ym0MjXC9ObVflmamAXfFY-TnuN5SM,30056
+modal/experimental/flash.pyi,sha256=EZiceVufvf7jsqKSQG-gd1hwqxzbEZxMkB1NtrK3AnE,14270
 modal/experimental/ipython.py,sha256=TrCfmol9LGsRZMeDoeMPx3Hv3BFqQhYnmD_iH0pqdhk,2904
-modal-1.1.5.dev3.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-1.1.5.dev5.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
 modal_docs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal_docs/gen_cli_docs.py,sha256=c1yfBS_x--gL5bs0N4ihMwqwX8l3IBWSkBAKNNIi6bQ,3801
 modal_docs/gen_reference_docs.py,sha256=d_CQUGQ0rfw28u75I2mov9AlS773z9rG40-yq5o7g2U,6359
@@ -176,10 +176,10 @@ modal_proto/options_pb2.pyi,sha256=l7DBrbLO7q3Ir-XDkWsajm0d0TQqqrfuX54i4BMpdQg,1
 modal_proto/options_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0yJSI,247
 modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-modal_version/__init__.py,sha256=0Y1tDsgw11T_vf_1T67OFGKn1Gtt4JMwDn3IKJeIETY,120
+modal_version/__init__.py,sha256=yuvvn6V0EFEDt_TQPIbeKwTnFZo9K96gcu6d1E6YerY,120
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal-1.1.5.dev3.dist-info/METADATA,sha256=mactKwIxNbCbeYMuYXywsEKRAnJgV6GEnURnLtmTI5o,2459
-modal-1.1.5.dev3.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-modal-1.1.5.dev3.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-1.1.5.dev3.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
-modal-1.1.5.dev3.dist-info/RECORD,,
+modal-1.1.5.dev5.dist-info/METADATA,sha256=cKYLGKaFAL0pfuBQKBnbnYbQVIY__1tC7C-fr9T7hk8,2459
+modal-1.1.5.dev5.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+modal-1.1.5.dev5.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-1.1.5.dev5.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
+modal-1.1.5.dev5.dist-info/RECORD,,

modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.1.5.dev3"
+__version__ = "1.1.5.dev5"