modal 1.1.5.dev2__tar.gz → 1.1.5.dev4__tar.gz

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.1.5.dev2
+Version: 1.1.5.dev4
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.5.dev2",
+        version: str = "1.1.5.dev4",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.5.dev2",
+        version: str = "1.1.5.dev4",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/modal/experimental/flash.py

@@ -21,7 +21,7 @@ from ..client import _Client
 from ..config import logger
 from ..exception import InvalidError
 
-MAX_FAILURES = 3
+_MAX_FAILURES = 3
 
 
 class _FlashManager:
@@ -42,7 +42,9 @@ class _FlashManager:
         self.num_failures = 0
         self.task_id = os.environ["MODAL_TASK_ID"]
 
-    async def check_port_connection(self, process: Optional[subprocess.Popen], timeout: int = 10):
+    async def is_port_connection_healthy(
+        self, process: Optional[subprocess.Popen], timeout: int = 5
+    ) -> tuple[bool, Optional[Exception]]:
         import socket
 
         start_time = time.monotonic()
@@ -50,13 +52,13 @@ class _FlashManager:
         while time.monotonic() - start_time < timeout:
             try:
                 if process is not None and process.poll() is not None:
-                    return Exception(f"Process {process.pid} exited with code {process.returncode}")
+                    return False, Exception(f"Process {process.pid} exited with code {process.returncode}")
                 with socket.create_connection(("localhost", self.port), timeout=1):
-                    return
+                    return True, None
             except (ConnectionRefusedError, OSError):
                 await asyncio.sleep(0.1)
 
-        return Exception(f"Waited too long for port {self.port} to start accepting connections")
+        return False, Exception(f"Waited too long for port {self.port} to start accepting connections")
 
     async def _start(self):
         self.tunnel = await self.tunnel_manager.__aenter__()
@@ -74,7 +76,7 @@ class _FlashManager:
         while True:
             try:
                 # Check if the container should be drained (e.g., too many failures)
-                if self.num_failures > MAX_FAILURES:
+                if self.num_failures > _MAX_FAILURES:
                     logger.warning(
                         f"[Modal Flash] Draining task {self.task_id} on {self.tunnel.url} due to too many failures."
                     )
@@ -101,35 +103,38 @@ class _FlashManager:
         first_registration = True
         while True:
             try:
-                await self.check_port_connection(process=self.process)
-                resp = await self.client.stub.FlashContainerRegister(
-                    api_pb2.FlashContainerRegisterRequest(
-                        priority=10,
-                        weight=5,
-                        host=host,
-                        port=port,
-                    ),
-                    timeout=10,
-                )
-                self.num_failures = 0
-                if first_registration:
-                    logger.warning(
-                        f"[Modal Flash] Listening at {resp.url} over {self.tunnel.url} for task_id {self.task_id}"
+                port_check_resp, port_check_error = await self.is_port_connection_healthy(process=self.process)
+                if port_check_resp:
+                    resp = await self.client.stub.FlashContainerRegister(
+                        api_pb2.FlashContainerRegisterRequest(
+                            priority=10,
+                            weight=5,
+                            host=host,
+                            port=port,
+                        ),
+                        timeout=10,
+                    )
+                    self.num_failures = 0
+                    if first_registration:
+                        logger.warning(
+                            f"[Modal Flash] Listening at {resp.url} over {self.tunnel.url} for task_id {self.task_id}"
+                        )
+                        first_registration = False
+                else:
+                    logger.error(
+                        f"[Modal Flash] Deregistering container {self.task_id} on {self.tunnel.url} "
+                        f"due to error: {port_check_error}, num_failures: {self.num_failures}"
+                    )
+                    self.num_failures += 1
+                    await retry_transient_errors(
+                        self.client.stub.FlashContainerDeregister,
+                        api_pb2.FlashContainerDeregisterRequest(),
                     )
-                    first_registration = False
             except asyncio.CancelledError:
                 logger.warning("[Modal Flash] Shutting down...")
                 break
             except Exception as e:
                 logger.error(f"[Modal Flash] Heartbeat failed: {e}")
-                self.num_failures += 1
-                logger.error(
-                    f"[Modal Flash] Deregistering container {self.tunnel.url}, num_failures: {self.num_failures}"
-                )
-                await retry_transient_errors(
-                    self.client.stub.FlashContainerDeregister,
-                    api_pb2.FlashContainerDeregisterRequest(),
-                )
 
             try:
                 await asyncio.sleep(1)
@@ -167,7 +172,9 @@ FlashManager = synchronize_api(_FlashManager)
 
 @synchronizer.create_blocking
 async def flash_forward(
-    port: int, process: Optional[subprocess.Popen] = None, health_check_url: Optional[str] = None
+    port: int,
+    process: Optional[subprocess.Popen] = None,
+    health_check_url: Optional[str] = None,
 ) -> _FlashManager:
     """
     Forward a port to the Modal Flash service, exposing that port as a stable web endpoint.
@@ -194,6 +201,7 @@ class _FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: Optional[int],
         max_containers: Optional[int],
+        buffer_containers: Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -221,6 +229,7 @@ class _FlashPrometheusAutoscaler:
         self.target_metric_value = target_metric_value
         self.min_containers = min_containers
         self.max_containers = max_containers
+        self.buffer_containers = buffer_containers
         self.scale_up_tolerance = scale_up_tolerance
         self.scale_down_tolerance = scale_down_tolerance
         self.scale_up_stabilization_window_seconds = scale_up_stabilization_window_seconds
@@ -286,6 +295,7 @@ class _FlashPrometheusAutoscaler:
                     scale_down_stabilization_window_seconds=self.scale_down_stabilization_window_seconds,
                     min_containers=self.min_containers,
                     max_containers=self.max_containers,
+                    buffer_containers=self.buffer_containers,
                 )
 
                 logger.warning(
@@ -395,6 +405,7 @@ class _FlashPrometheusAutoscaler:
         # Gets metrics from prometheus
         sum_metric = 0
         containers_with_metrics = 0
+        buffer_containers = self.buffer_containers or 0
         container_metrics_list = await asyncio.gather(
             *[
                 self._get_metrics(f"https://{container.host}:{container.port}/{self.metrics_endpoint}")
@@ -411,29 +422,36 @@ class _FlashPrometheusAutoscaler:
             sum_metric += container_metrics[target_metric][0].value
             containers_with_metrics += 1
 
-        # n_containers_missing_metric is the number of unhealthy containers + number of cold starting containers
+        # n_containers_missing = number of unhealthy containers + number of containers not registered in flash dns
         n_containers_missing_metric = current_replicas - containers_with_metrics
-        # n_containers_unhealthy is the number of live containers that are not emitting metrics i.e. unhealthy
+        # n_containers_unhealthy = number of dns registered containers that are not emitting metrics
         n_containers_unhealthy = len(containers) - containers_with_metrics
 
-        # Scale up assuming that every unhealthy container is at 2x the target metric value.
-        scale_up_target_metric_value = (sum_metric + n_containers_unhealthy * target_metric_value) / (
-            (containers_with_metrics + n_containers_unhealthy) or 1
-        )
+        # number of total containers - buffer containers
+        # This is used in 1) scale ratio denominators 2) provisioning base.
+        # Max is used to handle case when buffer_containers are first initialized.
+        num_provisioned_containers = max(current_replicas - buffer_containers, 1)
+
+        # Scale up assuming that every unhealthy container is at (1 + scale_up_tolerance)x the target metric value.
+        # This way if all containers are unhealthy, we will increase our number of containers.
+        scale_up_target_metric_value = (
+            sum_metric + (1 + self.scale_up_tolerance) * n_containers_unhealthy * target_metric_value
+        ) / (num_provisioned_containers)
 
         # Scale down assuming that every container (including cold starting containers) are at the target metric value.
+        # The denominator is just num_provisioned_containers because we don't want to account for the buffer containers.
         scale_down_target_metric_value = (sum_metric + n_containers_missing_metric * target_metric_value) / (
-            current_replicas or 1
+            num_provisioned_containers
         )
 
         scale_up_ratio = scale_up_target_metric_value / target_metric_value
         scale_down_ratio = scale_down_target_metric_value / target_metric_value
 
-        desired_replicas = current_replicas
+        desired_replicas = num_provisioned_containers
         if scale_up_ratio > 1 + self.scale_up_tolerance:
-            desired_replicas = math.ceil(current_replicas * scale_up_ratio)
+            desired_replicas = math.ceil(desired_replicas * scale_up_ratio)
         elif scale_down_ratio < 1 - self.scale_down_tolerance:
-            desired_replicas = math.ceil(current_replicas * scale_down_ratio)
+            desired_replicas = math.ceil(desired_replicas * scale_down_ratio)
 
         logger.warning(
             f"[Modal Flash] Current replicas: {current_replicas}, "
@@ -442,6 +460,7 @@ class _FlashPrometheusAutoscaler:
             f"number of containers with metrics: {containers_with_metrics}, "
             f"number of containers unhealthy: {n_containers_unhealthy}, "
             f"number of containers missing metric (includes unhealthy): {n_containers_missing_metric}, "
+            f"number of provisioned containers: {num_provisioned_containers}, "
             f"scale up ratio: {scale_up_ratio}, "
             f"scale down ratio: {scale_down_ratio}, "
             f"desired replicas: {desired_replicas}"
@@ -503,6 +522,7 @@ class _FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 60 * 5,
         min_containers: Optional[int] = None,
         max_containers: Optional[int] = None,
+        buffer_containers: Optional[int] = None,
     ) -> int:
         """
         Return the target number of containers following (simplified) Kubernetes HPA
@@ -553,6 +573,10 @@ class _FlashPrometheusAutoscaler:
             new_replicas = max(min_containers, new_replicas)
         if max_containers is not None:
             new_replicas = min(max_containers, new_replicas)
+
+        if buffer_containers is not None:
+            new_replicas += buffer_containers
+
         return new_replicas
 
     async def stop(self):
@@ -590,6 +614,8 @@ async def flash_prometheus_autoscaler(
     # How often to make autoscaling decisions.
     # Corresponds to --horizontal-pod-autoscaler-sync-period in Kubernetes.
     autoscaling_interval_seconds: int = 15,
+    # Whether to include overprovisioned containers in the scale up calculation.
+    buffer_containers: Optional[int] = None,
 ) -> _FlashPrometheusAutoscaler:
     """
     Autoscale a Flash service based on containers' Prometheus metrics.
@@ -607,19 +633,20 @@ async def flash_prometheus_autoscaler(
 
     client = await _Client.from_env()
     autoscaler = _FlashPrometheusAutoscaler(
-        client,
-        app_name,
-        cls_name,
-        metrics_endpoint,
-        target_metric,
-        target_metric_value,
-        min_containers,
-        max_containers,
-        scale_up_tolerance,
-        scale_down_tolerance,
-        scale_up_stabilization_window_seconds,
-        scale_down_stabilization_window_seconds,
-        autoscaling_interval_seconds,
+        client=client,
+        app_name=app_name,
+        cls_name=cls_name,
+        metrics_endpoint=metrics_endpoint,
+        target_metric=target_metric,
+        target_metric_value=target_metric_value,
+        min_containers=min_containers,
+        max_containers=max_containers,
+        buffer_containers=buffer_containers,
+        scale_up_tolerance=scale_up_tolerance,
+        scale_down_tolerance=scale_down_tolerance,
+        scale_up_stabilization_window_seconds=scale_up_stabilization_window_seconds,
+        scale_down_stabilization_window_seconds=scale_down_stabilization_window_seconds,
+        autoscaling_interval_seconds=autoscaling_interval_seconds,
     )
     await autoscaler.start()
     return autoscaler

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/modal/experimental/flash.pyi

@@ -15,7 +15,9 @@ class _FlashManager:
         """Initialize self.  See help(type(self)) for accurate signature."""
         ...
 
-    async def check_port_connection(self, process: typing.Optional[subprocess.Popen], timeout: int = 10): ...
+    async def is_port_connection_healthy(
+        self, process: typing.Optional[subprocess.Popen], timeout: int = 5
+    ) -> tuple[bool, typing.Optional[Exception]]: ...
     async def _start(self): ...
     async def _drain_container(self):
         """Background task that checks if we've encountered too many failures and drains the container if so."""
@@ -37,11 +39,15 @@ class FlashManager:
         health_check_url: typing.Optional[str] = None,
     ): ...
 
-    class __check_port_connection_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, process: typing.Optional[subprocess.Popen], timeout: int = 10): ...
-        async def aio(self, /, process: typing.Optional[subprocess.Popen], timeout: int = 10): ...
+    class __is_port_connection_healthy_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(
+            self, /, process: typing.Optional[subprocess.Popen], timeout: int = 5
+        ) -> tuple[bool, typing.Optional[Exception]]: ...
+        async def aio(
+            self, /, process: typing.Optional[subprocess.Popen], timeout: int = 5
+        ) -> tuple[bool, typing.Optional[Exception]]: ...
 
-    check_port_connection: __check_port_connection_spec[typing_extensions.Self]
+    is_port_connection_healthy: __is_port_connection_healthy_spec[typing_extensions.Self]
 
     class ___start_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /): ...
@@ -120,6 +126,7 @@ class _FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: typing.Optional[int],
         max_containers: typing.Optional[int],
+        buffer_containers: typing.Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -149,6 +156,7 @@ class _FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 300,
         min_containers: typing.Optional[int] = None,
         max_containers: typing.Optional[int] = None,
+        buffer_containers: typing.Optional[int] = None,
     ) -> int:
         """Return the target number of containers following (simplified) Kubernetes HPA
         stabilization-window semantics.
@@ -181,6 +189,7 @@ class FlashPrometheusAutoscaler:
         target_metric_value: float,
         min_containers: typing.Optional[int],
         max_containers: typing.Optional[int],
+        buffer_containers: typing.Optional[int],
         scale_up_tolerance: float,
         scale_down_tolerance: float,
         scale_up_stabilization_window_seconds: int,
@@ -247,6 +256,7 @@ class FlashPrometheusAutoscaler:
         scale_down_stabilization_window_seconds: int = 300,
         min_containers: typing.Optional[int] = None,
         max_containers: typing.Optional[int] = None,
+        buffer_containers: typing.Optional[int] = None,
     ) -> int:
         """Return the target number of containers following (simplified) Kubernetes HPA
         stabilization-window semantics.
@@ -288,6 +298,7 @@ class __flash_prometheus_autoscaler_spec(typing_extensions.Protocol):
         scale_up_stabilization_window_seconds: int = 0,
         scale_down_stabilization_window_seconds: int = 300,
         autoscaling_interval_seconds: int = 15,
+        buffer_containers: typing.Optional[int] = None,
     ) -> FlashPrometheusAutoscaler:
         """Autoscale a Flash service based on containers' Prometheus metrics.
 
@@ -313,6 +324,7 @@ class __flash_prometheus_autoscaler_spec(typing_extensions.Protocol):
         scale_up_stabilization_window_seconds: int = 0,
         scale_down_stabilization_window_seconds: int = 300,
         autoscaling_interval_seconds: int = 15,
+        buffer_containers: typing.Optional[int] = None,
     ) -> FlashPrometheusAutoscaler:
         """Autoscale a Flash service based on containers' Prometheus metrics.
 

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.1.5.dev2
+Version: 1.1.5.dev4
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.1.5.dev2 → modal-1.1.5.dev4}/modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.1.5.dev2"
+__version__ = "1.1.5.dev4"