modal 1.0.6.dev39__py3-none-any.whl → 1.0.6.dev42__py3-none-any.whl

modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.0.6.dev39",
+        version: str = "1.0.6.dev42",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -163,7 +163,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.0.6.dev39",
+        version: str = "1.0.6.dev42",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/experimental/__init__.py

@@ -1,7 +1,5 @@
 # Copyright Modal Labs 2025
-import asyncio
 import os
-import sys
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, Optional, Union
@@ -13,17 +11,16 @@ from .._functions import _Function
 from .._object import _get_environment_name
 from .._partial_function import _clustered
 from .._runtime.container_io_manager import _ContainerIOManager
-from .._tunnel import _forward as _forward_tunnel
 from .._utils.async_utils import synchronize_api, synchronizer
 from .._utils.deprecation import deprecation_warning
 from .._utils.grpc_utils import retry_transient_errors
 from ..app import _App
 from ..client import _Client
 from ..cls import _Cls, _Obj
-from ..config import logger
 from ..exception import InvalidError
 from ..image import DockerfileSpec, ImageBuilderVersion, _Image, _ImageRegistryConfig
 from ..secret import _Secret
+from .flash import flash_forward, flash_prometheus_autoscaler  # noqa: F401
 
 
 def stop_fetching_inputs():
@@ -259,89 +256,3 @@ async def update_autoscaler(
 
     request = api_pb2.FunctionUpdateSchedulingParamsRequest(function_id=f.object_id, settings=settings)
     await retry_transient_errors(client.stub.FunctionUpdateSchedulingParams, request)
-
-
-class _FlashManager:
-    def __init__(self, client: _Client, port: int):
-        self.client = client
-        self.port = port
-        self.tunnel_manager = _forward_tunnel(port, client=client)
-        self.stopped = False
-
-    async def _start(self):
-        self.tunnel = await self.tunnel_manager.__aenter__()
-
-        hostname = self.tunnel.url.split("://")[1]
-        if ":" in hostname:
-            host, port = hostname.split(":")
-        else:
-            host = hostname
-            port = "443"
-
-        self.heartbeat_task = asyncio.create_task(self._run_heartbeat(host, int(port)))
-
-    async def _run_heartbeat(self, host: str, port: int):
-        first_registration = True
-        while True:
-            try:
-                resp = await self.client.stub.FlashContainerRegister(
-                    api_pb2.FlashContainerRegisterRequest(
-                        priority=10,
-                        weight=5,
-                        host=host,
-                        port=port,
-                    ),
-                    timeout=10,
-                )
-                if first_registration:
-                    logger.warning(f"[Modal Flash] Listening at {resp.url}")
-                    first_registration = False
-            except asyncio.CancelledError:
-                logger.warning("[Modal Flash] Shutting down...")
-                break
-            except Exception as e:
-                logger.error(f"[Modal Flash] Heartbeat failed: {e}")
-
-            try:
-                await asyncio.sleep(1)
-            except asyncio.CancelledError:
-                logger.warning("[Modal Flash] Shutting down...")
-                break
-
-    async def stop(self):
-        self.heartbeat_task.cancel()
-        await retry_transient_errors(
-            self.client.stub.FlashContainerDeregister,
-            api_pb2.FlashContainerDeregisterRequest(),
-        )
-
-        self.stopped = True
-        logger.warning(f"[Modal Flash] No longer accepting new requests on {self.tunnel.url}.")
-
-        # NOTE(gongy): We skip calling TunnelStop to avoid interrupting in-flight requests.
-        # It is up to the user to wait after calling .stop() to drain in-flight requests.
-
-    async def close(self):
-        if not self.stopped:
-            await self.stop()
-
-        logger.warning(f"[Modal Flash] Closing tunnel on {self.tunnel.url}.")
-        await self.tunnel_manager.__aexit__(*sys.exc_info())
-
-
-FlashManager = synchronize_api(_FlashManager)
-
-
-@synchronizer.create_blocking
-async def flash_forward(port: int) -> _FlashManager:
-    """
-    Forward a port to the Modal Flash service, exposing that port as a stable web endpoint.
-
-    This is a highly experimental method that can break or be removed at any time without warning.
-    Do not use this method unless explicitly instructed to do so by Modal support.
-    """
-    client = await _Client.from_env()
-
-    manager = _FlashManager(client, port)
-    await manager._start()
-    return manager

modal/experimental/flash.py

@@ -0,0 +1,449 @@
+# Copyright Modal Labs 2025
+import asyncio
+import math
+import sys
+import time
+import traceback
+from collections import defaultdict
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+from modal.cls import _Cls
+from modal.dict import _Dict
+from modal_proto import api_pb2
+
+from .._tunnel import _forward as _forward_tunnel
+from .._utils.async_utils import synchronize_api, synchronizer
+from .._utils.grpc_utils import retry_transient_errors
+from ..client import _Client
+from ..config import logger
+from ..exception import InvalidError
+
+
+class _FlashManager:
+    def __init__(self, client: _Client, port: int, health_check_url: Optional[str] = None):
+        self.client = client
+        self.port = port
+        self.health_check_url = health_check_url
+        self.tunnel_manager = _forward_tunnel(port, client=client)
+        self.stopped = False
+
+    async def _start(self):
+        self.tunnel = await self.tunnel_manager.__aenter__()
+
+        parsed_url = urlparse(self.tunnel.url)
+        host = parsed_url.hostname
+        port = parsed_url.port or 443
+
+        self.heartbeat_task = asyncio.create_task(self._run_heartbeat(host, port))
+
+    async def _run_heartbeat(self, host: str, port: int):
+        first_registration = True
+        while True:
+            try:
+                resp = await self.client.stub.FlashContainerRegister(
+                    api_pb2.FlashContainerRegisterRequest(
+                        priority=10,
+                        weight=5,
+                        host=host,
+                        port=port,
+                    ),
+                    timeout=10,
+                )
+                if first_registration:
+                    logger.warning(f"[Modal Flash] Listening at {resp.url}")
+                    first_registration = False
+            except asyncio.CancelledError:
+                logger.warning("[Modal Flash] Shutting down...")
+                break
+            except Exception as e:
+                logger.error(f"[Modal Flash] Heartbeat failed: {e}")
+
+            try:
+                await asyncio.sleep(1)
+            except asyncio.CancelledError:
+                logger.warning("[Modal Flash] Shutting down...")
+                break
+
+    def get_container_url(self):
+        # WARNING: Try not to use this method; we aren't sure if we will keep it.
+        return self.tunnel.url
+
+    async def stop(self):
+        self.heartbeat_task.cancel()
+        await retry_transient_errors(
+            self.client.stub.FlashContainerDeregister,
+            api_pb2.FlashContainerDeregisterRequest(),
+        )
+
+        self.stopped = True
+        logger.warning(f"[Modal Flash] No longer accepting new requests on {self.tunnel.url}.")
+
+        # NOTE(gongy): We skip calling TunnelStop to avoid interrupting in-flight requests.
+        # It is up to the user to wait after calling .stop() to drain in-flight requests.
+
+    async def close(self):
+        if not self.stopped:
+            await self.stop()
+
+        logger.warning(f"[Modal Flash] Closing tunnel on {self.tunnel.url}.")
+        await self.tunnel_manager.__aexit__(*sys.exc_info())
+
+
+FlashManager = synchronize_api(_FlashManager)
+
+
+@synchronizer.create_blocking
+async def flash_forward(port: int, health_check_url: Optional[str] = None) -> _FlashManager:
+    """
+    Forward a port to the Modal Flash service, exposing that port as a stable web endpoint.
+
+    This is a highly experimental method that can break or be removed at any time without warning.
+    Do not use this method unless explicitly instructed to do so by Modal support.
+    """
+    client = await _Client.from_env()
+
+    manager = _FlashManager(client, port, health_check_url)
+    await manager._start()
+    return manager
+
+
+class _FlashPrometheusAutoscaler:
+    _max_window_seconds = 60 * 60
+
+    def __init__(
+        self,
+        client: _Client,
+        app_name: str,
+        cls_name: str,
+        metrics_endpoint: str,
+        target_metric: str,
+        target_metric_value: float,
+        min_containers: Optional[int],
+        max_containers: Optional[int],
+        scale_up_tolerance: float,
+        scale_down_tolerance: float,
+        scale_up_stabilization_window_seconds: int,
+        scale_down_stabilization_window_seconds: int,
+        autoscaling_interval_seconds: int,
+    ):
+        if scale_up_stabilization_window_seconds > self._max_window_seconds:
+            raise InvalidError(
+                f"scale_up_stabilization_window_seconds must be less than or equal to {self._max_window_seconds}"
+            )
+        if scale_down_stabilization_window_seconds > self._max_window_seconds:
+            raise InvalidError(
+                f"scale_down_stabilization_window_seconds must be less than or equal to {self._max_window_seconds}"
+            )
+        if target_metric_value <= 0:
+            raise InvalidError("target_metric_value must be greater than 0")
+
+        import aiohttp
+
+        self.client = client
+        self.app_name = app_name
+        self.cls_name = cls_name
+        self.metrics_endpoint = metrics_endpoint
+        self.target_metric = target_metric
+        self.target_metric_value = target_metric_value
+        self.min_containers = min_containers
+        self.max_containers = max_containers
+        self.scale_up_tolerance = scale_up_tolerance
+        self.scale_down_tolerance = scale_down_tolerance
+        self.scale_up_stabilization_window_seconds = scale_up_stabilization_window_seconds
+        self.scale_down_stabilization_window_seconds = scale_down_stabilization_window_seconds
+        self.autoscaling_interval_seconds = autoscaling_interval_seconds
+
+        FlashClass = _Cls.from_name(app_name, cls_name)
+        self.fn = FlashClass._class_service_function
+        self.cls = FlashClass()
+
+        self.http_client = aiohttp.ClientSession()
+        self.autoscaling_decisions_dict = _Dict.from_name(
+            f"{app_name}-{cls_name}-autoscaling-decisions",
+            create_if_missing=True,
+        )
+
+        self.autoscaler_thread = None
+
+    async def start(self):
+        await self.fn.hydrate(client=self.client)
+        self.autoscaler_thread = asyncio.create_task(self._run_autoscaler_loop())
+
+    async def _run_autoscaler_loop(self):
+        while True:
+            try:
+                autoscaling_time = time.time()
+
+                current_replicas = await self.autoscaling_decisions_dict.get("current_replicas", 0)
+                autoscaling_decisions = await self.autoscaling_decisions_dict.get("autoscaling_decisions", [])
+                if not isinstance(current_replicas, int):
+                    logger.warning(f"[Modal Flash] Invalid item in autoscaling decisions: {current_replicas}")
+                    current_replicas = 0
+                if not isinstance(autoscaling_decisions, list):
+                    logger.warning(f"[Modal Flash] Invalid item in autoscaling decisions: {autoscaling_decisions}")
+                    autoscaling_decisions = []
+                for item in autoscaling_decisions:
+                    if (
+                        not isinstance(item, tuple)
+                        or len(item) != 2
+                        or not isinstance(item[0], float)
+                        or not isinstance(item[1], int)
+                    ):
+                        logger.warning(f"[Modal Flash] Invalid item in autoscaling decisions: {item}")
+                        autoscaling_decisions = []
+                        break
+
+                autoscaling_decisions = [
+                    (timestamp, decision)
+                    for timestamp, decision in autoscaling_decisions
+                    if timestamp >= autoscaling_time - self._max_window_seconds
+                ]
+
+                current_target_containers = await self._compute_target_containers(current_replicas)
+                autoscaling_decisions.append((autoscaling_time, current_target_containers))
+
+                actual_target_containers = self._make_scaling_decision(
+                    current_replicas,
+                    autoscaling_decisions,
+                    scale_up_stabilization_window_seconds=self.scale_up_stabilization_window_seconds,
+                    scale_down_stabilization_window_seconds=self.scale_down_stabilization_window_seconds,
+                    min_containers=self.min_containers,
+                    max_containers=self.max_containers,
+                )
+
+                logger.warning(
+                    f"[Modal Flash] Scaling to {actual_target_containers} containers. Autoscaling decision "
+                    f"made in {time.time() - autoscaling_time} seconds."
+                )
+
+                await self.autoscaling_decisions_dict.put(
+                    "autoscaling_decisions",
+                    autoscaling_decisions,
+                )
+                await self.autoscaling_decisions_dict.put("current_replicas", actual_target_containers)
+
+                await self.cls.update_autoscaler(
+                    min_containers=actual_target_containers,
+                    max_containers=actual_target_containers,
+                )
+
+                if time.time() - autoscaling_time < self.autoscaling_interval_seconds:
+                    await asyncio.sleep(self.autoscaling_interval_seconds - (time.time() - autoscaling_time))
+            except asyncio.CancelledError:
+                logger.warning("[Modal Flash] Shutting down autoscaler...")
+                await self.http_client.close()
+                break
+            except Exception as e:
+                logger.error(f"[Modal Flash] Error in autoscaler: {e}")
+                logger.error(traceback.format_exc())
+                await asyncio.sleep(self.autoscaling_interval_seconds)
+
+    async def _compute_target_containers(self, current_replicas: int) -> int:
+        containers = await self._get_all_containers()
+        if len(containers) > current_replicas:
+            logger.info(
+                f"[Modal Flash] Current replicas {current_replicas} is less than the number of containers "
+                f"{len(containers)}. Setting current_replicas = num_containers."
+            )
+            current_replicas = len(containers)
+
+        if current_replicas == 0:
+            return 1
+
+        target_metric = self.target_metric
+        target_metric_value = float(self.target_metric_value)
+
+        sum_metric = 0
+        containers_with_metrics = 0
+        container_metrics_list = await asyncio.gather(
+            *[
+                self._get_metrics(f"https://{container.host}:{container.port}/{self.metrics_endpoint}")
+                for container in containers
+            ]
+        )
+        for container_metrics in container_metrics_list:
+            if (
+                container_metrics is None
+                or target_metric not in container_metrics
+                or len(container_metrics[target_metric]) == 0
+            ):
+                continue
+            sum_metric += container_metrics[target_metric][0].value
+            containers_with_metrics += 1
+
+        n_containers_missing_metric = current_replicas - containers_with_metrics
+
+        # Scale up / down conservatively: Any container that is missing the metric is assumed to be at the minimum
+        # value of the metric when scaling up and the maximum value of the metric when scaling down.
+        scale_up_target_metric_value = sum_metric / current_replicas
+        scale_down_target_metric_value = (
+            sum_metric + n_containers_missing_metric * target_metric_value
+        ) / current_replicas
+
+        scale_up_ratio = scale_up_target_metric_value / target_metric_value
+        scale_down_ratio = scale_down_target_metric_value / target_metric_value
+
+        desired_replicas = current_replicas
+        if scale_up_ratio > 1 + self.scale_up_tolerance:
+            desired_replicas = math.ceil(current_replicas * scale_up_ratio)
+        elif scale_down_ratio < 1 - self.scale_down_tolerance:
+            desired_replicas = math.ceil(current_replicas * scale_down_ratio)
+
+        logger.warning(
+            f"[Modal Flash] Current replicas: {current_replicas}, target metric value: {target_metric_value}, "
+            f"current sum of metric values: {sum_metric}, number of containers missing metric: "
+            f"{n_containers_missing_metric}, scale up ratio: {scale_up_ratio}, scale down ratio: {scale_down_ratio}, "
+            f"desired replicas: {desired_replicas}"
+        )
+
+        return desired_replicas
+
+    async def _get_metrics(self, url: str) -> Optional[dict[str, list[Any]]]:  # technically any should be Sample
+        from prometheus_client.parser import Sample, text_string_to_metric_families
+
+        # Fetch the metrics from the endpoint
+        try:
+            response = await self.http_client.get(url)
+            response.raise_for_status()
+        except Exception as e:
+            logger.warning(f"[Modal Flash] Error getting metrics from {url}: {e}")
+            return None
+
+        # Parse the text-based Prometheus metrics format
+        metrics: dict[str, list[Sample]] = defaultdict(list)
+        for family in text_string_to_metric_families(await response.text()):
+            for sample in family.samples:
+                metrics[sample.name] += [sample]
+
+        return metrics
+
+    async def _get_all_containers(self):
+        req = api_pb2.FlashContainerListRequest(function_id=self.fn.object_id)
+        resp = await retry_transient_errors(self.client.stub.FlashContainerList, req)
+        return resp.containers
+
+    def _make_scaling_decision(
+        self,
+        current_replicas: int,
+        autoscaling_decisions: list[tuple[float, int]],
+        scale_up_stabilization_window_seconds: int = 0,
+        scale_down_stabilization_window_seconds: int = 60 * 5,
+        min_containers: Optional[int] = None,
+        max_containers: Optional[int] = None,
+    ) -> int:
+        """
+        Return the target number of containers following (simplified) Kubernetes HPA
+        stabilization-window semantics.
+
+        Args:
+            current_replicas: Current number of running Pods/containers.
+            autoscaling_decisions: List of (timestamp, desired_replicas) pairs, where
+                                   timestamp is a UNIX epoch float (seconds).
+                                   The list *must* contain at least one entry and should
+                                   already include the most-recent measurement.
+            scale_up_stabilization_window_seconds: 0 disables the up-window.
+            scale_down_stabilization_window_seconds: 0 disables the down-window.
+            min_containers / max_containers: Clamp the final decision to this range.
+
+        Returns:
+            The target number of containers.
+        """
+        if not autoscaling_decisions:
+            # Without data we can’t make a new decision – stay where we are.
+            return current_replicas
+
+        # Sort just once in case the caller didn’t: newest record is last.
+        autoscaling_decisions.sort(key=lambda rec: rec[0])
+        now_ts, latest_desired = autoscaling_decisions[-1]
+
+        if latest_desired > current_replicas:
+            # ---- SCALE-UP path ----
+            window_start = now_ts - scale_up_stabilization_window_seconds
+            # Consider only records *inside* the window.
+            desired_candidates = [desired for ts, desired in autoscaling_decisions if ts >= window_start]
+            # Use the *minimum* so that any temporary dip blocks the scale-up.
+            candidate = min(desired_candidates) if desired_candidates else latest_desired
+            new_replicas = max(current_replicas, candidate)  # never scale *down* here
+        elif latest_desired < current_replicas:
+            # ---- SCALE-DOWN path ----
+            window_start = now_ts - scale_down_stabilization_window_seconds
+            desired_candidates = [desired for ts, desired in autoscaling_decisions if ts >= window_start]
+            # Use the *maximum* so that any temporary spike blocks the scale-down.
+            candidate = max(desired_candidates) if desired_candidates else latest_desired
+            new_replicas = min(current_replicas, candidate)  # never scale *up* here
+        else:
+            # No change requested.
+            new_replicas = current_replicas
+
+        # Clamp to [min_containers, max_containers].
+        if min_containers is not None:
+            new_replicas = max(min_containers, new_replicas)
+        if max_containers is not None:
+            new_replicas = min(max_containers, new_replicas)
+        return new_replicas
+
+    async def stop(self):
+        self.autoscaler_thread.cancel()
+        await self.autoscaler_thread
+
+
+FlashPrometheusAutoscaler = synchronize_api(_FlashPrometheusAutoscaler)
+
+
+@synchronizer.create_blocking
+async def flash_prometheus_autoscaler(
+    app_name: str,
+    cls_name: str,
+    # Endpoint to fetch metrics from. Must be in Prometheus format. Example: "/metrics"
+    metrics_endpoint: str,
+    # Target metric to autoscale on. Example: "vllm:num_requests_running"
+    target_metric: str,
+    # Target metric value. Example: 25
+    target_metric_value: float,
+    min_containers: Optional[int] = None,
+    max_containers: Optional[int] = None,
+    # Corresponds to https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#tolerance
+    scale_up_tolerance: float = 0.1,
+    # Corresponds to https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#tolerance
+    scale_down_tolerance: float = 0.1,
+    # Corresponds to https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#stabilization-window
+    scale_up_stabilization_window_seconds: int = 0,
+    # Corresponds to https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#stabilization-window
+    scale_down_stabilization_window_seconds: int = 300,
+    # How often to make autoscaling decisions.
+    # Corresponds to --horizontal-pod-autoscaler-sync-period in Kubernetes.
+    autoscaling_interval_seconds: int = 15,
+) -> _FlashPrometheusAutoscaler:
+    """
+    Autoscale a Flash service based on containers' Prometheus metrics.
+
+    The package `prometheus_client` is required to use this method.
+
+    This is a highly experimental method that can break or be removed at any time without warning.
+    Do not use this method unless explicitly instructed to do so by Modal support.
+    """
+
+    try:
+        import prometheus_client  # noqa: F401
+    except ImportError:
+        raise ImportError("The package `prometheus_client` is required to use this method.")
+
+    client = await _Client.from_env()
+    autoscaler = _FlashPrometheusAutoscaler(
+        client,
+        app_name,
+        cls_name,
+        metrics_endpoint,
+        target_metric,
+        target_metric_value,
+        min_containers,
+        max_containers,
+        scale_up_tolerance,
+        scale_down_tolerance,
+        scale_up_stabilization_window_seconds,
+        scale_down_stabilization_window_seconds,
+        autoscaling_interval_seconds,
+    )
+    await autoscaler.start()
+    return autoscaler

modal/experimental/flash.pyi

@@ -0,0 +1,252 @@
+import modal.client
+import typing
+import typing_extensions
+
+class _FlashManager:
+    def __init__(self, client: modal.client._Client, port: int, health_check_url: typing.Optional[str] = None):
+        """Initialize self.  See help(type(self)) for accurate signature."""
+        ...
+
+    async def _start(self): ...
+    async def _run_heartbeat(self, host: str, port: int): ...
+    def get_container_url(self): ...
+    async def stop(self): ...
+    async def close(self): ...
+
+SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
+
+class FlashManager:
+    def __init__(self, client: modal.client.Client, port: int, health_check_url: typing.Optional[str] = None): ...
+
+    class ___start_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    _start: ___start_spec[typing_extensions.Self]
+
+    class ___run_heartbeat_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /, host: str, port: int): ...
+        async def aio(self, /, host: str, port: int): ...
+
+    _run_heartbeat: ___run_heartbeat_spec[typing_extensions.Self]
+
+    def get_container_url(self): ...
+
+    class __stop_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    stop: __stop_spec[typing_extensions.Self]
+
+    class __close_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    close: __close_spec[typing_extensions.Self]
+
+class __flash_forward_spec(typing_extensions.Protocol):
+    def __call__(self, /, port: int, health_check_url: typing.Optional[str] = None) -> FlashManager:
+        """Forward a port to the Modal Flash service, exposing that port as a stable web endpoint.
+
+        This is a highly experimental method that can break or be removed at any time without warning.
+        Do not use this method unless explicitly instructed to do so by Modal support.
+        """
+        ...
+
+    async def aio(self, /, port: int, health_check_url: typing.Optional[str] = None) -> FlashManager:
+        """Forward a port to the Modal Flash service, exposing that port as a stable web endpoint.
+
+        This is a highly experimental method that can break or be removed at any time without warning.
+        Do not use this method unless explicitly instructed to do so by Modal support.
+        """
+        ...
+
+flash_forward: __flash_forward_spec
+
+class _FlashPrometheusAutoscaler:
+    def __init__(
+        self,
+        client: modal.client._Client,
+        app_name: str,
+        cls_name: str,
+        metrics_endpoint: str,
+        target_metric: str,
+        target_metric_value: float,
+        min_containers: typing.Optional[int],
+        max_containers: typing.Optional[int],
+        scale_up_tolerance: float,
+        scale_down_tolerance: float,
+        scale_up_stabilization_window_seconds: int,
+        scale_down_stabilization_window_seconds: int,
+        autoscaling_interval_seconds: int,
+    ):
+        """Initialize self.  See help(type(self)) for accurate signature."""
+        ...
+
+    async def start(self): ...
+    async def _run_autoscaler_loop(self): ...
+    async def _compute_target_containers(self, current_replicas: int) -> int: ...
+    async def _get_metrics(self, url: str) -> typing.Optional[dict[str, list[typing.Any]]]: ...
+    async def _get_all_containers(self): ...
+    def _make_scaling_decision(
+        self,
+        current_replicas: int,
+        autoscaling_decisions: list[tuple[float, int]],
+        scale_up_stabilization_window_seconds: int = 0,
+        scale_down_stabilization_window_seconds: int = 300,
+        min_containers: typing.Optional[int] = None,
+        max_containers: typing.Optional[int] = None,
+    ) -> int:
+        """Return the target number of containers following (simplified) Kubernetes HPA
+        stabilization-window semantics.
+
+        Args:
+            current_replicas: Current number of running Pods/containers.
+            autoscaling_decisions: List of (timestamp, desired_replicas) pairs, where
+                                   timestamp is a UNIX epoch float (seconds).
+                                   The list *must* contain at least one entry and should
+                                   already include the most-recent measurement.
+            scale_up_stabilization_window_seconds: 0 disables the up-window.
+            scale_down_stabilization_window_seconds: 0 disables the down-window.
+            min_containers / max_containers: Clamp the final decision to this range.
+
+        Returns:
+            The target number of containers.
+        """
+        ...
+
+    async def stop(self): ...
+
+class FlashPrometheusAutoscaler:
+    def __init__(
+        self,
+        client: modal.client.Client,
+        app_name: str,
+        cls_name: str,
+        metrics_endpoint: str,
+        target_metric: str,
+        target_metric_value: float,
+        min_containers: typing.Optional[int],
+        max_containers: typing.Optional[int],
+        scale_up_tolerance: float,
+        scale_down_tolerance: float,
+        scale_up_stabilization_window_seconds: int,
+        scale_down_stabilization_window_seconds: int,
+        autoscaling_interval_seconds: int,
+    ): ...
+
+    class __start_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    start: __start_spec[typing_extensions.Self]
+
+    class ___run_autoscaler_loop_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    _run_autoscaler_loop: ___run_autoscaler_loop_spec[typing_extensions.Self]
+
+    class ___compute_target_containers_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /, current_replicas: int) -> int: ...
+        async def aio(self, /, current_replicas: int) -> int: ...
+
+    _compute_target_containers: ___compute_target_containers_spec[typing_extensions.Self]
+
+    class ___get_metrics_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /, url: str) -> typing.Optional[dict[str, list[typing.Any]]]: ...
+        async def aio(self, /, url: str) -> typing.Optional[dict[str, list[typing.Any]]]: ...
+
+    _get_metrics: ___get_metrics_spec[typing_extensions.Self]
+
+    class ___get_all_containers_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    _get_all_containers: ___get_all_containers_spec[typing_extensions.Self]
+
+    def _make_scaling_decision(
+        self,
+        current_replicas: int,
+        autoscaling_decisions: list[tuple[float, int]],
+        scale_up_stabilization_window_seconds: int = 0,
+        scale_down_stabilization_window_seconds: int = 300,
+        min_containers: typing.Optional[int] = None,
+        max_containers: typing.Optional[int] = None,
+    ) -> int:
+        """Return the target number of containers following (simplified) Kubernetes HPA
+        stabilization-window semantics.
+
+        Args:
+            current_replicas: Current number of running Pods/containers.
+            autoscaling_decisions: List of (timestamp, desired_replicas) pairs, where
+                                   timestamp is a UNIX epoch float (seconds).
+                                   The list *must* contain at least one entry and should
+                                   already include the most-recent measurement.
+            scale_up_stabilization_window_seconds: 0 disables the up-window.
+            scale_down_stabilization_window_seconds: 0 disables the down-window.
+            min_containers / max_containers: Clamp the final decision to this range.
+
+        Returns:
+            The target number of containers.
+        """
+        ...
+
+    class __stop_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /): ...
+        async def aio(self, /): ...
+
+    stop: __stop_spec[typing_extensions.Self]
+
+class __flash_prometheus_autoscaler_spec(typing_extensions.Protocol):
+    def __call__(
+        self,
+        /,
+        app_name: str,
+        cls_name: str,
+        metrics_endpoint: str,
+        target_metric: str,
+        target_metric_value: float,
+        min_containers: typing.Optional[int] = None,
+        max_containers: typing.Optional[int] = None,
+        scale_up_tolerance: float = 0.1,
+        scale_down_tolerance: float = 0.1,
+        scale_up_stabilization_window_seconds: int = 0,
+        scale_down_stabilization_window_seconds: int = 300,
+        autoscaling_interval_seconds: int = 15,
+    ) -> FlashPrometheusAutoscaler:
+        """Autoscale a Flash service based on containers' Prometheus metrics.
+
+        The package `prometheus_client` is required to use this method.
+
+        This is a highly experimental method that can break or be removed at any time without warning.
+        Do not use this method unless explicitly instructed to do so by Modal support.
+        """
+        ...
+
+    async def aio(
+        self,
+        /,
+        app_name: str,
+        cls_name: str,
+        metrics_endpoint: str,
+        target_metric: str,
+        target_metric_value: float,
+        min_containers: typing.Optional[int] = None,
+        max_containers: typing.Optional[int] = None,
+        scale_up_tolerance: float = 0.1,
+        scale_down_tolerance: float = 0.1,
+        scale_up_stabilization_window_seconds: int = 0,
+        scale_down_stabilization_window_seconds: int = 300,
+        autoscaling_interval_seconds: int = 15,
+    ) -> FlashPrometheusAutoscaler:
+        """Autoscale a Flash service based on containers' Prometheus metrics.
+
+        The package `prometheus_client` is required to use this method.
+
+        This is a highly experimental method that can break or be removed at any time without warning.
+        Do not use this method unless explicitly instructed to do so by Modal support.
+        """
+        ...
+
+flash_prometheus_autoscaler: __flash_prometheus_autoscaler_spec

modal/functions.pyi

@@ -428,7 +428,7 @@ class Function(
 
     _call_generator: ___call_generator_spec[typing_extensions.Self]
 
-    class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER:
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
@@ -437,7 +437,7 @@ class Function(
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
 
-    remote: __remote_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    remote: __remote_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     class __remote_gen_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, *args, **kwargs) -> typing.Generator[typing.Any, None, None]:
@@ -464,7 +464,7 @@ class Function(
         """
         ...
 
-    class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """[Experimental] Calls the function with the given arguments, without waiting for the results.
 
@@ -488,7 +488,7 @@ class Function(
             ...
 
     _experimental_spawn: ___experimental_spawn_spec[
-        modal._functions.ReturnType, modal._functions.P, typing_extensions.Self
+        modal._functions.P, modal._functions.ReturnType, typing_extensions.Self
     ]
 
     class ___spawn_map_inner_spec(typing_extensions.Protocol[P_INNER, SUPERSELF]):
@@ -497,7 +497,7 @@ class Function(
 
     _spawn_map_inner: ___spawn_map_inner_spec[modal._functions.P, typing_extensions.Self]
 
-    class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
+    class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """Calls the function with the given arguments, without waiting for the results.
 
@@ -518,7 +518,7 @@ class Function(
             """
             ...
 
-    spawn: __spawn_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
+    spawn: __spawn_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
 
     def get_raw_f(self) -> collections.abc.Callable[..., typing.Any]:
         """Return the inner Python object wrapped by this Modal Function."""

modal/image.py

@@ -23,6 +23,7 @@ from typing import (
 
 from google.protobuf.message import Message
 from grpclib.exceptions import GRPCError, StreamTerminatedError
+from typing_extensions import Self
 
 from modal_proto import api_pb2
 
@@ -42,7 +43,7 @@ from .client import _Client
 from .cloud_bucket_mount import _CloudBucketMount
 from .config import config, logger, user_config_path
 from .environments import _get_environment_cached
-from .exception import InvalidError, NotFoundError, RemoteError, VersionError
+from .exception import ExecutionError, InvalidError, NotFoundError, RemoteError, VersionError
 from .file_pattern_matcher import NON_PYTHON_FILES, FilePatternMatcher, _ignore_fn
 from .gpu import GPU_T, parse_gpu_config
 from .mount import _Mount, python_standalone_mount_name
@@ -2295,5 +2296,15 @@ class _Image(_Object, type_prefix="im"):
                 if task_log.data:
                     yield task_log.data
 
+    async def hydrate(self, client: Optional[_Client] = None) -> Self:
+        """mdmd:hidden"""
+        # Image inherits hydrate() from Object but can't be hydrated on demand
+        # Overriding the method lets us hide it from the docs and raise a better error message
+        if not self.is_hydrated:
+            raise ExecutionError(
+                "Images cannot currently be hydrated on demand; you can build an Image by running an App that uses it."
+            )
+        return self
+
 
 Image = synchronize_api(_Image)

modal/image.pyi

@@ -940,6 +940,10 @@ class _Image(modal._object._Object):
         """
         ...
 
+    async def hydrate(self, client: typing.Optional[modal.client._Client] = None) -> typing_extensions.Self:
+        """mdmd:hidden"""
+        ...
+
 SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
 
 class Image(modal.object.Object):
@@ -1781,4 +1785,15 @@ class Image(modal.object.Object):
 
     _logs: ___logs_spec[typing_extensions.Self]
 
+    class __hydrate_spec(typing_extensions.Protocol[SUPERSELF]):
+        def __call__(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF:
+            """mdmd:hidden"""
+            ...
+
+        async def aio(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF:
+            """mdmd:hidden"""
+            ...
+
+    hydrate: __hydrate_spec[typing_extensions.Self]
+
 SUPPORTED_PYTHON_SERIES: dict[typing.Literal["2023.12", "2024.04", "2024.10", "2025.06", "PREVIEW"], list[str]]

{modal-1.0.6.dev39.dist-info → modal-1.0.6.dev42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.0.6.dev39
+Version: 1.0.6.dev42
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.0.6.dev39.dist-info → modal-1.0.6.dev42.dist-info}/RECORD

@@ -22,7 +22,7 @@ modal/app.py,sha256=U0sPiHpphcRHLnoLYh2IrU2RSpRFX9BE5uHb7h42STs,47478
 modal/app.pyi,sha256=cXiSTu2bwu6csAUdkOlh7mr9tPvtaS2qWSEhlC1UxAg,43787
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=5QyM7VJjsFbHf6E91ar3A2KY9mx03wdtGlNJvfTKUVs,17087
-modal/client.pyi,sha256=COGGfsMoG-WzeMliY9b683xB8h_t74zyWbrTEfs3AUI,15270
+modal/client.pyi,sha256=1npkSvHzSBChsHg2sIUks5RbJ4fAQg3Uw4ZnScGzjDg,15270
 modal/cloud_bucket_mount.py,sha256=YOe9nnvSr4ZbeCn587d7_VhE9IioZYRvF9VYQTQux08,5914
 modal/cloud_bucket_mount.pyi,sha256=-qSfYAQvIoO_l2wsCCGTG5ZUwQieNKXdAO00yP1-LYU,7394
 modal/cls.py,sha256=B5EtzpBXemH718YvgXaYjuTKvairvqfXJ7IwLZ_6vVA,40034
@@ -39,10 +39,10 @@ modal/file_io.py,sha256=BVqAJ0sgPUfN8QsYztWiGB4j56he60TncM02KsylnCw,21449
 modal/file_io.pyi,sha256=cPT_hsplE5iLCXhYOLn1Sp9eDdk7DxdFmicQHanJZyg,15918
 modal/file_pattern_matcher.py,sha256=urAue8es8jxqX94k9EYoZxxhtfgOlsEES8lbFHOorzc,7734
 modal/functions.py,sha256=kcNHvqeGBxPI7Cgd57NIBBghkfbeFJzXO44WW0jSmao,325
-modal/functions.pyi,sha256=FJe_91dSrMCRNVT-YV1UhtxFKzIvL_C5q8xdk08-wT8,34840
+modal/functions.pyi,sha256=ffW_kkU8AxMuV77ltmjK3nslXW_2iwEjKsT-Cgd4Trs,34840
 modal/gpu.py,sha256=Fe5ORvVPDIstSq1xjmM6OoNgLYFWvogP9r5BgmD3hYg,6769
-modal/image.py,sha256=dqiVVxKTh77pM5I9M7ty5nFSBtd3JQME0_RAROIBJuw,101717
-modal/image.pyi,sha256=oJYcnVAY-hSLWlA_B4pFNDh3b6ss4wqGuHj9puPPnV4,67552
+modal/image.py,sha256=qTJ6pTcLfYRh112wId7CCNWWmm077w6JoIqxE8BiCoo,102261
+modal/image.pyi,sha256=TVy-rnSAP2WgQ5zf_sQLFzb-99Qg9LiQNGXR9psFA_o,68107
 modal/io_streams.py,sha256=FUDpBsVK8isqwyC7DtAcQZhaHlMFSaNZGhYJOg-SFW0,15590
 modal/io_streams.pyi,sha256=5b3b93ztZeR8IpJtNIGffX24QLPgocE4-gAps8y7CKU,13824
 modal/mount.py,sha256=q-pPeVxAmte-G_LDpbFwaNs2Rb2MIpscfnCXzkhxrOI,36734
@@ -139,7 +139,9 @@ modal/cli/volume.py,sha256=KJ4WKQYjRGsTERkwHE1HcRia9rWzLIDDnlc89QmTLvE,10960
 modal/cli/programs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal/cli/programs/run_jupyter.py,sha256=44Lpvqk2l3hH-uOkmAOzw60NEsfB5uaRDWDKVshvQhs,2682
 modal/cli/programs/vscode.py,sha256=KbTAaIXyQBVCDXxXjmBHmKpgXkUw0q4R4KkJvUjCYgk,3380
-modal/experimental/__init__.py,sha256=IT7Tq7nBR59_v5YjKoA5U0KXvMbv_kl5Cgw1jsnm-No,13836
+modal/experimental/__init__.py,sha256=XhRr0QQds4fEAoILQFa0CQWUtQ76Gxioo88CupOxDvI,10847
+modal/experimental/flash.py,sha256=xqfJLpdqaa7mgW8OgAFBfZV95PKzuTzevh7SOSVesnA,19055
+modal/experimental/flash.pyi,sha256=1Nd31nYD8Eqi0BI63XuK6owXPGA0s9CgU_WOAzCQSQs,9957
 modal/experimental/ipython.py,sha256=TrCfmol9LGsRZMeDoeMPx3Hv3BFqQhYnmD_iH0pqdhk,2904
 modal/requirements/2023.12.312.txt,sha256=zWWUVgVQ92GXBKNYYr2-5vn9rlnXcmkqlwlX5u1eTYw,400
 modal/requirements/2023.12.txt,sha256=OjsbXFkCSdkzzryZP82Q73osr5wxQ6EUzmGcK7twfkA,502
@@ -149,7 +151,7 @@ modal/requirements/2025.06.txt,sha256=KxDaVTOwatHvboDo4lorlgJ7-n-MfAwbPwxJ0zcJqr
 modal/requirements/PREVIEW.txt,sha256=KxDaVTOwatHvboDo4lorlgJ7-n-MfAwbPwxJ0zcJqrs,312
 modal/requirements/README.md,sha256=9tK76KP0Uph7O0M5oUgsSwEZDj5y-dcUPsnpR0Sc-Ik,854
 modal/requirements/base-images.json,sha256=JYSDAgHTl-WrV_TZW5icY-IJEnbe2eQ4CZ_KN6EOZKU,1304
-modal-1.0.6.dev39.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-1.0.6.dev42.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
 modal_docs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal_docs/gen_cli_docs.py,sha256=c1yfBS_x--gL5bs0N4ihMwqwX8l3IBWSkBAKNNIi6bQ,3801
 modal_docs/gen_reference_docs.py,sha256=d_CQUGQ0rfw28u75I2mov9AlS773z9rG40-yq5o7g2U,6359
@@ -172,10 +174,10 @@ modal_proto/options_pb2.pyi,sha256=l7DBrbLO7q3Ir-XDkWsajm0d0TQqqrfuX54i4BMpdQg,1
 modal_proto/options_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0yJSI,247
 modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-modal_version/__init__.py,sha256=WRzzO6MqiJi9zlkIt0yrWHuE0OutPWptBfyQDvvYlNA,121
+modal_version/__init__.py,sha256=6aQs8eQpP-g3lZcjz4g62zHviQ50MB--advXf8eOryg,121
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal-1.0.6.dev39.dist-info/METADATA,sha256=iJ9m5KNPXJmvuj-le8NAR0nS6s1CikbQnynAvi4LyPg,2462
-modal-1.0.6.dev39.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-modal-1.0.6.dev39.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-1.0.6.dev39.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
-modal-1.0.6.dev39.dist-info/RECORD,,
+modal-1.0.6.dev42.dist-info/METADATA,sha256=FNTsJdFH_KZwx-SLKVTHs2M992nVzuII_iYxH4RXj6E,2462
+modal-1.0.6.dev42.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+modal-1.0.6.dev42.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-1.0.6.dev42.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
+modal-1.0.6.dev42.dist-info/RECORD,,

modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.0.6.dev39"
+__version__ = "1.0.6.dev42"