modal 1.1.1.dev38__py3-none-any.whl → 1.1.1.dev40__py3-none-any.whl

modal/_functions.py

@@ -1513,8 +1513,7 @@ Use the `Function.get_web_url()` method instead.
         else:
             count_update_callback = None
 
-        # TODO(ben-okeefe): Feature gating for input plane map until feature is enabled.
-        if self._input_plane_url and False:
+        if self._input_plane_url:
             async with aclosing(
                 _map_invocation_inputplane(
                     self,

modal/_output.py

@@ -31,7 +31,7 @@ from rich.progress import (
 from rich.spinner import Spinner
 from rich.text import Text
 
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal_proto import api_pb2
 
 from ._utils.grpc_utils import RETRYABLE_GRPC_STATUS_CODES, retry_transient_errors
@@ -91,7 +91,7 @@ class LineBufferedOutput:
 
         if self._show_timestamps:
             for i in range(0, len(chunks) - 1, 2):
-                chunks[i] = f"{timestamp_to_local(log.timestamp)} {chunks[i]}"
+                chunks[i] = f"{timestamp_to_localized_str(log.timestamp)} {chunks[i]}"
 
         completed_lines = "".join(chunks[:-1])
         remainder = chunks[-1]

modal/_utils/async_utils.py

@@ -279,7 +279,9 @@ class TimestampPriorityQueue(Generic[T]):
 
     def __init__(self, maxsize: int = 0):
         self.condition = asyncio.Condition()
-        self._queue: asyncio.PriorityQueue[tuple[float, Union[T, None]]] = asyncio.PriorityQueue(maxsize=maxsize)
+        self._queue: asyncio.PriorityQueue[tuple[float, int, Union[T, None]]] = asyncio.PriorityQueue(maxsize=maxsize)
+        # Used to tiebreak items with the same timestamp that are not comparable. (eg. protos)
+        self._counter = itertools.count()
 
     async def close(self):
         await self.put(self._MAX_PRIORITY, None)
@@ -288,7 +290,7 @@ class TimestampPriorityQueue(Generic[T]):
         """
         Add an item to the queue to be processed at a specific timestamp.
         """
-        await self._queue.put((timestamp, item))
+        await self._queue.put((timestamp, next(self._counter), item))
         async with self.condition:
             self.condition.notify_all()  # notify any waiting coroutines
 
@@ -301,7 +303,7 @@ class TimestampPriorityQueue(Generic[T]):
                 while self.empty():
                     await self.condition.wait()
                 # peek at the next item
-                timestamp, item = await self._queue.get()
+                timestamp, counter, item = await self._queue.get()
                 now = time.time()
                 if timestamp < now:
                     return item
@@ -309,7 +311,7 @@ class TimestampPriorityQueue(Generic[T]):
                     return None
                 # not ready yet, calculate sleep time
                 sleep_time = timestamp - now
-                self._queue.put_nowait((timestamp, item))  # put it back
+                self._queue.put_nowait((timestamp, counter, item))  # put it back
                 # wait until either the timeout or a new item is added
                 try:
                     await asyncio.wait_for(self.condition.wait(), timeout=sleep_time)

modal/_utils/time_utils.py

@@ -3,13 +3,17 @@ from datetime import datetime
 from typing import Optional
 
 
-def timestamp_to_local(ts: float, isotz: bool = True) -> Optional[str]:
+def timestamp_to_localized_dt(ts: float) -> datetime:
+    locale_tz = datetime.now().astimezone().tzinfo
+    return datetime.fromtimestamp(ts, tz=locale_tz)
+
+
+def timestamp_to_localized_str(ts: float, isotz: bool = True) -> Optional[str]:
     if ts > 0:
-        locale_tz = datetime.now().astimezone().tzinfo
-        dt = datetime.fromtimestamp(ts, tz=locale_tz)
+        dt = timestamp_to_localized_dt(ts)
         if isotz:
             return dt.isoformat(sep=" ", timespec="seconds")
         else:
-            return f"{datetime.strftime(dt, '%Y-%m-%d %H:%M')} {locale_tz.tzname(dt)}"
+            return f"{dt:%Y-%m-%d %H:%M %Z}"
     else:
         return None

modal/cli/app.py

@@ -15,7 +15,7 @@ from modal.client import _Client
 from modal.environments import ensure_env
 from modal_proto import api_pb2
 
-from .._utils.time_utils import timestamp_to_local
+from .._utils.time_utils import timestamp_to_localized_str
 from .utils import ENV_OPTION, display_table, get_app_id_from_name, stream_app_logs
 
 APP_IDENTIFIER = Argument("", help="App name or ID")
@@ -71,8 +71,8 @@ async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
                 app_stats.description,
                 state,
                 str(app_stats.n_running_tasks),
-                timestamp_to_local(app_stats.created_at, json),
-                timestamp_to_local(app_stats.stopped_at, json),
+                timestamp_to_localized_str(app_stats.created_at, json),
+                timestamp_to_localized_str(app_stats.stopped_at, json),
             ]
         )
 
@@ -217,7 +217,7 @@ async def history(
 
         row = [
             Text(f"v{app_stats.version}", style=style),
-            Text(timestamp_to_local(app_stats.deployed_at, json), style=style),
+            Text(timestamp_to_localized_str(app_stats.deployed_at, json), style=style),
             Text(app_stats.client_version, style=style),
             Text(app_stats.deployed_by, style=style),
         ]

modal/cli/cluster.py

@@ -8,7 +8,7 @@ from modal._object import _get_environment_name
 from modal._output import make_console
 from modal._pty import get_pty_info
 from modal._utils.async_utils import synchronizer
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, display_table, is_tty
 from modal.client import _Client
 from modal.config import config
@@ -42,7 +42,7 @@ async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
             [
                 c.cluster_id,
                 c.app_id,
-                timestamp_to_local(c.started_at, json) if c.started_at else "Pending",
+                timestamp_to_localized_str(c.started_at, json) if c.started_at else "Pending",
                 str(len(c.task_ids)),
             ]
         )

modal/cli/container.py

@@ -8,7 +8,7 @@ from modal._object import _get_environment_name
 from modal._pty import get_pty_info
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, display_table, is_tty, stream_app_logs
 from modal.client import _Client
 from modal.config import config
@@ -40,7 +40,7 @@ async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
                 task_stats.task_id,
                 task_stats.app_id,
                 task_stats.app_description,
-                timestamp_to_local(task_stats.started_at, json) if task_stats.started_at else "Pending",
+                timestamp_to_localized_str(task_stats.started_at, json) if task_stats.started_at else "Pending",
             ]
         )
 

modal/cli/dict.py

@@ -8,7 +8,7 @@ from modal._output import make_console
 from modal._resolver import Resolver
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
 from modal.dict import _Dict
@@ -44,7 +44,7 @@ async def list_(*, json: bool = False, env: Optional[str] = ENV_OPTION):
     request = api_pb2.DictListRequest(environment_name=env)
     response = await retry_transient_errors(client.stub.DictList, request)
 
-    rows = [(d.name, timestamp_to_local(d.created_at, json)) for d in response.dicts]
+    rows = [(d.name, timestamp_to_localized_str(d.created_at, json)) for d in response.dicts]
     display_table(["Name", "Created at"], rows, json)
 
 

modal/cli/network_file_system.py

@@ -16,7 +16,7 @@ from modal._location import display_location
 from modal._output import OutputManager, ProgressHandler, make_console
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli._download import _volume_download
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
@@ -44,7 +44,7 @@ async def list_(env: Optional[str] = ENV_OPTION, json: Optional[bool] = False):
             [
                 item.label,
                 display_location(item.cloud_provider),
-                timestamp_to_local(item.created_at, json),
+                timestamp_to_localized_str(item.created_at, json),
             ]
         )
     display_table(column_names, rows, json, title=f"Shared Volumes{env_part}")

modal/cli/queues.py

@@ -8,7 +8,7 @@ from modal._output import make_console
 from modal._resolver import Resolver
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
 from modal.environments import ensure_env
@@ -71,7 +71,7 @@ async def list_(*, json: bool = False, env: Optional[str] = ENV_OPTION):
     rows = [
         (
             q.name,
-            timestamp_to_local(q.created_at, json),
+            timestamp_to_localized_str(q.created_at, json),
             str(q.num_partitions),
             str(q.total_size) if q.total_size <= max_total_size else f">{max_total_size}",
         )

modal/cli/secret.py

@@ -15,7 +15,7 @@ from typer import Argument
 from modal._output import make_console
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
 from modal.environments import ensure_env
@@ -38,8 +38,8 @@ async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
         rows.append(
             [
                 item.label,
-                timestamp_to_local(item.created_at, json),
-                timestamp_to_local(item.last_used_at, json) if item.last_used_at else "-",
+                timestamp_to_localized_str(item.created_at, json),
+                timestamp_to_localized_str(item.last_used_at, json) if item.last_used_at else "-",
             ]
         )
 

modal/cli/volume.py

@@ -14,7 +14,7 @@ import modal
 from modal._output import OutputManager, ProgressHandler, make_console
 from modal._utils.async_utils import synchronizer
 from modal._utils.grpc_utils import retry_transient_errors
-from modal._utils.time_utils import timestamp_to_local
+from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli._download import _volume_download
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
@@ -116,7 +116,7 @@ async def list_(env: Optional[str] = ENV_OPTION, json: Optional[bool] = False):
     column_names = ["Name", "Created at"]
     rows = []
     for item in response.items:
-        rows.append([item.label, timestamp_to_local(item.created_at, json)])
+        rows.append([item.label, timestamp_to_localized_str(item.created_at, json)])
     display_table(column_names, rows, json, title=f"Volumes{env_part}")
 
 
@@ -163,7 +163,7 @@ async def ls(
                 (
                     entry.path.encode("unicode_escape").decode("utf-8"),
                     filetype,
-                    timestamp_to_local(entry.mtime, False),
+                    timestamp_to_localized_str(entry.mtime, False),
                     humanize_filesize(entry.size),
                 )
             )

modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.1.dev38",
+        version: str = "1.1.1.dev40",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.1.dev38",
+        version: str = "1.1.1.dev40",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/dict.py

@@ -17,6 +17,7 @@ from ._utils.async_utils import TaskContext, synchronize_api
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.name_utils import check_object_name
+from ._utils.time_utils import timestamp_to_localized_dt
 from .client import _Client
 from .config import logger
 from .exception import RequestSizeError
@@ -248,7 +249,7 @@ class _Dict(_Object, type_prefix="di"):
         creation_info = metadata.creation_info
         return DictInfo(
             name=metadata.name or None,
-            created_at=datetime.fromtimestamp(creation_info.created_at),
+            created_at=timestamp_to_localized_dt(creation_info.created_at),
             created_by=creation_info.created_by or None,
         )
 

modal/parallel_map.py

@@ -6,7 +6,7 @@ import time
 import typing
 from asyncio import FIRST_COMPLETED
 from dataclasses import dataclass
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, Union
 
 from grpclib import Status
 
@@ -437,17 +437,14 @@ async def _map_invocation_inputplane(
 
     This is analogous to `_map_invocation`, but instead of the control-plane
     `FunctionMap` / `FunctionPutInputs` / `FunctionGetOutputs` RPCs it speaks
-    the input-plane protocol consisting of `MapStartOrContinue` and `MapAwait`.
-
-    The implementation purposefully ignores retry handling for now - a stub is
-    left in place so that a future change can add support for the retry path
-    without re-structuring the surrounding code.
+    the input-plane protocol consisting of `MapStartOrContinue`, `MapAwait`, and `MapCheckInputs`.
     """
 
     assert function._input_plane_url, "_map_invocation_inputplane should only be used for input-plane backed functions"
 
     input_plane_stub = await client.get_stub(function._input_plane_url)
 
+    # Required for _create_input.
     assert client.stub, "Client must be hydrated with a stub for _map_invocation_inputplane"
 
     # ------------------------------------------------------------
@@ -459,15 +456,19 @@ async def _map_invocation_inputplane(
 
     inputs_created = 0
     outputs_completed = 0
+    successful_completions = 0
+    failed_completions = 0
+    no_context_duplicates = 0
+    stale_retry_duplicates = 0
+    already_complete_duplicates = 0
+    retried_outputs = 0
+    input_queue_size = 0
+    last_entry_id = ""
 
     # The input-plane server returns this after the first request.
-    function_call_id: str | None = None
+    function_call_id = None
     function_call_id_received = asyncio.Event()
 
-    # Map of idx -> attempt_token returned by the server.  This will be needed
-    # for a future client-side retry implementation.
-    attempt_tokens: dict[int, str] = {}
-
     # Single priority queue that holds *both* fresh inputs (timestamp == now)
     # and future retries (timestamp > now).
     queue: TimestampPriorityQueue[api_pb2.MapStartOrContinueItem] = TimestampPriorityQueue()
@@ -477,11 +478,25 @@ async def _map_invocation_inputplane(
     # any reason).
     max_inputs_outstanding = MAX_INPUTS_OUTSTANDING_DEFAULT
 
-    # ------------------------------------------------------------
-    # Helper functions
-    # ------------------------------------------------------------
+    # Input plane does not yet return a retry policy. So we currently disable retries.
+    retry_policy = api_pb2.FunctionRetryPolicy(
+        retries=0,  # Input plane does not yet return a retry policy. So only retry server failures for now.
+        initial_delay_ms=1000,
+        max_delay_ms=1000,
+        backoff_coefficient=1.0,
+    )
+    map_items_manager = _MapItemsManager(
+        retry_policy=retry_policy,
+        function_call_invocation_type=api_pb2.FUNCTION_CALL_INVOCATION_TYPE_SYNC,
+        retry_queue=queue,
+        sync_client_retries_enabled=True,
+        max_inputs_outstanding=MAX_INPUTS_OUTSTANDING_DEFAULT,
+        is_input_plane_instance=True,
+    )
 
-    def update_counters(created_delta: int = 0, completed_delta: int = 0, set_have_all_inputs: bool | None = None):
+    def update_counters(
+        created_delta: int = 0, completed_delta: int = 0, set_have_all_inputs: Union[bool, None] = None
+    ):
         nonlocal inputs_created, outputs_completed, have_all_inputs
 
         if created_delta:
@@ -511,10 +526,6 @@ async def _map_invocation_inputplane(
         )
         return api_pb2.MapStartOrContinueItem(input=put_item)
 
-    # ------------------------------------------------------------
-    # Coroutine: drain user input iterator, upload blobs, enqueue for sending
-    # ------------------------------------------------------------
-
     async def input_iter():
         while True:
             raw_input = await raw_input_queue.get()
@@ -530,22 +541,19 @@ async def _map_invocation_inputplane(
                 await queue.put(time.time(), q_item)
 
         # All inputs have been read.
-        await queue.close()
         update_counters(set_have_all_inputs=True)
         yield
 
-    # ------------------------------------------------------------
-    # Coroutine: send queued items to the input-plane server
-    # ------------------------------------------------------------
-
     async def pump_inputs():
         nonlocal function_call_id, max_inputs_outstanding
-
         async for batch in queue_batch_iterator(queue, max_batch_size=MAP_INVOCATION_CHUNK_SIZE):
             # Convert the queued items into the proto format expected by the RPC.
             request_items: list[api_pb2.MapStartOrContinueItem] = [
                 api_pb2.MapStartOrContinueItem(input=qi.input, attempt_token=qi.attempt_token) for qi in batch
             ]
+
+            await map_items_manager.add_items_inputplane(request_items)
+
             # Build request
             request = api_pb2.MapStartOrContinueRequest(
                 function_id=function.object_id,
@@ -560,43 +568,62 @@ async def _map_invocation_inputplane(
                 input_plane_stub.MapStartOrContinue, request, metadata=metadata
             )
 
-            # TODO(ben-okeefe): Understand if an input could be lost at this step and not registered
+            # match response items to the corresponding request item index
+            response_items_idx_tuple = [
+                (request_items[idx].input.idx, attempt_token)
+                for idx, attempt_token in enumerate(response.attempt_tokens)
+            ]
+
+            map_items_manager.handle_put_continue_response(response_items_idx_tuple)
 
             if function_call_id is None:
                 function_call_id = response.function_call_id
                 function_call_id_received.set()
                 max_inputs_outstanding = response.max_inputs_outstanding or MAX_INPUTS_OUTSTANDING_DEFAULT
-
-            # Record attempt tokens for future retries; also release semaphore slots now that the
-            # inputs are officially registered on the server.
-            for idx, attempt_token in enumerate(response.attempt_tokens):
-                # Client expects the server to return the attempt tokens in the same order as the inputs we sent.
-                attempt_tokens[request_items[idx].input.idx] = attempt_token
-
         yield
 
-    # ------------------------------------------------------------
-    # Coroutine: **stub** – retry handling will be added in the future
-    # ------------------------------------------------------------
-
-    async def retry_inputs():
-        """Temporary stub for retrying inputs. Retry handling will be added in the future."""
-
+    async def check_lost_inputs():
+        nonlocal last_entry_id  # shared with get_all_outputs
         try:
             while not map_done_event.is_set():
+                if function_call_id is None:
+                    await function_call_id_received.wait()
+                    continue
+
                 await asyncio.sleep(1)
-                if False:
-                    yield
+
+                # check_inputs = [(idx, attempt_token), ...]
+                check_inputs = map_items_manager.get_input_idxs_waiting_for_output()
+                attempt_tokens = [attempt_token for _, attempt_token in check_inputs]
+                request = api_pb2.MapCheckInputsRequest(
+                    last_entry_id=last_entry_id,
+                    timeout=0,  # Non-blocking read
+                    attempt_tokens=attempt_tokens,
+                )
+
+                metadata = await client.get_input_plane_metadata(function._input_plane_region)
+                response: api_pb2.MapCheckInputsResponse = await retry_transient_errors(
+                    input_plane_stub.MapCheckInputs, request, metadata=metadata
+                )
+                check_inputs_response = [
+                    (check_inputs[resp_idx][0], response.lost[resp_idx]) for resp_idx, _ in enumerate(response.lost)
+                ]
+                # check_inputs_response = [(idx, lost: bool), ...]
+                await map_items_manager.handle_check_inputs_response(check_inputs_response)
+            yield
         except asyncio.CancelledError:
             pass
 
-    # ------------------------------------------------------------
-    # Coroutine: stream outputs via MapAwait
-    # ------------------------------------------------------------
-
     async def get_all_outputs():
-        """Continuously fetch outputs until the map is complete."""
-        last_entry_id = ""
+        nonlocal \
+            successful_completions, \
+            failed_completions, \
+            no_context_duplicates, \
+            stale_retry_duplicates, \
+            already_complete_duplicates, \
+            retried_outputs, \
+            last_entry_id
+
         while not map_done_event.is_set():
             if function_call_id is None:
                 await function_call_id_received.wait()
@@ -609,21 +636,51 @@ async def _map_invocation_inputplane(
                 timeout=OUTPUTS_TIMEOUT,
             )
             metadata = await client.get_input_plane_metadata(function._input_plane_region)
-            response: api_pb2.MapAwaitResponse = await retry_transient_errors(
-                input_plane_stub.MapAwait,
-                request,
-                max_retries=20,
-                attempt_timeout=OUTPUTS_TIMEOUT + ATTEMPT_TIMEOUT_GRACE_PERIOD,
-                metadata=metadata,
+            get_response_task = asyncio.create_task(
+                retry_transient_errors(
+                    input_plane_stub.MapAwait,
+                    request,
+                    max_retries=20,
+                    attempt_timeout=OUTPUTS_TIMEOUT + ATTEMPT_TIMEOUT_GRACE_PERIOD,
+                    metadata=metadata,
+                )
             )
+            map_done_task = asyncio.create_task(map_done_event.wait())
+            try:
+                done, pending = await asyncio.wait([get_response_task, map_done_task], return_when=FIRST_COMPLETED)
+                if get_response_task in done:
+                    map_done_task.cancel()
+                    response = get_response_task.result()
+                else:
+                    assert map_done_event.is_set()
+                    # map is done - no more outputs, so return early
+                    return
+            finally:
+                # clean up tasks, in case of cancellations etc.
+                get_response_task.cancel()
+                map_done_task.cancel()
             last_entry_id = response.last_entry_id
 
             for output_item in response.outputs:
-                yield output_item
-
-                update_counters(completed_delta=1)
+                output_type = await map_items_manager.handle_get_outputs_response(output_item, int(time.time()))
+                if output_type == _OutputType.SUCCESSFUL_COMPLETION:
+                    successful_completions += 1
+                elif output_type == _OutputType.FAILED_COMPLETION:
+                    failed_completions += 1
+                elif output_type == _OutputType.RETRYING:
+                    retried_outputs += 1
+                elif output_type == _OutputType.NO_CONTEXT_DUPLICATE:
+                    no_context_duplicates += 1
+                elif output_type == _OutputType.STALE_RETRY_DUPLICATE:
+                    stale_retry_duplicates += 1
+                elif output_type == _OutputType.ALREADY_COMPLETE_DUPLICATE:
+                    already_complete_duplicates += 1
+                else:
+                    raise Exception(f"Unknown output type: {output_type}")
 
-            # The loop condition will exit when map_done_event is set from update_counters.
+                if output_type == _OutputType.SUCCESSFUL_COMPLETION or output_type == _OutputType.FAILED_COMPLETION:
+                    update_counters(completed_delta=1)
+                    yield output_item
 
     async def get_all_outputs_and_clean_up():
         try:
@@ -631,23 +688,24 @@ async def _map_invocation_inputplane(
                 async for item in stream:
                     yield item
         finally:
-            # We could signal server we are done with outputs so it can clean up.
+            await queue.close()
             pass
 
-    # ------------------------------------------------------------
-    # Coroutine: convert FunctionGetOutputsItem → actual result value
-    # ------------------------------------------------------------
-
     async def fetch_output(item: api_pb2.FunctionGetOutputsItem) -> tuple[int, Any]:
         try:
-            output_val = await _process_result(item.result, item.data_format, input_plane_stub, client)
-        except Exception as exc:
+            output = await _process_result(item.result, item.data_format, input_plane_stub, client)
+        except Exception as e:
             if return_exceptions:
-                output_val = exc
+                if wrap_returned_exceptions:
+                    # Prior to client 1.0.4 there was a bug where return_exceptions would wrap
+                    # any returned exceptions in a synchronicity.UserCodeException. This adds
+                    # deprecated non-breaking compatibility bandaid for migrating away from that:
+                    output = modal.exception.UserCodeException(e)
+                else:
+                    output = e
             else:
-                raise exc
-
-        return (item.idx, output_val)
+                raise e
+        return (item.idx, output)
 
     async def poll_outputs():
         # map to store out-of-order outputs received
@@ -677,17 +735,14 @@ async def _map_invocation_inputplane(
 
         assert len(received_outputs) == 0
 
-    # ------------------------------------------------------------
-    # Debug-logging helper
-    # ------------------------------------------------------------
     async def log_debug_stats():
         def log_stats():
             logger.debug(
-                "Map-IP stats: have_all_inputs=%s inputs_created=%d outputs_completed=%d queue_size=%d",
-                have_all_inputs,
-                inputs_created,
-                outputs_completed,
-                queue.qsize(),
+                f"Map stats:\nsuccessful_completions={successful_completions} failed_completions={failed_completions} "
+                f"no_context_duplicates={no_context_duplicates} stale_retry_duplicates={stale_retry_duplicates} "
+                f"already_complete_duplicates={already_complete_duplicates} retried_outputs={retried_outputs} "
+                f"function_call_id={function_call_id} max_inputs_outstanding={max_inputs_outstanding} "
+                f"map_items_manager_size={len(map_items_manager)} input_queue_size={input_queue_size}"
             )
 
         while True:
@@ -699,13 +754,11 @@ async def _map_invocation_inputplane(
                 log_stats()
                 break
 
-    # ------------------------------------------------------------
-    # Run the four coroutines concurrently and yield results as they arrive
-    # ------------------------------------------------------------
-
     log_task = asyncio.create_task(log_debug_stats())
 
-    async with aclosing(async_merge(drain_input_generator(), pump_inputs(), poll_outputs(), retry_inputs())) as merged:
+    async with aclosing(
+        async_merge(drain_input_generator(), pump_inputs(), poll_outputs(), check_lost_inputs())
+    ) as merged:
         async for maybe_output in merged:
             if maybe_output is not None:  # ignore None sentinels
                 yield maybe_output.value
@@ -1045,12 +1098,19 @@ class _MapItemContext:
     sync_client_retries_enabled: bool
     # Both these futures are strings. Omitting generic type because
     # it causes an error when running `inv protoc type-stubs`.
+    # Unused. But important, input_id is not set for inputplane invocations.
     input_id: asyncio.Future
     input_jwt: asyncio.Future
     previous_input_jwt: Optional[str]
     _event_loop: asyncio.AbstractEventLoop
 
-    def __init__(self, input: api_pb2.FunctionInput, retry_manager: RetryManager, sync_client_retries_enabled: bool):
+    def __init__(
+        self,
+        input: api_pb2.FunctionInput,
+        retry_manager: RetryManager,
+        sync_client_retries_enabled: bool,
+        is_input_plane_instance: bool = False,
+    ):
         self.state = _MapItemState.SENDING
         self.input = input
         self.retry_manager = retry_manager
@@ -1061,7 +1121,22 @@ class _MapItemContext:
         # a race condition where we could receive outputs before we have
         # recorded the input ID and JWT in `pending_outputs`.
         self.input_jwt = self._event_loop.create_future()
+        # Unused. But important, this is not set for inputplane invocations.
         self.input_id = self._event_loop.create_future()
+        self._is_input_plane_instance = is_input_plane_instance
+
+    def handle_map_start_or_continue_response(self, attempt_token: str):
+        if not self.input_jwt.done():
+            self.input_jwt.set_result(attempt_token)
+        else:
+            # Create a new future for the next value
+            self.input_jwt = asyncio.Future()
+            self.input_jwt.set_result(attempt_token)
+
+        # Set state to WAITING_FOR_OUTPUT only if current state is SENDING. If state is
+        # RETRYING, WAITING_TO_RETRY, or COMPLETE, then we already got the output.
+        if self.state == _MapItemState.SENDING:
+            self.state = _MapItemState.WAITING_FOR_OUTPUT
 
     def handle_put_inputs_response(self, item: api_pb2.FunctionPutInputsResponseItem):
         self.input_jwt.set_result(item.input_jwt)
@@ -1088,7 +1163,7 @@ class _MapItemContext:
         if self.state == _MapItemState.COMPLETE:
             logger.debug(
                 f"Received output for input marked as complete. Must be duplicate, so ignoring. "
-                f"idx={item.idx} input_id={item.input_id}, retry_count={item.retry_count}"
+                f"idx={item.idx} input_id={item.input_id} retry_count={item.retry_count}"
             )
             return _OutputType.ALREADY_COMPLETE_DUPLICATE
         # If the item's retry count doesn't match our retry count, this is probably a duplicate of an old output.
@@ -1136,7 +1211,11 @@ class _MapItemContext:
 
         self.state = _MapItemState.WAITING_TO_RETRY
 
-        await retry_queue.put(now_seconds + (delay_ms / 1000), item.idx)
+        if self._is_input_plane_instance:
+            retry_item = await self.create_map_start_or_continue_item(item.idx)
+            await retry_queue.put(now_seconds + delay_ms / 1_000, retry_item)
+        else:
+            await retry_queue.put(now_seconds + delay_ms / 1_000, item.idx)
 
         return _OutputType.RETRYING
 
@@ -1155,6 +1234,16 @@ class _MapItemContext:
         self.input_jwt.set_result(input_jwt)
         self.state = _MapItemState.WAITING_FOR_OUTPUT
 
+    async def create_map_start_or_continue_item(self, idx: int) -> api_pb2.MapStartOrContinueItem:
+        attempt_token = await self.input_jwt
+        return api_pb2.MapStartOrContinueItem(
+            input=api_pb2.FunctionPutInputsItem(
+                input=self.input,
+                idx=idx,
+            ),
+            attempt_token=attempt_token,
+        )
+
 
 class _MapItemsManager:
     def __init__(
@@ -1164,6 +1253,7 @@ class _MapItemsManager:
         retry_queue: TimestampPriorityQueue,
         sync_client_retries_enabled: bool,
         max_inputs_outstanding: int,
+        is_input_plane_instance: bool = False,
     ):
         self._retry_policy = retry_policy
         self.function_call_invocation_type = function_call_invocation_type
@@ -1174,6 +1264,7 @@ class _MapItemsManager:
         self._inputs_outstanding = asyncio.BoundedSemaphore(max_inputs_outstanding)
         self._item_context: dict[int, _MapItemContext] = {}
         self._sync_client_retries_enabled = sync_client_retries_enabled
+        self._is_input_plane_instance = is_input_plane_instance
 
     async def add_items(self, items: list[api_pb2.FunctionPutInputsItem]):
         for item in items:
@@ -1186,6 +1277,21 @@ class _MapItemsManager:
                 sync_client_retries_enabled=self._sync_client_retries_enabled,
             )
 
+    async def add_items_inputplane(self, items: list[api_pb2.MapStartOrContinueItem]):
+        for item in items:
+            # acquire semaphore to limit the number of inputs in progress
+            # (either queued to be sent, waiting for completion, or retrying)
+            if item.attempt_token != "":  # if it is a retry item
+                self._item_context[item.input.idx].state = _MapItemState.SENDING
+                continue
+            await self._inputs_outstanding.acquire()
+            self._item_context[item.input.idx] = _MapItemContext(
+                input=item.input.input,
+                retry_manager=RetryManager(self._retry_policy),
+                sync_client_retries_enabled=self._sync_client_retries_enabled,
+                is_input_plane_instance=self._is_input_plane_instance,
+            )
+
     async def prepare_items_for_retry(self, retriable_idxs: list[int]) -> list[api_pb2.FunctionRetryInputsItem]:
         return [await self._item_context[idx].prepare_item_for_retry() for idx in retriable_idxs]
 
@@ -1200,6 +1306,17 @@ class _MapItemsManager:
             if ctx.state == _MapItemState.WAITING_FOR_OUTPUT and ctx.input_jwt.done()
         ]
 
+    def get_input_idxs_waiting_for_output(self) -> list[tuple[int, str]]:
+        """
+        Returns a list of input_idxs for inputs that are waiting for output.
+        """
+        # Idx doesn't need a future because it is set by client and not server.
+        return [
+            (idx, ctx.input_jwt.result())
+            for idx, ctx in self._item_context.items()
+            if ctx.state == _MapItemState.WAITING_FOR_OUTPUT and ctx.input_jwt.done()
+        ]
+
     def _remove_item(self, item_idx: int):
         del self._item_context[item_idx]
         self._inputs_outstanding.release()
@@ -1207,6 +1324,18 @@ class _MapItemsManager:
     def get_item_context(self, item_idx: int) -> _MapItemContext:
         return self._item_context.get(item_idx)
 
+    def handle_put_continue_response(
+        self,
+        items: list[tuple[int, str]],  # idx, input_jwt
+    ):
+        for index, item in items:
+            ctx = self._item_context.get(index, None)
+            # If the context is None, then get_all_outputs() has already received a successful
+            # output, and deleted the context. This happens if FunctionGetOutputs completes
+            # before MapStartOrContinueResponse is received.
+            if ctx is not None:
+                ctx.handle_map_start_or_continue_response(item)
+
     def handle_put_inputs_response(self, items: list[api_pb2.FunctionPutInputsResponseItem]):
         for item in items:
             ctx = self._item_context.get(item.idx, None)
@@ -1226,6 +1355,16 @@ class _MapItemsManager:
             if ctx is not None:
                 ctx.handle_retry_response(input_jwt)
 
+    async def handle_check_inputs_response(self, response: list[tuple[int, bool]]):
+        for idx, lost in response:
+            ctx = self._item_context.get(idx, None)
+            if ctx is not None:
+                if lost:
+                    ctx.state = _MapItemState.WAITING_TO_RETRY
+                    retry_item = await ctx.create_map_start_or_continue_item(idx)
+                    _ = ctx.retry_manager.get_delay_ms()  # increment retry count but instant retry for lost inputs
+                    await self._retry_queue.put(time.time(), retry_item)
+
     async def handle_get_outputs_response(self, item: api_pb2.FunctionGetOutputsItem, now_seconds: int) -> _OutputType:
         ctx = self._item_context.get(item.idx, None)
         if ctx is None:

modal/parallel_map.pyi

@@ -83,11 +83,7 @@ def _map_invocation_inputplane(
 
     This is analogous to `_map_invocation`, but instead of the control-plane
     `FunctionMap` / `FunctionPutInputs` / `FunctionGetOutputs` RPCs it speaks
-    the input-plane protocol consisting of `MapStartOrContinue` and `MapAwait`.
-
-    The implementation purposefully ignores retry handling for now - a stub is
-    left in place so that a future change can add support for the retry path
-    without re-structuring the surrounding code.
+    the input-plane protocol consisting of `MapStartOrContinue`, `MapAwait`, and `MapCheckInputs`.
     """
     ...
 
@@ -281,10 +277,12 @@ class _MapItemContext:
         input: modal_proto.api_pb2.FunctionInput,
         retry_manager: modal.retries.RetryManager,
         sync_client_retries_enabled: bool,
+        is_input_plane_instance: bool = False,
     ):
         """Initialize self.  See help(type(self)) for accurate signature."""
         ...
 
+    def handle_map_start_or_continue_response(self, attempt_token: str): ...
     def handle_put_inputs_response(self, item: modal_proto.api_pb2.FunctionPutInputsResponseItem): ...
     async def handle_get_outputs_response(
         self,
@@ -301,6 +299,7 @@ class _MapItemContext:
 
     async def prepare_item_for_retry(self) -> modal_proto.api_pb2.FunctionRetryInputsItem: ...
     def handle_retry_response(self, input_jwt: str): ...
+    async def create_map_start_or_continue_item(self, idx: int) -> modal_proto.api_pb2.MapStartOrContinueItem: ...
 
 class _MapItemsManager:
     def __init__(
@@ -310,11 +309,13 @@ class _MapItemsManager:
         retry_queue: modal._utils.async_utils.TimestampPriorityQueue,
         sync_client_retries_enabled: bool,
         max_inputs_outstanding: int,
+        is_input_plane_instance: bool = False,
     ):
         """Initialize self.  See help(type(self)) for accurate signature."""
         ...
 
     async def add_items(self, items: list[modal_proto.api_pb2.FunctionPutInputsItem]): ...
+    async def add_items_inputplane(self, items: list[modal_proto.api_pb2.MapStartOrContinueItem]): ...
     async def prepare_items_for_retry(
         self, retriable_idxs: list[int]
     ) -> list[modal_proto.api_pb2.FunctionRetryInputsItem]: ...
@@ -322,10 +323,16 @@ class _MapItemsManager:
         """Returns a list of input_jwts for inputs that are waiting for output."""
         ...
 
+    def get_input_idxs_waiting_for_output(self) -> list[tuple[int, str]]:
+        """Returns a list of input_idxs for inputs that are waiting for output."""
+        ...
+
     def _remove_item(self, item_idx: int): ...
     def get_item_context(self, item_idx: int) -> _MapItemContext: ...
+    def handle_put_continue_response(self, items: list[tuple[int, str]]): ...
     def handle_put_inputs_response(self, items: list[modal_proto.api_pb2.FunctionPutInputsResponseItem]): ...
     def handle_retry_response(self, input_jwts: list[str]): ...
+    async def handle_check_inputs_response(self, response: list[tuple[int, bool]]): ...
     async def handle_get_outputs_response(
         self, item: modal_proto.api_pb2.FunctionGetOutputsItem, now_seconds: int
     ) -> _OutputType: ...

modal/queue.py

@@ -20,6 +20,7 @@ from ._utils.async_utils import TaskContext, synchronize_api, warn_if_generator_
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.name_utils import check_object_name
+from ._utils.time_utils import timestamp_to_localized_dt
 from .client import _Client
 from .exception import InvalidError, RequestSizeError
 
@@ -260,7 +261,7 @@ class _Queue(_Object, type_prefix="qu"):
         creation_info = metadata.creation_info
         return QueueInfo(
             name=metadata.name or None,
-            created_at=datetime.fromtimestamp(creation_info.created_at),
+            created_at=timestamp_to_localized_dt(creation_info.created_at),
             created_by=creation_info.created_by or None,
         )
 

modal/secret.py

@@ -16,6 +16,7 @@ from ._utils.async_utils import synchronize_api
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.name_utils import check_object_name
+from ._utils.time_utils import timestamp_to_localized_dt
 from .client import _Client
 from .exception import InvalidError, NotFoundError
 
@@ -299,7 +300,7 @@ class _Secret(_Object, type_prefix="st"):
         creation_info = metadata.creation_info
         return SecretInfo(
             name=metadata.name or None,
-            created_at=datetime.fromtimestamp(creation_info.created_at),
+            created_at=timestamp_to_localized_dt(creation_info.created_at),
             created_by=creation_info.created_by or None,
         )
 

modal/volume.py

@@ -55,6 +55,7 @@ from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.http_utils import ClientSessionRegistry
 from ._utils.name_utils import check_object_name
+from ._utils.time_utils import timestamp_to_localized_dt
 from .client import _Client
 from .config import logger
 
@@ -364,7 +365,7 @@ class _Volume(_Object, type_prefix="vo"):
         creation_info = metadata.creation_info
         return VolumeInfo(
             name=metadata.name or None,
-            created_at=datetime.fromtimestamp(creation_info.created_at) if creation_info.created_at else None,
+            created_at=timestamp_to_localized_dt(creation_info.created_at),
             created_by=creation_info.created_by or None,
         )
 

{modal-1.1.1.dev38.dist-info → modal-1.1.1.dev40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.1.1.dev38
+Version: 1.1.1.dev40
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.1.1.dev38.dist-info → modal-1.1.1.dev40.dist-info}/RECORD

@@ -3,11 +3,11 @@ modal/__main__.py,sha256=uBjSb_5cdlxmr9AwLkznYsW2tGEJRBjEcCGILvgR_1s,2844
 modal/_clustered_functions.py,sha256=zmrKbptRbqp4euS3LWncKaLXb8Kjj4YreusOzpEpRMk,2856
 modal/_clustered_functions.pyi,sha256=_wtFjWocGf1WgI-qYBpbJPArNkg2H9JV7BVaGgMesEQ,1103
 modal/_container_entrypoint.py,sha256=1qBMNY_E9ICC_sRCtillMxmKPsmxJl1J0_qOAG8rH-0,28288
-modal/_functions.py,sha256=MVQEkih76OLsmy-QbphuyTH1gien8hTwjGs3Jxwx6cA,82826
+modal/_functions.py,sha256=n4Tjvf2Gw4DMYw5-O4OUeTUt2GucbuK3RA1orseQJEc,82727
 modal/_ipython.py,sha256=TW1fkVOmZL3YYqdS2YlM1hqpf654Yf8ZyybHdBnlhSw,301
 modal/_location.py,sha256=joiX-0ZeutEUDTrrqLF1GHXCdVLF-rHzstocbMcd_-k,366
 modal/_object.py,sha256=nCkQeLibSuvVAEIheGaLnUfN5PIh1CGpJCnzPIXymGY,11563
-modal/_output.py,sha256=T7CRq90W09d-WD4ko7T4PBe26JNeAXE1-8HNO9xpNPI,25787
+modal/_output.py,sha256=G9CeSQEBzjhveWWEzWmYa5Uwbu4lZf8N8IFH1UM4fU0,25803
 modal/_partial_function.py,sha256=B1J4S9W-La0NHaVmY1aCuH0E3QxJHIX6ZWY5eNTQ7io,37142
 modal/_pty.py,sha256=JZfPDDpzqICZqtyPI_oMJf_9w-p_lLNuzHhwhodUXio,1329
 modal/_resolver.py,sha256=2RWvm34cNSnbv1v7izJMNZgfvpLDD6LzaBlr0lIrLnY,7364
@@ -22,7 +22,7 @@ modal/app.py,sha256=kpq4kXp7pch688y6g55QYAC10wqPTU5FXKoWPMirA3E,47899
 modal/app.pyi,sha256=-jKXlGDBWRPVsuenBhdMRqawK-L2eiJ7gHbmSblhltg,43525
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=kyAIVB3Ay-XKJizQ_1ufUFB__EagV0MLmHJpyYyJ7J0,18636
-modal/client.pyi,sha256=2gT-xgWVAmS-XdFWtLiQSI42V8JiZuRagrBadMPN9FA,15831
+modal/client.pyi,sha256=PNk58EEgTKJjy4hA9vN9sClYMF9gkjfbuFmCfJ3Ujfs,15831
 modal/cloud_bucket_mount.py,sha256=YOe9nnvSr4ZbeCn587d7_VhE9IioZYRvF9VYQTQux08,5914
 modal/cloud_bucket_mount.pyi,sha256=-qSfYAQvIoO_l2wsCCGTG5ZUwQieNKXdAO00yP1-LYU,7394
 modal/cls.py,sha256=7A0xGnugQzm8dOfnKMjLjtqekRlRtQ0jPFRYgq6xdUM,40018
@@ -30,7 +30,7 @@ modal/cls.pyi,sha256=_tZ5qrlL-ZDEcD-mf9BZkkNH5XPr4SmGTEQ-RVmqF3I,27772
 modal/config.py,sha256=FqVewLPVVR4feq_46JBENiCzqTuXKpnvQZxaeWbS39g,12009
 modal/container_process.py,sha256=XkPwNIW-iD_GB9u9yqv9q8y-i5cQ8eBbLZZ_GvEw9t8,6858
 modal/container_process.pyi,sha256=9m-st3hCUlNN1GOTctfPPvIvoLtEl7FbuGWwif5-7YU,6037
-modal/dict.py,sha256=IyhKwQPM-HX10ZT-0ouSxpM-oAWqrT5waXFHmunmtyo,15804
+modal/dict.py,sha256=IWpPQtBwR96TJN7ogpIZvL9Ge9rxY4KJ2CjkUKfWr6g,15864
 modal/dict.pyi,sha256=vUrNmCKWZqiPIQSdbMT6fCq9q1QV3qkGVdz2B_yld34,22578
 modal/environments.py,sha256=gHFNLG78bqgizpQ4w_elz27QOqmcgAonFsmLs7NjUJ4,6804
 modal/environments.pyi,sha256=9-KtrzAcUe55cCP4020lSUD7-fWS7OPakAHssq4-bro,4219
@@ -52,14 +52,14 @@ modal/network_file_system.pyi,sha256=Td_IobHr84iLo_9LZKQ4tNdUB60yjX8QWBaFiUvhfi8
 modal/object.py,sha256=bTeskuY8JFrESjU4_UL_nTwYlBQdOLmVaOX3X6EMxsg,164
 modal/object.pyi,sha256=sgbaq_d3QSmnPKg5jRbMG3dOceKs0l54kHUAhAyZKAE,6796
 modal/output.py,sha256=q4T9uHduunj4NwY-YSwkHGgjZlCXMuJbfQ5UFaAGRAc,1968
-modal/parallel_map.py,sha256=-9nS9s1jbx1Iqh_5HQRK4xTdhnXF4AGIXwT4UGJ8R78,52666
-modal/parallel_map.pyi,sha256=fCugFsGup4Cflesb10_uR-nt5_eguuvhvtvavus_F98,11186
+modal/parallel_map.py,sha256=qZjvo33YAifqCVGz-d_PCRhA70sAF01EbqxQHBAdVsg,59293
+modal/parallel_map.pyi,sha256=T2HsEJVYT0KpDy8kqGz98WgH3HnIqz4kvIhZXs7c3Dw,11724
 modal/partial_function.py,sha256=aIdlGfTjjgqY6Fpr-biCjvRU9W542_S5N2xkNN_rYGM,1127
 modal/partial_function.pyi,sha256=lqqOzZ9-QvHTDWKQ_oAYYOvsXgTOBKhO9u-RI98JbUk,13986
 modal/proxy.py,sha256=NQJJMGo-D2IfmeU0vb10WWaE4oTLcuf9jTeEJvactOg,1446
 modal/proxy.pyi,sha256=yWGWwADCRGrC2w81B7671UTH4Uv3HMZKy5vVqlJUZoA,1417
 modal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-modal/queue.py,sha256=19Xri5HzY0VgOxWMZ4y_mNco8jhNgn9klegXrSHWmGc,20306
+modal/queue.py,sha256=6oKACWiFus03vx5-cDxowxX80a1J7sAwNpBBHLJM8sk,20366
 modal/queue.pyi,sha256=Pv4OtY7j17Yb89HGKaMQRiIv0yol-aV-ZtelxQ9GrlU,28330
 modal/retries.py,sha256=IvNLDM0f_GLUDD5VgEDoN09C88yoxSrCquinAuxT1Sc,5205
 modal/runner.py,sha256=ostdzYpQb-20tlD6dIq7bpWTkZkOhjJBNuMNektqnJA,24068
@@ -69,7 +69,7 @@ modal/sandbox.py,sha256=eQd0Cf9yTFCNshnj7oH8WvecbhVIwsEsmuXB9O-REis,40927
 modal/sandbox.pyi,sha256=_ddnvZGauSRG-WelsMB5oPil8KVWb0PSvmuAzAzrLIw,41713
 modal/schedule.py,sha256=ng0g0AqNY5GQI9KhkXZQ5Wam5G42glbkqVQsNpBtbDE,3078
 modal/scheduler_placement.py,sha256=BAREdOY5HzHpzSBqt6jDVR6YC_jYfHMVqOzkyqQfngU,1235
-modal/secret.py,sha256=UaUmwYmT52VarDh92b0QzAa8_BNUAgBs-wE4eMQ6-B8,11967
+modal/secret.py,sha256=_d_OQUE1S0v_wO5Ck728dsC_1v8-B_4ku4TS4BgC4Bc,12027
 modal/secret.pyi,sha256=zcC_OM0JzIF1ccnhNvVIlL6sY3xVjq3t0s3fE1ZDDVs,9732
 modal/serving.py,sha256=3I3WBeVbzZY258u9PXBCW_dZBgypq3OhwBuTVvlgubE,4423
 modal/serving.pyi,sha256=YfixTaWikyYpwhnNxCHMZnDDQiPmV1xJ87QF91U_WGU,1924
@@ -78,7 +78,7 @@ modal/snapshot.pyi,sha256=0q83hlmWxAhDu8xwZyL5VmYh0i8Tigf7S60or2k30L8,1682
 modal/stream_type.py,sha256=A6320qoAAWhEfwOCZfGtymQTu5AfLfJXXgARqooTPvY,417
 modal/token_flow.py,sha256=GWpar0gANs71vm9Bd_Cj87UG1K3ljTURbkEjG3JLsrY,7616
 modal/token_flow.pyi,sha256=eirYjyqbRiT3GCKMIPHJPpkvBTu8WxDKqSHehWaJI_4,2533
-modal/volume.py,sha256=b3VYD-0D8rZWxoeIIYpjqm2lgxrVOjFEgTyW2btUVnw,45396
+modal/volume.py,sha256=6sMyykbz9lvwzClAOW9Pdbl9naXo9CipYf65t-eJdrs,45418
 modal/volume.pyi,sha256=lMXzeyeC85ji8g2j0Ghy1WQrk2A2J0LPVpLFpabbr6A,41933
 modal/_runtime/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
 modal/_runtime/asgi.py,sha256=_2xSTsDD27Cit7xnMs4lzkJA2wzer2_N4Oa3BkXFzVA,22521
@@ -91,7 +91,7 @@ modal/_runtime/telemetry.py,sha256=T1RoAGyjBDr1swiM6pPsGRSITm7LI5FDK18oNXxY08U,5
 modal/_runtime/user_code_imports.py,sha256=78wJyleqY2RVibqcpbDQyfWVBVT9BjyHPeoV9WdwV5Y,17720
 modal/_utils/__init__.py,sha256=waLjl5c6IPDhSsdWAm9Bji4e2PVxamYABKAze6CHVXY,28
 modal/_utils/app_utils.py,sha256=88BT4TPLWfYAQwKTHcyzNQRHg8n9B-QE2UyJs96iV-0,108
-modal/_utils/async_utils.py,sha256=ot8NiPGZ5bRJhY5ilZyjNgx24VI-1BIpCu054oLHDf0,29556
+modal/_utils/async_utils.py,sha256=7uA4KJV7XRgak5nXZSGRE-RN1h91UOyNwK6v_ilUQMQ,29737
 modal/_utils/auth_token_manager.py,sha256=i-kfLgDd4BMAw6wouO5aKfNGHo27VAZoVOsbEWqDr2I,5252
 modal/_utils/blob_utils.py,sha256=bySVr9M7hlFzZo-u4ikovxMdcdEE8yfGOs94Zex2k4o,20913
 modal/_utils/bytes_io_segment_payload.py,sha256=vaXPq8b52-x6G2hwE7SrjS58pg_aRm7gV3bn3yjmTzQ,4261
@@ -111,7 +111,7 @@ modal/_utils/package_utils.py,sha256=LcL2olGN4xaUzu2Tbv-C-Ft9Qp6bsLxEfETOAVd-mjU
 modal/_utils/pattern_utils.py,sha256=ZUffaECfe2iYBhH6cvCB-0-UWhmEBTZEl_TwG_So3ag,6714
 modal/_utils/rand_pb_testing.py,sha256=mmVPk1rZldHwHZx0DnHTuHQlRLAiiAYdxjwEJpxvT9c,3900
 modal/_utils/shell_utils.py,sha256=hWHzv730Br2Xyj6cGPiMZ-198Z3RZuOu3pDXhFSZ22c,2157
-modal/_utils/time_utils.py,sha256=THhRz59gez8jNV1B_eNS2gJJVPPGQSFVlr1esBGQoqg,494
+modal/_utils/time_utils.py,sha256=Un_nCG9ZXPMPKK5kJayrFVl1eFckVikPyqrWtI2553M,553
 modal/_vendor/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
 modal/_vendor/a2wsgi_wsgi.py,sha256=Q1AsjpV_Q_vzQsz_cSqmP9jWzsGsB-ARFU6vpQYml8k,21878
 modal/_vendor/cloudpickle.py,sha256=avxOIgNKqL9KyPNuIOVQzBm0D1l9ipeB4RrcUMUGmeQ,55216
@@ -127,23 +127,23 @@ modal/builder/base-images.json,sha256=JYSDAgHTl-WrV_TZW5icY-IJEnbe2eQ4CZ_KN6EOZK
 modal/cli/__init__.py,sha256=6FRleWQxBDT19y7OayO4lBOzuL6Bs9r0rLINYYYbHwQ,769
 modal/cli/_download.py,sha256=tV8JFkncTtQKh85bSguQg6AW5aRRlynf-rvyN7ruigc,4337
 modal/cli/_traceback.py,sha256=IKj9xtc6LjAxyhGJWolNIXEX3MhAIulnRqywZNOFmkU,7324
-modal/cli/app.py,sha256=Q4yoPGuNqdWMwIIbjJQflp9RvmgNQQRWBNhCg_Cvi9g,7800
-modal/cli/cluster.py,sha256=GZniNlzH1QMlHNpzGurfEdxNqwaH0BznMc4i-j7uhe4,3152
+modal/cli/app.py,sha256=rbuAG92my-1eZN0olk6p2eD4oBnyBliUsrCOUW-U-9k,7832
+modal/cli/cluster.py,sha256=8pQurDUvLP_HdSeHH5ZB6WIoDh48FR8qP9vGOtSsFXI,3168
 modal/cli/config.py,sha256=lhp2Pq4RbTDhaZJ-ZJvhrMqJj8c-WjuRX6gjE3TrvXc,1691
-modal/cli/container.py,sha256=mRYRCGsP6DiWzm3Az4W5Fcc5Tbl58zOIc62HDzS9TvQ,3703
-modal/cli/dict.py,sha256=_UiF8G2aRJwZDNSFYiHFt_Xy8K9Gtrx4qKzbRgHri5I,4639
+modal/cli/container.py,sha256=9Ti-TIZ6vjDSmn9mk9h6SRwyhkQjtwirBN18LjpLyvE,3719
+modal/cli/dict.py,sha256=XsySnxSOcfF9ZehHO3whRgFuxZGGNE_I87Hiye36wE4,4655
 modal/cli/entry_point.py,sha256=M9ZeIsYx7rxdc6XP2iOIptVzmpj39D3rU8nfW7Dc3CQ,4388
 modal/cli/environment.py,sha256=Ayddkiq9jdj3XYDJ8ZmUqFpPPH8xajYlbexRkzGtUcg,4334
 modal/cli/import_refs.py,sha256=X59Z5JwgliRO6C-cIFto2Pr7o3SwlZMKQPKA0aI4ZK4,13927
 modal/cli/launch.py,sha256=0_sBu6bv2xJEPWi-rbGS6Ri9ggnkWQvrGlgpYSUBMyY,3097
-modal/cli/network_file_system.py,sha256=hj_Cfh-xjz9uWdrXrZxxPScepaZYZjJ7aHa5B93pBGM,8110
+modal/cli/network_file_system.py,sha256=I9IqTpVfk32uKYwGd8LTldkQx6UKYrQYNZ26q7Ab5Oo,8126
 modal/cli/profile.py,sha256=r5hnA_GPe_2zwgv6n0Mi8XQXyejQgShb17yjD4dPXcw,3212
-modal/cli/queues.py,sha256=As6WXkkqWbicu52uljqbmC86woejcpTdzSjU-hYpU24,4549
+modal/cli/queues.py,sha256=6Ck7B-Eu3ZEV6oOX0GxnVMQ5k9DWrIZPYXxaHVqUhKU,4565
 modal/cli/run.py,sha256=96m6fpJKbjtva4xzJut0pxS36Z5WCMq0umpAry96im0,24946
-modal/cli/secret.py,sha256=bxp4qeooYUgKoSOgiBC86lw_5niZhw56tSzX7tuICGQ,6622
+modal/cli/secret.py,sha256=PqAvgaiIHTYDbXsKLLQcDvwQn283QnmtgfIr5jH4Prw,6646
 modal/cli/token.py,sha256=NAmQzKBfEHkcldWKeFxAVIqQBoo1RTp7_A4yc7-8qM0,1911
 modal/cli/utils.py,sha256=aUXDU9_VgcJrGaGRy4bGf4dqwKYXHCpoO27x4m_bpuo,3293
-modal/cli/volume.py,sha256=Ju9Hy6UkSe4XGThuMOGSFq9Ba-t10rx8YqXM5Po7sP0,10961
+modal/cli/volume.py,sha256=L4ryL-_yjgUyr0Zwy390DYsFCLtMxeOW2elm7J76y7w,10985
 modal/cli/programs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal/cli/programs/run_jupyter.py,sha256=44Lpvqk2l3hH-uOkmAOzw60NEsfB5uaRDWDKVshvQhs,2682
 modal/cli/programs/vscode.py,sha256=KbTAaIXyQBVCDXxXjmBHmKpgXkUw0q4R4KkJvUjCYgk,3380
@@ -151,7 +151,7 @@ modal/experimental/__init__.py,sha256=nuc7AL4r_Fs08DD5dciWFZhrV1nanwoClOfdTcudU0
 modal/experimental/flash.py,sha256=viXQumCIFp5VFsPFURdFTBTjP_QnsAi8nSWXAMmfjeQ,19744
 modal/experimental/flash.pyi,sha256=A8_qJGtGoXEzKDdHbvhmCw7oqfneFEvJQK3ZdTOvUdU,10830
 modal/experimental/ipython.py,sha256=TrCfmol9LGsRZMeDoeMPx3Hv3BFqQhYnmD_iH0pqdhk,2904
-modal-1.1.1.dev38.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-1.1.1.dev40.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
 modal_docs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal_docs/gen_cli_docs.py,sha256=c1yfBS_x--gL5bs0N4ihMwqwX8l3IBWSkBAKNNIi6bQ,3801
 modal_docs/gen_reference_docs.py,sha256=d_CQUGQ0rfw28u75I2mov9AlS773z9rG40-yq5o7g2U,6359
@@ -174,10 +174,10 @@ modal_proto/options_pb2.pyi,sha256=l7DBrbLO7q3Ir-XDkWsajm0d0TQqqrfuX54i4BMpdQg,1
 modal_proto/options_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0yJSI,247
 modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-modal_version/__init__.py,sha256=ZJUyKWUb4kha2XnqTQCiSenTliHEzKuLWSwDrkH8iqQ,121
+modal_version/__init__.py,sha256=yKVPZgQsYtXUtD_7xH36mRiDGgXyJAA7FA7cDCXtMQI,121
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal-1.1.1.dev38.dist-info/METADATA,sha256=DfDzvLTc_tYh0IubHK1YY3YWgYsRtuBpsIBoDT-Ky6A,2460
-modal-1.1.1.dev38.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-modal-1.1.1.dev38.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-1.1.1.dev38.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
-modal-1.1.1.dev38.dist-info/RECORD,,
+modal-1.1.1.dev40.dist-info/METADATA,sha256=G_cZZm2OijpQdzkE97eYr1DIr8doI9Q1_4VSQsl37rA,2460
+modal-1.1.1.dev40.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+modal-1.1.1.dev40.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-1.1.1.dev40.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
+modal-1.1.1.dev40.dist-info/RECORD,,

modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2025
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.1.1.dev38"
+__version__ = "1.1.1.dev40"