modal 1.0.5.dev1__py3-none-any.whl → 1.0.5.dev3__py3-none-any.whl

modal/network_file_system.pyi

@@ -14,17 +14,84 @@ def network_file_system_mount_protos(
 ) -> list[modal_proto.api_pb2.SharedVolumeMount]: ...
 
 class _NetworkFileSystem(modal._object._Object):
+    """A shared, writable file system accessible by one or more Modal functions.
+
+    By attaching this file system as a mount to one or more functions, they can
+    share and persist data with each other.
+
+    **Usage**
+
+    ```python
+    import modal
+
+    nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
+    app = modal.App()
+
+    @app.function(network_file_systems={"/root/foo": nfs})
+    def f():
+        pass
+
+    @app.function(network_file_systems={"/root/goo": nfs})
+    def g():
+        pass
+    ```
+
+    Also see the CLI methods for accessing network file systems:
+
+    ```
+    modal nfs --help
+    ```
+
+    A `NetworkFileSystem` can also be useful for some local scripting scenarios, e.g.:
+
+    ```python notest
+    nfs = modal.NetworkFileSystem.from_name("my-network-file-system")
+    for chunk in nfs.read_file("my_db_dump.csv"):
+        ...
+    ```
+    """
     @staticmethod
     def from_name(
         name: str, *, namespace=1, environment_name: typing.Optional[str] = None, create_if_missing: bool = False
-    ) -> _NetworkFileSystem: ...
+    ) -> _NetworkFileSystem:
+        """Reference a NetworkFileSystem by its name, creating if necessary.
+
+        In contrast to `modal.NetworkFileSystem.lookup`, this is a lazy method
+        that defers hydrating the local object with metadata from Modal servers
+        until the first time it is actually used.
+
+        ```python notest
+        nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
+
+        @app.function(network_file_systems={"/data": nfs})
+        def f():
+            pass
+        ```
+        """
+        ...
+
     @classmethod
     def ephemeral(
         cls: type[_NetworkFileSystem],
         client: typing.Optional[modal.client._Client] = None,
         environment_name: typing.Optional[str] = None,
         _heartbeat_sleep: float = 300,
-    ) -> typing.AsyncContextManager[_NetworkFileSystem]: ...
+    ) -> typing.AsyncContextManager[_NetworkFileSystem]:
+        """Creates a new ephemeral network filesystem within a context manager:
+
+        Usage:
+        ```python
+        with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert nfs.listdir("/") == []
+        ```
+
+        ```python notest
+        async with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert await nfs.listdir("/") == []
+        ```
+        """
+        ...
+
     @staticmethod
     async def lookup(
         name: str,
@@ -32,14 +99,32 @@ class _NetworkFileSystem(modal._object._Object):
         client: typing.Optional[modal.client._Client] = None,
         environment_name: typing.Optional[str] = None,
         create_if_missing: bool = False,
-    ) -> _NetworkFileSystem: ...
+    ) -> _NetworkFileSystem:
+        """mdmd:hidden
+        Lookup a named NetworkFileSystem.
+
+        DEPRECATED: This method is deprecated in favor of `modal.NetworkFileSystem.from_name`.
+
+        In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
+        that will hydrate the local object with metadata from Modal servers.
+
+        ```python notest
+        nfs = modal.NetworkFileSystem.lookup("my-nfs")
+        print(nfs.listdir("/"))
+        ```
+        """
+        ...
+
     @staticmethod
     async def create_deployed(
         deployment_name: str,
         namespace=1,
         client: typing.Optional[modal.client._Client] = None,
         environment_name: typing.Optional[str] = None,
-    ) -> str: ...
+    ) -> str:
+        """mdmd:hidden"""
+        ...
+
     @staticmethod
     async def delete(
         name: str, client: typing.Optional[modal.client._Client] = None, environment_name: typing.Optional[str] = None
@@ -49,9 +134,30 @@ class _NetworkFileSystem(modal._object._Object):
         remote_path: str,
         fp: typing.BinaryIO,
         progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
-    ) -> int: ...
-    def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
-    def iterdir(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
+    ) -> int:
+        """Write from a file object to a path on the network file system, atomically.
+
+        Will create any needed parent directories automatically.
+
+        If remote_path ends with `/` it's assumed to be a directory and the
+        file will be uploaded with its current name to that directory.
+        """
+        ...
+
+    def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]:
+        """Read a file from the network file system"""
+        ...
+
+    def iterdir(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]:
+        """Iterate over all files in a directory in the network file system.
+
+        * Passing a directory path lists all files in the directory (names are relative to the directory)
+        * Passing a file path returns a list containing only that file's listing description
+        * Passing a glob path (including at least one * or ** sequence) returns all files matching
+        that glob path (using absolute paths)
+        """
+        ...
+
     async def add_local_file(
         self,
         local_path: typing.Union[pathlib.Path, str],
@@ -64,24 +170,104 @@ class _NetworkFileSystem(modal._object._Object):
         remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
         progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
     ): ...
-    async def listdir(self, path: str) -> list[modal.volume.FileEntry]: ...
-    async def remove_file(self, path: str, recursive=False): ...
+    async def listdir(self, path: str) -> list[modal.volume.FileEntry]:
+        """List all files in a directory in the network file system.
+
+        * Passing a directory path lists all files in the directory (names are relative to the directory)
+        * Passing a file path returns a list containing only that file's listing description
+        * Passing a glob path (including at least one * or ** sequence) returns all files matching
+        that glob path (using absolute paths)
+        """
+        ...
+
+    async def remove_file(self, path: str, recursive=False):
+        """Remove a file in a network file system."""
+        ...
 
 SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
 
 class NetworkFileSystem(modal.object.Object):
-    def __init__(self, *args, **kwargs): ...
+    """A shared, writable file system accessible by one or more Modal functions.
+
+    By attaching this file system as a mount to one or more functions, they can
+    share and persist data with each other.
+
+    **Usage**
+
+    ```python
+    import modal
+
+    nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
+    app = modal.App()
+
+    @app.function(network_file_systems={"/root/foo": nfs})
+    def f():
+        pass
+
+    @app.function(network_file_systems={"/root/goo": nfs})
+    def g():
+        pass
+    ```
+
+    Also see the CLI methods for accessing network file systems:
+
+    ```
+    modal nfs --help
+    ```
+
+    A `NetworkFileSystem` can also be useful for some local scripting scenarios, e.g.:
+
+    ```python notest
+    nfs = modal.NetworkFileSystem.from_name("my-network-file-system")
+    for chunk in nfs.read_file("my_db_dump.csv"):
+        ...
+    ```
+    """
+    def __init__(self, *args, **kwargs):
+        """mdmd:hidden"""
+        ...
+
     @staticmethod
     def from_name(
         name: str, *, namespace=1, environment_name: typing.Optional[str] = None, create_if_missing: bool = False
-    ) -> NetworkFileSystem: ...
+    ) -> NetworkFileSystem:
+        """Reference a NetworkFileSystem by its name, creating if necessary.
+
+        In contrast to `modal.NetworkFileSystem.lookup`, this is a lazy method
+        that defers hydrating the local object with metadata from Modal servers
+        until the first time it is actually used.
+
+        ```python notest
+        nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
+
+        @app.function(network_file_systems={"/data": nfs})
+        def f():
+            pass
+        ```
+        """
+        ...
+
     @classmethod
     def ephemeral(
         cls: type[NetworkFileSystem],
         client: typing.Optional[modal.client.Client] = None,
         environment_name: typing.Optional[str] = None,
         _heartbeat_sleep: float = 300,
-    ) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]: ...
+    ) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]:
+        """Creates a new ephemeral network filesystem within a context manager:
+
+        Usage:
+        ```python
+        with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert nfs.listdir("/") == []
+        ```
+
+        ```python notest
+        async with modal.NetworkFileSystem.ephemeral() as nfs:
+            assert await nfs.listdir("/") == []
+        ```
+        """
+        ...
 
     class __lookup_spec(typing_extensions.Protocol):
         def __call__(
@@ -92,7 +278,22 @@ class NetworkFileSystem(modal.object.Object):
             client: typing.Optional[modal.client.Client] = None,
             environment_name: typing.Optional[str] = None,
             create_if_missing: bool = False,
-        ) -> NetworkFileSystem: ...
+        ) -> NetworkFileSystem:
+            """mdmd:hidden
+            Lookup a named NetworkFileSystem.
+
+            DEPRECATED: This method is deprecated in favor of `modal.NetworkFileSystem.from_name`.
+
+            In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
+            that will hydrate the local object with metadata from Modal servers.
+
+            ```python notest
+            nfs = modal.NetworkFileSystem.lookup("my-nfs")
+            print(nfs.listdir("/"))
+            ```
+            """
+            ...
+
         async def aio(
             self,
             /,
@@ -101,7 +302,21 @@ class NetworkFileSystem(modal.object.Object):
             client: typing.Optional[modal.client.Client] = None,
             environment_name: typing.Optional[str] = None,
             create_if_missing: bool = False,
-        ) -> NetworkFileSystem: ...
+        ) -> NetworkFileSystem:
+            """mdmd:hidden
+            Lookup a named NetworkFileSystem.
+
+            DEPRECATED: This method is deprecated in favor of `modal.NetworkFileSystem.from_name`.
+
+            In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
+            that will hydrate the local object with metadata from Modal servers.
+
+            ```python notest
+            nfs = modal.NetworkFileSystem.lookup("my-nfs")
+            print(nfs.listdir("/"))
+            ```
+            """
+            ...
 
     lookup: __lookup_spec
 
@@ -113,7 +328,10 @@ class NetworkFileSystem(modal.object.Object):
             namespace=1,
             client: typing.Optional[modal.client.Client] = None,
             environment_name: typing.Optional[str] = None,
-        ) -> str: ...
+        ) -> str:
+            """mdmd:hidden"""
+            ...
+
         async def aio(
             self,
             /,
@@ -121,7 +339,9 @@ class NetworkFileSystem(modal.object.Object):
             namespace=1,
             client: typing.Optional[modal.client.Client] = None,
             environment_name: typing.Optional[str] = None,
-        ) -> str: ...
+        ) -> str:
+            """mdmd:hidden"""
+            ...
 
     create_deployed: __create_deployed_spec
 
@@ -150,26 +370,65 @@ class NetworkFileSystem(modal.object.Object):
             remote_path: str,
             fp: typing.BinaryIO,
             progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
-        ) -> int: ...
+        ) -> int:
+            """Write from a file object to a path on the network file system, atomically.
+
+            Will create any needed parent directories automatically.
+
+            If remote_path ends with `/` it's assumed to be a directory and the
+            file will be uploaded with its current name to that directory.
+            """
+            ...
+
         async def aio(
             self,
             /,
             remote_path: str,
             fp: typing.BinaryIO,
             progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
-        ) -> int: ...
+        ) -> int:
+            """Write from a file object to a path on the network file system, atomically.
+
+            Will create any needed parent directories automatically.
+
+            If remote_path ends with `/` it's assumed to be a directory and the
+            file will be uploaded with its current name to that directory.
+            """
+            ...
 
     write_file: __write_file_spec[typing_extensions.Self]
 
     class __read_file_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, path: str) -> typing.Iterator[bytes]: ...
-        def aio(self, /, path: str) -> collections.abc.AsyncIterator[bytes]: ...
+        def __call__(self, /, path: str) -> typing.Iterator[bytes]:
+            """Read a file from the network file system"""
+            ...
+
+        def aio(self, /, path: str) -> collections.abc.AsyncIterator[bytes]:
+            """Read a file from the network file system"""
+            ...
 
     read_file: __read_file_spec[typing_extensions.Self]
 
     class __iterdir_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, path: str) -> typing.Iterator[modal.volume.FileEntry]: ...
-        def aio(self, /, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
+        def __call__(self, /, path: str) -> typing.Iterator[modal.volume.FileEntry]:
+            """Iterate over all files in a directory in the network file system.
+
+            * Passing a directory path lists all files in the directory (names are relative to the directory)
+            * Passing a file path returns a list containing only that file's listing description
+            * Passing a glob path (including at least one * or ** sequence) returns all files matching
+            that glob path (using absolute paths)
+            """
+            ...
+
+        def aio(self, /, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]:
+            """Iterate over all files in a directory in the network file system.
+
+            * Passing a directory path lists all files in the directory (names are relative to the directory)
+            * Passing a file path returns a list containing only that file's listing description
+            * Passing a glob path (including at least one * or ** sequence) returns all files matching
+            that glob path (using absolute paths)
+            """
+            ...
 
     iterdir: __iterdir_spec[typing_extensions.Self]
 
@@ -210,13 +469,35 @@ class NetworkFileSystem(modal.object.Object):
     add_local_dir: __add_local_dir_spec[typing_extensions.Self]
 
     class __listdir_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, path: str) -> list[modal.volume.FileEntry]: ...
-        async def aio(self, /, path: str) -> list[modal.volume.FileEntry]: ...
+        def __call__(self, /, path: str) -> list[modal.volume.FileEntry]:
+            """List all files in a directory in the network file system.
+
+            * Passing a directory path lists all files in the directory (names are relative to the directory)
+            * Passing a file path returns a list containing only that file's listing description
+            * Passing a glob path (including at least one * or ** sequence) returns all files matching
+            that glob path (using absolute paths)
+            """
+            ...
+
+        async def aio(self, /, path: str) -> list[modal.volume.FileEntry]:
+            """List all files in a directory in the network file system.
+
+            * Passing a directory path lists all files in the directory (names are relative to the directory)
+            * Passing a file path returns a list containing only that file's listing description
+            * Passing a glob path (including at least one * or ** sequence) returns all files matching
+            that glob path (using absolute paths)
+            """
+            ...
 
     listdir: __listdir_spec[typing_extensions.Self]
 
     class __remove_file_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, path: str, recursive=False): ...
-        async def aio(self, /, path: str, recursive=False): ...
+        def __call__(self, /, path: str, recursive=False):
+            """Remove a file in a network file system."""
+            ...
+
+        async def aio(self, /, path: str, recursive=False):
+            """Remove a file in a network file system."""
+            ...
 
     remove_file: __remove_file_spec[typing_extensions.Self]

modal/object.pyi

@@ -32,7 +32,10 @@ class Object:
     _is_hydrated: bool
     _is_rehydrated: bool
 
-    def __init__(self, *args, **kwargs): ...
+    def __init__(self, *args, **kwargs):
+        """mdmd:hidden"""
+        ...
+
     @classmethod
     def __init_subclass__(cls, type_prefix: typing.Optional[str] = None): ...
 
@@ -85,7 +88,10 @@ class Object:
     def _hydrate_metadata(self, metadata: typing.Optional[google.protobuf.message.Message]): ...
     def _get_metadata(self) -> typing.Optional[google.protobuf.message.Message]: ...
     def _validate_is_hydrated(self): ...
-    def clone(self) -> typing_extensions.Self: ...
+    def clone(self) -> typing_extensions.Self:
+        """mdmd:hidden Clone a given hydrated object."""
+        ...
+
     @classmethod
     def _from_loader(
         cls,
@@ -114,18 +120,51 @@ class Object:
     def _hydrate_from_other(self, other: typing_extensions.Self): ...
     def __repr__(self): ...
     @property
-    def local_uuid(self): ...
+    def local_uuid(self):
+        """mdmd:hidden"""
+        ...
+
     @property
-    def object_id(self) -> str: ...
+    def object_id(self) -> str:
+        """mdmd:hidden"""
+        ...
+
     @property
-    def client(self) -> modal.client.Client: ...
+    def client(self) -> modal.client.Client:
+        """mdmd:hidden"""
+        ...
+
     @property
-    def is_hydrated(self) -> bool: ...
+    def is_hydrated(self) -> bool:
+        """mdmd:hidden"""
+        ...
+
     @property
-    def deps(self) -> collections.abc.Callable[..., collections.abc.Sequence[Object]]: ...
+    def deps(self) -> collections.abc.Callable[..., collections.abc.Sequence[Object]]:
+        """mdmd:hidden"""
+        ...
 
     class __hydrate_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF: ...
-        async def aio(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF: ...
+        def __call__(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF:
+            """Synchronize the local object with its identity on the Modal server.
+
+            It is rarely necessary to call this method explicitly, as most operations
+            will lazily hydrate when needed. The main use case is when you need to
+            access object metadata, such as its ID.
+
+            *Added in v0.72.39*: This method replaces the deprecated `.resolve()` method.
+            """
+            ...
+
+        async def aio(self, /, client: typing.Optional[modal.client.Client] = None) -> SUPERSELF:
+            """Synchronize the local object with its identity on the Modal server.
+
+            It is rarely necessary to call this method explicitly, as most operations
+            will lazily hydrate when needed. The main use case is when you need to
+            access object metadata, such as its ID.
+
+            *Added in v0.72.39*: This method replaces the deprecated `.resolve()` method.
+            """
+            ...
 
     hydrate: __hydrate_spec[typing_extensions.Self]

modal/parallel_map.py

@@ -9,6 +9,7 @@ from typing import Any, Callable, Optional
 
 from grpclib import Status
 
+import modal.exception
 from modal._runtime.execution_context import current_input_id
 from modal._utils.async_utils import (
     AsyncOrSyncIterable,
@@ -89,6 +90,7 @@ async def _map_invocation(
     client: "modal.client._Client",
     order_outputs: bool,
     return_exceptions: bool,
+    wrap_returned_exceptions: bool,
     count_update_callback: Optional[Callable[[int, int], None]],
     function_call_invocation_type: "api_pb2.FunctionCallInvocationType.ValueType",
 ):
@@ -341,7 +343,13 @@ async def _map_invocation(
             output = await _process_result(item.result, item.data_format, client.stub, client)
         except Exception as e:
             if return_exceptions:
-                output = e
+                if wrap_returned_exceptions:
+                    # Prior to client 1.0.4 there was a bug where return_exceptions would wrap
+                    # any returned exceptions in a synchronicity.UserCodeException. This adds
+                    # deprecated non-breaking compatibility bandaid for migrating away from that:
+                    output = modal.exception.UserCodeException(e)
+                else:
+                    output = e
             else:
                 raise e
         return (item.idx, output)
@@ -413,6 +421,7 @@ async def _map_helper(
     kwargs={},  # any extra keyword arguments for the function
     order_outputs: bool = True,  # return outputs in order
     return_exceptions: bool = False,  # propagate exceptions (False) or aggregate them in the results list (True)
+    wrap_returned_exceptions: bool = True,
 ) -> typing.AsyncGenerator[Any, None]:
     """Core implementation that supports `_map_async()`, `_starmap_async()` and `_for_each_async()`.
 
@@ -443,7 +452,9 @@ async def _map_helper(
     # synchronicity-wrapped, since they accept executable code in the form of iterators that we don't want to run inside
     # the synchronicity thread. Instead, we delegate to `._map()` with a safer Queue as input.
     async with aclosing(
-        async_merge(self._map.aio(raw_input_queue, order_outputs, return_exceptions), feed_queue())
+        async_merge(
+            self._map.aio(raw_input_queue, order_outputs, return_exceptions, wrap_returned_exceptions), feed_queue()
+        )
     ) as map_output_stream:
         async for output in map_output_stream:
             yield output
@@ -458,10 +469,16 @@ async def _map_async(
     kwargs={},  # any extra keyword arguments for the function
     order_outputs: bool = True,  # return outputs in order
     return_exceptions: bool = False,  # propagate exceptions (False) or aggregate them in the results list (True)
+    wrap_returned_exceptions: bool = True,  # wrap returned exceptions in modal.exception.UserCodeException
 ) -> typing.AsyncGenerator[Any, None]:
     async_input_gen = async_zip(*[sync_or_async_iter(it) for it in input_iterators])
     async for output in _map_helper(
-        self, async_input_gen, kwargs=kwargs, order_outputs=order_outputs, return_exceptions=return_exceptions
+        self,
+        async_input_gen,
+        kwargs=kwargs,
+        order_outputs=order_outputs,
+        return_exceptions=return_exceptions,
+        wrap_returned_exceptions=wrap_returned_exceptions,
     ):
         yield output
 
@@ -474,6 +491,7 @@ async def _starmap_async(
     kwargs={},
     order_outputs: bool = True,
     return_exceptions: bool = False,
+    wrap_returned_exceptions: bool = True,
 ) -> typing.AsyncIterable[Any]:
     async for output in _map_helper(
         self,
@@ -481,6 +499,7 @@ async def _starmap_async(
         kwargs=kwargs,
         order_outputs=order_outputs,
         return_exceptions=return_exceptions,
+        wrap_returned_exceptions=wrap_returned_exceptions,
     ):
         yield output
 
@@ -502,6 +521,7 @@ def _map_sync(
     kwargs={},  # any extra keyword arguments for the function
     order_outputs: bool = True,  # return outputs in order
     return_exceptions: bool = False,  # propagate exceptions (False) or aggregate them in the results list (True)
+    wrap_returned_exceptions: bool = True,
 ) -> AsyncOrSyncIterable:
     """Parallel map over a set of inputs.
 
@@ -542,7 +562,12 @@ def _map_sync(
 
     return AsyncOrSyncIterable(
         _map_async(
-            self, *input_iterators, kwargs=kwargs, order_outputs=order_outputs, return_exceptions=return_exceptions
+            self,
+            *input_iterators,
+            kwargs=kwargs,
+            order_outputs=order_outputs,
+            return_exceptions=return_exceptions,
+            wrap_returned_exceptions=wrap_returned_exceptions,
         ),
         nested_async_message=(
             "You can't iter(Function.map()) from an async function. Use async for ... in Function.map.aio() instead."
@@ -622,6 +647,7 @@ def _starmap_sync(
     kwargs={},
     order_outputs: bool = True,
     return_exceptions: bool = False,
+    wrap_returned_exceptions: bool = True,
 ) -> AsyncOrSyncIterable:
     """Like `map`, but spreads arguments over multiple function arguments.
 
@@ -641,7 +667,12 @@ def _starmap_sync(
     """
     return AsyncOrSyncIterable(
         _starmap_async(
-            self, input_iterator, kwargs=kwargs, order_outputs=order_outputs, return_exceptions=return_exceptions
+            self,
+            input_iterator,
+            kwargs=kwargs,
+            order_outputs=order_outputs,
+            return_exceptions=return_exceptions,
+            wrap_returned_exceptions=wrap_returned_exceptions,
         ),
         nested_async_message=(
             "You can't `iter(Function.starmap())` from an async function. "
@@ -764,11 +795,12 @@ class _MapItemContext:
             delay_ms = 0
 
         # None means the maximum number of retries has been reached, so output the error
-        if delay_ms is None:
+        if delay_ms is None or item.result.status == api_pb2.GenericResult.GENERIC_STATUS_TERMINATED:
             self.state = _MapItemState.COMPLETE
             return _OutputType.FAILED_COMPLETION
 
         self.state = _MapItemState.WAITING_TO_RETRY
+
         await retry_queue.put(now_seconds + (delay_ms / 1000), item.idx)
 
         return _OutputType.RETRYING