modal 1.0.6.dev58__py3-none-any.whl → 1.2.3.dev7__py3-none-any.whl

modal/queue.py

@@ -3,22 +3,213 @@ import queue  # The system library
 import time
 import warnings
 from collections.abc import AsyncGenerator, AsyncIterator
-from typing import Any, Optional
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Optional, Union
 
+from google.protobuf.message import Message
 from grpclib import GRPCError, Status
+from synchronicity import classproperty
 from synchronicity.async_wrap import asynccontextmanager
 
 from modal_proto import api_pb2
 
-from ._object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
+from ._load_context import LoadContext
+from ._object import (
+    EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
+    _get_environment_name,
+    _Object,
+    live_method,
+    live_method_gen,
+)
 from ._resolver import Resolver
 from ._serialization import deserialize, serialize
 from ._utils.async_utils import TaskContext, synchronize_api, warn_if_generator_is_not_consumed
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
-from ._utils.grpc_utils import retry_transient_errors
+from ._utils.grpc_utils import Retry
 from ._utils.name_utils import check_object_name
+from ._utils.time_utils import as_timestamp, timestamp_to_localized_dt
 from .client import _Client
-from .exception import InvalidError, RequestSizeError
+from .exception import AlreadyExistsError, InvalidError, NotFoundError, RequestSizeError
+
+
+@dataclass
+class QueueInfo:
+    """Information about the Queue object."""
+
+    # This dataclass should be limited to information that is unchanging over the lifetime of the Queue,
+    # since it is transmitted from the server when the object is hydrated and could be stale when accessed.
+
+    name: Optional[str]
+    created_at: datetime
+    created_by: Optional[str]
+
+
+class _QueueManager:
+    """Namespace with methods for managing named Queue objects."""
+
+    @staticmethod
+    async def create(
+        name: str,  # Name to use for the new Queue
+        *,
+        allow_existing: bool = False,  # If True, no-op when the Queue already exists
+        environment_name: Optional[str] = None,  # Uses active environment if not specified
+        client: Optional[_Client] = None,  # Optional client with Modal credentials
+    ) -> None:
+        """Create a new Queue object.
+
+        **Examples:**
+
+        ```python notest
+        modal.Queue.objects.create("my-queue")
+        ```
+
+        Queues will be created in the active environment, or another one can be specified:
+
+        ```python notest
+        modal.Queue.objects.create("my-queue", environment_name="dev")
+        ```
+
+        By default, an error will be raised if the Queue already exists, but passing
+        `allow_existing=True` will make the creation attempt a no-op in this case.
+
+        ```python notest
+        modal.Queue.objects.create("my-queue", allow_existing=True)
+        ```
+
+        Note that this method does not return a local instance of the Queue. You can use
+        `modal.Queue.from_name` to perform a lookup after creation.
+
+        Added in v1.1.2.
+
+        """
+        check_object_name(name, "Queue")
+        client = await _Client.from_env() if client is None else client
+        object_creation_type = (
+            api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING
+            if allow_existing
+            else api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS
+        )
+        req = api_pb2.QueueGetOrCreateRequest(
+            deployment_name=name,
+            environment_name=_get_environment_name(environment_name),
+            object_creation_type=object_creation_type,
+        )
+        try:
+            await client.stub.QueueGetOrCreate(req)
+        except GRPCError as exc:
+            if exc.status == Status.ALREADY_EXISTS and not allow_existing:
+                raise AlreadyExistsError(exc.message)
+            else:
+                raise
+
+    @staticmethod
+    async def list(
+        *,
+        max_objects: Optional[int] = None,  # Limit results to this size
+        created_before: Optional[Union[datetime, str]] = None,  # Limit based on creation date
+        environment_name: str = "",  # Uses active environment if not specified
+        client: Optional[_Client] = None,  # Optional client with Modal credentials
+    ) -> list["_Queue"]:
+        """Return a list of hydrated Queue objects.
+
+        **Examples:**
+
+        ```python
+        queues = modal.Queue.objects.list()
+        print([q.name for q in queues])
+        ```
+
+        Queues will be retreived from the active environment, or another one can be specified:
+
+        ```python notest
+        dev_queues = modal.Queue.objects.list(environment_name="dev")
+        ```
+
+        By default, all named Queues are returned, newest to oldest. It's also possible to limit the
+        number of results and to filter by creation date:
+
+        ```python
+        queues = modal.Queue.objects.list(max_objects=10, created_before="2025-01-01")
+        ```
+
+        Added in v1.1.2.
+
+        """
+        client = await _Client.from_env() if client is None else client
+        if max_objects is not None and max_objects < 0:
+            raise InvalidError("max_objects cannot be negative")
+
+        items: list[api_pb2.QueueListResponse.QueueInfo] = []
+
+        async def retrieve_page(created_before: float) -> bool:
+            max_page_size = 100 if max_objects is None else min(100, max_objects - len(items))
+            pagination = api_pb2.ListPagination(max_objects=max_page_size, created_before=created_before)
+            req = api_pb2.QueueListRequest(
+                environment_name=_get_environment_name(environment_name), pagination=pagination
+            )
+            resp = await client.stub.QueueList(req)
+            items.extend(resp.queues)
+            finished = (len(resp.queues) < max_page_size) or (max_objects is not None and len(items) >= max_objects)
+            return finished
+
+        finished = await retrieve_page(as_timestamp(created_before))
+        while True:
+            if finished:
+                break
+            finished = await retrieve_page(items[-1].metadata.creation_info.created_at)
+
+        queues = [
+            _Queue._new_hydrated(
+                item.queue_id,
+                client,
+                item.metadata,
+                is_another_app=True,
+                rep=_Queue._repr(item.name, environment_name),
+            )
+            for item in items
+        ]
+        return queues[:max_objects] if max_objects is not None else queues
+
+    @staticmethod
+    async def delete(
+        name: str,  # Name of the Queue to delete
+        *,
+        allow_missing: bool = False,  # If True, don't raise an error if the Queue doesn't exist
+        environment_name: Optional[str] = None,  # Uses active environment if not specified
+        client: Optional[_Client] = None,  # Optional client with Modal credentials
+    ):
+        """Delete a named Queue.
+
+        Warning: This deletes an *entire Queue*, not just a specific entry or partition.
+        Deletion is irreversible and will affect any Apps currently using the Queue.
+
+        **Examples:**
+
+        ```python notest
+        await modal.Queue.objects.delete("my-queue")
+        ```
+
+        Queues will be deleted from the active environment, or another one can be specified:
+
+        ```python notest
+        await modal.Queue.objects.delete("my-queue", environment_name="dev")
+        ```
+
+        Added in v1.1.2.
+
+        """
+        try:
+            obj = await _Queue.from_name(name, environment_name=environment_name).hydrate(client)
+        except NotFoundError:
+            if not allow_missing:
+                raise
+        else:
+            req = api_pb2.QueueDeleteRequest(queue_id=obj.object_id)
+            await obj._client.stub.QueueDelete(req)
+
+
+QueueManager = synchronize_api(_QueueManager)
 
 
 class _Queue(_Object, type_prefix="qu"):
@@ -94,10 +285,30 @@ class _Queue(_Object, type_prefix="qu"):
     Partition keys must be non-empty and must not exceed 64 bytes.
     """
 
+    _metadata: Optional[api_pb2.QueueMetadata] = None
+
     def __init__(self):
         """mdmd:hidden"""
         raise RuntimeError("Queue() is not allowed. Please use `Queue.from_name(...)` or `Queue.ephemeral()` instead.")
 
+    @classproperty
+    def objects(cls) -> _QueueManager:
+        return _QueueManager
+
+    @property
+    def name(self) -> Optional[str]:
+        return self._name
+
+    def _hydrate_metadata(self, metadata: Optional[Message]):
+        if metadata:
+            assert isinstance(metadata, api_pb2.QueueMetadata)
+            self._metadata = metadata
+            self._name = metadata.name
+
+    def _get_metadata(self) -> api_pb2.QueueMetadata:
+        assert self._metadata
+        return self._metadata
+
     @staticmethod
     def validate_partition_key(partition: Optional[str]) -> bytes:
         if partition is not None:
@@ -115,7 +326,7 @@ class _Queue(_Object, type_prefix="qu"):
         cls: type["_Queue"],
         client: Optional[_Client] = None,
         environment_name: Optional[str] = None,
-        _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
+        _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,  # mdmd:line-hidden
     ) -> AsyncIterator["_Queue"]:
         """Creates a new ephemeral queue within a context manager:
 
@@ -142,7 +353,7 @@ class _Queue(_Object, type_prefix="qu"):
         async with TaskContext() as tc:
             request = api_pb2.QueueHeartbeatRequest(queue_id=response.queue_id)
             tc.infinite_loop(lambda: client.stub.QueueHeartbeat(request), sleep=_heartbeat_sleep)
-            yield cls._new_hydrated(response.queue_id, client, None, is_another_app=True)
+            yield cls._new_hydrated(response.queue_id, client, response.metadata, is_another_app=True)
 
     @staticmethod
     def from_name(
@@ -151,6 +362,7 @@ class _Queue(_Object, type_prefix="qu"):
         namespace=None,  # mdmd:line-hidden
         environment_name: Optional[str] = None,
         create_if_missing: bool = False,
+        client: Optional[_Client] = None,
     ) -> "_Queue":
         """Reference a named Queue, creating if necessary.
 
@@ -166,61 +378,52 @@ class _Queue(_Object, type_prefix="qu"):
         check_object_name(name, "Queue")
         warn_if_passing_namespace(namespace, "modal.Queue.from_name")
 
-        async def _load(self: _Queue, resolver: Resolver, existing_object_id: Optional[str]):
+        async def _load(self: _Queue, resolver: Resolver, load_context: LoadContext, existing_object_id: Optional[str]):
             req = api_pb2.QueueGetOrCreateRequest(
                 deployment_name=name,
-                environment_name=_get_environment_name(environment_name, resolver),
+                environment_name=load_context.environment_name,
                 object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
             )
-            response = await resolver.client.stub.QueueGetOrCreate(req)
-            self._hydrate(response.queue_id, resolver.client, None)
-
-        return _Queue._from_loader(_load, "Queue()", is_another_app=True, hydrate_lazily=True)
+            response = await load_context.client.stub.QueueGetOrCreate(req)
+            self._hydrate(response.queue_id, load_context.client, response.metadata)
+
+        rep = _Queue._repr(name, environment_name)
+        return _Queue._from_loader(
+            _load,
+            rep,
+            is_another_app=True,
+            hydrate_lazily=True,
+            name=name,
+            load_context_overrides=LoadContext(environment_name=environment_name, client=client),
+        )
 
     @staticmethod
-    async def lookup(
-        name: str,
-        namespace=None,  # mdmd:line-hidden
-        client: Optional[_Client] = None,
-        environment_name: Optional[str] = None,
-        create_if_missing: bool = False,
-    ) -> "_Queue":
+    async def delete(name: str, *, client: Optional[_Client] = None, environment_name: Optional[str] = None):
         """mdmd:hidden
-        Lookup a named Queue.
+        Delete a named Queue.
 
-        DEPRECATED: This method is deprecated in favor of `modal.Queue.from_name`.
+        Warning: This deletes an *entire Queue*, not just a specific entry or partition.
+        Deletion is irreversible and will affect any Apps currently using the Queue.
 
-        In contrast to `modal.Queue.from_name`, this is an eager method
-        that will hydrate the local object with metadata from Modal servers.
+        DEPRECATED: This method is deprecated; we recommend using `modal.Queue.objects.delete` instead.
 
-        ```python notest
-        q = modal.Queue.lookup("my-queue")
-        q.put(123)
-        ```
         """
         deprecation_warning(
-            (2025, 1, 27),
-            "`modal.Queue.lookup` is deprecated and will be removed in a future release."
-            " It can be replaced with `modal.Queue.from_name`."
-            "\n\nSee https://modal.com/docs/guide/modal-1-0-migration for more information.",
-        )
-        warn_if_passing_namespace(namespace, "modal.Queue.lookup")
-        obj = _Queue.from_name(
-            name,
-            environment_name=environment_name,
-            create_if_missing=create_if_missing,
+            (2025, 8, 6),
+            "`modal.Queue.delete` is deprecated; we recommend using `modal.Queue.objects.delete` instead.",
         )
-        if client is None:
-            client = await _Client.from_env()
-        resolver = Resolver(client=client)
-        await resolver.load(obj)
-        return obj
+        await _Queue.objects.delete(name, environment_name=environment_name, client=client)
 
-    @staticmethod
-    async def delete(name: str, *, client: Optional[_Client] = None, environment_name: Optional[str] = None):
-        obj = await _Queue.from_name(name, environment_name=environment_name).hydrate(client)
-        req = api_pb2.QueueDeleteRequest(queue_id=obj.object_id)
-        await retry_transient_errors(obj._client.stub.QueueDelete, req)
+    @live_method
+    async def info(self) -> QueueInfo:
+        """Return information about the Queue object."""
+        metadata = self._get_metadata()
+        creation_info = metadata.creation_info
+        return QueueInfo(
+            name=metadata.name or None,
+            created_at=timestamp_to_localized_dt(creation_info.created_at),
+            created_by=creation_info.created_by or None,
+        )
 
     async def _get_nonblocking(self, partition: Optional[str], n_values: int) -> list[Any]:
         request = api_pb2.QueueGetRequest(
@@ -230,7 +433,7 @@ class _Queue(_Object, type_prefix="qu"):
             n_values=n_values,
         )
 
-        response = await retry_transient_errors(self._client.stub.QueueGet, request)
+        response = await self._client.stub.QueueGet(request)
         if response.values:
             return [deserialize(value, self._client) for value in response.values]
         else:
@@ -255,7 +458,7 @@ class _Queue(_Object, type_prefix="qu"):
                 n_values=n_values,
             )
 
-            response = await retry_transient_errors(self._client.stub.QueueGet, request)
+            response = await self._client.stub.QueueGet(request)
 
             if response.values:
                 return [deserialize(value, self._client) for value in response.values]
@@ -275,7 +478,7 @@ class _Queue(_Object, type_prefix="qu"):
             partition_key=self.validate_partition_key(partition),
             all_partitions=all,
         )
-        await retry_transient_errors(self._client.stub.QueueClear, request)
+        await self._client.stub.QueueClear(request)
 
     @live_method
     async def get(
@@ -384,14 +587,15 @@ class _Queue(_Object, type_prefix="qu"):
             partition_ttl_seconds=partition_ttl,
         )
         try:
-            await retry_transient_errors(
-                self._client.stub.QueuePut,
+            await self._client.stub.QueuePut(
                 request,
                 # A full queue will return this status.
-                additional_status_codes=[Status.RESOURCE_EXHAUSTED],
-                max_delay=30.0,
-                max_retries=None,
-                total_timeout=timeout,
+                retry=Retry(
+                    additional_status_codes=[Status.RESOURCE_EXHAUSTED],
+                    max_delay=30.0,
+                    max_retries=None,
+                    total_timeout=timeout,
+                ),
             )
         except GRPCError as exc:
             if exc.status == Status.RESOURCE_EXHAUSTED:
@@ -411,7 +615,7 @@ class _Queue(_Object, type_prefix="qu"):
             partition_ttl_seconds=partition_ttl,
         )
         try:
-            await retry_transient_errors(self._client.stub.QueuePut, request)
+            await self._client.stub.QueuePut(request)
         except GRPCError as exc:
             if exc.status == Status.RESOURCE_EXHAUSTED:
                 raise queue.Full(exc.message)
@@ -431,7 +635,7 @@ class _Queue(_Object, type_prefix="qu"):
             partition_key=self.validate_partition_key(partition),
             total=total,
         )
-        response = await retry_transient_errors(self._client.stub.QueueLen, request)
+        response = await self._client.stub.QueueLen(request)
         return response.len
 
     @warn_if_generator_is_not_consumed()
@@ -457,9 +661,7 @@ class _Queue(_Object, type_prefix="qu"):
                 item_poll_timeout=poll_duration,
             )
 
-            response: api_pb2.QueueNextItemsResponse = await retry_transient_errors(
-                self._client.stub.QueueNextItems, request
-            )
+            response: api_pb2.QueueNextItemsResponse = await self._client.stub.QueueNextItems(request)
             if response.items:
                 for item in response.items:
                     yield deserialize(item.value, self._client)