modal 1.1.2.dev11__py3-none-any.whl → 1.1.2.dev12__py3-none-any.whl

modal/_utils/time_utils.py

@@ -1,11 +1,35 @@
 # Copyright Modal Labs 2025
-from datetime import datetime
-from typing import Optional
+from datetime import datetime, tzinfo
+from typing import Optional, Union
+
+
+def locale_tz() -> tzinfo:
+    return datetime.now().astimezone().tzinfo
+
+
+def as_timestamp(arg: Optional[Union[datetime, str]]) -> float:
+    """Coerce a user-provided argument to a timestamp.
+
+    An argument provided without timezone information will be treated as local time.
+
+    When the argument is null, returns the current time.
+    """
+    if arg is None:
+        dt = datetime.now().astimezone()
+    elif isinstance(arg, str):
+        dt = datetime.fromisoformat(arg)
+    elif isinstance(arg, datetime):
+        dt = arg
+    else:
+        raise TypeError(f"Invalid argument: {arg}")
+
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=locale_tz())
+    return dt.timestamp()
 
 
 def timestamp_to_localized_dt(ts: float) -> datetime:
-    locale_tz = datetime.now().astimezone().tzinfo
-    return datetime.fromtimestamp(ts, tz=locale_tz)
+    return datetime.fromtimestamp(ts, tz=locale_tz())
 
 
 def timestamp_to_localized_str(ts: float, isotz: bool = True) -> Optional[str]:

modal/cli/dict.py

@@ -7,13 +7,11 @@ from typer import Argument, Option, Typer
 from modal._output import make_console
 from modal._resolver import Resolver
 from modal._utils.async_utils import synchronizer
-from modal._utils.grpc_utils import retry_transient_errors
 from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
 from modal.client import _Client
 from modal.dict import _Dict
 from modal.environments import ensure_env
-from modal_proto import api_pb2
 
 dict_cli = Typer(
     name="dict",
@@ -40,12 +38,13 @@ async def create(name: str, *, env: Optional[str] = ENV_OPTION):
 async def list_(*, json: bool = False, env: Optional[str] = ENV_OPTION):
     """List all named Dicts."""
     env = ensure_env(env)
-    client = await _Client.from_env()
-    request = api_pb2.DictListRequest(environment_name=env)
-    response = await retry_transient_errors(client.stub.DictList, request)
+    dicts = await _Dict.objects.list(environment_name=env)
+    rows = []
+    for obj in dicts:
+        info = await obj.info()
+        rows.append((info.name, timestamp_to_localized_str(info.created_at.timestamp(), json), info.created_by))
 
-    rows = [(d.name, timestamp_to_localized_str(d.created_at, json)) for d in response.dicts]
-    display_table(["Name", "Created at"], rows, json)
+    display_table(["Name", "Created at", "Created by"], rows, json)
 
 
 @dict_cli.command("clear", rich_help_panel="Management")

modal/cli/queues.py

@@ -1,4 +1,5 @@
 # Copyright Modal Labs 2024
+from datetime import datetime
 from typing import Optional
 
 import typer
@@ -62,22 +63,46 @@ async def delete(name: str, *, yes: bool = YES_OPTION, env: Optional[str] = ENV_
 async def list_(*, json: bool = False, env: Optional[str] = ENV_OPTION):
     """List all named Queues."""
     env = ensure_env(env)
-
-    max_total_size = 100_000
     client = await _Client.from_env()
-    request = api_pb2.QueueListRequest(environment_name=env, total_size_limit=max_total_size + 1)
-    response = await retry_transient_errors(client.stub.QueueList, request)
-
-    rows = [
-        (
-            q.name,
-            timestamp_to_localized_str(q.created_at, json),
-            str(q.num_partitions),
-            str(q.total_size) if q.total_size <= max_total_size else f">{max_total_size}",
+    max_total_size = 100_000  # Limit on the *Queue size* that we report
+
+    items: list[api_pb2.QueueListResponse.QueueInfo] = []
+
+    # Note that we need to continue using the gRPC API directly here rather than using Queue.objects.list.
+    # There is some metadata that historically appears in the CLI output (num_partitions, total_size) that
+    # doesn't make sense to transmit as hydration metadata, because the values can change over time and
+    # the metadata retrieved at hydration time could get stale. Alternatively, we could rewrite this using
+    # only public API by sequentially retrieving the queues and then querying their dynamic metadata, but
+    # that would require multiple round trips and would add lag to the CLI.
+    async def retrieve_page(created_before: float) -> bool:
+        max_page_size = 100
+        pagination = api_pb2.ListPagination(max_objects=max_page_size, created_before=created_before)
+        req = api_pb2.QueueListRequest(environment_name=env, pagination=pagination, total_size_limit=max_total_size)
+        resp = await retry_transient_errors(client.stub.QueueList, req)
+        items.extend(resp.queues)
+        return len(resp.queues) < max_page_size
+
+    finished = await retrieve_page(datetime.now().timestamp())
+    while True:
+        if finished:
+            break
+        finished = await retrieve_page(items[-1].metadata.creation_info.created_at)
+
+    queues = [_Queue._new_hydrated(item.queue_id, client, item.metadata, is_another_app=True) for item in items]
+
+    rows = []
+    for obj, resp_data in zip(queues, items):
+        info = await obj.info()
+        rows.append(
+            (
+                obj.name,
+                timestamp_to_localized_str(info.created_at.timestamp(), json),
+                info.created_by,
+                str(resp_data.num_partitions),
+                str(resp_data.total_size) if resp_data.total_size <= max_total_size else f">{max_total_size}",
+            )
         )
-        for q in response.queues
-    ]
-    display_table(["Name", "Created at", "Partitions", "Total size"], rows, json)
+    display_table(["Name", "Created at", "Created by", "Partitions", "Total size"], rows, json)
 
 
 @queue_cli.command(name="clear", rich_help_panel="Management")
@@ -119,7 +144,7 @@ async def peek(
 
 @queue_cli.command(name="len", rich_help_panel="Inspection")
 @synchronizer.create_blocking
-async def len(
+async def len_(
     name: str,
     partition: Optional[str] = PARTITION_OPTION,
     total: bool = Option(False, "-t", "--total", help="Compute the sum of the queue lengths across all partitions"),

modal/cli/secret.py

@@ -3,6 +3,7 @@ import json
 import os
 import platform
 import subprocess
+from datetime import datetime
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Optional
@@ -30,20 +31,45 @@ secret_cli = typer.Typer(name="secret", help="Manage secrets.", no_args_is_help=
 async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
     env = ensure_env(env)
     client = await _Client.from_env()
-    response = await retry_transient_errors(client.stub.SecretList, api_pb2.SecretListRequest(environment_name=env))
-    column_names = ["Name", "Created at", "Last used at"]
-    rows = []
 
-    for item in response.items:
+    items: list[api_pb2.SecretListItem] = []
+
+    # Note that we need to continue using the gRPC API directly here rather than using Secret.objects.list.
+    # There is some metadata that historically appears in the CLI output (last_used_at) that
+    # doesn't make sense to transmit as hydration metadata, because the value can change over time and
+    # the metadata retrieved at hydration time could get stale. Alternatively, we could rewrite this using
+    # only public API by sequentially retrieving the secrets and then querying their dynamic metadata, but
+    # that would require multiple round trips and would add lag to the CLI.
+    async def retrieve_page(created_before: float) -> bool:
+        max_page_size = 100
+        pagination = api_pb2.ListPagination(max_objects=max_page_size, created_before=created_before)
+        req = api_pb2.SecretListRequest(environment_name=env, pagination=pagination)
+        resp = await retry_transient_errors(client.stub.SecretList, req)
+        items.extend(resp.items)
+        return len(resp.items) < max_page_size
+
+    finished = await retrieve_page(datetime.now().timestamp())
+    while True:
+        if finished:
+            break
+        finished = await retrieve_page(items[-1].metadata.creation_info.created_at)
+
+    secrets = [_Secret._new_hydrated(item.secret_id, client, item.metadata, is_another_app=True) for item in items]
+
+    rows = []
+    for obj, resp_data in zip(secrets, items):
+        info = await obj.info()
         rows.append(
             [
-                item.label,
-                timestamp_to_localized_str(item.created_at, json),
-                timestamp_to_localized_str(item.last_used_at, json) if item.last_used_at else "-",
+                obj.name,
+                timestamp_to_localized_str(info.created_at.timestamp(), json),
+                info.created_by,
+                timestamp_to_localized_str(resp_data.last_used_at, json) if resp_data.last_used_at else "-",
             ]
         )
 
     env_part = f" in environment '{env}'" if env else ""
+    column_names = ["Name", "Created at", "Created by", "Last used at"]
     display_table(column_names, rows, json, title=f"Secrets{env_part}")
 
 

modal/cli/volume.py

@@ -13,11 +13,9 @@ from typer import Argument, Option, Typer
 import modal
 from modal._output import OutputManager, ProgressHandler, make_console
 from modal._utils.async_utils import synchronizer
-from modal._utils.grpc_utils import retry_transient_errors
 from modal._utils.time_utils import timestamp_to_localized_str
 from modal.cli._download import _volume_download
 from modal.cli.utils import ENV_OPTION, YES_OPTION, display_table
-from modal.client import _Client
 from modal.environments import ensure_env
 from modal.volume import _AbstractVolumeUploadContextManager, _Volume
 from modal_proto import api_pb2
@@ -110,14 +108,13 @@ async def get(
 @synchronizer.create_blocking
 async def list_(env: Optional[str] = ENV_OPTION, json: Optional[bool] = False):
     env = ensure_env(env)
-    client = await _Client.from_env()
-    response = await retry_transient_errors(client.stub.VolumeList, api_pb2.VolumeListRequest(environment_name=env))
-    env_part = f" in environment '{env}'" if env else ""
-    column_names = ["Name", "Created at"]
+    volumes = await _Volume.objects.list(environment_name=env)
     rows = []
-    for item in response.items:
-        rows.append([item.label, timestamp_to_localized_str(item.created_at, json)])
-    display_table(column_names, rows, json, title=f"Volumes{env_part}")
+    for obj in volumes:
+        info = await obj.info()
+        rows.append((info.name, timestamp_to_localized_str(info.created_at.timestamp(), json), info.created_by))
+
+    display_table(["Name", "Created at", "Created by"], rows, json)
 
 
 @volume_cli.command(

modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.2.dev11",
+        version: str = "1.1.2.dev12",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.2.dev11",
+        version: str = "1.1.2.dev12",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/dict.py

@@ -2,25 +2,32 @@
 from collections.abc import AsyncIterator, Mapping
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from google.protobuf.message import Message
 from grpclib import GRPCError
+from synchronicity import classproperty
 from synchronicity.async_wrap import asynccontextmanager
 
 from modal_proto import api_pb2
 
-from ._object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
+from ._object import (
+    EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
+    _get_environment_name,
+    _Object,
+    live_method,
+    live_method_gen,
+)
 from ._resolver import Resolver
 from ._serialization import deserialize, serialize
 from ._utils.async_utils import TaskContext, synchronize_api
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.name_utils import check_object_name
-from ._utils.time_utils import timestamp_to_localized_dt
+from ._utils.time_utils import as_timestamp, timestamp_to_localized_dt
 from .client import _Client
 from .config import logger
-from .exception import RequestSizeError
+from .exception import InvalidError, RequestSizeError
 
 
 def _serialize_dict(data):
@@ -29,7 +36,7 @@ def _serialize_dict(data):
 
 @dataclass
 class DictInfo:
-    """Information about the Dict object."""
+    """Information about a Dict object."""
 
     # This dataclass should be limited to information that is unchanging over the lifetime of the Dict,
     # since it is transmitted from the server when the object is hydrated and could be stale when accessed.
@@ -39,6 +46,68 @@ class DictInfo:
     created_by: Optional[str]
 
 
+class _DictManager:
+    """Namespace with methods for managing named Dict objects."""
+
+    @staticmethod
+    async def list(
+        *,
+        max_objects: Optional[int] = None,  # Limit results to this size
+        created_before: Optional[Union[datetime, str]] = None,  # Limit based on creation date
+        environment_name: str = "",  # Uses active environment if not specified
+        client: Optional[_Client] = None,  # Optional client with Modal credentials
+    ) -> list["_Dict"]:
+        """Return a list of hydrated Dict objects.
+
+        **Examples:**
+
+        ```python
+        dicts = modal.Dict.objects.list()
+        print([d.name for d in dicts])
+        ```
+
+        Dicts will be retreived from the active environment, or another one can be specified:
+
+        ```python notest
+        dev_dicts = modal.Dict.objects.list(environment_name="dev")
+        ```
+
+        By default, all named Dict are returned, newest to oldest. It's also possible to limit the
+        number of results and to filter by creation date:
+
+        ```python
+        dicts = modal.Dict.objects.list(max_objects=10, created_before="2025-01-01")
+        ```
+
+        """
+        client = await _Client.from_env() if client is None else client
+        if max_objects is not None and max_objects < 0:
+            raise InvalidError("max_objects cannot be negative")
+
+        items: list[api_pb2.DictListResponse.DictInfo] = []
+
+        async def retrieve_page(created_before: float) -> bool:
+            max_page_size = 100 if max_objects is None else min(100, max_objects - len(items))
+            pagination = api_pb2.ListPagination(max_objects=max_page_size, created_before=created_before)
+            req = api_pb2.DictListRequest(environment_name=environment_name, pagination=pagination)
+            resp = await retry_transient_errors(client.stub.DictList, req)
+            items.extend(resp.dicts)
+            finished = (len(resp.dicts) < max_page_size) or (max_objects is not None and len(items) >= max_objects)
+            return finished
+
+        finished = await retrieve_page(as_timestamp(created_before))
+        while True:
+            if finished:
+                break
+            finished = await retrieve_page(items[-1].metadata.creation_info.created_at)
+
+        dicts = [_Dict._new_hydrated(item.dict_id, client, item.metadata, is_another_app=True) for item in items]
+        return dicts[:max_objects] if max_objects is not None else dicts
+
+
+DictManager = synchronize_api(_DictManager)
+
+
 class _Dict(_Object, type_prefix="di"):
     """Distributed dictionary for storage in Modal apps.
 
@@ -90,6 +159,10 @@ class _Dict(_Object, type_prefix="di"):
             "`Dict(...)` constructor is not allowed. Please use `Dict.from_name` or `Dict.ephemeral` instead"
         )
 
+    @classproperty
+    def objects(cls) -> _DictManager:
+        return _DictManager
+
     @property
     def name(self) -> Optional[str]:
         return self._name

modal/dict.pyi

@@ -5,6 +5,7 @@ import modal._object
 import modal.client
 import modal.object
 import modal_proto.api_pb2
+import synchronicity
 import synchronicity.combined_types
 import typing
 import typing_extensions
@@ -12,7 +13,7 @@ import typing_extensions
 def _serialize_dict(data): ...
 
 class DictInfo:
-    """Information about the Dict object."""
+    """Information about a Dict object."""
 
     name: typing.Optional[str]
     created_at: datetime.datetime
@@ -32,6 +33,115 @@ class DictInfo:
         """Return self==value."""
         ...
 
+class _DictManager:
+    """Namespace with methods for managing named Dict objects."""
+    @staticmethod
+    async def list(
+        *,
+        max_objects: typing.Optional[int] = None,
+        created_before: typing.Union[datetime.datetime, str, None] = None,
+        environment_name: str = "",
+        client: typing.Optional[modal.client._Client] = None,
+    ) -> list[_Dict]:
+        """Return a list of hydrated Dict objects.
+
+        **Examples:**
+
+        ```python
+        dicts = modal.Dict.objects.list()
+        print([d.name for d in dicts])
+        ```
+
+        Dicts will be retreived from the active environment, or another one can be specified:
+
+        ```python notest
+        dev_dicts = modal.Dict.objects.list(environment_name="dev")
+        ```
+
+        By default, all named Dict are returned, newest to oldest. It's also possible to limit the
+        number of results and to filter by creation date:
+
+        ```python
+        dicts = modal.Dict.objects.list(max_objects=10, created_before="2025-01-01")
+        ```
+        """
+        ...
+
+class DictManager:
+    """Namespace with methods for managing named Dict objects."""
+    def __init__(self, /, *args, **kwargs):
+        """Initialize self.  See help(type(self)) for accurate signature."""
+        ...
+
+    class __list_spec(typing_extensions.Protocol):
+        def __call__(
+            self,
+            /,
+            *,
+            max_objects: typing.Optional[int] = None,
+            created_before: typing.Union[datetime.datetime, str, None] = None,
+            environment_name: str = "",
+            client: typing.Optional[modal.client.Client] = None,
+        ) -> list[Dict]:
+            """Return a list of hydrated Dict objects.
+
+            **Examples:**
+
+            ```python
+            dicts = modal.Dict.objects.list()
+            print([d.name for d in dicts])
+            ```
+
+            Dicts will be retreived from the active environment, or another one can be specified:
+
+            ```python notest
+            dev_dicts = modal.Dict.objects.list(environment_name="dev")
+            ```
+
+            By default, all named Dict are returned, newest to oldest. It's also possible to limit the
+            number of results and to filter by creation date:
+
+            ```python
+            dicts = modal.Dict.objects.list(max_objects=10, created_before="2025-01-01")
+            ```
+            """
+            ...
+
+        async def aio(
+            self,
+            /,
+            *,
+            max_objects: typing.Optional[int] = None,
+            created_before: typing.Union[datetime.datetime, str, None] = None,
+            environment_name: str = "",
+            client: typing.Optional[modal.client.Client] = None,
+        ) -> list[Dict]:
+            """Return a list of hydrated Dict objects.
+
+            **Examples:**
+
+            ```python
+            dicts = modal.Dict.objects.list()
+            print([d.name for d in dicts])
+            ```
+
+            Dicts will be retreived from the active environment, or another one can be specified:
+
+            ```python notest
+            dev_dicts = modal.Dict.objects.list(environment_name="dev")
+            ```
+
+            By default, all named Dict are returned, newest to oldest. It's also possible to limit the
+            number of results and to filter by creation date:
+
+            ```python
+            dicts = modal.Dict.objects.list(max_objects=10, created_before="2025-01-01")
+            ```
+            """
+            ...
+
+    list: __list_spec
+
 class _Dict(modal._object._Object):
     """Distributed dictionary for storage in Modal apps.
 
@@ -81,6 +191,8 @@ class _Dict(modal._object._Object):
         """mdmd:hidden"""
         ...
 
+    @synchronicity.classproperty
+    def objects(cls) -> _DictManager: ...
     @property
     def name(self) -> typing.Optional[str]: ...
     def _hydrate_metadata(self, metadata: typing.Optional[google.protobuf.message.Message]): ...
@@ -308,6 +420,8 @@ class Dict(modal.object.Object):
         """mdmd:hidden"""
         ...
 
+    @synchronicity.classproperty
+    def objects(cls) -> DictManager: ...
     @property
     def name(self) -> typing.Optional[str]: ...
     def _hydrate_metadata(self, metadata: typing.Optional[google.protobuf.message.Message]): ...

modal/queue.py

@@ -5,22 +5,29 @@ import warnings
 from collections.abc import AsyncGenerator, AsyncIterator
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from google.protobuf.message import Message
 from grpclib import GRPCError, Status
+from synchronicity import classproperty
 from synchronicity.async_wrap import asynccontextmanager
 
 from modal_proto import api_pb2
 
-from ._object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
+from ._object import (
+    EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
+    _get_environment_name,
+    _Object,
+    live_method,
+    live_method_gen,
+)
 from ._resolver import Resolver
 from ._serialization import deserialize, serialize
 from ._utils.async_utils import TaskContext, synchronize_api, warn_if_generator_is_not_consumed
 from ._utils.deprecation import deprecation_warning, warn_if_passing_namespace
 from ._utils.grpc_utils import retry_transient_errors
 from ._utils.name_utils import check_object_name
-from ._utils.time_utils import timestamp_to_localized_dt
+from ._utils.time_utils import as_timestamp, timestamp_to_localized_dt
 from .client import _Client
 from .exception import InvalidError, RequestSizeError
 
@@ -37,6 +44,68 @@ class QueueInfo:
     created_by: Optional[str]
 
 
+class _QueueManager:
+    """Namespace with methods for managing named Queue objects."""
+
+    @staticmethod
+    async def list(
+        *,
+        max_objects: Optional[int] = None,  # Limit requests to this size
+        created_before: Optional[Union[datetime, str]] = None,  # Limit based on creation date
+        environment_name: str = "",  # Uses active environment if not specified
+        client: Optional[_Client] = None,  # Optional client with Modal credentials
+    ) -> list["_Queue"]:
+        """Return a list of hydrated Queue objects.
+
+        **Examples:**
+
+        ```python
+        queues = modal.Queue.objects.list()
+        print([q.name for q in queues])
+        ```
+
+        Queues will be retreived from the active environment, or another one can be specified:
+
+        ```python notest
+        dev_queues = modal.Queue.objects.list(environment_name="dev")
+        ```
+
+        By default, all named Queues are returned, newest to oldest. It's also possible to limit the
+        number of results and to filter by creation date:
+
+        ```python
+        queues = modal.Queue.objects.list(max_objects=10, created_before="2025-01-01")
+        ```
+
+        """
+        client = await _Client.from_env() if client is None else client
+        if max_objects is not None and max_objects < 0:
+            raise InvalidError("max_objects cannot be negative")
+
+        items: list[api_pb2.QueueListResponse.QueueInfo] = []
+
+        async def retrieve_page(created_before: float) -> bool:
+            max_page_size = 100 if max_objects is None else min(100, max_objects - len(items))
+            pagination = api_pb2.ListPagination(max_objects=max_page_size, created_before=created_before)
+            req = api_pb2.QueueListRequest(environment_name=environment_name, pagination=pagination)
+            resp = await retry_transient_errors(client.stub.QueueList, req)
+            items.extend(resp.queues)
+            finished = (len(resp.queues) < max_page_size) or (max_objects is not None and len(items) >= max_objects)
+            return finished
+
+        finished = await retrieve_page(as_timestamp(created_before))
+        while True:
+            if finished:
+                break
+            finished = await retrieve_page(items[-1].metadata.creation_info.created_at)
+
+        queues = [_Queue._new_hydrated(item.queue_id, client, item.metadata, is_another_app=True) for item in items]
+        return queues[:max_objects] if max_objects is not None else queues
+
+
+QueueManager = synchronize_api(_QueueManager)
+
+
 class _Queue(_Object, type_prefix="qu"):
     """Distributed, FIFO queue for data flow in Modal apps.
 
@@ -116,6 +185,10 @@ class _Queue(_Object, type_prefix="qu"):
         """mdmd:hidden"""
         raise RuntimeError("Queue() is not allowed. Please use `Queue.from_name(...)` or `Queue.ephemeral()` instead.")
 
+    @classproperty
+    def objects(cls) -> _QueueManager:
+        return _QueueManager
+
     @property
     def name(self) -> Optional[str]:
         return self._name