modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/call_graph.py

@@ -1,7 +1,7 @@
 # Copyright Modal Labs 2022
 from dataclasses import dataclass
 from enum import IntEnum
-from typing import Dict, List, Optional
+from typing import Optional
 
 from modal_proto import api_pb2
 
@@ -12,6 +12,7 @@ class InputStatus(IntEnum):
     PENDING = 0
     SUCCESS = api_pb2.GenericResult.GENERIC_STATUS_SUCCESS
     FAILURE = api_pb2.GenericResult.GENERIC_STATUS_FAILURE
+    INIT_FAILURE = api_pb2.GenericResult.GENERIC_STATUS_INIT_FAILURE
     TERMINATED = api_pb2.GenericResult.GENERIC_STATUS_TERMINATED
     TIMEOUT = api_pb2.GenericResult.GENERIC_STATUS_TIMEOUT
 
@@ -30,12 +31,12 @@ class InputInfo:
     status: InputStatus
     function_name: str
     module_name: str
-    children: List["InputInfo"]
+    children: list["InputInfo"]
 
 
-def _reconstruct_call_graph(ser_graph: api_pb2.FunctionGetCallGraphResponse) -> List[InputInfo]:
-    function_calls_by_id: Dict[str, api_pb2.FunctionCallCallGraphInfo] = {}
-    inputs_by_id: Dict[str, api_pb2.InputCallGraphInfo] = {}
+def _reconstruct_call_graph(ser_graph: api_pb2.FunctionGetCallGraphResponse) -> list[InputInfo]:
+    function_calls_by_id: dict[str, api_pb2.FunctionCallCallGraphInfo] = {}
+    inputs_by_id: dict[str, api_pb2.InputCallGraphInfo] = {}
 
     for function_call in ser_graph.function_calls:
         function_calls_by_id[function_call.function_call_id] = function_call
@@ -43,7 +44,7 @@ def _reconstruct_call_graph(ser_graph: api_pb2.FunctionGetCallGraphResponse) ->
     for input in ser_graph.inputs:
         inputs_by_id[input.input_id] = input
 
-    input_info_by_id: Dict[str, InputInfo] = {}
+    input_info_by_id: dict[str, InputInfo] = {}
     result = []
 
     def _reconstruct(input_id: str) -> Optional[InputInfo]:

modal/cli/_download.py

@@ -3,64 +3,60 @@ import asyncio
 import os
 import shutil
 import sys
-from pathlib import Path
-from typing import Callable, Optional, Tuple, Union, overload
+from collections.abc import AsyncIterator
+from pathlib import Path, PurePosixPath
+from typing import Callable, Optional, Union
 
 from click import UsageError
 
+from modal._utils.async_utils import TaskContext
+from modal.config import logger
 from modal.network_file_system import _NetworkFileSystem
-from modal.volume import _Volume
-from modal_proto import api_pb2
+from modal.volume import FileEntry, FileEntryType, _Volume
 
-_Entry = Union[api_pb2.SharedVolumeListFilesEntry, api_pb2.VolumeListFilesEntry]
+PIPE_PATH = Path("-")
 
 
-@overload
-def _glob_download(
-    volume: _Volume,
-    is_file_fn: Callable[[api_pb2.VolumeListFilesEntry], bool],
-    remote_glob_path: str,
+async def _volume_download(
+    volume: Union[_NetworkFileSystem, _Volume],
+    remote_path: str,
     local_destination: Path,
     overwrite: bool,
+    progress_cb: Callable,
 ):
-    ...
+    is_pipe = local_destination == PIPE_PATH
 
-
-@overload
-def _glob_download(
-    volume: _NetworkFileSystem,
-    is_file_fn: Callable[[api_pb2.SharedVolumeListFilesEntry], bool],
-    remote_glob_path: str,
-    local_destination: Path,
-    overwrite: bool,
-):
-    ...
-
-
-async def _glob_download(
-    volume,
-    is_file_fn,
-    remote_glob_path: str,
-    local_destination: Path,
-    overwrite: bool,
-):
-    q: asyncio.Queue[Tuple[Optional[Path], Optional[_Entry]]] = asyncio.Queue()
-    num_consumers = 10  # concurrency limit
+    q: asyncio.Queue[tuple[Optional[Path], Optional[FileEntry]]] = asyncio.Queue()
+    num_consumers = 1 if is_pipe else 10  # concurrency limit for downloading files
 
     async def producer():
-        async for entry in volume.iterdir(remote_glob_path):
-            output_path = local_destination / entry.path
-            if output_path.exists():
-                if overwrite:
-                    if is_file_fn(entry):
-                        os.remove(output_path)
-                    else:
-                        shutil.rmtree(output_path)
+        iterator: AsyncIterator[FileEntry]
+        if isinstance(volume, _Volume):
+            iterator = volume.iterdir(remote_path, recursive=True)
+        else:
+            iterator = volume.iterdir(remote_path)  # NFS still supports "glob" paths
+
+        async for entry in iterator:
+            if is_pipe:
+                await q.put((None, entry))
+            else:
+                start_path = Path(remote_path).parent.as_posix().split("*")[0]
+                rel_path = PurePosixPath(entry.path).relative_to(start_path.lstrip("/"))
+                if local_destination.is_dir():
+                    output_path = local_destination / rel_path
                 else:
-                    raise UsageError(
-                        f"Output path '{output_path}' already exists. Use --force to overwrite the output directory"
-                    )
-            await q.put((output_path, entry))
+                    output_path = local_destination
+                if output_path.exists():
+                    if overwrite:
+                        if output_path.is_file():
+                            os.remove(output_path)
+                        else:
+                            shutil.rmtree(output_path)
+                    else:
+                        raise UsageError(
+                            f"Output path '{output_path}' already exists. Use --force to overwrite the output directory"
+                        )
+                await q.put((output_path, entry))
         # No more entries to process; issue one shutdown message for each consumer.
         for _ in range(num_consumers):
             await q.put((None, None))
@@ -68,19 +64,33 @@ async def _glob_download(
     async def consumer():
         while True:
             output_path, entry = await q.get()
-            if output_path is None:
+            if entry is None:
                 return
             try:
-                if is_file_fn(entry):
-                    output_path.parent.mkdir(parents=True, exist_ok=True)
-                    with output_path.open("wb") as fp:
-                        b = 0
+                if is_pipe:
+                    if entry.type == FileEntryType.FILE:
+                        progress_task_id = progress_cb(name=entry.path, size=entry.size)
                         async for chunk in volume.read_file(entry.path):
-                            b += fp.write(chunk)
-
-                    print(f"Wrote {b} bytes to {output_path}", file=sys.stderr)
+                            sys.stdout.buffer.write(chunk)
+                            progress_cb(task_id=progress_task_id, advance=len(chunk))
+                        progress_cb(task_id=progress_task_id, complete=True)
+                else:
+                    if entry.type == FileEntryType.FILE:
+                        progress_task_id = progress_cb(name=entry.path, size=entry.size)
+                        output_path.parent.mkdir(parents=True, exist_ok=True)
+                        with output_path.open("wb") as fp:
+                            b = 0
+                            async for chunk in volume.read_file(entry.path):
+                                b += fp.write(chunk)
+                                progress_cb(task_id=progress_task_id, advance=len(chunk))
+                        logger.debug(f"Wrote {b} bytes to {output_path}")
+                        progress_cb(task_id=progress_task_id, complete=True)
+                    elif entry.type == FileEntryType.DIRECTORY:
+                        output_path.mkdir(parents=True, exist_ok=True)
             finally:
                 q.task_done()
 
     consumers = [consumer() for _ in range(num_consumers)]
-    await asyncio.gather(producer(), *consumers)
+    await TaskContext.gather(producer(), *consumers)
+    progress_cb(complete=True)
+    sys.stdout.flush()

modal/cli/_traceback.py

@@ -0,0 +1,200 @@
+# Copyright Modal Labs 2024
+"""Helper functions related to displaying tracebacks in the CLI."""
+
+import functools
+import re
+import warnings
+from typing import Optional
+
+from rich.console import Console, RenderResult, group
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.text import Text
+from rich.traceback import PathHighlighter, Stack, Traceback, install
+
+from ..exception import DeprecationError, PendingDeprecationError, ServerWarning
+
+
+@group()
+def _render_stack(self, stack: Stack) -> RenderResult:
+    """Patched variant of rich.Traceback._render_stack that uses the line from the modal StackSummary,
+    when the file isn't available to be read locally."""
+
+    path_highlighter = PathHighlighter()
+    theme = self.theme
+    code_cache: dict[str, str] = {}
+    line_cache = getattr(stack, "line_cache", {})
+    task_id = None
+
+    def read_code(filename: str) -> str:
+        code = code_cache.get(filename)
+        if code is None:
+            with open(filename, encoding="utf-8", errors="replace") as code_file:
+                code = code_file.read()
+            code_cache[filename] = code
+        return code
+
+    exclude_frames: Optional[range] = None
+    if self.max_frames != 0:
+        exclude_frames = range(
+            self.max_frames // 2,
+            len(stack.frames) - self.max_frames // 2,
+        )
+
+    excluded = False
+    for frame_index, frame in enumerate(stack.frames):
+        if exclude_frames and frame_index in exclude_frames:
+            excluded = True
+            continue
+
+        if excluded:
+            assert exclude_frames is not None
+            yield Text(
+                f"\n... {len(exclude_frames)} frames hidden ...",
+                justify="center",
+                style="traceback.error",
+            )
+            excluded = False
+
+        first = frame_index == 0
+        # Patched Modal-specific code.
+        if frame.filename.startswith("<") and ":" in frame.filename:
+            next_task_id, frame_filename = frame.filename.split(":", 1)
+            next_task_id = next_task_id.strip("<>")
+        else:
+            frame_filename = frame.filename
+            next_task_id = None
+        suppressed = any(frame_filename.startswith(path) for path in self.suppress)
+
+        if next_task_id != task_id:
+            task_id = next_task_id
+            yield ""
+            yield Text(
+                f"...Remote call to Modal Function ({task_id})...",
+                justify="center",
+                style="green",
+            )
+
+        text = Text.assemble(
+            path_highlighter(Text(frame_filename, style="pygments.string")),
+            (":", "pygments.text"),
+            (str(frame.lineno), "pygments.number"),
+            " in ",
+            (frame.name, "pygments.function"),
+            style="pygments.text",
+        )
+        if not frame_filename.startswith("<") and not first:
+            yield ""
+
+        yield text
+        if not suppressed:
+            try:
+                code = read_code(frame_filename)
+                lexer_name = self._guess_lexer(frame_filename, code)
+                syntax = Syntax(
+                    code,
+                    lexer_name,
+                    theme=theme,
+                    line_numbers=True,
+                    line_range=(
+                        frame.lineno - self.extra_lines,
+                        frame.lineno + self.extra_lines,
+                    ),
+                    highlight_lines={frame.lineno},
+                    word_wrap=self.word_wrap,
+                    code_width=88,
+                    indent_guides=self.indent_guides,
+                    dedent=False,
+                )
+                yield ""
+            except Exception as error:
+                # Patched Modal-specific code.
+                line = line_cache.get((frame_filename, frame.lineno))
+                if line:
+                    try:
+                        lexer_name = self._guess_lexer(frame_filename, line)
+                        yield ""
+                        yield Syntax(
+                            line,
+                            lexer_name,
+                            theme=theme,
+                            line_numbers=True,
+                            line_range=(0, 1),
+                            highlight_lines={frame.lineno},
+                            word_wrap=self.word_wrap,
+                            code_width=88,
+                            indent_guides=self.indent_guides,
+                            dedent=False,
+                            start_line=frame.lineno,
+                        )
+                    except Exception:
+                        yield Text.assemble(
+                            (f"\n{error}", "traceback.error"),
+                        )
+                yield ""
+            else:
+                yield syntax
+
+
+def setup_rich_traceback() -> None:
+    from_exception = Traceback.from_exception
+
+    @functools.wraps(Traceback.from_exception)
+    def _from_exception(exc_type, exc_value, *args, **kwargs):
+        """Patch from_exception to grab the Modal line_cache and store it with the
+        Stack object, so it's available to render_stack at display time."""
+
+        line_cache = getattr(exc_value, "__line_cache__", {})
+        tb = from_exception(exc_type, exc_value, *args, **kwargs)
+        for stack in tb.trace.stacks:
+            stack.line_cache = line_cache  # type: ignore
+        return tb
+
+    Traceback._render_stack = _render_stack  # type: ignore
+    Traceback.from_exception = _from_exception  # type: ignore
+
+    import click
+    import grpclib
+    import synchronicity
+    import typer
+
+    install(suppress=[synchronicity, grpclib, click, typer], extra_lines=1)
+
+
+def highlight_modal_deprecation_warnings() -> None:
+    """Patch the warnings module to make client deprecation warnings more salient in the CLI."""
+    base_showwarning = warnings.showwarning
+
+    def showwarning(warning, category, filename, lineno, file=None, line=None):
+        if issubclass(category, (DeprecationError, PendingDeprecationError, ServerWarning)):
+            content = str(warning)
+            if re.match(r"^\d{4}-\d{2}-\d{2}", content):
+                date = content[:10]
+                message = content[11:].strip()
+            else:
+                date = ""
+                message = content
+            try:
+                with open(filename, encoding="utf-8", errors="replace") as code_file:
+                    source = code_file.readlines()[lineno - 1].strip()
+                message = f"{message}\n\nSource: {filename}:{lineno}\n  {source}"
+            except OSError:
+                # e.g., when filename is "<unknown>"; raises FileNotFoundError on posix but OSError on windows
+                pass
+            if issubclass(category, ServerWarning):
+                title = "Modal Warning"
+            else:
+                title = "Modal Deprecation Warning"
+            if date:
+                title += f" ({date})"
+            panel = Panel(
+                message,
+                border_style="yellow",
+                title=title,
+                title_align="left",
+            )
+            Console().print(panel)
+        else:
+            base_showwarning(warning, category, filename, lineno, file=None, line=None)
+
+    warnings.showwarning = showwarning

modal/cli/app.py

@@ -1,89 +1,249 @@
 # Copyright Modal Labs 2022
-import asyncio
-from typing import List, Optional, Union
+import re
+from typing import Optional, Union
 
+import rich
 import typer
 from click import UsageError
-from grpclib import GRPCError, Status
+from rich.table import Column
 from rich.text import Text
+from typer import Argument
 
-from modal._output import OutputManager, get_app_logs_loop
 from modal._utils.async_utils import synchronizer
-from modal.app import _list_apps
-from modal.cli.utils import ENV_OPTION, display_table, timestamp_to_local
+from modal._utils.deprecation import deprecation_warning
 from modal.client import _Client
 from modal.environments import ensure_env
+from modal.object import _get_environment_name
 from modal_proto import api_pb2
 
+from .utils import ENV_OPTION, display_table, get_app_id_from_name, stream_app_logs, timestamp_to_local
+
+APP_IDENTIFIER = Argument("", help="App name or ID")
+NAME_OPTION = typer.Option("", "-n", "--name", help="Deprecated: Pass App name as a positional argument")
+
 app_cli = typer.Typer(name="app", help="Manage deployed and running apps.", no_args_is_help=True)
 
 APP_STATE_TO_MESSAGE = {
     api_pb2.APP_STATE_DEPLOYED: Text("deployed", style="green"),
-    api_pb2.APP_STATE_DETACHED: Text("running (detached)", style="green"),
+    api_pb2.APP_STATE_DETACHED: Text("ephemeral (detached)", style="green"),
     api_pb2.APP_STATE_DISABLED: Text("disabled", style="dim"),
-    api_pb2.APP_STATE_EPHEMERAL: Text("running", style="green"),
+    api_pb2.APP_STATE_EPHEMERAL: Text("ephemeral", style="green"),
     api_pb2.APP_STATE_INITIALIZING: Text("initializing...", style="green"),
     api_pb2.APP_STATE_STOPPED: Text("stopped", style="blue"),
     api_pb2.APP_STATE_STOPPING: Text("stopping...", style="blue"),
 }
 
 
+@synchronizer.create_blocking
+async def get_app_id(app_identifier: str, env: Optional[str], client: Optional[_Client] = None) -> str:
+    """Resolve an app_identifier that may be a name or an ID into an ID."""
+    if re.match(r"^ap-[a-zA-Z0-9]{22}$", app_identifier):
+        return app_identifier
+    return await get_app_id_from_name.aio(app_identifier, env, client)
+
+
+def warn_on_name_option(command: str, app_identifier: str, name: str) -> str:
+    if name:
+        message = (
+            "Passing an App name using --name is deprecated;"
+            " App names can now be passed directly as positional arguments:"
+            f"\n\n    modal app {command} {name} ..."
+        )
+        deprecation_warning((2024, 8, 15), message, show_source=False)
+        return name
+    return app_identifier
+
+
 @app_cli.command("list")
 @synchronizer.create_blocking
-async def list(env: Optional[str] = ENV_OPTION, json: Optional[bool] = False):
-    """List all running or recently running Modal apps for the current account"""
-    client = await _Client.from_env()
+async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
+    """List Modal apps that are currently deployed/running or recently stopped."""
     env = ensure_env(env)
+    client = await _Client.from_env()
 
-    column_names = ["App ID", "Name", "State", "Creation time", "Stop time"]
-    rows: List[List[Union[Text, str]]] = []
-    apps = await _list_apps(env=env, client=client)
-    for app_stats in apps:
-        state = APP_STATE_TO_MESSAGE.get(app_stats.state, Text("unknown", style="gray"))
+    resp: api_pb2.AppListResponse = await client.stub.AppList(
+        api_pb2.AppListRequest(environment_name=_get_environment_name(env))
+    )
 
+    columns: list[Union[Column, str]] = [
+        Column("App ID", min_width=25),  # Ensure that App ID is not truncated in slim terminals
+        "Description",
+        "State",
+        "Tasks",
+        "Created at",
+        "Stopped at",
+    ]
+    rows: list[list[Union[Text, str]]] = []
+    for app_stats in resp.apps:
+        state = APP_STATE_TO_MESSAGE.get(app_stats.state, Text("unknown", style="gray"))
         rows.append(
             [
                 app_stats.app_id,
                 app_stats.description,
                 state,
-                timestamp_to_local(app_stats.created_at),
-                timestamp_to_local(app_stats.stopped_at),
+                str(app_stats.n_running_tasks),
+                timestamp_to_local(app_stats.created_at, json),
+                timestamp_to_local(app_stats.stopped_at, json),
             ]
         )
 
     env_part = f" in environment '{env}'" if env else ""
-    display_table(column_names, rows, json, title=f"Apps{env_part}")
-
-
-@app_cli.command("logs")
-def app_logs(app_id: str):
-    """Output logs for a running app."""
-
-    @synchronizer.create_blocking
-    async def sync_command():
-        client = await _Client.from_env()
-        output_mgr = OutputManager(None, None, "Tailing logs for {app_id}")
-        try:
-            with output_mgr.show_status_spinner():
-                await get_app_logs_loop(app_id, client, output_mgr)
-        except asyncio.CancelledError:
-            pass
-
-    try:
-        sync_command()
-    except GRPCError as exc:
-        if exc.status in (Status.INVALID_ARGUMENT, Status.NOT_FOUND):
-            raise UsageError(exc.message)
+    display_table(columns, rows, json, title=f"Apps{env_part}")
+
+
+@app_cli.command("logs", no_args_is_help=True)
+def logs(
+    app_identifier: str = APP_IDENTIFIER,
+    *,
+    name: str = NAME_OPTION,
+    env: Optional[str] = ENV_OPTION,
+):
+    """Show App logs, streaming while active.
+
+    **Examples:**
+
+    Get the logs based on an app ID:
+
+    ```
+    modal app logs ap-123456
+    ```
+
+    Get the logs for a currently deployed App based on its name:
+
+    ```
+    modal app logs my-app
+    ```
+
+    """
+    app_identifier = warn_on_name_option("logs", app_identifier, name)
+    app_id = get_app_id(app_identifier, env)
+    stream_app_logs(app_id)
+
+
+@app_cli.command("rollback", no_args_is_help=True, context_settings={"ignore_unknown_options": True})
+@synchronizer.create_blocking
+async def rollback(
+    app_identifier: str = APP_IDENTIFIER,
+    version: str = typer.Argument("", help="Target version for rollback."),
+    *,
+    env: Optional[str] = ENV_OPTION,
+):
+    """Redeploy a previous version of an App.
+
+    Note that the App must currently be in a "deployed" state.
+    Rollbacks will appear as a new deployment in the App history, although
+    the App state will be reset to the state at the time of the previous deployment.
+
+    **Examples:**
+
+    Rollback an App to its previous version:
+
+    ```
+    modal app rollback my-app
+    ```
+
+    Rollback an App to a specific version:
+
+    ```
+    modal app rollback my-app v3
+    ```
+
+    Rollback an App using its App ID instead of its name:
+
+    ```
+    modal app rollback ap-abcdefghABCDEFGH123456
+    ```
+
+    """
+    env = ensure_env(env)
+    client = await _Client.from_env()
+    app_id = await get_app_id.aio(app_identifier, env, client)
+    if not version:
+        version_number = -1
+    else:
+        if m := re.match(r"v(\d+)", version):
+            version_number = int(m.group(1))
         else:
-            raise
-    except KeyboardInterrupt:
-        pass
+            raise UsageError(f"Invalid version specifer: {version}")
+    req = api_pb2.AppRollbackRequest(app_id=app_id, version=version_number)
+    await client.stub.AppRollback(req)
+    rich.print("[green]✓[/green] Deployment rollback successful!")
 
 
-@app_cli.command("stop")
+@app_cli.command("stop", no_args_is_help=True)
 @synchronizer.create_blocking
-async def stop(app_id: str):
+async def stop(
+    app_identifier: str = APP_IDENTIFIER,
+    *,
+    name: str = NAME_OPTION,
+    env: Optional[str] = ENV_OPTION,
+):
     """Stop an app."""
+    app_identifier = warn_on_name_option("stop", app_identifier, name)
     client = await _Client.from_env()
+    app_id = await get_app_id.aio(app_identifier, env)
     req = api_pb2.AppStopRequest(app_id=app_id, source=api_pb2.APP_STOP_SOURCE_CLI)
     await client.stub.AppStop(req)
+
+
+@app_cli.command("history", no_args_is_help=True)
+@synchronizer.create_blocking
+async def history(
+    app_identifier: str = APP_IDENTIFIER,
+    *,
+    env: Optional[str] = ENV_OPTION,
+    name: str = NAME_OPTION,
+    json: bool = False,
+):
+    """Show App deployment history, for a currently deployed app
+
+    **Examples:**
+
+    Get the history based on an app ID:
+
+    ```
+    modal app history ap-123456
+    ```
+
+    Get the history for a currently deployed App based on its name:
+
+    ```
+    modal app history my-app
+    ```
+
+    """
+    app_identifier = warn_on_name_option("history", app_identifier, name)
+    env = ensure_env(env)
+    client = await _Client.from_env()
+    app_id = await get_app_id.aio(app_identifier, env, client)
+    resp = await client.stub.AppDeploymentHistory(api_pb2.AppDeploymentHistoryRequest(app_id=app_id))
+
+    columns = [
+        "Version",
+        "Time deployed",
+        "Client",
+        "Deployed by",
+    ]
+    rows = []
+    deployments_with_tags = False
+    for idx, app_stats in enumerate(resp.app_deployment_histories):
+        style = "bold green" if idx == 0 else ""
+
+        row = [
+            Text(f"v{app_stats.version}", style=style),
+            Text(timestamp_to_local(app_stats.deployed_at, json), style=style),
+            Text(app_stats.client_version, style=style),
+            Text(app_stats.deployed_by, style=style),
+        ]
+
+        if app_stats.tag:
+            deployments_with_tags = True
+            row.append(Text(app_stats.tag, style=style))
+
+        rows.append(row)
+
+    if deployments_with_tags:
+        columns.append("Tag")
+
+    rows = sorted(rows, key=lambda x: int(str(x[0])[1:]), reverse=True)
+    display_table(columns, rows, json)