levers9 0.2.2__tar.gz → 0.3.0__tar.gz

{levers9-0.2.2 → levers9-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: levers9
-Version: 0.2.2
+Version: 0.3.0
 Summary: levers9 SDK and CLI: run jobs, endpoints, queues, crons, boxes and GPU work on a levers9 cluster
 Author-email: Levers AI Lab <hello@uselevers.com>
 Requires-Python: <4.0,>=3.8

{levers9-0.2.2 → levers9-0.3.0}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
     "pyyaml<7.0.0,>=6.0.0",
 ]
 name = "levers9"
-version = "0.2.2"
+version = "0.3.0"
 description = "levers9 SDK and CLI: run jobs, endpoints, queues, crons, boxes and GPU work on a levers9 cluster"
 
 [project.scripts]

{levers9-0.2.2 → levers9-0.3.0}/src/levers9/abstractions/integrations/fastmcp.py

@@ -153,6 +153,7 @@ class MCPServer(ASGI):
         name: Optional[str] = None,
         context: Optional[ConfigContext] = None,
         invocation_details_func: Optional[Callable[..., None]] = None,
+        rollout: str = "auto",
         **invocation_details_options: Any,
     ) -> Tuple[Dict[str, Any], bool]:
         self.name = name or self.name
@@ -172,19 +173,28 @@ class MCPServer(ASGI):
 
         terminal.header("Deploying")
         deploy_response: DeployStubResponse = self.controlplane_stub.deploy_stub(
-            DeployStubRequest(stub_id=self.stub_id, name=self.name)
+            DeployStubRequest(stub_id=self.stub_id, name=self.name, rollout=rollout)
         )
 
         self.deployment_id = deploy_response.deployment_id
+        warn_msg = deploy_response.warn_msg if isinstance(deploy_response.warn_msg, str) else ""
         if deploy_response.ok:
+            # A rollout can succeed and still have something to say - that a revision was replaced, or
+            # that downtime is expected. Swallowing it is how a rollout that quietly did nothing stayed
+            # invisible for nine versions.
+            if warn_msg:
+                terminal.warn(warn_msg)
             terminal.header("Deployed 🎉")
             if invocation_details_func:
                 invocation_details_func(**invocation_details_options)
             else:
                 self.print_invocation_snippet(**invocation_details_options)
+        elif deploy_response.err_msg:
+            terminal.error(deploy_response.err_msg, exit=False)
 
         return {
             "deployment_id": deploy_response.deployment_id,
+            "warning": warn_msg,
         }, deploy_response.ok
 
     @with_grpc_error_handling

{levers9-0.2.2 → levers9-0.3.0}/src/levers9/abstractions/integrations/vllm.py

@@ -10,6 +10,7 @@ from ...abstractions.endpoint import ASGI
 from ...abstractions.blueprint import Image
 from ...abstractions.volume import CloudBucket, Volume
 from ...channel import with_grpc_error_handling
+from ...checkpoint_memory import VLLM_PROCESS_RSS_GIB, advise
 from ...clients.endpoint import StartEndpointServeRequest, StartEndpointServeResponse
 from ...clients.controlplane import DeployStubRequest, DeployStubResponse
 from ...config import ConfigContext
@@ -35,6 +36,47 @@ def _served_model_name(value: Optional[Union[str, List[str]]]) -> str:
     return value or ""
 
 
+class _ServerArgs(SimpleNamespace):
+    """A namespace for vLLM's OpenAI server arguments that cannot raise AttributeError.
+
+    `init_app_state` reads these by name and vLLM adds to them every release, so a hand-listed namespace
+    breaks on the first argument a new version starts reading. Known values are set explicitly; anything
+    else falls back to the installed release's own default if it publishes one, and to None otherwise -
+    which is what vLLM itself defaults almost all of these to.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        if name.startswith("__"):
+            raise AttributeError(name)
+        value = _server_arg_default(name)
+        # cache it, so the same lookup does not re-import on every access
+        setattr(self, name, value)
+        return value
+
+
+def _server_arg_default(name: str) -> Any:
+    """This vLLM's own default for a server argument, or None if it does not publish one."""
+    import dataclasses
+
+    for module, cls_name in (
+        ("vllm.entrypoints.launchers.cli_args", "FrontendArgs"),      # 0.29+
+        ("vllm.entrypoints.openai.cli_args", "FrontendArgs"),
+    ):
+        try:
+            mod = __import__(module, fromlist=[cls_name])
+            spec = getattr(mod, cls_name)
+            for field in dataclasses.fields(spec):
+                if field.name != name:
+                    continue
+                if field.default is not dataclasses.MISSING:
+                    return field.default
+                if field.default_factory is not dataclasses.MISSING:  # type: ignore[misc]
+                    return field.default_factory()                    # type: ignore[misc]
+        except Exception:
+            continue
+    return None
+
+
 # vllm/engine/arg_utils.py:EngineArgs
 @dataclass
 class VLLMArgs:
@@ -167,6 +209,10 @@ class VLLMArgs:
     data_parallel_size: int = 1
     compilation_config: Optional[Any] = None
     vllm_cache_root: Optional[str] = DEFAULT_VLLM_CACHE_ROOT
+    # Any engine argument this dataclass does not name, applied by name onto the installed vLLM's own
+    # arguments. The fields above mirror one release; vLLM adds and removes arguments every release, so
+    # without this a flag newer than this SDK cannot be set at all except by subclassing.
+    extra_args: Optional[Dict[str, Any]] = None
 
 
 class VLLM(ASGI):
@@ -186,9 +232,13 @@ class VLLM(ASGI):
             The container image used for the task execution. Whatever you pass here will have an additional `add_python_packages` call
             with `["fastapi", "vllm", "huggingface_hub"]` added to it to ensure that we can run vLLM in the container.
         vllm_version (str):
-            The version of vLLM that will be installed from PyPI. As the configuration of the vLLM engine depends on the version of vLLM, using a non-default vllm_version might require subclassing VLLMArgs in order to add the missing configuration options. Default is version 0.8.4.
-        huggingface_hub_version (str):
-            The version of huggingface_hub that will be installed from PyPI. Different versions of vLLM require different versions of huggingface_hub, thus using a non-default vLLM version might require using a non-default version of huggingface_hub.  Default is version 0.30.2.
+            The version of vLLM installed from PyPI. Engine arguments are built from whatever that
+            release declares, so a non-default version no longer needs VLLMArgs subclassed: arguments it
+            does not have are ignored, arguments it adds are reachable through VLLMArgs.extra_args.
+            Default is 0.29.0.
+        huggingface_hub_version (Optional[str]):
+            Pin huggingface_hub rather than letting vLLM's own dependency bound choose it. Default None,
+            which is almost always right: the bound differs sharply between vLLM releases.
         workers (int):
             The number of workers to run in the container. Default is 1.
         concurrent_requests (int):
@@ -232,14 +282,16 @@ class VLLM(ASGI):
         gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU,
         gpu_count: int = 0,
         image: Image = Image(python_version="python3.11"),
-        vllm_version: str = "0.8.4",
-        huggingface_hub_version: str = "0.30.2",
+        vllm_version: str = "0.29.0",
+        huggingface_hub_version: Optional[str] = None,
         workers: int = 1,
         concurrent_requests: int = 1,
         keep_warm_seconds: int = 60,
         max_pending_tasks: int = 100,
         timeout: int = 3600,
         authorized: bool = True,
+        checkpoint_enabled: bool = False,
+        env: Optional[Dict[str, str]] = None,
         name: Optional[str] = None,
         volumes: Optional[List[Union[Volume, CloudBucket]]] = [],
         disks: Optional[List[DurableDisk]] = None,
@@ -253,14 +305,18 @@ class VLLM(ASGI):
 
         volumes.append(Volume(name="vllm_cache_root", mount_path=vllm_args.vllm_cache_root))
 
-        image = image.add_python_packages(
-            [
-                "fastapi",
-                "numpy",
-                f"vllm=={vllm_version}",
-                f"huggingface_hub=={huggingface_hub_version}",
-            ]
-        )
+        # vLLM already depends on fastapi and numpy, with the bounds its own dependencies need. Asking for
+        # them unconstrained alongside it made pip take the newest of each first, find that numba wants
+        # numpy<2.3, and backtrack - walking dozens of transformers releases to get back out. Naming only
+        # what vLLM does not bring lets the resolver go straight to a working set.
+        #
+        # huggingface_hub is one vLLM does bring, and pinning it alongside is how this broke: 0.8.4 wants
+        # <1.0 and 0.29 wants >=1.28, so a single pinned version makes one of them unresolvable outright.
+        # Left unset, vLLM's own bound decides; pass huggingface_hub_version only to override that.
+        packages = [f"vllm=={vllm_version}"]
+        if huggingface_hub_version:
+            packages.append(f"huggingface_hub=={huggingface_hub_version}")
+        image = image.add_python_packages(packages)
 
         super().__init__(
             cpu=cpu,
@@ -274,6 +330,8 @@ class VLLM(ASGI):
             max_pending_tasks=max_pending_tasks,
             timeout=timeout,
             authorized=authorized,
+            checkpoint_enabled=checkpoint_enabled,
+            env=env or {},
             name=name,
             volumes=volumes,
             disks=disks,
@@ -281,6 +339,24 @@ class VLLM(ASGI):
             autoscaler=autoscaler,
         )
 
+        # A GPU checkpoint has to fit a copy of the card inside the container. vLLM reserves
+        # gpu_memory_utilization of the device up front and holds it, so the footprint is known here -
+        # which means an impossible deployment can be named now rather than after a ten-minute cold start
+        # and a dump log that blames CUDA for what is really an out-of-memory kill.
+        if checkpoint_enabled and gpu_count > 0:
+            gpu_name = gpu[0] if isinstance(gpu, list) and gpu else gpu
+            sizing = advise(
+                gpu=str(getattr(gpu_name, "value", gpu_name)),
+                gpu_count=gpu_count,
+                requested_memory_gib=self.parse_memory(memory) / 1024
+                if isinstance(memory, str)
+                else float(memory) / 1024,
+                gpu_memory_utilization=vllm_args.gpu_memory_utilization,
+                process_rss_gib=VLLM_PROCESS_RSS_GIB,
+            )
+            for warning in sizing.warnings:
+                terminal.warn(warning)
+
         self.serving = ServingConfig(
             app_kind=LLM_APP_KIND,
             serving_protocol=OPENAI_SERVING_PROTOCOL,
@@ -296,7 +372,21 @@ class VLLM(ASGI):
         )
         self.chat_template_url = vllm_args.chat_template_url
         self.engine_args = vllm_args
-        self.app_args = SimpleNamespace(
+        # vLLM's --served-model-name is nargs="+", so both the engine and the API server's model registry
+        # expect a list. A bare string is iterated character by character: "qwen15" is registered as six
+        # models named q, w, e, n, 1 and 5, and the name you actually asked for matches none of them.
+        # engine_args below is this same object, so normalising here settles both.
+        if isinstance(vllm_args.served_model_name, str):
+            vllm_args.served_model_name = (
+                [vllm_args.served_model_name] if vllm_args.served_model_name else None
+            )
+
+        # The server arguments the OpenAI app reads off this namespace. init_app_state accesses them by
+        # name, so anything it expects and this does not set is an AttributeError at startup - the same
+        # failure the engine bridge fixed, in a second place. _ServerArgs seeds the installed version's own
+        # defaults where it publishes them and answers None for the rest, so a release that starts reading
+        # a new argument gets a sane value instead of a crash.
+        self.app_args = _ServerArgs(
             model=vllm_args.model,
             served_model_name=vllm_args.served_model_name,
             disable_log_requests=vllm_args.disable_log_requests,
@@ -325,6 +415,37 @@ class VLLM(ASGI):
     def func(self, *args: Any, **kwargs: Any):
         pass
 
+    def _engine_args_for(self, cls: Any) -> Any:
+        """Build the engine's arguments from whatever version of vLLM is installed.
+
+        `AsyncEngineArgs.from_cli_args` does `getattr(args, name)` with no default for every field the
+        installed vLLM declares, so a VLLMArgs missing even one of them raises AttributeError. That is why
+        VLLMArgs mirrors a single release by hand, and why changing vllm_version used to mean subclassing
+        it - vLLM's EngineArgs has gone from 104 fields to 236 across releases, and 32 of the ones named
+        here no longer exist at all.
+
+        Starting from the installed version's own defaults inverts that: fields it knows take our value
+        when we have one and its default when we do not, fields it has dropped are ignored, and anything
+        in extra_args is applied by name, so a flag added after this SDK was written can still be set.
+        """
+        import dataclasses
+
+        missing = object()
+        engine = cls()                                # this vLLM's defaults, whatever version it is
+        known = {f.name for f in dataclasses.fields(cls)}
+        for name in known:
+            value = getattr(self.engine_args, name, missing)
+            if value is not missing:
+                setattr(engine, name, value)
+
+        for name, value in (getattr(self.engine_args, "extra_args", None) or {}).items():
+            if name not in known:
+                # Not an argument of the installed vLLM. Saying so beats a setting that does nothing.
+                print(f"[levers9] ignoring unknown vLLM engine argument {name!r}", flush=True)
+                continue
+            setattr(engine, name, value)
+        return engine
+
     def __call__(self, *args: Any, **kwargs: Any):
         import asyncio
 
@@ -332,9 +453,16 @@ class VLLM(ASGI):
         from fastapi import FastAPI
         from vllm.engine.arg_utils import AsyncEngineArgs
         from vllm.engine.async_llm_engine import AsyncLLMEngine
-        from vllm.entrypoints.openai.tool_parsers import ToolParserManager
         from vllm.usage.usage_lib import UsageContext
 
+        # 0.29 reorganised vllm/entrypoints/openai into subpackages and moved the tool parsers out of it
+        # entirely. The class and its API are unchanged, so both paths are tried rather than pinning the
+        # SDK to one layout - an endpoint can pin its own vllm_version, and the default has not moved yet.
+        try:
+            from vllm.tool_parsers import ToolParserManager          # 0.29+
+        except ImportError:
+            from vllm.entrypoints.openai.tool_parsers import ToolParserManager
+
         if self.engine_args.vllm_cache_root:
             os.environ["VLLM_CACHE_ROOT"] = self.engine_args.vllm_cache_root
 
@@ -379,23 +507,28 @@ class VLLM(ASGI):
         async def health_check():
             return {"status": "healthy"}
 
-        app.include_router(api_server.router)
+        # 0.29 replaced the module-level router with a function that registers them onto the app.
+        if hasattr(api_server, "register_api_routers"):
+            api_server.register_api_routers(app)
+        else:
+            app.include_router(api_server.router)
 
-        engine_args = AsyncEngineArgs.from_cli_args(self.engine_args)
+        engine_args = self._engine_args_for(AsyncEngineArgs)
 
         engine_client = AsyncLLMEngine.from_engine_args(
             engine_args, usage_context=UsageContext.OPENAI_API_SERVER
         )
 
-        model_config = asyncio.run(engine_client.get_model_config())
-        asyncio.run(
-            api_server.init_app_state(
-                engine_client,
-                model_config,
-                app.state,
-                self.app_args,
-            )
-        )
+        # 0.8.4 wanted the model config passed in; 0.29 reads it off the engine client and dropped the
+        # parameter. Ask the function what it takes rather than branching on a version string.
+        import inspect
+
+        takes_model_config = "model_config" in inspect.signature(api_server.init_app_state).parameters
+        if takes_model_config:
+            state_args = (engine_client, asyncio.run(engine_client.get_model_config()), app.state, self.app_args)
+        else:
+            state_args = (engine_client, app.state, self.app_args)
+        asyncio.run(api_server.init_app_state(*state_args))
 
         return app
 
@@ -404,6 +537,7 @@ class VLLM(ASGI):
         name: Optional[str] = None,
         context: Optional[ConfigContext] = None,
         invocation_details_func: Optional[Callable[..., None]] = None,
+        rollout: str = "auto",
         **invocation_details_options: Any,
     ) -> Tuple[Dict[str, Any], bool]:
         self.name = name or self.name
@@ -431,19 +565,28 @@ class VLLM(ASGI):
 
         terminal.header("Deploying")
         deploy_response: DeployStubResponse = self.controlplane_stub.deploy_stub(
-            DeployStubRequest(stub_id=self.stub_id, name=self.name)
+            DeployStubRequest(stub_id=self.stub_id, name=self.name, rollout=rollout)
         )
 
         self.deployment_id = deploy_response.deployment_id
+        warn_msg = deploy_response.warn_msg if isinstance(deploy_response.warn_msg, str) else ""
         if deploy_response.ok:
+            # A rollout can succeed and still have something to say - that a revision was replaced, or
+            # that downtime is expected. Swallowing it is how a rollout that quietly did nothing stayed
+            # invisible for nine versions.
+            if warn_msg:
+                terminal.warn(warn_msg)
             terminal.header("Deployed 🎉")
             if invocation_details_func:
                 invocation_details_func(**invocation_details_options)
             else:
                 self.print_invocation_snippet(**invocation_details_options)
+        elif deploy_response.err_msg:
+            terminal.error(deploy_response.err_msg, exit=False)
 
         return {
             "deployment_id": deploy_response.deployment_id,
+            "warning": warn_msg,
         }, deploy_response.ok
 
     @with_grpc_error_handling

levers9-0.3.0/src/levers9/checkpoint_memory.py

@@ -0,0 +1,128 @@
+"""How much container memory a GPU checkpoint needs, and why.
+
+A GPU checkpoint copies device memory into shmem-backed host allocations that are charged to the
+container's memory cgroup, at roughly 1.2x the live device footprint. So a container that will be
+checkpointed has to hold its own resident memory *and* a copy of everything on the card:
+
+    container_memory  >  process_rss + HOST_COPY_RATIO * device_footprint + slack
+
+Get this wrong and the kernel OOM-kills the process part way through the dump. What surfaces is not an
+out-of-memory error - CRIU's NVIDIA plugin reports
+
+    cuda_plugin: CHECKPOINT_DEVICES failed ... "OS call failed or operation not supported on this OS"
+
+which reads like the card or the driver cannot do checkpointing at all. It can. The tell is in the kernel
+log rather than the container log: `Memory cgroup out of memory` with a large `shmem-rss`, identical
+across runs with very different device footprints, because the copy is killed on reaching the ceiling
+while it is still growing.
+
+The point of this module is that nobody should have to learn that twice. A deployment that cannot
+possibly checkpoint is worth saying so at deploy time, in a second, rather than after a ten-minute cold
+start and an unreadable dump log.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+# Usable VRAM per card in GiB - a little under the nameplate figure, which is what the driver reports
+# once its own reservations are taken out (a "16GB" T4 profiles at 14.76GiB).
+GPU_VRAM_GIB = {
+    "T4": 14.7,
+    "L4": 22.3,
+    "A10": 22.3,
+    "A10G": 22.3,
+    "A16": 14.7,
+    "A30": 23.5,
+    "A40": 44.3,
+    "L40": 44.3,
+    "L40S": 44.3,
+    "A100": 39.5,
+    "A100-40": 39.5,
+    "A100-80": 79.2,
+    "H100": 79.2,
+    "H200": 140.0,
+    "A4000": 15.7,
+    "A5000": 23.5,
+    "A6000": 47.5,
+    "RTX4000Ada": 19.5,
+    "RTXPro6000": 95.0,
+}
+
+# Measured: a dump of an 8GiB device buffer fitted under an 11Gi limit alongside ~1.5GiB of interpreter,
+# and vLLM at 12.5GiB device needed ~18.8GiB. Both put the host copy at ~1.2x the device footprint.
+HOST_COPY_RATIO = 1.2
+
+# What the runtime itself costs before any model. A bare Python endpoint with torch sits near 1.5GiB;
+# vLLM carries ~3.8GiB before a single weight is loaded.
+DEFAULT_PROCESS_RSS_GIB = 1.5
+VLLM_PROCESS_RSS_GIB = 3.8
+
+# Calibrated against the runs, not guessed. A T4 at 0.85 (12.5GiB device) checkpointed inside 20Gi, so the
+# figure has to stay under that or it warns about a configuration known to work; a T4 at 0.45 inside 11Gi
+# failed, so it has to stay above that. 1GiB sits between them.
+SLACK_GIB = 1.0
+
+
+@dataclass
+class MemoryAdvice:
+    """What a checkpointed GPU deployment needs, and whether what was asked for is enough."""
+
+    required_gib: float = 0.0
+    device_gib: float = 0.0
+    process_rss_gib: float = 0.0
+    fits: bool = True
+    warnings: list = field(default_factory=list)
+
+    @property
+    def suggestion(self) -> str:
+        return f'memory="{int(self.required_gib + 0.999)}Gi"'
+
+
+def device_footprint_gib(
+    gpu: str, gpu_count: int = 1, gpu_memory_utilization: Optional[float] = None
+) -> float:
+    """How much of the card will actually be live when the snapshot is taken.
+
+    For vLLM that is `gpu_memory_utilization` of the card, because it reserves that up front and holds it.
+    Without a utilisation figure the whole card is the only safe assumption: a process is free to fill it.
+    """
+    vram = GPU_VRAM_GIB.get(gpu, 0.0) * max(gpu_count, 1)
+    if not vram:
+        return 0.0
+    return vram * (gpu_memory_utilization if gpu_memory_utilization else 1.0)
+
+
+def advise(
+    gpu: str,
+    gpu_count: int = 1,
+    requested_memory_gib: Optional[float] = None,
+    gpu_memory_utilization: Optional[float] = None,
+    process_rss_gib: float = DEFAULT_PROCESS_RSS_GIB,
+) -> MemoryAdvice:
+    """Size a checkpointed GPU container, and say plainly when what was asked for cannot work."""
+    device = device_footprint_gib(gpu, gpu_count, gpu_memory_utilization)
+    if not device:
+        # An unknown card: no basis to advise on, and a wrong number is worse than none.
+        return MemoryAdvice()
+
+    required = process_rss_gib + HOST_COPY_RATIO * device + SLACK_GIB
+    advice = MemoryAdvice(
+        required_gib=required,
+        device_gib=device,
+        process_rss_gib=process_rss_gib,
+    )
+
+    if requested_memory_gib is not None and requested_memory_gib < required:
+        advice.fits = False
+        advice.warnings.append(
+            f"checkpoint_enabled needs about {required:.0f}Gi of container memory on "
+            f"{gpu}{f' x{gpu_count}' if gpu_count > 1 else ''}, but this asks for "
+            f"{requested_memory_gib:.0f}Gi. A GPU checkpoint copies device memory into the container "
+            f"({device:.1f}GiB live on the card, about {HOST_COPY_RATIO * device:.1f}GiB of host memory) "
+            f"on top of the process itself (~{process_rss_gib:.1f}GiB). Too little and the dump is "
+            f"OOM-killed half way through, which is reported as a CUDA error rather than an out-of-memory "
+            f"one. Set {advice.suggestion}, lower gpu_memory_utilization, or drop checkpoint_enabled."
+        )
+    return advice

{levers9-0.2.2 → levers9-0.3.0}/src/levers9/cli/main.py

@@ -32,6 +32,7 @@ from . import (
     token,
     volume,
     worker,
+    workspace,
 )
 from .extraclick import CLICK_CONTEXT_SETTINGS, ClickCommonGroup, CommandGroupCollection
 
@@ -125,6 +126,7 @@ def load_cli(check_config=True, **kwargs: Any) -> CLI:
     cli.register(secret)
     cli.register(token)
     cli.register(worker)
+    cli.register(workspace)
     cli.register(shell)
     cli.register(run)
     cli.register(dev)

levers9-0.3.0/src/levers9/cli/workspace.py

@@ -0,0 +1,160 @@
+from typing import Any, Dict
+
+import click
+from rich.table import Column, Table, box
+
+from .. import terminal
+from ..client import get as rest_get
+from ..client import post as rest_post
+from ..client.client import Client
+from .extraclick import ClickManagementGroup
+
+
+@click.group(
+    name="workspace",
+    help="Inspect and configure the workspace.",
+    cls=ClickManagementGroup,
+)
+def management():
+    pass
+
+
+@management.group(name="storage", help="The object storage this workspace keeps its data in.")
+def storage():
+    pass
+
+
+def _client() -> Client:
+    return Client()
+
+
+def _get(client: Client, path: str) -> Dict[str, Any]:
+    r = rest_get(token=client.token, url=client.base_url, path=path)
+    if r.status_code >= 400:
+        terminal.error(f"{r.status_code}: {_message(r)}")
+    return r.json()
+
+
+def _message(response) -> str:
+    try:
+        body = response.json()
+    except ValueError:
+        return response.text.strip() or "no detail"
+    if isinstance(body, dict):
+        return str(body.get("message") or body.get("detail") or body)
+    return str(body)
+
+
+@storage.command(
+    name="show",
+    help="Show which bucket this workspace uses.",
+    epilog="""
+    Examples:
+
+      {cli_name} workspace storage show
+      {cli_name} workspace storage show --format json
+      \b
+    """,
+)
+@click.option("--format", type=click.Choice(("table", "json")), default="table", show_default=True)
+def show(format: str):
+    client = _client()
+    info = _get(client, f"/api/v1/workspace/{client.workspace_id}/storage")
+
+    if format == "json":
+        terminal.print_json(info)
+        return
+
+    if not info.get("configured"):
+        terminal.warn("This workspace has no storage of its own.")
+        terminal.print(
+            "Checkpoints need it, and objects fall back to shared storage without it.\n"
+            "Attach your own bucket with: levers9 workspace storage set --help"
+        )
+        return
+
+    table = Table(Column("Field"), Column("Value"), box=box.SIMPLE)
+    for label, key in (
+        ("Bucket", "bucket_name"),
+        ("Endpoint", "endpoint_url"),
+        ("Region", "region"),
+        ("Access key", "access_key"),
+        ("Created", "created_at"),
+    ):
+        if info.get(key):
+            table.add_row(label, str(info[key]))
+    terminal.print(table)
+
+
+@storage.command(
+    name="set",
+    help="Point this workspace at an S3-compatible bucket you own.",
+    epilog="""
+    The credentials are checked against the bucket before anything is saved: it must exist and accept a
+    write. They are then stored encrypted and handed to workers to mount, so scope them to this bucket.
+
+    Examples:
+
+      # AWS S3
+      {cli_name} workspace storage set --bucket my-bucket --region us-east-2 \\
+          --endpoint-url https://s3.us-east-2.amazonaws.com \\
+          --access-key AKIA... --secret-key ...
+
+      # Google Cloud Storage, via its S3-compatible API. Use an HMAC key
+      # (Cloud Storage -> Settings -> Interoperability), not a service account JSON.
+      {cli_name} workspace storage set --bucket my-bucket --region us-central1 \\
+          --endpoint-url https://storage.googleapis.com \\
+          --access-key GOOG1E... --secret-key ...
+
+      # Cloudflare R2, MinIO, or anything else that speaks S3
+      {cli_name} workspace storage set --bucket my-bucket --region auto \\
+          --endpoint-url https://<account>.r2.cloudflarestorage.com \\
+          --access-key ... --secret-key ...
+      \b
+    """,
+)
+@click.option("--bucket", required=True, help="Bucket name. It must already exist.")
+@click.option("--endpoint-url", required=True, help="e.g. https://s3.us-east-2.amazonaws.com")
+@click.option("--region", required=True, help="e.g. us-east-2, or auto for R2")
+@click.option(
+    "--access-key", required=True, help="For GCS this is an HMAC key id, not a service account."
+)
+@click.option("--secret-key", required=True, help="Stored encrypted.")
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace the bucket this workspace already uses. Nothing is copied: whatever is in the old bucket stops being visible.",
+)
+def set_storage(
+    bucket: str, endpoint_url: str, region: str, access_key: str, secret_key: str, replace: bool
+):
+    client = _client()
+
+    if (
+        replace
+        and not terminal.prompt(
+            text=f"Point this workspace at {bucket}? Data in its current bucket will not be moved and will stop being visible. Type the bucket name to confirm",
+            default="",
+        )
+        == bucket
+    ):
+        terminal.error("Cancelled: the bucket name did not match.")
+
+    r = rest_post(
+        token=client.token,
+        url=client.base_url,
+        path=f"/api/v1/workspace/{client.workspace_id}/set-external-storage",
+        data={
+            "bucket_name": bucket,
+            "endpoint_url": endpoint_url,
+            "region": region,
+            "access_key": access_key,
+            "secret_key": secret_key,
+            "replace": replace,
+        },
+    )
+    if r.status_code >= 400:
+        terminal.error(f"{r.status_code}: {_message(r)}")
+
+    terminal.success(f"This workspace now stores its data in {bucket}.")
+    terminal.print("Existing containers keep their current mount until they restart.")