modal 1.1.1.dev40__py3-none-any.whl → 1.1.1.dev44__py3-none-any.whl

modal/_runtime/gpu_memory_snapshot.py

@@ -1,17 +1,18 @@
 # Copyright Modal Labs 2022
 #
 # This module provides a simple interface for creating GPU memory snapshots,
-# provising a convenient interface to `cuda-checkpoint` [1]. This is intended
+# providing a convenient interface to `cuda-checkpoint` [1]. This is intended
 # to be used in conjunction with memory snapshots.
 #
 # [1] https://github.com/NVIDIA/cuda-checkpoint
 
 import subprocess
 import time
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
+from typing import List, Optional
 
 from modal.config import config, logger
 
@@ -19,7 +20,9 @@ CUDA_CHECKPOINT_PATH: str = config.get("cuda_checkpoint_path")
 
 
 class CudaCheckpointState(Enum):
-    """State representation from the CUDA API: https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TYPES.html#group__CUDA__TYPES_1gc96cdda177a2b8c296144567cbea4f23"""
+    """State representation from the CUDA API [1].
+
+    [1] https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TYPES.html"""
 
     RUNNING = "running"
     LOCKED = "locked"
@@ -28,6 +31,8 @@ class CudaCheckpointState(Enum):
 
 
 class CudaCheckpointException(Exception):
+    """Exception raised for CUDA checkpoint operations."""
+
     pass
 
 
@@ -39,16 +44,31 @@ class CudaCheckpointProcess:
     pid: int
     state: CudaCheckpointState
 
-    def toggle(self, target_state: CudaCheckpointState, timeout_secs: float = 5 * 60.0):
+    def toggle(self, target_state: CudaCheckpointState, timeout_secs: float = 5 * 60.0) -> None:
         """Toggle CUDA checkpoint state for current process, moving GPU memory to the
-        CPU and back depending on the current process state when called."""
+        CPU and back depending on the current process state when called.
+        """
         logger.debug(f"PID: {self.pid} Toggling CUDA checkpoint state to {target_state.value}")
 
         start_time = time.monotonic()
+        retry_count = 0
+        max_retries = 3
 
         while self._should_continue_toggle(target_state, start_time, timeout_secs):
-            self._execute_toggle_command()
-            time.sleep(0.1)
+            try:
+                self._execute_toggle_command()
+                # Use exponential backoff for retries
+                sleep_time = min(0.1 * (2**retry_count), 1.0)
+                time.sleep(sleep_time)
+                retry_count = 0
+            except CudaCheckpointException as e:
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise CudaCheckpointException(
+                        f"PID: {self.pid} Failed to toggle state after {max_retries} retries: {e}"
+                    )
+                logger.debug(f"PID: {self.pid} Retry {retry_count}/{max_retries} after error: {e}")
+                time.sleep(0.5 * retry_count)
 
         logger.debug(f"PID: {self.pid} Target state {target_state.value} reached")
 
@@ -73,19 +93,25 @@ class CudaCheckpointProcess:
 
         return True
 
-    def _execute_toggle_command(self):
+    def _execute_toggle_command(self) -> None:
         """Execute the cuda-checkpoint toggle command."""
         try:
-            subprocess.run(
+            _ = subprocess.run(
                 [CUDA_CHECKPOINT_PATH, "--toggle", "--pid", str(self.pid)],
                 check=True,
                 capture_output=True,
                 text=True,
+                timeout=30,
             )
             logger.debug(f"PID: {self.pid} Successfully toggled CUDA checkpoint state")
         except subprocess.CalledProcessError as e:
-            logger.debug(f"PID: {self.pid} Failed to toggle CUDA checkpoint state: {e.stderr}")
-            raise CudaCheckpointException(e.stderr)
+            error_msg = f"PID: {self.pid} Failed to toggle CUDA checkpoint state: {e.stderr}"
+            logger.debug(error_msg)
+            raise CudaCheckpointException(error_msg)
+        except subprocess.TimeoutExpired:
+            error_msg = f"PID: {self.pid} Toggle command timed out"
+            logger.debug(error_msg)
+            raise CudaCheckpointException(error_msg)
 
     def refresh_state(self) -> None:
         """Refreshes the current CUDA checkpoint state for this process."""
@@ -95,15 +121,20 @@ class CudaCheckpointProcess:
                 check=True,
                 capture_output=True,
                 text=True,
-                timeout=5,
+                timeout=10,
             )
 
             state_str = result.stdout.strip().lower()
             self.state = CudaCheckpointState(state_str)
 
         except subprocess.CalledProcessError as e:
-            logger.debug(f"PID: {self.pid} Failed to get CUDA checkpoint state: {e.stderr}")
-            raise CudaCheckpointException(e.stderr)
+            error_msg = f"PID: {self.pid} Failed to get CUDA checkpoint state: {e.stderr}"
+            logger.debug(error_msg)
+            raise CudaCheckpointException(error_msg)
+        except subprocess.TimeoutExpired:
+            error_msg = f"PID: {self.pid} Get state command timed out"
+            logger.debug(error_msg)
+            raise CudaCheckpointException(error_msg)
 
 
 class CudaCheckpointSession:
@@ -111,12 +142,17 @@ class CudaCheckpointSession:
 
     def __init__(self):
         self.cuda_processes = self._get_cuda_pids()
-        logger.debug(f"PIDs with CUDA sessions: {[c.pid for c in self.cuda_processes]}")
+        if self.cuda_processes:
+            logger.debug(
+                f"Found {len(self.cuda_processes)} PID(s) with CUDA sessions: {[c.pid for c in self.cuda_processes]}"
+            )
+        else:
+            logger.debug("No CUDA sessions found.")
 
-    def _get_cuda_pids(self) -> list[CudaCheckpointProcess]:
+    def _get_cuda_pids(self) -> List[CudaCheckpointProcess]:
         """Iterates over all PIDs and identifies the ones that have running
         CUDA sessions."""
-        cuda_pids: list[CudaCheckpointProcess] = []
+        cuda_pids: List[CudaCheckpointProcess] = []
 
         # Get all active process IDs from /proc directory
         proc_dir = Path("/proc")
@@ -125,75 +161,143 @@ class CudaCheckpointSession:
                 "OS does not have /proc path rendering it incompatible with GPU memory snapshots."
             )
 
-        for entry in proc_dir.iterdir():
-            if not entry.name.isdigit():
-                continue
-
-            pid = int(entry.name)
-            try:
-                # Call cuda-checkpoint to check if this PID has a CUDA session
-                result = subprocess.run(
-                    [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)],
-                    capture_output=True,
-                    text=True,
-                    timeout=10,
-                )
-
-                # If the command succeeds (return code 0), this PID has a CUDA session
-                if result.returncode == 0:
-                    state_str = result.stdout.strip().lower()
-                    state = CudaCheckpointState(state_str)
-
-                    cuda_checkpoint_process = CudaCheckpointProcess(pid=pid, state=state)
-                    cuda_pids.append(cuda_checkpoint_process)
+        # Get all numeric directories (PIDs) from /proc
+        pid_dirs = [entry for entry in proc_dir.iterdir() if entry.name.isdigit()]
 
-            # Command failed, which is expected for PIDs without CUDA sessions
-            except subprocess.CalledProcessError:
-                continue
+        # Use ThreadPoolExecutor to check PIDs in parallel for better performance
+        with ThreadPoolExecutor(max_workers=min(50, len(pid_dirs))) as executor:
+            future_to_pid = {
+                executor.submit(self._check_cuda_session, int(entry.name)): int(entry.name) for entry in pid_dirs
+            }
 
-            # Raise other exceptions
-            except subprocess.TimeoutExpired:
-                raise CudaCheckpointException(f"Failed to get CUDA state for PID {pid}")
-            except Exception as e:
-                raise CudaCheckpointException(e)
+            for future in as_completed(future_to_pid):
+                pid = future_to_pid[future]
+                try:
+                    cuda_process = future.result()
+                    if cuda_process:
+                        cuda_pids.append(cuda_process)
+                except Exception as e:
+                    logger.debug(f"Error checking PID {pid}: {e}")
 
         # Sort PIDs for ordered checkpointing
         cuda_pids.sort(key=lambda x: x.pid)
         return cuda_pids
 
+    def _check_cuda_session(self, pid: int) -> Optional[CudaCheckpointProcess]:
+        """Check if a specific PID has a CUDA session."""
+        try:
+            result = subprocess.run(
+                [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+
+            # If the command succeeds (return code 0), this PID has a CUDA session
+            if result.returncode == 0:
+                state_str = result.stdout.strip().lower()
+                state = CudaCheckpointState(state_str)
+                return CudaCheckpointProcess(pid=pid, state=state)
+
+        except subprocess.CalledProcessError:
+            # Command failed, which is expected for PIDs without CUDA sessions
+            pass
+        except subprocess.TimeoutExpired:
+            logger.debug(f"Timeout checking CUDA state for PID {pid}")
+        except Exception as e:
+            logger.debug(f"Error checking PID {pid}: {e}")
+
+        return None
+
     def checkpoint(self) -> None:
+        """Checkpoint all CUDA processes, moving GPU memory to CPU."""
+        if not self.cuda_processes:
+            logger.debug("No CUDA processes to checkpoint.")
+            return
+
         # Validate all states first
         for proc in self.cuda_processes:
+            proc.refresh_state()  # Refresh state before validation
             if proc.state != CudaCheckpointState.RUNNING:
-                raise CudaCheckpointException(f"CUDA session not in {CudaCheckpointState.RUNNING} state.")
+                raise CudaCheckpointException(
+                    f"PID {proc.pid}: CUDA session not in {CudaCheckpointState.RUNNING.value} state. "
+                    f"Current state: {proc.state.value}"
+                )
 
         # Moving state from GPU to CPU can take several seconds per CUDA session.
         # Make a parallel call per CUDA session.
         start = time.perf_counter()
 
-        def checkpoint_impl(proc: CudaCheckpointProcess):
+        def checkpoint_impl(proc: CudaCheckpointProcess) -> None:
             proc.toggle(CudaCheckpointState.CHECKPOINTED)
 
         with ThreadPoolExecutor() as executor:
-            list(executor.map(checkpoint_impl, self.cuda_processes))
+            futures = [executor.submit(checkpoint_impl, proc) for proc in self.cuda_processes]
+
+            # Wait for all futures and collect any exceptions
+            exceptions = []
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    exceptions.append(e)
+
+            if exceptions:
+                raise CudaCheckpointException(
+                    f"Failed to checkpoint {len(exceptions)} processes: {'; '.join(str(e) for e in exceptions)}"
+                )
 
         elapsed = time.perf_counter() - start
-        logger.debug(f"Checkpointing CUDA sessions took => {elapsed:.3f}s")
+        logger.debug(f"Checkpointing {len(self.cuda_processes)} CUDA sessions took => {elapsed:.3f}s")
 
     def restore(self) -> None:
+        """Restore all CUDA processes, moving memory back from CPU to GPU."""
+        if not self.cuda_processes:
+            logger.debug("No CUDA sessions to restore.")
+            return
+
         # Validate all states first
         for proc in self.cuda_processes:
+            proc.refresh_state()  # Refresh state before validation
             if proc.state != CudaCheckpointState.CHECKPOINTED:
-                raise CudaCheckpointException(f"CUDA session not in {CudaCheckpointState.CHECKPOINTED} state.")
+                raise CudaCheckpointException(
+                    f"PID {proc.pid}: CUDA session not in {CudaCheckpointState.CHECKPOINTED.value} state. "
+                    f"Current state: {proc.state.value}"
+                )
 
         # See checkpoint() for rationale about parallelism.
         start = time.perf_counter()
 
-        def restore_process(proc: CudaCheckpointProcess):
+        def restore_process(proc: CudaCheckpointProcess) -> None:
             proc.toggle(CudaCheckpointState.RUNNING)
 
         with ThreadPoolExecutor() as executor:
-            list(executor.map(restore_process, self.cuda_processes))
+            futures = [executor.submit(restore_process, proc) for proc in self.cuda_processes]
+
+            # Wait for all futures and collect any exceptions
+            exceptions = []
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    exceptions.append(e)
+
+            if exceptions:
+                raise CudaCheckpointException(
+                    f"Failed to restore {len(exceptions)} processes: {'; '.join(str(e) for e in exceptions)}"
+                )
 
         elapsed = time.perf_counter() - start
-        logger.debug(f"Restoring CUDA sessions took => {elapsed:.3f}s")
+        logger.debug(f"Restoring {len(self.cuda_processes)} CUDA session(s) took => {elapsed:.3f}s")
+
+    def get_process_count(self) -> int:
+        """Get the number of CUDA processes managed by this session."""
+        return len(self.cuda_processes)
+
+    def get_process_states(self) -> List[tuple[int, CudaCheckpointState]]:
+        """Get current states of all managed processes."""
+        states = []
+        for proc in self.cuda_processes:
+            proc.refresh_state()
+            states.append((proc.pid, proc.state))
+        return states

modal/client.pyi

@@ -33,7 +33,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.1.dev40",
+        version: str = "1.1.1.dev44",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.
@@ -164,7 +164,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.1.1.dev40",
+        version: str = "1.1.1.dev44",
     ):
         """mdmd:hidden
         The Modal client object is not intended to be instantiated directly by users.

modal/functions.pyi

@@ -427,7 +427,7 @@ class Function(
 
     _call_generator: ___call_generator_spec[typing_extensions.Self]
 
-    class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
+    class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER:
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
@@ -436,7 +436,7 @@ class Function(
             """Calls the function remotely, executing it with the given arguments and returning the execution's result."""
             ...
 
-    remote: __remote_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
+    remote: __remote_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
 
     class __remote_gen_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, *args, **kwargs) -> typing.Generator[typing.Any, None, None]:
@@ -463,7 +463,7 @@ class Function(
         """
         ...
 
-    class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
+    class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """[Experimental] Calls the function with the given arguments, without waiting for the results.
 
@@ -487,7 +487,7 @@ class Function(
             ...
 
     _experimental_spawn: ___experimental_spawn_spec[
-        modal._functions.P, modal._functions.ReturnType, typing_extensions.Self
+        modal._functions.ReturnType, modal._functions.P, typing_extensions.Self
     ]
 
     class ___spawn_map_inner_spec(typing_extensions.Protocol[P_INNER, SUPERSELF]):
@@ -496,7 +496,7 @@ class Function(
 
     _spawn_map_inner: ___spawn_map_inner_spec[modal._functions.P, typing_extensions.Self]
 
-    class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER, SUPERSELF]):
+    class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER, SUPERSELF]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """Calls the function with the given arguments, without waiting for the results.
 
@@ -517,7 +517,7 @@ class Function(
             """
             ...
 
-    spawn: __spawn_spec[modal._functions.P, modal._functions.ReturnType, typing_extensions.Self]
+    spawn: __spawn_spec[modal._functions.ReturnType, modal._functions.P, typing_extensions.Self]
 
     def get_raw_f(self) -> collections.abc.Callable[..., typing.Any]:
         """Return the inner Python object wrapped by this Modal Function."""

modal/image.py

@@ -1393,7 +1393,13 @@ class _Image(_Object, type_prefix="im"):
                 # a requirement in `uv.lock`
                 return
 
-            dependencies = pyproject_toml_content["project"]["dependencies"]
+            try:
+                dependencies = pyproject_toml_content["project"]["dependencies"]
+            except KeyError as e:
+                raise InvalidError(
+                    f"Invalid pyproject.toml file: missing key {e} in {pyproject_toml}. "
+                    "See https://packaging.python.org/en/latest/guides/writing-pyproject-toml for guidelines."
+                )
 
             for group in groups:
                 if (
@@ -1459,7 +1465,7 @@ class _Image(_Object, type_prefix="im"):
                 commands.append(f"COPY /.uv.lock {UV_ROOT}/uv.lock")
 
                 if frozen:
-                    # Do not update `uv.lock` when we have one when `frozen=True`. This it ehd efault because this
+                    # Do not update `uv.lock` when we have one when `frozen=True`. This is the default because this
                     # ensures that the runtime environment matches the local `uv.lock`.
                     #
                     # If `frozen=False`, then `uv sync` will update the the dependencies in the `uv.lock` file

{modal-1.1.1.dev40.dist-info → modal-1.1.1.dev44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.1.1.dev40
+Version: 1.1.1.dev44
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License: Apache-2.0

{modal-1.1.1.dev40.dist-info → modal-1.1.1.dev44.dist-info}/RECORD

@@ -22,7 +22,7 @@ modal/app.py,sha256=kpq4kXp7pch688y6g55QYAC10wqPTU5FXKoWPMirA3E,47899
 modal/app.pyi,sha256=-jKXlGDBWRPVsuenBhdMRqawK-L2eiJ7gHbmSblhltg,43525
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=kyAIVB3Ay-XKJizQ_1ufUFB__EagV0MLmHJpyYyJ7J0,18636
-modal/client.pyi,sha256=PNk58EEgTKJjy4hA9vN9sClYMF9gkjfbuFmCfJ3Ujfs,15831
+modal/client.pyi,sha256=vIhdpiz_vkRCPexKcG97KNhPQ20G-QFeJRcPeLYiQfQ,15831
 modal/cloud_bucket_mount.py,sha256=YOe9nnvSr4ZbeCn587d7_VhE9IioZYRvF9VYQTQux08,5914
 modal/cloud_bucket_mount.pyi,sha256=-qSfYAQvIoO_l2wsCCGTG5ZUwQieNKXdAO00yP1-LYU,7394
 modal/cls.py,sha256=7A0xGnugQzm8dOfnKMjLjtqekRlRtQ0jPFRYgq6xdUM,40018
@@ -39,9 +39,9 @@ modal/file_io.py,sha256=BVqAJ0sgPUfN8QsYztWiGB4j56he60TncM02KsylnCw,21449
 modal/file_io.pyi,sha256=cPT_hsplE5iLCXhYOLn1Sp9eDdk7DxdFmicQHanJZyg,15918
 modal/file_pattern_matcher.py,sha256=urAue8es8jxqX94k9EYoZxxhtfgOlsEES8lbFHOorzc,7734
 modal/functions.py,sha256=kcNHvqeGBxPI7Cgd57NIBBghkfbeFJzXO44WW0jSmao,325
-modal/functions.pyi,sha256=cS7QYPHcFD_p95hrAU-TWxBIraIzzG-uO9KyPAwfxdw,34766
+modal/functions.pyi,sha256=VYPnfnCLfHRDAl6t5AaVHmZEz0T5f2igjTeMOtHPie8,34766
 modal/gpu.py,sha256=Fe5ORvVPDIstSq1xjmM6OoNgLYFWvogP9r5BgmD3hYg,6769
-modal/image.py,sha256=Gm1TGIy1IzaNaYG659C6Jl_flPe6YYzGdSRpf0epDAg,102833
+modal/image.py,sha256=A83nmo0zfCUwgvJh0LZ7Yc1sYvDnZLl_phbKxN-9HIw,103144
 modal/image.pyi,sha256=mlGwmpkKdwNK_VRrCa0WMDURmQPSOohm7hgiiFTfdXI,68541
 modal/io_streams.py,sha256=ut9tY_yEtiBsgQ40u_72Ns87IZHfbMxfnh8t6U9RSGA,16204
 modal/io_streams.pyi,sha256=aOun_jUFKHSJyUY6-7gKvNoxzcULsa8_hxdtEO7v-gk,13980
@@ -86,7 +86,7 @@ modal/_runtime/container_io_manager.py,sha256=hjkK4gke_A8_mULSfm3F1hZKR0C61lKbRU
 modal/_runtime/container_io_manager.pyi,sha256=_HvYZzpXX-msFDFuOvk1z6L5DBbv5Dfly16PgYDOojY,23065
 modal/_runtime/execution_context.py,sha256=73Y5zH_o-MhVCrkJXakYVlFkKqCa2CWvqoHjOfJrJGg,3034
 modal/_runtime/execution_context.pyi,sha256=IFcW1jphqTchX4fy-45rqfz91RhkZPWtIhIvLvGsNGM,2294
-modal/_runtime/gpu_memory_snapshot.py,sha256=HXgqPHQj0LARhmie_h62V95L-M2R1Kg21INUm_IStn8,7574
+modal/_runtime/gpu_memory_snapshot.py,sha256=BWIMKkH-UXTQOJJuXbM15UWCHHSYlJ0XxGlZunKb0Ug,11877
 modal/_runtime/telemetry.py,sha256=T1RoAGyjBDr1swiM6pPsGRSITm7LI5FDK18oNXxY08U,5163
 modal/_runtime/user_code_imports.py,sha256=78wJyleqY2RVibqcpbDQyfWVBVT9BjyHPeoV9WdwV5Y,17720
 modal/_utils/__init__.py,sha256=waLjl5c6IPDhSsdWAm9Bji4e2PVxamYABKAze6CHVXY,28
@@ -151,7 +151,7 @@ modal/experimental/__init__.py,sha256=nuc7AL4r_Fs08DD5dciWFZhrV1nanwoClOfdTcudU0
 modal/experimental/flash.py,sha256=viXQumCIFp5VFsPFURdFTBTjP_QnsAi8nSWXAMmfjeQ,19744
 modal/experimental/flash.pyi,sha256=A8_qJGtGoXEzKDdHbvhmCw7oqfneFEvJQK3ZdTOvUdU,10830
 modal/experimental/ipython.py,sha256=TrCfmol9LGsRZMeDoeMPx3Hv3BFqQhYnmD_iH0pqdhk,2904
-modal-1.1.1.dev40.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-1.1.1.dev44.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
 modal_docs/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,28
 modal_docs/gen_cli_docs.py,sha256=c1yfBS_x--gL5bs0N4ihMwqwX8l3IBWSkBAKNNIi6bQ,3801
 modal_docs/gen_reference_docs.py,sha256=d_CQUGQ0rfw28u75I2mov9AlS773z9rG40-yq5o7g2U,6359
@@ -159,10 +159,10 @@ modal_docs/mdmd/__init__.py,sha256=svYKtV8HDwDCN86zbdWqyq5T8sMdGDj0PVlzc2tIxDM,2
 modal_docs/mdmd/mdmd.py,sha256=eW5MzrEl7mSclDo4Uv64sQ1-4IyLggldbgUJdBVLDdI,6449
 modal_docs/mdmd/signatures.py,sha256=XJaZrK7Mdepk5fdX51A8uENiLFNil85Ud0d4MH8H5f0,3218
 modal_proto/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
-modal_proto/api.proto,sha256=BhH6QQvIbZ-08MmZUxxK_BuU9718ruM29SX8NGqLV9s,103769
+modal_proto/api.proto,sha256=b9CGi-zi7EqegJKq1UiHLp18DXQF0cDAU9lCky2iOGU,103797
 modal_proto/api_grpc.py,sha256=1mDcIexGtoVq0675-sqlYVr_M2ncGETpmKsOP7VwE3Y,126369
-modal_proto/api_pb2.py,sha256=fk7VQgT6VAuQrP-caTU1Wxx2stTGOGPABD9pHxb_c8E,363956
-modal_proto/api_pb2.pyi,sha256=7CzSDCotsAG22rOhA01TOrP1A2Unjo69ypiI_Q1Fzfw,500527
+modal_proto/api_pb2.py,sha256=oW3_AuZmIxZcE9RoFGjr-eCOKJxHeQSL_Fhqsnm339Q,364022
+modal_proto/api_pb2.pyi,sha256=fy34bN5FBEum7zLiMw6PUJkLWjh1VuIE35l8eB9iQRc,500746
 modal_proto/api_pb2_grpc.py,sha256=tug2WESqUM6l_M8mCkumK0RkDpD_GnMbx-3cH8-Waig,272807
 modal_proto/api_pb2_grpc.pyi,sha256=w1bj43sKHDFOqZnR28_jWWvkFjSFBcRNY8HLGqKO__g,63991
 modal_proto/modal_api_grpc.py,sha256=KL5Nw4AS9hJNxfL6VIeuxHz4jIUN7Unz7hYnoSsqyx0,19071
@@ -174,10 +174,10 @@ modal_proto/options_pb2.pyi,sha256=l7DBrbLO7q3Ir-XDkWsajm0d0TQqqrfuX54i4BMpdQg,1
 modal_proto/options_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0yJSI,247
 modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-modal_version/__init__.py,sha256=yKVPZgQsYtXUtD_7xH36mRiDGgXyJAA7FA7cDCXtMQI,121
+modal_version/__init__.py,sha256=hR_nl7HUS3EYXDfStfBHO0ndbZEzuHyHLbJvQvXR3_g,121
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal-1.1.1.dev40.dist-info/METADATA,sha256=G_cZZm2OijpQdzkE97eYr1DIr8doI9Q1_4VSQsl37rA,2460
-modal-1.1.1.dev40.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-modal-1.1.1.dev40.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-1.1.1.dev40.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
-modal-1.1.1.dev40.dist-info/RECORD,,
+modal-1.1.1.dev44.dist-info/METADATA,sha256=VUIWbFXSw8XKvOrzYBS9rUfJv67fk-axKhZT9FiKS-w,2460
+modal-1.1.1.dev44.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+modal-1.1.1.dev44.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-1.1.1.dev44.dist-info/top_level.txt,sha256=4BWzoKYREKUZ5iyPzZpjqx4G8uB5TWxXPDwibLcVa7k,43
+modal-1.1.1.dev44.dist-info/RECORD,,

modal_proto/api.proto

@@ -3008,6 +3008,7 @@ message TaskInfo {
   string gpu_type = 6;
   string sandbox_id = 7;
   TaskSnapshotBehavior snapshot_behavior = 8;
+  GPUConfig gpu_config = 9;
 }
 
 message TaskListRequest {