modal 1.5.6.dev2__tar.gz → 1.5.6.dev3__tar.gz

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.5.6.dev2
+Version: 1.5.6.dev3
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License-Expression: Apache-2.0

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/_runtime/gpu_memory_snapshot.py

@@ -22,15 +22,9 @@ CUDA_CHECKPOINT_PATH: str = config.get("cuda_checkpoint_path")
 # stay in sync with `GPU_SNAPSHOT_RESTORE_FAILED_EXIT_CODE` in runner.rs.
 SNAPSHOT_RESTORE_FAILED_EXIT_CODE: int = 222
 
-# Maximum total duration for each individual `cuda-checkpoint` invocation.
-CUDA_CHECKPOINT_TIMEOUT: float = 3 * 60.0
-
 # Number of retries for each individual `cuda-checkpoint --toggle` invocation.
 CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES: int = 3
 
-# Maximum total duration for an entire toggle operation.
-CUDA_CHECKPOINT_TOGGLE_TIMEOUT: float = CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES * CUDA_CHECKPOINT_TIMEOUT
-
 
 class CudaCheckpointState(Enum):
     """State representation from the CUDA API [1].
@@ -63,14 +57,11 @@ class CudaCheckpointProcess:
         """
         logger.debug(f"PID: {self.pid} Toggling CUDA checkpoint state to {target_state.value}")
 
-        start_time = time.monotonic()
         retry_count = 0
         max_retries = CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES
 
         attempts = 0
-        while self._should_continue_toggle(
-            target_state, start_time, refresh=not (skip_first_refresh and attempts == 0)
-        ):
+        while self._should_continue_toggle(target_state, refresh=not (skip_first_refresh and attempts == 0)):
             attempts += 1
             try:
                 self._execute_toggle_command()
@@ -89,10 +80,8 @@ class CudaCheckpointProcess:
 
         logger.debug(f"PID: {self.pid} Target state {target_state.value} reached")
 
-    def _should_continue_toggle(
-        self, target_state: CudaCheckpointState, start_time: float, refresh: bool = True
-    ) -> bool:
-        """Check if toggle operation should continue based on current state and timeout."""
+    def _should_continue_toggle(self, target_state: CudaCheckpointState, refresh: bool = True) -> bool:
+        """Check if toggle operation should continue based on current state."""
         if refresh:
             self.refresh_state()
 
@@ -102,13 +91,6 @@ class CudaCheckpointProcess:
         if self.state == CudaCheckpointState.FAILED:
             raise CudaCheckpointException(f"PID: {self.pid} CUDA process state is {self.state}")
 
-        elapsed = time.monotonic() - start_time
-        if elapsed >= CUDA_CHECKPOINT_TOGGLE_TIMEOUT:
-            raise CudaCheckpointException(
-                f"PID: {self.pid} Timeout after {elapsed:.2f}s waiting for state {target_state.value}. "
-                f"Current state: {self.state}"
-            )
-
         return True
 
     def _execute_toggle_command(self) -> None:
@@ -119,17 +101,12 @@ class CudaCheckpointProcess:
                 check=True,
                 capture_output=True,
                 text=True,
-                timeout=CUDA_CHECKPOINT_TIMEOUT,
             )
             logger.debug(f"PID: {self.pid} Successfully toggled CUDA checkpoint state")
         except subprocess.CalledProcessError as e:
             error_msg = f"PID: {self.pid} Failed to toggle CUDA checkpoint state: {e.stderr}"
             logger.debug(error_msg)
             raise CudaCheckpointException(error_msg)
-        except subprocess.TimeoutExpired:
-            error_msg = f"PID: {self.pid} Toggle command timed out"
-            logger.debug(error_msg)
-            raise CudaCheckpointException(error_msg)
 
     def refresh_state(self) -> None:
         """Refreshes the current CUDA checkpoint state for this process."""
@@ -139,7 +116,6 @@ class CudaCheckpointProcess:
                 check=True,
                 capture_output=True,
                 text=True,
-                timeout=CUDA_CHECKPOINT_TIMEOUT,
             )
 
             state_str = result.stdout.strip().lower()
@@ -149,10 +125,6 @@ class CudaCheckpointProcess:
             error_msg = f"PID: {self.pid} Failed to get CUDA checkpoint state: {e.stderr}"
             logger.debug(error_msg)
             raise CudaCheckpointException(error_msg)
-        except subprocess.TimeoutExpired:
-            error_msg = f"PID: {self.pid} Get state command timed out"
-            logger.debug(error_msg)
-            raise CudaCheckpointException(error_msg)
 
 
 class CudaCheckpointSession:
@@ -208,7 +180,6 @@ class CudaCheckpointSession:
                 [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)],
                 capture_output=True,
                 text=True,
-                timeout=CUDA_CHECKPOINT_TIMEOUT,
             )
 
             # If the command succeeds (return code 0), this PID has a CUDA session
@@ -220,8 +191,6 @@ class CudaCheckpointSession:
         except subprocess.CalledProcessError:
             # Command failed, which is expected for PIDs without CUDA sessions
             pass
-        except subprocess.TimeoutExpired:
-            logger.debug(f"Timeout checking CUDA state for PID {pid}")
         except Exception as e:
             logger.debug(f"Error checking PID {pid}: {e}")
 

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/_runtime/task_lifecycle_manager.py

@@ -143,6 +143,12 @@ class _TaskLifecycleManager:
             else:
                 logger.debug(f"modal.Volume background commit success for {volume_id}.")
 
+    @property
+    def _gpu_snapshot_enabled(self) -> bool:
+        return bool(
+            self.function_def._experimental_enable_gpu_snapshot and self.function_def.resources.gpu_config.gpu_type
+        )
+
     async def memory_restore(self) -> None:
         # Busy-wait for restore. `/__modal/restore-state.json` is created
         # by the worker process with updates to the container config.
@@ -177,18 +183,21 @@ class _TaskLifecycleManager:
                 config.override_locally(key, value)
 
         # Restore GPU memory.
-        if self.function_def._experimental_enable_gpu_snapshot and self.function_def.resources.gpu_config.gpu_type:
-            logger.debug("GPU memory snapshot enabled. Attempting to restore GPU memory.")
-
-            try:
-                if self._cuda_checkpoint_session is None:
-                    raise gpu_memory_snapshot.CudaCheckpointException("CudaCheckpointSession not found")
-                self._cuda_checkpoint_session.restore()
-            except gpu_memory_snapshot.CudaCheckpointException as exc:
-                logger.warning(f"GPU memory snapshot restore failed; retrying task without snapshot. Error: {exc}")
-                sys.stderr.flush()
-                # Exit with a sentinel code that the runtime will use to retry the task without a snapshot.
-                os._exit(gpu_memory_snapshot.SNAPSHOT_RESTORE_FAILED_EXIT_CODE)
+        if self._gpu_snapshot_enabled:
+            if config.get("runtime_managed_cuda_checkpoint"):
+                logger.debug("GPU memory restore is managed by the container runtime.")
+            else:
+                logger.debug("GPU memory snapshot enabled. Attempting to restore GPU memory.")
+
+                try:
+                    if self._cuda_checkpoint_session is None:
+                        raise gpu_memory_snapshot.CudaCheckpointException("CudaCheckpointSession not found")
+                    self._cuda_checkpoint_session.restore()
+                except gpu_memory_snapshot.CudaCheckpointException as exc:
+                    logger.warning(f"GPU memory snapshot restore failed; retrying task without snapshot. Error: {exc}")
+                    sys.stderr.flush()
+                    # Exit with a sentinel code that the runtime will use to retry the task without a snapshot.
+                    os._exit(gpu_memory_snapshot.SNAPSHOT_RESTORE_FAILED_EXIT_CODE)
 
         self._client = await _Client.from_env()
 
@@ -199,11 +208,14 @@ class _TaskLifecycleManager:
         else:
             raise ValueError("No checkpoint ID provided for memory snapshot")
 
-        if self.function_def._experimental_enable_gpu_snapshot and self.function_def.resources.gpu_config.gpu_type:
-            logger.debug("GPU memory snapshot enabled. Attempting to snapshot GPU memory.")
+        if self._gpu_snapshot_enabled:
+            if config.get("runtime_managed_cuda_checkpoint"):
+                logger.debug("GPU memory snapshot is managed by the container runtime.")
+            else:
+                logger.debug("GPU memory snapshot enabled. Attempting to snapshot GPU memory.")
 
-            self._cuda_checkpoint_session = gpu_memory_snapshot.CudaCheckpointSession()
-            self._cuda_checkpoint_session.checkpoint()
+                self._cuda_checkpoint_session = gpu_memory_snapshot.CudaCheckpointSession()
+                self._cuda_checkpoint_session.checkpoint()
 
         await self._client.stub.ContainerCheckpoint(
             api_pb2.ContainerCheckpointRequest(checkpoint_id=self.checkpoint_id)

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/_runtime/task_lifecycle_manager.pyi

@@ -54,6 +54,8 @@ class _TaskLifecycleManager:
         """
         ...
 
+    @property
+    def _gpu_snapshot_enabled(self) -> bool: ...
     async def memory_restore(self) -> None: ...
     async def memory_snapshot(self) -> None:
         """Message server indicating that function is ready to be checkpointed."""
@@ -111,6 +113,9 @@ class TaskLifecycleManager:
 
     volume_commit: __volume_commit_spec
 
+    @property
+    def _gpu_snapshot_enabled(self) -> bool: ...
+
     class __memory_restore_spec(typing_extensions.Protocol):
         def __call__(self, /) -> None: ...
         async def aio(self, /) -> None: ...

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/client.pyi

@@ -77,7 +77,7 @@ class _Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.5.6.dev2",
+        version: str = "1.5.6.dev3",
         *,
         oauth_credentials: typing.Optional[_OAuthCredentials] = None,
     ):
@@ -277,7 +277,7 @@ class Client:
         server_url: str,
         client_type: int,
         credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.5.6.dev2",
+        version: str = "1.5.6.dev3",
         *,
         oauth_credentials: typing.Optional[_OAuthCredentials] = None,
     ):

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/config.py

@@ -339,6 +339,8 @@ _SETTINGS = {
     "sandbox_v2": _Setting(None, transform=_to_boolean),
     "snapshot_debug": _Setting(False, transform=_to_boolean),
     "cuda_checkpoint_path": _Setting("/__modal/.bin/cuda-checkpoint"),  # Used for snapshotting GPU memory.
+    # Used to indicate gVisor cuda-checkpoint automation.
+    "runtime_managed_cuda_checkpoint": _Setting(False, transform=_to_boolean),
     "build_validation": _Setting("error", transform=_check_value(["error", "warn", "ignore"])),
     # Payload format for function inputs/outputs: 'pickle' (default) or 'cbor'
     "payload_format": _Setting(

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal/functions.pyi

@@ -619,7 +619,7 @@ class Function(
 
     _call_generator: ___call_generator_spec
 
-    class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
+    class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER:
             """Calls the function remotely, executing it with the given arguments and returning the execution's result.
 
@@ -644,7 +644,7 @@ class Function(
             """
             ...
 
-    remote: __remote_spec[modal._functions.P, modal._functions.ReturnType]
+    remote: __remote_spec[modal._functions.ReturnType, modal._functions.P]
 
     class __remote_gen_spec(typing_extensions.Protocol):
         def __call__(self, /, *args, **kwargs) -> typing.Generator[typing.Any, None, None]:
@@ -694,7 +694,7 @@ class Function(
         """
         ...
 
-    class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
+    class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """[Experimental] Calls the function with the given arguments, without waiting for the results.
 
@@ -727,7 +727,7 @@ class Function(
             """
             ...
 
-    _experimental_spawn: ___experimental_spawn_spec[modal._functions.P, modal._functions.ReturnType]
+    _experimental_spawn: ___experimental_spawn_spec[modal._functions.ReturnType, modal._functions.P]
 
     class ___spawn_map_inner_spec(typing_extensions.Protocol[P_INNER]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> None: ...
@@ -735,7 +735,7 @@ class Function(
 
     _spawn_map_inner: ___spawn_map_inner_spec[modal._functions.P]
 
-    class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
+    class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
         def __call__(self, /, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]:
             """Calls the function with the given arguments, without waiting for the results.
 
@@ -768,7 +768,7 @@ class Function(
             """
             ...
 
-    spawn: __spawn_spec[modal._functions.P, modal._functions.ReturnType]
+    spawn: __spawn_spec[modal._functions.ReturnType, modal._functions.P]
 
     def get_raw_f(self) -> collections.abc.Callable[..., typing.Any]:
         """Return the inner Python object wrapped by this Modal Function.

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modal
-Version: 1.5.6.dev2
+Version: 1.5.6.dev3
 Summary: Python client library for Modal
 Author-email: Modal Labs <support@modal.com>
 License-Expression: Apache-2.0

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/modal_version/__init__.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2026
 """Supplies the current version of the modal client library."""
 
-__version__ = "1.5.6.dev2"
+__version__ = "1.5.6.dev3"

{modal-1.5.6.dev2 → modal-1.5.6.dev3}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools~=77.0.3", "wheel"]
+requires = ["setuptools>=83,<84", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -53,10 +53,10 @@ dev = [
     "pytest-env~=1.1.5",
     "pytest-markdown-docs==0.9.2",
     "pytest-timeout~=2.4.0",
-    "python-dotenv~=1.0.0",
-    "requests~=2.32.4",
+    "python-dotenv~=1.2.2",
+    "requests==2.33.0",
     "ruff~=0.9.6",
-    "setuptools<82",                  # Required by grpcio-tools for pkg_resources (v82+ removed pkg_resources)"
+    "setuptools>=83,<84",
     "types-croniter~=1.0.8",
     "types-python-dateutil~=2.8.10",
     "types-requests~=2.31.0",
@@ -98,10 +98,6 @@ dev = [
     "aiohttp-socks>=0.8",
 ]
 
-[tool.uv.extra-build-dependencies]
-grpcio-tools = ["setuptools<82"]
-
-
 [project.urls]
 Homepage = "https://modal.com"
 Source = "https://github.com/modal-labs/modal-client"