modal 1.0.2.dev6__py3-none-any.whl → 1.0.3__py3-none-any.whl

modal/_functions.py

@@ -101,6 +101,7 @@ if TYPE_CHECKING:
 
 MAX_INTERNAL_FAILURE_COUNT = 8
 
+
 @dataclasses.dataclass
 class _RetryContext:
     function_call_invocation_type: "api_pb2.FunctionCallInvocationType.ValueType"
@@ -1661,8 +1662,8 @@ Use the `Function.get_web_url()` method instead.
     async def spawn(self, *args: P.args, **kwargs: P.kwargs) -> "_FunctionCall[ReturnType]":
         """Calls the function with the given arguments, without waiting for the results.
 
-        Returns a `modal.FunctionCall` object, that can later be polled or
-        waited for using `.get(timeout=...)`.
+        Returns a [`modal.FunctionCall`](/docs/reference/modal.FunctionCall) object, that can later be polled or
+        waited for using [`.get(timeout=...)`](/docs/reference/modal.FunctionCall#get).
         Conceptually similar to `multiprocessing.pool.apply_async`, or a Future/Promise in other contexts.
         """
         self._check_no_web_url("spawn")

modal/_runtime/container_io_manager.py

@@ -323,6 +323,7 @@ class _ContainerIOManager:
         self._heartbeat_loop = None
         self._heartbeat_condition = None
         self._waiting_for_memory_snapshot = False
+        self._cuda_checkpoint_session = None
 
         self._is_interactivity_enabled = False
         self._fetching_inputs = True
@@ -881,13 +882,11 @@ class _ContainerIOManager:
         # Restore GPU memory.
         if self.function_def._experimental_enable_gpu_snapshot and self.function_def.resources.gpu_config.gpu_type:
             logger.debug("GPU memory snapshot enabled. Attempting to restore GPU memory.")
-            gpu_process_state = gpu_memory_snapshot.get_state()
-            if gpu_process_state != gpu_memory_snapshot.CudaCheckpointState.CHECKPOINTED:
-                raise ValueError(
-                    "Cannot restore GPU state if GPU isn't in a 'checkpointed' state. "
-                    f"Current GPU state: {gpu_process_state}"
-                )
-            gpu_memory_snapshot.toggle()
+
+            assert self._cuda_checkpoint_session, (
+                "CudaCheckpointSession not found when attempting to restore GPU memory"
+            )
+            self._cuda_checkpoint_session.restore()
 
         # Restore input to default state.
         self.current_input_id = None
@@ -907,14 +906,9 @@ class _ContainerIOManager:
             # Snapshot GPU memory.
             if self.function_def._experimental_enable_gpu_snapshot and self.function_def.resources.gpu_config.gpu_type:
                 logger.debug("GPU memory snapshot enabled. Attempting to snapshot GPU memory.")
-                gpu_process_state = gpu_memory_snapshot.get_state()
-                if gpu_process_state != gpu_memory_snapshot.CudaCheckpointState.RUNNING:
-                    raise ValueError(
-                        f"Cannot snapshot GPU state if it isn't running. Current GPU state: {gpu_process_state}"
-                    )
 
-                gpu_memory_snapshot.toggle()
-                gpu_memory_snapshot.wait_for_state(gpu_memory_snapshot.CudaCheckpointState.CHECKPOINTED)
+                self._cuda_checkpoint_session = gpu_memory_snapshot.CudaCheckpointSession()
+                self._cuda_checkpoint_session.checkpoint()
 
             # Notify the heartbeat loop that the snapshot phase has begun in order to
             # prevent it from sending heartbeat RPCs

modal/_runtime/gpu_memory_snapshot.py

@@ -6,10 +6,12 @@
 #
 # [1] https://github.com/NVIDIA/cuda-checkpoint
 
-import os
 import subprocess
 import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from enum import Enum
+from pathlib import Path
 
 from modal.config import config, logger
 
@@ -29,73 +31,169 @@ class CudaCheckpointException(Exception):
     pass
 
 
-def toggle():
-    """Toggle CUDA checkpoint state for current process, moving GPU memory to the
-    CPU and back depending on the current process state when called."""
-    pid = get_own_pid()
-    logger.debug(f"Toggling CUDA checkpoint state for PID {pid}")
+@dataclass
+class CudaCheckpointProcess:
+    """Contains a reference to a PID with active CUDA session. This also provides
+    methods for checkpointing and restoring GPU memory."""
 
-    try:
-        subprocess.run(
-            [
-                CUDA_CHECKPOINT_PATH,
-                "--toggle",
-                "--pid",
-                str(pid),
-            ],
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-        logger.debug("Successfully toggled CUDA checkpoint state")
+    pid: int
+    state: CudaCheckpointState
 
-    except subprocess.CalledProcessError as e:
-        logger.debug(f"Failed to toggle CUDA checkpoint state: {e.stderr}")
-        raise CudaCheckpointException(e.stderr)
+    def toggle(self, target_state: CudaCheckpointState, timeout_secs: float = 5 * 60.0):
+        """Toggle CUDA checkpoint state for current process, moving GPU memory to the
+        CPU and back depending on the current process state when called."""
+        logger.debug(f"PID: {self.pid} Toggling CUDA checkpoint state to {target_state.value}")
 
+        start_time = time.monotonic()
 
-def get_state() -> CudaCheckpointState:
-    """Get current CUDA checkpoint state for this process."""
-    pid = get_own_pid()
+        while self._should_continue_toggle(target_state, start_time, timeout_secs):
+            self._execute_toggle_command()
+            time.sleep(0.1)
 
-    try:
-        result = subprocess.run(
-            [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)], check=True, capture_output=True, text=True
-        )
+        logger.debug(f"PID: {self.pid} Target state {target_state.value} reached")
 
-        # Parse output to get state
-        state_str = result.stdout.strip().lower()
-        return CudaCheckpointState(state_str)
+    def _should_continue_toggle(
+        self, target_state: CudaCheckpointState, start_time: float, timeout_secs: float
+    ) -> bool:
+        """Check if toggle operation should continue based on current state and timeout."""
+        self.refresh_state()
 
-    except subprocess.CalledProcessError as e:
-        logger.debug(f"Failed to get CUDA checkpoint state: {e.stderr}")
-        raise CudaCheckpointException(e.stderr)
+        if self.state == target_state:
+            return False
 
-
-def wait_for_state(target_state: CudaCheckpointState, timeout_secs: float = 5.0):
-    """Wait for CUDA checkpoint to reach a specific state."""
-    logger.debug(f"Waiting for CUDA checkpoint state {target_state.value}")
-    start_time = time.monotonic()
-
-    while True:
-        current_state = get_state()
-
-        if current_state == target_state:
-            logger.debug(f"Target state {target_state.value} reached")
-            break
-
-        if current_state == CudaCheckpointState.FAILED:
-            raise CudaCheckpointException(f"CUDA process state is {current_state}")
+        if self.state == CudaCheckpointState.FAILED:
+            raise CudaCheckpointException(f"PID: {self.pid} CUDA process state is {self.state}")
 
         elapsed = time.monotonic() - start_time
         if elapsed >= timeout_secs:
-            raise CudaCheckpointException(f"Timeout after {elapsed:.2f}s waiting for state {target_state.value}")
-
-        time.sleep(0.1)
-
-
-def get_own_pid():
-    """Returns the Process ID (PID) of the current Python process
-    using only the standard library.
-    """
-    return os.getpid()
+            raise CudaCheckpointException(
+                f"PID: {self.pid} Timeout after {elapsed:.2f}s waiting for state {target_state.value}. "
+                f"Current state: {self.state}"
+            )
+
+        return True
+
+    def _execute_toggle_command(self):
+        """Execute the cuda-checkpoint toggle command."""
+        try:
+            subprocess.run(
+                [CUDA_CHECKPOINT_PATH, "--toggle", "--pid", str(self.pid)],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            logger.debug(f"PID: {self.pid} Successfully toggled CUDA checkpoint state")
+        except subprocess.CalledProcessError as e:
+            logger.debug(f"PID: {self.pid} Failed to toggle CUDA checkpoint state: {e.stderr}")
+            raise CudaCheckpointException(e.stderr)
+
+    def refresh_state(self) -> None:
+        """Refreshes the current CUDA checkpoint state for this process."""
+        try:
+            result = subprocess.run(
+                [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(self.pid)],
+                check=True,
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+
+            state_str = result.stdout.strip().lower()
+            self.state = CudaCheckpointState(state_str)
+
+        except subprocess.CalledProcessError as e:
+            logger.debug(f"PID: {self.pid} Failed to get CUDA checkpoint state: {e.stderr}")
+            raise CudaCheckpointException(e.stderr)
+
+
+class CudaCheckpointSession:
+    """Manages the checkpointing state of processes with active CUDA sessions."""
+
+    def __init__(self):
+        self.cuda_processes = self._get_cuda_pids()
+        logger.debug(f"PIDs with CUDA sessions: {[c.pid for c in self.cuda_processes]}")
+
+    def _get_cuda_pids(self) -> list[CudaCheckpointProcess]:
+        """Iterates over all PIDs and identifies the ones that have running
+        CUDA sessions."""
+        cuda_pids: list[CudaCheckpointProcess] = []
+
+        # Get all active process IDs from /proc directory
+        proc_dir = Path("/proc")
+        if not proc_dir.exists():
+            raise CudaCheckpointException(
+                "OS does not have /proc path rendering it incompatible with GPU memory snapshots."
+            )
+
+        for entry in proc_dir.iterdir():
+            if not entry.name.isdigit():
+                continue
+
+            pid = int(entry.name)
+            try:
+                # Call cuda-checkpoint to check if this PID has a CUDA session
+                result = subprocess.run(
+                    [CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+
+                # If the command succeeds (return code 0), this PID has a CUDA session
+                if result.returncode == 0:
+                    state_str = result.stdout.strip().lower()
+                    state = CudaCheckpointState(state_str)
+
+                    cuda_checkpoint_process = CudaCheckpointProcess(pid=pid, state=state)
+                    cuda_pids.append(cuda_checkpoint_process)
+
+            # Command failed, which is expected for PIDs without CUDA sessions
+            except subprocess.CalledProcessError:
+                continue
+
+            # Raise other exceptions
+            except subprocess.TimeoutExpired:
+                raise CudaCheckpointException(f"Failed to get CUDA state for PID {pid}")
+            except Exception as e:
+                raise CudaCheckpointException(e)
+
+        # Sort PIDs for ordered checkpointing
+        cuda_pids.sort(key=lambda x: x.pid)
+        return cuda_pids
+
+    def checkpoint(self) -> None:
+        # Validate all states first
+        for proc in self.cuda_processes:
+            if proc.state != CudaCheckpointState.RUNNING:
+                raise CudaCheckpointException(f"CUDA session not in {CudaCheckpointState.RUNNING} state.")
+
+        # Moving state from GPU to CPU can take several seconds per CUDA session.
+        # Make a parallel call per CUDA session.
+        start = time.perf_counter()
+
+        def checkpoint_impl(proc: CudaCheckpointProcess):
+            proc.toggle(CudaCheckpointState.CHECKPOINTED)
+
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(checkpoint_impl, self.cuda_processes))
+
+        elapsed = time.perf_counter() - start
+        logger.debug(f"Checkpointing CUDA sessions took => {elapsed:.3f}s")
+
+    def restore(self) -> None:
+        # Validate all states first
+        for proc in self.cuda_processes:
+            if proc.state != CudaCheckpointState.CHECKPOINTED:
+                raise CudaCheckpointException(f"CUDA session not in {CudaCheckpointState.CHECKPOINTED} state.")
+
+        # See checkpoint() for rationale about parallelism.
+        start = time.perf_counter()
+
+        def restore_process(proc: CudaCheckpointProcess):
+            proc.toggle(CudaCheckpointState.RUNNING)
+
+        with ThreadPoolExecutor() as executor:
+            list(executor.map(restore_process, self.cuda_processes))
+
+        elapsed = time.perf_counter() - start
+        logger.debug(f"Restoring CUDA sessions took => {elapsed:.3f}s")

modal/_utils/bytes_io_segment_payload.py

@@ -8,14 +8,14 @@ from typing import BinaryIO, Callable, Optional
 # Note: this module needs to import aiohttp in global scope
 # This takes about 50ms and isn't needed in many cases for Modal execution
 # To avoid this, we import it in local scope when needed (blob_utils.py)
-from aiohttp import BytesIOPayload
+from aiohttp import Payload
 from aiohttp.abc import AbstractStreamWriter
 
 # read ~16MiB chunks by default
 DEFAULT_SEGMENT_CHUNK_SIZE = 2**24
 
 
-class BytesIOSegmentPayload(BytesIOPayload):
+class BytesIOSegmentPayload(Payload):
     """Modified bytes payload for concurrent sends of chunks from the same file.
 
     Adds:
@@ -26,6 +26,8 @@ class BytesIOSegmentPayload(BytesIOPayload):
     Feels like this should be in some standard lib...
     """
 
+    _value: BinaryIO
+
     def __init__(
         self,
         bytes_io: BinaryIO,  # should *not* be shared as IO position modification is not locked
@@ -36,6 +38,7 @@ class BytesIOSegmentPayload(BytesIOPayload):
     ):
         # not thread safe constructor!
         super().__init__(bytes_io)
+        self._size = segment_length
         self.initial_seek_pos = bytes_io.tell()
         self.segment_start = segment_start
         self.segment_length = segment_length
@@ -46,6 +49,10 @@ class BytesIOSegmentPayload(BytesIOPayload):
         self.progress_report_cb = progress_report_cb or (lambda *_, **__: None)
         self.reset_state()
 
+    def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str:
+        self._value.seek(self.initial_seek_pos)
+        return self._value.read().decode(encoding, errors)
+
     def reset_state(self):
         self._md5_checksum = hashlib.md5()
         self.num_bytes_read = 0
@@ -76,14 +83,21 @@ class BytesIOSegmentPayload(BytesIOPayload):
         return self._md5_checksum
 
     async def write(self, writer: "AbstractStreamWriter"):
+        # On aiohttp < 3.12.0 - this is the method that's being called on a custom payload,
+        # but on aiohttp 3.12+ `write_with_length` is called directly.
+        await self.write_with_length(writer, None)
+
+    async def write_with_length(self, writer: AbstractStreamWriter, content_length: Optional[int]):
         loop = asyncio.get_event_loop()
 
         async def safe_read():
             read_start = self.initial_seek_pos + self.segment_start + self.num_bytes_read
             self._value.seek(read_start)
             num_bytes = min(self.chunk_size, self.remaining_bytes())
-            chunk = await loop.run_in_executor(None, self._value.read, num_bytes)
+            if content_length is not None:
+                num_bytes = min(num_bytes, content_length)
 
+            chunk = await loop.run_in_executor(None, self._value.read, num_bytes)
             await loop.run_in_executor(None, self._md5_checksum.update, chunk)
             self.num_bytes_read += len(chunk)
             return chunk

modal/cli/run.py

@@ -171,6 +171,14 @@ def _write_local_result(result_path: str, res: Any):
         fid.write(res)
 
 
+def _validate_interactive_quiet_params(ctx):
+    interactive = ctx.obj["interactive"]
+    show_progress = ctx.obj["show_progress"]
+
+    if not show_progress and interactive:
+        raise InvalidError("To use interactive mode, remove the --quiet flag")
+
+
 def _make_click_function(app, signature: CliRunnableSignature, inner: Callable[[tuple[str, ...], dict[str, Any]], Any]):
     @click.pass_context
     def f(ctx, **kwargs):
@@ -180,6 +188,8 @@ def _make_click_function(app, signature: CliRunnableSignature, inner: Callable[[
         else:
             args = ()
 
+        _validate_interactive_quiet_params(ctx)
+
         show_progress: bool = ctx.obj["show_progress"]
         with enable_output(show_progress):
             with run_app(
@@ -291,6 +301,8 @@ def _get_click_command_for_local_entrypoint(app: App, entrypoint: LocalEntrypoin
             assert len(args) == 0 and len(kwargs) == 0
             args = ctx.args
 
+        _validate_interactive_quiet_params(ctx)
+
         show_progress: bool = ctx.obj["show_progress"]
         with enable_output(show_progress):
             with run_app(

modal/cli/secret.py

@@ -1,7 +1,9 @@
 # Copyright Modal Labs 2022
+import json
 import os
 import platform
 import subprocess
+from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Optional
 
@@ -48,14 +50,17 @@ async def list_(env: Optional[str] = ENV_OPTION, json: bool = False):
 @secret_cli.command("create", help="Create a new secret.")
 @synchronizer.create_blocking
 async def create(
-    secret_name,
-    keyvalues: list[str] = typer.Argument(..., help="Space-separated KEY=VALUE items"),
+    secret_name: str,
+    keyvalues: Optional[list[str]] = typer.Argument(default=None, help="Space-separated KEY=VALUE items."),
     env: Optional[str] = ENV_OPTION,
+    from_dotenv: Optional[Path] = typer.Option(default=None, help="Path to a .env file to load secrets from."),
+    from_json: Optional[Path] = typer.Option(default=None, help="Path to a JSON file to load secrets from."),
     force: bool = typer.Option(False, "--force", help="Overwrite the secret if it already exists."),
 ):
     env = ensure_env(env)
     env_dict = {}
-    for arg in keyvalues:
+
+    for arg in keyvalues or []:
         if "=" in arg:
             key, value = arg.split("=", 1)
             if value == "-":
@@ -63,17 +68,51 @@ async def create(
             env_dict[key] = value
         else:
             raise click.UsageError(
-                """Each item should be of the form <KEY>=VALUE. To enter secrets using your $EDITOR, use `<KEY>=-`.
+                """Each item should be of the form <KEY>=VALUE. To enter secrets using your $EDITOR, use `<KEY>=-`. To
+enter secrets from environment variables, use `<KEY>="$ENV_VAR"`.
 
 E.g.
 
 modal secret create my-credentials username=john password=-
+modal secret create my-credentials username=john password="$PASSWORD"
 """
             )
 
+    if from_dotenv:
+        if not from_dotenv.is_file():
+            raise click.UsageError(f"Could not read .env file at {from_dotenv}")
+
+        try:
+            from dotenv import dotenv_values
+        except ImportError:
+            raise ImportError(
+                "Need the `python-dotenv` package installed. You can install it by running `pip install python-dotenv`."
+            )
+
+        try:
+            env_dict.update(dotenv_values(from_dotenv))
+        except Exception as e:
+            raise click.UsageError(f"Could not parse .env file at {from_dotenv}: {e}")
+
+    if from_json:
+        if not from_json.is_file():
+            raise click.UsageError(f"Could not read JSON file at {from_json}")
+
+        try:
+            with from_json.open("r") as f:
+                env_dict.update(json.load(f))
+        except Exception as e:
+            raise click.UsageError(f"Could not parse JSON file at {from_json}: {e}")
+
     if not env_dict:
         raise click.UsageError("You need to specify at least one key for your secret")
 
+    for k, v in env_dict.items():
+        if not isinstance(k, str) or not k:
+            raise click.UsageError(f"Invalid key: '{k}'")
+        if not isinstance(v, str):
+            raise click.UsageError(f"Non-string value for secret '{k}'")
+
     # Create secret
     await _Secret.create_deployed(secret_name, env_dict, overwrite=force)
 

modal/cli/volume.py

@@ -154,6 +154,10 @@ async def ls(
                 filetype = "dir"
             elif entry.type == api_pb2.FileEntry.FileType.SYMLINK:
                 filetype = "link"
+            elif entry.type == api_pb2.FileEntry.FileType.FIFO:
+                filetype = "fifo"
+            elif entry.type == api_pb2.FileEntry.FileType.SOCKET:
+                filetype = "socket"
             else:
                 filetype = "file"
             rows.append(
@@ -261,12 +265,13 @@ async def rm(
 async def cp(
     volume_name: str,
     paths: list[str],  # accepts multiple paths, last path is treated as destination path
+    recursive: bool = Option(False, "-r", "--recursive", help="Copy directories recursively"),
     env: Optional[str] = ENV_OPTION,
 ):
     ensure_env(env)
     volume = _Volume.from_name(volume_name, environment_name=env)
     *src_paths, dst_path = paths
-    await volume.copy_files(src_paths, dst_path)
+    await volume.copy_files(src_paths, dst_path, recursive)
 
 
 @volume_cli.command(

modal/client.pyi

@@ -27,11 +27,7 @@ class _Client:
     _snapshotted: bool
 
     def __init__(
-        self,
-        server_url: str,
-        client_type: int,
-        credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.0.2.dev6",
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "1.0.3"
     ): ...
     def is_closed(self) -> bool: ...
     @property
@@ -90,11 +86,7 @@ class Client:
     _snapshotted: bool
 
     def __init__(
-        self,
-        server_url: str,
-        client_type: int,
-        credentials: typing.Optional[tuple[str, str]],
-        version: str = "1.0.2.dev6",
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "1.0.3"
     ): ...
     def is_closed(self) -> bool: ...
     @property

modal/experimental/__init__.py

@@ -94,8 +94,9 @@ async def raw_dockerfile_image(
 
     Unlike for `modal.Image.from_dockerfile`, the provided recipe will not be embellished with
     steps to install dependencies for the Modal client package. As a consequence, the resulting
-    Image cannot be used with a modal Function unless those dependencies are added in a subsequent
-    layer. It _can_ be directly used with a modal Sandbox, which does not need the Modal client.
+    Image cannot be used with a modal Function unless those dependencies are already included
+    as part of the base Dockerfile recipe or are added in a subsequent layer. The Image _can_ be
+    directly used with a modal Sandbox, which does not need the Modal client.
 
     We expect to support this experimental function until the `2025.04` Modal Image Builder is
     stable, at which point Modal Image recipes will no longer install the client dependencies
@@ -127,8 +128,9 @@ async def raw_registry_image(
 
     Unlike for `modal.Image.from_registry`, the provided recipe will not be embellished with
     steps to install dependencies for the Modal client package. As a consequence, the resulting
-    Image cannot be used with a modal Function unless those dependencies are added in a subsequent
-    layer. It _can_ be directly used with a modal Sandbox, which does not need the Modal client.
+    Image cannot be used with a modal Function unless those dependencies are already included
+    as part of the registry Image or are added in a subsequent layer. The Image _can_ be
+    directly used with a modal Sandbox, which does not need the Modal client.
 
     We expect to support this experimental function until the `2025.04` Modal Image Builder is
     stable, at which point Modal Image recipes will no longer install the client dependencies

modal/image.py

@@ -647,7 +647,13 @@ class _Image(_Object, type_prefix="im"):
                     metadata = resp.metadata
 
             if result.status == api_pb2.GenericResult.GENERIC_STATUS_FAILURE:
-                raise RemoteError(f"Image build for {image_id} failed with the exception:\n{result.exception}")
+                if result.exception:
+                    raise RemoteError(f"Image build for {image_id} failed with the exception:\n{result.exception}")
+                else:
+                    msg = f"Image build for {image_id} failed. See build logs for more details."
+                    if not _get_output_manager():
+                        msg += " (Hint: Use `modal.enable_output()` to see logs from the process building the Image.)"
+                    raise RemoteError(msg)
             elif result.status == api_pb2.GenericResult.GENERIC_STATUS_TERMINATED:
                 raise RemoteError(f"Image build for {image_id} terminated due to external shut-down. Please try again.")
             elif result.status == api_pb2.GenericResult.GENERIC_STATUS_TIMEOUT:
@@ -1522,12 +1528,14 @@ class _Image(_Object, type_prefix="im"):
         modal_requirements_commands = []
         if builder_version <= "2024.10":
             # past 2024.10, client dependencies are mounted at runtime
-            modal_requirements_commands.extend([
-                f"COPY {CONTAINER_REQUIREMENTS_PATH} {CONTAINER_REQUIREMENTS_PATH}",
-                f"RUN python -m pip install --upgrade {_base_image_config('package_tools', builder_version)}",
-                f"RUN {requirements_prefix}{_get_modal_requirements_command(builder_version)}",
-            ])
-        if  "2024.10" >= builder_version > "2023.12":
+            modal_requirements_commands.extend(
+                [
+                    f"COPY {CONTAINER_REQUIREMENTS_PATH} {CONTAINER_REQUIREMENTS_PATH}",
+                    f"RUN python -m pip install --upgrade {_base_image_config('package_tools', builder_version)}",
+                    f"RUN {requirements_prefix}{_get_modal_requirements_command(builder_version)}",
+                ]
+            )
+        if "2024.10" >= builder_version > "2023.12":
             modal_requirements_commands.append(f"RUN rm {CONTAINER_REQUIREMENTS_PATH}")
 
         return [
@@ -1830,23 +1838,31 @@ class _Image(_Object, type_prefix="im"):
                 f"FROM python:{full_python_version}-slim-{debian_codename}",
             ]
             if version <= "2024.10":
-                commands.extend([
-                    f"COPY {CONTAINER_REQUIREMENTS_PATH} {CONTAINER_REQUIREMENTS_PATH}",
-                ])
-            commands.extend([
-                "RUN apt-get update",
-                "RUN apt-get install -y gcc gfortran build-essential",
-                f"RUN pip install --upgrade {_base_image_config('package_tools', version)}",
-            ])
+                commands.extend(
+                    [
+                        f"COPY {CONTAINER_REQUIREMENTS_PATH} {CONTAINER_REQUIREMENTS_PATH}",
+                    ]
+                )
+            commands.extend(
+                [
+                    "RUN apt-get update",
+                    "RUN apt-get install -y gcc gfortran build-essential",
+                    f"RUN pip install --upgrade {_base_image_config('package_tools', version)}",
+                ]
+            )
             if version <= "2024.10":
                 # after 2024.10, modal requirements are mounted at runtime
-                commands.extend([
-                    f"RUN {_get_modal_requirements_command(version)}",
-                ])
-            commands.extend([
-                # Set debian front-end to non-interactive to avoid users getting stuck with input prompts.
-                "RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections",
-            ])
+                commands.extend(
+                    [
+                        f"RUN {_get_modal_requirements_command(version)}",
+                    ]
+                )
+            commands.extend(
+                [
+                    # Set debian front-end to non-interactive to avoid users getting stuck with input prompts.
+                    "RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections",
+                ]
+            )
             if "2024.10" >= version > "2023.12":
                 commands.append(f"RUN rm {CONTAINER_REQUIREMENTS_PATH}")
             if version > "2024.10":