modal 1.0.3.dev10__py3-none-any.whl → 1.2.3.dev7__py3-none-any.whl

modal/mount.py

@@ -8,23 +8,24 @@ import re
 import time
 import typing
 import warnings
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Generator
 from pathlib import Path, PurePosixPath
 from typing import Callable, Optional, Sequence, Union
 
 from google.protobuf.message import Message
+from grpclib import GRPCError
 
 import modal.exception
 import modal.file_pattern_matcher
 from modal_proto import api_pb2
 from modal_version import __version__
 
-from ._object import _get_environment_name, _Object
+from ._load_context import LoadContext
+from ._object import _Object
 from ._resolver import Resolver
-from ._utils.async_utils import aclosing, async_map, synchronize_api
+from ._utils.async_utils import TaskContext, aclosing, async_map, synchronize_api
 from ._utils.blob_utils import FileUploadSpec, blob_upload_file, get_file_upload_spec_from_path
-from ._utils.deprecation import deprecation_warning
-from ._utils.grpc_utils import retry_transient_errors
+from ._utils.grpc_utils import Retry
 from ._utils.name_utils import check_object_name
 from ._utils.package_utils import get_module_mount_info
 from .client import _Client
@@ -115,7 +116,8 @@ class _MountFile(_MountEntry):
     def get_files_to_upload(self):
         local_file = self.local_file.resolve()
         if not local_file.exists():
-            raise FileNotFoundError(local_file)
+            msg = f"local file {local_file} does not exist"
+            raise FileNotFoundError(msg)
 
         rel_filename = self.remote_path
         yield local_file, rel_filename
@@ -132,25 +134,48 @@ class _MountFile(_MountEntry):
 class _MountDir(_MountEntry):
     local_dir: Path
     remote_path: PurePosixPath
-    ignore: Callable[[Path], bool]
+    ignore: Union[Callable[[Path], bool], modal.file_pattern_matcher._AbstractPatternMatcher]
     recursive: bool
 
     def description(self):
         return str(self.local_dir.expanduser().absolute())
 
+    def _walk_and_prune(self, top_dir: Path) -> Generator[str, None, None]:
+        """Walk directories and prune ignored directories early."""
+        for root, dirs, files in os.walk(top_dir, topdown=True):
+            # with topdown=True, os.walk allows modifying the dirs list in-place, and will only
+            # recurse into dirs that are not ignored.
+            dirs[:] = [d for d in dirs if not self.ignore(Path(os.path.join(root, d)).relative_to(top_dir))]
+            for file in files:
+                yield os.path.join(root, file)
+
+    def _walk_all(self, top_dir: Path) -> Generator[str, None, None]:
+        """Walk all directories without early pruning - safe for complex/inverted ignore patterns."""
+        for root, _, files in os.walk(top_dir):
+            for file in files:
+                yield os.path.join(root, file)
+
     def get_files_to_upload(self):
         # we can't use .resolve() eagerly here since that could end up "renaming" symlinked files
         # see test_mount_directory_with_symlinked_file
         local_dir = self.local_dir.expanduser().absolute()
 
         if not local_dir.exists():
-            raise FileNotFoundError(local_dir)
+            msg = f"local dir {local_dir} does not exist"
+            raise FileNotFoundError(msg)
 
         if not local_dir.is_dir():
-            raise NotADirectoryError(local_dir)
+            msg = f"local dir {local_dir} is not a directory"
+            raise NotADirectoryError(msg)
 
         if self.recursive:
-            gen = (os.path.join(root, name) for root, dirs, files in os.walk(local_dir) for name in files)
+            if (
+                isinstance(self.ignore, modal.file_pattern_matcher._AbstractPatternMatcher)
+                and self.ignore.can_prune_directories()
+            ):
+                gen = self._walk_and_prune(local_dir)
+            else:
+                gen = self._walk_all(local_dir)
         else:
             gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
 
@@ -286,7 +311,8 @@ class _Mount(_Object, type_prefix="mo"):
     _entries: Optional[list[_MountEntry]] = None
     _deployment_name: Optional[str] = None
     _namespace: Optional[int] = None
-    _environment_name: Optional[str] = None
+
+    _allow_overwrite: bool = False
     _content_checksum_sha256_hex: Optional[str] = None
 
     @staticmethod
@@ -300,7 +326,12 @@ class _Mount(_Object, type_prefix="mo"):
                 return None
             return (_Mount._type_prefix, "local", frozenset(included_files))
 
-        obj = _Mount._from_loader(_Mount._load_mount, rep, deduplication_key=mount_content_deduplication_key)
+        obj = _Mount._from_loader(
+            _Mount._load_mount,
+            rep,
+            deduplication_key=mount_content_deduplication_key,
+            load_context_overrides=LoadContext.empty(),
+        )
         obj._entries = entries
         obj._is_local = True
         return obj
@@ -386,39 +417,6 @@ class _Mount(_Object, type_prefix="mo"):
             ),
         )
 
-    @staticmethod
-    def from_local_dir(
-        local_path: Union[str, Path],
-        *,
-        # Where the directory is placed within in the mount
-        remote_path: Union[str, PurePosixPath, None] = None,
-        # Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
-        # Defaults to including all files.
-        condition: Optional[Callable[[str], bool]] = None,
-        # add files from subdirectories as well
-        recursive: bool = True,
-    ) -> "_Mount":
-        """
-        **Deprecated:** Use image.add_local_dir() instead
-
-        Create a `Mount` from a local directory.
-
-        **Usage**
-
-        ```python notest
-        assets = modal.Mount.from_local_dir(
-            "~/assets",
-            condition=lambda pth: not ".venv" in pth,
-            remote_path="/assets",
-        )
-        ```
-        """
-        deprecation_warning(
-            (2025, 1, 8),
-            MOUNT_DEPRECATION_MESSAGE_PATTERN.format(replacement="image.add_local_dir"),
-        )
-        return _Mount._from_local_dir(local_path, remote_path=remote_path, condition=condition, recursive=recursive)
-
     @staticmethod
     def _from_local_dir(
         local_path: Union[str, Path],
@@ -454,29 +452,6 @@ class _Mount(_Object, type_prefix="mo"):
             ),
         )
 
-    @staticmethod
-    def from_local_file(local_path: Union[str, Path], remote_path: Union[str, PurePosixPath, None] = None) -> "_Mount":
-        """
-        **Deprecated**: Use image.add_local_file() instead
-
-        Create a `Mount` mounting a single local file.
-
-        **Usage**
-
-        ```python notest
-        # Mount the DBT profile in user's home directory into container.
-        dbt_profiles = modal.Mount.from_local_file(
-            local_path="~/profiles.yml",
-            remote_path="/root/dbt_profile/profiles.yml",
-        )
-        ```
-        """
-        deprecation_warning(
-            (2025, 1, 8),
-            MOUNT_DEPRECATION_MESSAGE_PATTERN.format(replacement="image.add_local_file"),
-        )
-        return _Mount._from_local_file(local_path, remote_path)
-
     @staticmethod
     def _from_local_file(local_path: Union[str, Path], remote_path: Union[str, PurePosixPath, None] = None) -> "_Mount":
         return _Mount._new().add_local_file(local_path, remote_path=remote_path)
@@ -508,6 +483,7 @@ class _Mount(_Object, type_prefix="mo"):
     async def _load_mount(
         self: "_Mount",
         resolver: Resolver,
+        load_context: LoadContext,
         existing_object_id: Optional[str],
     ):
         t0 = time.monotonic()
@@ -549,7 +525,7 @@ class _Mount(_Object, type_prefix="mo"):
 
             request = api_pb2.MountPutFileRequest(sha256_hex=file_spec.sha256_hex)
             accounted_hashes.add(file_spec.sha256_hex)
-            response = await retry_transient_errors(resolver.client.stub.MountPutFile, request, base_delay=1)
+            response = await load_context.client.stub.MountPutFile(request, retry=Retry(base_delay=1))
 
             if response.exists:
                 n_finished += 1
@@ -563,7 +539,7 @@ class _Mount(_Object, type_prefix="mo"):
                 async with blob_upload_concurrency:
                     with file_spec.source() as fp:
                         blob_id = await blob_upload_file(
-                            fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
+                            fp, load_context.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
                         )
                 logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
                 request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
@@ -575,7 +551,7 @@ class _Mount(_Object, type_prefix="mo"):
 
             start_time = time.monotonic()
             while time.monotonic() - start_time < MOUNT_PUT_FILE_CLIENT_TIMEOUT:
-                response = await retry_transient_errors(resolver.client.stub.MountPutFile, request2, base_delay=1)
+                response = await load_context.client.stub.MountPutFile(request2, retry=Retry(base_delay=1))
                 if response.exists:
                     n_finished += 1
                     return mount_file
@@ -583,7 +559,7 @@ class _Mount(_Object, type_prefix="mo"):
             raise modal.exception.MountUploadTimeoutError(f"Mounting of {file_spec.source_description} timed out")
 
         # Upload files, or check if they already exist.
-        n_concurrent_uploads = 512
+        n_concurrent_uploads = 64
         files: list[api_pb2.MountFile] = []
         async with aclosing(
             async_map(_Mount._get_files(self._entries), _put_file, concurrency=n_concurrent_uploads)
@@ -597,70 +573,36 @@ class _Mount(_Object, type_prefix="mo"):
         # Build the mount.
         status_row.message(f"Creating mount {message_label}: Finalizing index of {len(files)} files")
         if self._deployment_name:
+            creation_type = (
+                api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING
+                if self._allow_overwrite
+                else api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS
+            )
             req = api_pb2.MountGetOrCreateRequest(
                 deployment_name=self._deployment_name,
                 namespace=self._namespace,
-                environment_name=self._environment_name,
-                object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
+                environment_name=load_context.environment_name,
+                object_creation_type=creation_type,
                 files=files,
             )
-        elif resolver.app_id is not None:
+        elif load_context.app_id is not None:
             req = api_pb2.MountGetOrCreateRequest(
                 object_creation_type=api_pb2.OBJECT_CREATION_TYPE_ANONYMOUS_OWNED_BY_APP,
                 files=files,
-                app_id=resolver.app_id,
+                app_id=load_context.app_id,
             )
         else:
             req = api_pb2.MountGetOrCreateRequest(
                 object_creation_type=api_pb2.OBJECT_CREATION_TYPE_EPHEMERAL,
                 files=files,
-                environment_name=resolver.environment_name,
+                environment_name=load_context.environment_name,
             )
 
-        resp = await retry_transient_errors(resolver.client.stub.MountGetOrCreate, req, base_delay=1)
+        resp = await load_context.client.stub.MountGetOrCreate(req, retry=Retry(base_delay=1))
         status_row.finish(f"Created mount {message_label}")
 
         logger.debug(f"Uploaded {total_uploads} new files and {total_bytes} bytes in {time.monotonic() - t0}s")
-        self._hydrate(resp.mount_id, resolver.client, resp.handle_metadata)
-
-    @staticmethod
-    def from_local_python_packages(
-        *module_names: str,
-        remote_dir: Union[str, PurePosixPath] = ROOT_DIR.as_posix(),
-        # Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
-        # Defaults to including all files.
-        condition: Optional[Callable[[str], bool]] = None,
-        ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
-    ) -> "_Mount":
-        """
-        **Deprecated**: Use image.add_local_python_source instead
-
-        Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
-        This works by mounting the local path of each module's package to a directory inside the container
-        that's on `PYTHONPATH`.
-
-        **Usage**
-
-        ```python notest
-        import modal
-        import my_local_module
-
-        app = modal.App()
-
-        @app.function(mounts=[
-            modal.Mount.from_local_python_packages("my_local_module", "my_other_module"),
-        ])
-        def f():
-            my_local_module.do_stuff()
-        ```
-        """
-        deprecation_warning(
-            (2025, 1, 8),
-            MOUNT_DEPRECATION_MESSAGE_PATTERN.format(replacement="image.add_local_python_source"),
-        )
-        return _Mount._from_local_python_packages(
-            *module_names, remote_dir=remote_dir, condition=condition, ignore=ignore
-        )
+        self._hydrate(resp.mount_id, load_context.client, resp.handle_metadata)
 
     @staticmethod
     def _from_local_python_packages(
@@ -693,58 +635,42 @@ class _Mount(_Object, type_prefix="mo"):
         *,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
         environment_name: Optional[str] = None,
+        client: Optional[_Client] = None,
     ) -> "_Mount":
         """mdmd:hidden"""
 
-        async def _load(provider: _Mount, resolver: Resolver, existing_object_id: Optional[str]):
+        async def _load(provider: _Mount, resolver: Resolver, load_context, existing_object_id: Optional[str]):
             req = api_pb2.MountGetOrCreateRequest(
                 deployment_name=name,
                 namespace=namespace,
-                environment_name=_get_environment_name(environment_name, resolver),
+                environment_name=load_context.environment_name,
             )
-            response = await resolver.client.stub.MountGetOrCreate(req)
-            provider._hydrate(response.mount_id, resolver.client, response.handle_metadata)
-
-        return _Mount._from_loader(_load, "Mount()", hydrate_lazily=True)
-
-    @classmethod
-    async def lookup(
-        cls: type["_Mount"],
-        name: str,
-        namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
-        client: Optional[_Client] = None,
-        environment_name: Optional[str] = None,
-    ) -> "_Mount":
-        """mdmd:hidden"""
-        deprecation_warning(
-            (2025, 1, 27),
-            "`modal.Mount.lookup` is deprecated and will be removed in a future release."
-            " It can be replaced with `modal.Mount.from_name`."
-            "\n\nSee https://modal.com/docs/guide/modal-1-0-migration for more information.",
+            response = await load_context.client.stub.MountGetOrCreate(req)
+            provider._hydrate(response.mount_id, load_context.client, response.handle_metadata)
+
+        return _Mount._from_loader(
+            _load,
+            "Mount()",
+            hydrate_lazily=True,
+            load_context_overrides=LoadContext(environment_name=environment_name, client=client),
         )
-        obj = _Mount.from_name(name, namespace=namespace, environment_name=environment_name)
-        if client is None:
-            client = await _Client.from_env()
-        resolver = Resolver(client=client)
-        await resolver.load(obj)
-        return obj
 
     async def _deploy(
         self: "_Mount",
         deployment_name: Optional[str] = None,
         namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
+        *,
         environment_name: Optional[str] = None,
+        allow_overwrite: bool = False,
         client: Optional[_Client] = None,
     ) -> None:
         check_object_name(deployment_name, "Mount")
-        environment_name = _get_environment_name(environment_name, resolver=None)
         self._deployment_name = deployment_name
         self._namespace = namespace
-        self._environment_name = environment_name
-        if client is None:
-            client = await _Client.from_env()
-        resolver = Resolver(client=client, environment_name=environment_name)
-        await resolver.load(self)
+        self._allow_overwrite = allow_overwrite
+        resolver = Resolver()
+        root_metadata = LoadContext(client=client, environment_name=environment_name)
+        await resolver.load(self, root_metadata)
 
     def _get_metadata(self) -> api_pb2.MountHandleMetadata:
         if self._content_checksum_sha256_hex is None:
@@ -810,3 +736,142 @@ def _is_modal_path(remote_path: PurePosixPath):
         if is_modal_path:
             return True
     return False
+
+
+REMOTE_PACKAGES_PATH = "/__modal/deps"
+REMOTE_SITECUSTOMIZE_PATH = "/pkg/sitecustomize.py"
+
+SITECUSTOMIZE_CONTENT = f"""
+# This file is automatically generated by Modal.
+# It ensures that Modal's python dependencies are available in the Python PATH,
+# while prioritizing user-installed packages.
+import sys; sys.path.append('{REMOTE_PACKAGES_PATH}')
+""".strip()
+
+
+async def _create_single_client_dependency_mount(
+    client: _Client,
+    builder_version: str,
+    python_version: str,
+    arch: str,
+    platform: str,
+    uv_python_platform: str,
+    check_if_exists: bool = True,
+    allow_overwrite: bool = False,
+    dry_run: bool = False,
+):
+    import tempfile
+
+    profile_environment = config.get("environment")
+    abi_tag = "cp" + python_version.replace(".", "")
+    mount_name = f"{builder_version}-{abi_tag}-{platform}-{arch}"
+
+    if check_if_exists:
+        try:
+            await Mount.from_name(mount_name, namespace=api_pb2.DEPLOYMENT_NAMESPACE_GLOBAL).hydrate.aio(client)
+            print(f"➖ Found existing mount {mount_name} in global namespace.")
+            return
+        except modal.exception.NotFoundError:
+            pass
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpd:
+        print(f"📦 Building {mount_name}.")
+        requirements = os.path.join(os.path.dirname(__file__), f"builder/{builder_version}.txt")
+        cmd = " ".join(
+            [
+                "uv",
+                "pip",
+                "install",
+                "--strict",
+                "--no-deps",
+                "--no-cache",
+                "-r",
+                requirements,
+                "--compile-bytecode",
+                "--target",
+                tmpd,
+                "--python-platform",
+                uv_python_platform,
+                "--python-version",
+                python_version,
+            ]
+        )
+        proc = await asyncio.create_subprocess_shell(
+            cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.wait()
+        if proc.returncode:
+            stdout, stderr = await proc.communicate()
+            print(stdout.decode("utf-8"))
+            print(stderr.decode("utf-8"))
+            raise RuntimeError(f"Subprocess failed with {proc.returncode}")
+
+        print(f"🌐 Downloaded and unpacked {mount_name} packages to {tmpd}.")
+
+        python_mount = Mount._from_local_dir(tmpd, remote_path=REMOTE_PACKAGES_PATH)
+
+        with tempfile.NamedTemporaryFile() as sitecustomize:
+            sitecustomize.write(
+                SITECUSTOMIZE_CONTENT.encode("utf-8"),
+            )
+            sitecustomize.flush()
+
+            python_mount = python_mount.add_local_file(
+                sitecustomize.name,
+                remote_path=REMOTE_SITECUSTOMIZE_PATH,
+            )
+
+            if not dry_run:
+                try:
+                    await python_mount._deploy.aio(
+                        mount_name,
+                        api_pb2.DEPLOYMENT_NAMESPACE_GLOBAL,
+                        environment_name=profile_environment,
+                        allow_overwrite=allow_overwrite,
+                        client=client,
+                    )
+                    print(f"✅ Deployed mount {mount_name} to global namespace.")
+                except GRPCError as e:
+                    print(f"⚠️ Mount creation failed with {e.status}: {e.message}")
+            else:
+                print(f"Dry run - skipping deployment of mount {mount_name}")
+
+
+async def _create_client_dependency_mounts(
+    client=None,
+    python_versions: list[str] = list(PYTHON_STANDALONE_VERSIONS),
+    builder_versions: list[str] = ["2025.06"],  # Reenable "PREVIEW" during testing
+    check_if_exists=True,
+    dry_run=False,
+):
+    arch = "x86_64"
+    platform_tags = [
+        ("manylinux_2_17", f"{arch}-manylinux_2_17"),  # glibc >= 2.17
+        ("musllinux_1_2", f"{arch}-unknown-linux-musl"),  # musl >= 1.2
+    ]
+    coros = []
+    for python_version in python_versions:
+        for builder_version in builder_versions:
+            for platform, uv_python_platform in platform_tags:
+                coros.append(
+                    _create_single_client_dependency_mount(
+                        client,
+                        builder_version,
+                        python_version,
+                        arch,
+                        platform,
+                        uv_python_platform,
+                        # This check_if_exists / allow_overwrite parameterization is very awkward
+                        # Also it doesn't provide a hook for overwriting a non-preview version, which
+                        # in theory we may need to do at some point (hopefully not, but...)
+                        check_if_exists=check_if_exists and builder_version != "PREVIEW",
+                        allow_overwrite=builder_version == "PREVIEW",
+                        dry_run=dry_run,
+                    )
+                )
+    await TaskContext.gather(*coros)
+
+
+create_client_dependency_mounts = synchronize_api(_create_client_dependency_mounts)