modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

modal/_utils/mount_utils.py

@@ -1,26 +1,29 @@
 # Copyright Modal Labs 2022
 import posixpath
 import typing
+from collections.abc import Mapping, Sequence
 from pathlib import PurePath, PurePosixPath
-from typing import TYPE_CHECKING, Dict, List, Mapping, Tuple, Union
+from typing import Union
 
+from ..cloud_bucket_mount import _CloudBucketMount
 from ..exception import InvalidError
+from ..network_file_system import _NetworkFileSystem
 from ..volume import _Volume
 
-if TYPE_CHECKING:
-    from ..cloud_bucket_mount import _CloudBucketMount
-    from ..network_file_system import _NetworkFileSystem
-
-
-T = typing.TypeVar("T", bound=Union["_Volume", "_NetworkFileSystem", "_CloudBucketMount"])
+T = typing.TypeVar("T", bound=Union[_Volume, _NetworkFileSystem, _CloudBucketMount])
 
 
 def validate_mount_points(
     display_name: str,
     volume_likes: Mapping[Union[str, PurePosixPath], T],
-) -> List[Tuple[str, T]]:
+) -> list[tuple[str, T]]:
     """Mount point path validation for volumes and network file systems."""
 
+    if not isinstance(volume_likes, dict):
+        raise InvalidError(
+            f"`volume_likes` should be a dict[str | PurePosixPath, {display_name}], got {type(volume_likes)} instead"
+        )
+
     validated = []
     for path, vol in volume_likes.items():
         path = PurePath(path).as_posix()
@@ -38,17 +41,32 @@ def validate_mount_points(
     return validated
 
 
-def validate_volumes(
-    volumes: Mapping[Union[str, PurePosixPath], Union["_Volume", "_CloudBucketMount"]],
-) -> List[Tuple[str, Union["_Volume", "_NetworkFileSystem", "_CloudBucketMount"]]]:
-    if not isinstance(volumes, dict):
-        raise InvalidError("volumes must be a dict[str, Volume] where the keys are paths")
+def validate_network_file_systems(
+    network_file_systems: Mapping[Union[str, PurePosixPath], _NetworkFileSystem],
+):
+    validated_network_file_systems = validate_mount_points("NetworkFileSystem", network_file_systems)
+
+    for path, network_file_system in validated_network_file_systems:
+        if not isinstance(network_file_system, (_NetworkFileSystem)):
+            raise InvalidError(
+                f"Object of type {type(network_file_system)} mounted at '{path}' "
+                + "is not useable as a network file system."
+            )
+
+    return validated_network_file_systems
 
+
+def validate_volumes(
+    volumes: Mapping[Union[str, PurePosixPath], Union[_Volume, _CloudBucketMount]],
+) -> Sequence[tuple[str, Union[_Volume, _CloudBucketMount]]]:
     validated_volumes = validate_mount_points("Volume", volumes)
-    # We don't support mounting a volume in more than one location
-    volume_to_paths: Dict["_Volume", List[str]] = {}
+    # We don't support mounting a modal.Volume in more than one location,
+    # but the same CloudBucketMount object can be used in more than one location.
+    volume_to_paths: dict[_Volume, list[str]] = {}
     for path, volume in validated_volumes:
-        if isinstance(volume, _Volume):
+        if not isinstance(volume, (_Volume, _CloudBucketMount)):
+            raise InvalidError(f"Object of type {type(volume)} mounted at '{path}' is not useable as a volume.")
+        elif isinstance(volume, (_Volume)):
             volume_to_paths.setdefault(volume, []).append(path)
     for paths in volume_to_paths.values():
         if len(paths) > 1:

modal/_utils/name_utils.py

@@ -0,0 +1,58 @@
+# Copyright Modal Labs 2022
+import re
+
+from ..exception import InvalidError
+
+# https://www.rfc-editor.org/rfc/rfc1035
+subdomain_regex = re.compile("^(?![0-9]+$)(?!-)[a-z0-9-]{,63}(?<!-)$")
+
+
+def is_valid_subdomain_label(label: str) -> bool:
+    return subdomain_regex.match(label) is not None
+
+
+def replace_invalid_subdomain_chars(label: str) -> str:
+    return re.sub("[^a-z0-9-]", "-", label.lower())
+
+
+def is_valid_object_name(name: str) -> bool:
+    return (
+        # Limit object name length
+        len(name) <= 64
+        # Limit character set
+        and re.match("^[a-zA-Z0-9-_.]+$", name) is not None
+        # Avoid collisions with App IDs
+        and re.match("^ap-[a-zA-Z0-9]{22}$", name) is None
+    )
+
+
+def is_valid_environment_name(name: str) -> bool:
+    # first char is alnum, the rest allows other chars
+    return len(name) <= 64 and re.match(r"^[a-zA-Z0-9][a-zA-Z0-9-_.]+$", name) is not None
+
+
+def is_valid_tag(tag: str) -> bool:
+    """Tags are alphanumeric, dashes, periods, and underscores, and must be 50 characters or less"""
+    pattern = r"^[a-zA-Z0-9._-]{1,50}$"
+    return bool(re.match(pattern, tag))
+
+
+def check_object_name(name: str, object_type: str) -> None:
+    message = (
+        f"Invalid {object_type} name: '{name}'."
+        "\n\nNames may contain only alphanumeric characters, dashes, periods, and underscores,"
+        " must be shorter than 64 characters, and cannot conflict with App ID strings."
+    )
+    if not is_valid_object_name(name):
+        raise InvalidError(message)
+
+
+def check_environment_name(name: str) -> None:
+    message = (
+        f"Invalid environment name: '{name}'."
+        "\n\nEnvironment names can only start with alphanumeric characters,"
+        " may contain only alphanumeric characters, dashes, periods, and underscores,"
+        " and must be shorter than 64 characters."
+    )
+    if not is_valid_environment_name(name):
+        raise InvalidError(message)

modal/_utils/package_utils.py

@@ -23,7 +23,7 @@ def get_file_formats(module):
 BINARY_FORMATS = ["so", "S", "s", "asm"]  # TODO
 
 
-def get_module_mount_info(module_name: str) -> typing.Sequence[typing.Tuple[bool, Path]]:
+def get_module_mount_info(module_name: str) -> typing.Sequence[tuple[bool, Path]]:
     """Returns a list of tuples [(is_dir, path)] describing how to mount a given module."""
     file_formats = get_file_formats(module_name)
     if set(BINARY_FORMATS) & set(file_formats):
@@ -46,3 +46,16 @@ def get_module_mount_info(module_name: str) -> typing.Sequence[typing.Tuple[bool
     if not entries:
         raise ModuleNotMountable(f"{module_name} has no mountable paths")
     return entries
+
+
+def parse_major_minor_version(version_string: str) -> tuple[int, int]:
+    parts = version_string.split(".")
+    if len(parts) < 2:
+        raise ValueError("version_string must have at least an 'X.Y' format")
+    try:
+        major = int(parts[0])
+        minor = int(parts[1])
+    except ValueError:
+        raise ValueError("version_string must have at least an 'X.Y' format with integral major/minor values")
+
+    return major, minor

modal/_utils/pattern_utils.py

@@ -0,0 +1,205 @@
+# Copyright Modal Labs 2024
+"""Pattern matching library ported from https://github.com/moby/patternmatcher.
+
+This is the same pattern-matching logic used by Docker, except it is written in
+Python rather than Go. Also, the original Go library has a couple deprecated
+functions that we don't implement in this port.
+
+The main way to use this library is by constructing a `FilePatternMatcher` object,
+then asking it whether file paths match any of its patterns.
+"""
+
+import enum
+import os
+import re
+from typing import Optional, TextIO
+
+escape_chars = frozenset(".+()|{}$")
+
+
+class MatchType(enum.IntEnum):
+    UNKNOWN = 0
+    EXACT = 1
+    PREFIX = 2
+    SUFFIX = 3
+    REGEXP = 4
+
+
+class Pattern:
+    """Defines a single regex pattern used to filter file paths."""
+
+    def __init__(self) -> None:
+        """Initialize a new Pattern instance."""
+        self.match_type = MatchType.UNKNOWN
+        self.cleaned_pattern = ""
+        self.dirs: list[str] = []
+        self.regexp: Optional[re.Pattern] = None
+        self.exclusion = False
+
+    def __str__(self) -> str:
+        """Return the cleaned pattern as the string representation."""
+        return self.cleaned_pattern
+
+    def compile(self, separator: str) -> None:
+        """Compile the pattern into a regular expression.
+
+        Args:
+            separator (str): The path separator (e.g., '/' or '\\').
+
+        Raises:
+            ValueError: If the pattern is invalid.
+        """
+        reg_str = "^"
+        pattern = self.cleaned_pattern
+
+        esc_separator = separator
+        if separator == "\\":
+            esc_separator = "\\\\"
+
+        self.match_type = MatchType.EXACT
+        i = 0
+        pattern_length = len(pattern)
+        while i < pattern_length:
+            ch = pattern[i]
+            if ch == "*":
+                if (i + 1) < pattern_length and pattern[i + 1] == "*":
+                    # Handle '**'
+                    i += 1  # Skip the second '*'
+                    # Treat '**/' as '**' so eat the '/'
+                    if (i + 1) < pattern_length and pattern[i + 1] == separator:
+                        i += 1  # Skip the '/'
+                    if i + 1 == pattern_length:
+                        # Pattern ends with '**'
+                        if self.match_type == MatchType.EXACT:
+                            self.match_type = MatchType.PREFIX
+                        else:
+                            reg_str += ".*"
+                            self.match_type = MatchType.REGEXP
+                    else:
+                        # '**' in the middle
+                        reg_str += f"(.*{esc_separator})?"
+                        self.match_type = MatchType.REGEXP
+
+                    if i == 1:
+                        self.match_type = MatchType.SUFFIX
+                else:
+                    # Single '*'
+                    reg_str += f"[^{esc_separator}]*"
+                    self.match_type = MatchType.REGEXP
+            elif ch == "?":
+                # Single '?'
+                reg_str += f"[^{esc_separator}]"
+                self.match_type = MatchType.REGEXP
+            elif ch in escape_chars:
+                reg_str += "\\" + ch
+            elif ch == "\\":
+                # Escape next character
+                if separator == "\\":
+                    reg_str += esc_separator
+                    i += 1
+                    continue
+                if (i + 1) < pattern_length:
+                    reg_str += "\\" + pattern[i + 1]
+                    i += 1  # Skip the escaped character
+                    self.match_type = MatchType.REGEXP
+                else:
+                    reg_str += "\\"
+            elif ch == "[" or ch == "]":
+                reg_str += ch
+                self.match_type = MatchType.REGEXP
+            else:
+                reg_str += ch
+            i += 1
+
+        if self.match_type != MatchType.REGEXP:
+            return
+
+        reg_str += "$"
+
+        try:
+            self.regexp = re.compile(reg_str)
+            self.match_type = MatchType.REGEXP
+        except re.error as e:
+            raise ValueError(f"Bad pattern: {pattern}") from e
+
+    def match(self, path: str) -> bool:
+        """Check if the path matches the pattern."""
+        if self.match_type == MatchType.UNKNOWN:
+            self.compile(os.path.sep)
+
+        if self.match_type == MatchType.EXACT:
+            return path == self.cleaned_pattern
+        elif self.match_type == MatchType.PREFIX:
+            # Strip trailing '**'
+            return path.startswith(self.cleaned_pattern[:-2])
+        elif self.match_type == MatchType.SUFFIX:
+            # Strip leading '**'
+            suffix = self.cleaned_pattern[2:]
+            if path.endswith(suffix):
+                return True
+            # '**/foo' matches 'foo'
+            if suffix[0] == os.path.sep and path == suffix[1:]:
+                return True
+            else:
+                return False
+        elif self.match_type == MatchType.REGEXP:
+            return self.regexp.match(path) is not None
+        else:
+            return False
+
+
+def read_ignorefile(reader: TextIO) -> list[str]:
+    """Read an ignore file from a reader and return the list of file patterns to
+    ignore, applying the following rules:
+
+    - An UTF8 BOM header (if present) is stripped. (Python does this already)
+    - Lines starting with "#" are considered comments and are skipped.
+
+    For remaining lines:
+
+    - Leading and trailing whitespace is removed from each ignore pattern.
+    - It uses `os.path.normpath` to get the shortest/cleanest path for ignore
+      patterns.
+    - Leading forward-slashes ("/") are removed from ignore patterns, so
+      "/some/path" and "some/path" are considered equivalent.
+
+    Args:
+        reader (file-like object): The input stream to read from.
+
+    Returns:
+        list: A list of patterns to ignore.
+    """
+    if reader is None:
+        return []
+
+    excludes: list[str] = []
+
+    for line in reader:
+        pattern = line.rstrip("\n\r")
+
+        # Lines starting with "#" are ignored
+        if pattern.startswith("#"):
+            continue
+
+        pattern = pattern.strip()
+        if pattern == "":
+            continue
+
+        # Normalize absolute paths to paths relative to the context
+        # (taking care of '!' prefix)
+        invert = pattern[0] == "!"
+        if invert:
+            pattern = pattern[1:].strip()
+
+        if len(pattern) > 0:
+            pattern = os.path.normpath(pattern)
+            pattern = pattern.replace(os.sep, "/")
+            if len(pattern) > 1 and pattern[0] == "/":
+                pattern = pattern[1:]
+
+        if invert:
+            pattern = "!" + pattern
+
+        excludes.append(pattern)
+
+    return excludes

modal/_utils/rand_pb_testing.py

@@ -7,13 +7,13 @@ Modal, with random seeds, and it supports oneofs, and Protobuf v4.
 
 import string
 from random import Random
-from typing import Any, Callable, Dict, Optional, Type, TypeVar
+from typing import Any, Callable, Optional, TypeVar
 
 from google.protobuf.descriptor import Descriptor, FieldDescriptor
 
 T = TypeVar("T")
 
-_FIELD_RANDOM_GENERATOR: Dict[int, Callable[[Random], Any]] = {
+_FIELD_RANDOM_GENERATOR: dict[int, Callable[[Random], Any]] = {
     FieldDescriptor.TYPE_DOUBLE: lambda rand: rand.normalvariate(0, 1),
     FieldDescriptor.TYPE_FLOAT: lambda rand: rand.normalvariate(0, 1),
     FieldDescriptor.TYPE_INT32: lambda rand: int.from_bytes(rand.randbytes(4), "little", signed=True),
@@ -58,12 +58,10 @@ def _fill(msg, desc: Descriptor, rand: Random) -> None:
         else:
             if field.type == FieldDescriptor.TYPE_ENUM:
                 enum_values = [x.number for x in field.enum_type.values]
-
-                def generator(rand):
-                    return rand.choice(enum_values)
+                generator = lambda rand: rand.choice(enum_values)  # noqa: E731
 
             else:
-                generator = _FIELD_RANDOM_GENERATOR.get(field.type)
+                generator = _FIELD_RANDOM_GENERATOR[field.type]
             if is_repeated:
                 num = rand.randint(0, 2)
                 msg_field = getattr(msg, field.name)
@@ -73,7 +71,7 @@ def _fill(msg, desc: Descriptor, rand: Random) -> None:
                 setattr(msg, field.name, generator(rand))
 
 
-def rand_pb(proto: Type[T], rand: Optional[Random] = None) -> T:
+def rand_pb(proto: type[T], rand: Optional[Random] = None) -> T:
     """Generate a pseudorandom protobuf message.
 
     ```python notest

modal/_utils/shell_utils.py

@@ -1,18 +1,17 @@
 # Copyright Modal Labs 2024
+
 import asyncio
 import contextlib
 import errno
 import os
 import select
 import sys
-from typing import Callable, Coroutine, Optional
-
-import rich.status
+from collections.abc import Coroutine
+from typing import Callable, Optional
 
 from modal._pty import raw_terminal, set_nonblocking
-from modal.exception import ExecutionError, InteractiveTimeoutError
 
-from .async_utils import TaskContext, asyncify
+from .async_utils import asyncify
 
 
 def write_to_fd(fd: int, data: bytes):
@@ -20,14 +19,20 @@ def write_to_fd(fd: int, data: bytes):
     future = loop.create_future()
 
     def try_write():
+        nonlocal data
         try:
             nbytes = os.write(fd, data)
-            loop.remove_writer(fd)
-            future.set_result(nbytes)
+            data = data[nbytes:]
+            if not data:
+                loop.remove_writer(fd)
+                future.set_result(None)
         except OSError as e:
-            if e.errno != errno.EAGAIN:
-                future.set_exception(e)
-                raise
+            if e.errno == errno.EAGAIN:
+                # Wait for the next write notification
+                return
+            # Fail if it's not EAGAIN
+            loop.remove_writer(fd)
+            future.set_exception(e)
 
     loop.add_writer(fd, try_write)
     return future
@@ -72,42 +77,3 @@ async def stream_from_stdin(handle_input: Callable[[bytes, int], Coroutine], use
         yield
     os.write(quit_pipe_write, b"\n")
     write_task.cancel()
-
-
-async def connect_to_terminal(
-    # Handles data read from stdin. Inputs are the stdin data and message index.
-    handle_stdin: Callable[[bytes, int], Coroutine],
-    # Creates a coroutine that streams data to stdout/stderr. Returns the exit status.
-    stream_to_stdio: Callable[[asyncio.Event], Coroutine[None, None, int]],
-    pty: bool = False,
-    connecting_status: Optional[rich.status.Status] = None,
-) -> None:
-    """
-    Connect to the current terminal by streaming data from terminal's stdin to the running process
-    and streaming output from running process into terminal's stdout.
-
-    If connecting_status is given, this function will stop the status spinner upon connection or error.
-    """
-
-    def stop_connecting_status():
-        if connecting_status:
-            connecting_status.stop()
-
-    on_connect = asyncio.Event()
-    async with TaskContext() as tc:
-        exec_output_task = tc.create_task(stream_to_stdio(on_connect))
-        try:
-            # time out if we can't connect to the server fast enough
-            await asyncio.wait_for(on_connect.wait(), timeout=15)
-            stop_connecting_status()
-
-            async with stream_from_stdin(handle_stdin, use_raw_terminal=pty):
-                exit_status = await exec_output_task
-
-            if exit_status != 0:
-                raise ExecutionError(f"Process exited with status code {exit_status}")
-
-        except (asyncio.TimeoutError, TimeoutError):
-            stop_connecting_status()
-            exec_output_task.cancel()
-            raise InteractiveTimeoutError("Failed to establish connection to container.")