modal 1.0.3.dev10__py3-none-any.whl → 1.2.3.dev7__py3-none-any.whl

modal/file_io.pyi

@@ -1,18 +1,29 @@
 import _typeshed
 import enum
 import modal.client
-import modal_proto.api_pb2
 import typing
 import typing_extensions
 
 T = typing.TypeVar("T")
 
-async def _delete_bytes(
-    file: _FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-) -> None: ...
+async def _delete_bytes(file: _FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None:
+    """Delete a range of bytes from the file.
+
+    `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+    If either is None, the start or end of the file is used, respectively.
+    """
+    ...
+
 async def _replace_bytes(
     file: _FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-) -> None: ...
+) -> None:
+    """Replace a range of bytes in the file with new data. The length of the data does not
+    have to be the same as the length of the range being replaced.
+
+    `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+    If either is None, the start or end of the file is used, respectively.
+    """
+    ...
 
 class FileWatchEventType(enum.Enum):
     Unknown = "Unknown"
@@ -22,23 +33,58 @@ class FileWatchEventType(enum.Enum):
     Remove = "Remove"
 
 class FileWatchEvent:
+    """FileWatchEvent(paths: list[str], type: modal.file_io.FileWatchEventType)"""
+
     paths: list[str]
     type: FileWatchEventType
 
-    def __init__(self, paths: list[str], type: FileWatchEventType) -> None: ...
-    def __repr__(self): ...
-    def __eq__(self, other): ...
+    def __init__(self, paths: list[str], type: FileWatchEventType) -> None:
+        """Initialize self.  See help(type(self)) for accurate signature."""
+        ...
+
+    def __repr__(self):
+        """Return repr(self)."""
+        ...
+
+    def __eq__(self, other):
+        """Return self==value."""
+        ...
 
 class _FileIO(typing.Generic[T]):
+    """[Alpha] FileIO handle, used in the Sandbox filesystem API.
+
+    The API is designed to mimic Python's io.FileIO.
+
+    Currently this API is in Alpha and is subject to change. File I/O operations
+    may be limited in size to 100 MiB, and the throughput of requests is
+    restricted in the current implementation. For our recommendations on large file transfers
+    see the Sandbox [filesystem access guide](https://modal.com/docs/guide/sandbox-files).
+
+    **Usage**
+
+    ```python notest
+    import modal
+
+    app = modal.App.lookup("my-app", create_if_missing=True)
+
+    sb = modal.Sandbox.create(app=app)
+    f = sb.open("/tmp/foo.txt", "w")
+    f.write("hello")
+    f.close()
+    ```
+    """
+
     _task_id: str
     _file_descriptor: str
     _client: modal.client._Client
-    _watch_output_buffer: list[typing.Optional[bytes]]
+    _watch_output_buffer: list[typing.Union[bytes, None, Exception]]
+
+    def __init__(self, client: modal.client._Client, task_id: str) -> None:
+        """Initialize self.  See help(type(self)) for accurate signature."""
+        ...
 
-    def __init__(self, client: modal.client._Client, task_id: str) -> None: ...
     def _validate_mode(self, mode: str) -> None: ...
-    def _handle_error(self, error: modal_proto.api_pb2.SystemErrorMessage) -> None: ...
-    def _consume_output(self, exec_id: str) -> typing.AsyncIterator[typing.Optional[bytes]]: ...
+    def _consume_output(self, exec_id: str) -> typing.AsyncIterator[typing.Union[bytes, None, Exception]]: ...
     async def _consume_watch_output(self, exec_id: str) -> None: ...
     async def _parse_watch_output(self, event: bytes) -> typing.Optional[FileWatchEvent]: ...
     async def _wait(self, exec_id: str) -> bytes: ...
@@ -51,21 +97,60 @@ class _FileIO(typing.Generic[T]):
         mode: typing.Union[_typeshed.OpenTextMode, _typeshed.OpenBinaryMode],
         client: modal.client._Client,
         task_id: str,
-    ) -> _FileIO: ...
+    ) -> _FileIO:
+        """Create a new FileIO handle."""
+        ...
+
     async def _make_read_request(self, n: typing.Optional[int]) -> bytes: ...
-    async def read(self, n: typing.Optional[int] = None) -> T: ...
-    async def readline(self) -> T: ...
-    async def readlines(self) -> typing.Sequence[T]: ...
-    async def write(self, data: typing.Union[bytes, str]) -> None: ...
-    async def flush(self) -> None: ...
+    async def read(self, n: typing.Optional[int] = None) -> T:
+        """Read n bytes from the current position, or the entire remaining file if n is None."""
+        ...
+
+    async def readline(self) -> T:
+        """Read a single line from the current position."""
+        ...
+
+    async def readlines(self) -> typing.Sequence[T]:
+        """Read all lines from the current position."""
+        ...
+
+    async def write(self, data: typing.Union[bytes, str]) -> None:
+        """Write data to the current position.
+
+        Writes may not appear until the entire buffer is flushed, which
+        can be done manually with `flush()` or automatically when the file is
+        closed.
+        """
+        ...
+
+    async def flush(self) -> None:
+        """Flush the buffer to disk."""
+        ...
+
     def _get_whence(self, whence: int): ...
-    async def seek(self, offset: int, whence: int = 0) -> None: ...
+    async def seek(self, offset: int, whence: int = 0) -> None:
+        """Move to a new position in the file.
+
+        `whence` defaults to 0 (absolute file positioning); other values are 1
+        (relative to the current position) and 2 (relative to the file's end).
+        """
+        ...
+
     @classmethod
-    async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]: ...
+    async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]:
+        """List the contents of the provided directory."""
+        ...
+
     @classmethod
-    async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None: ...
+    async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None:
+        """Create a new directory."""
+        ...
+
     @classmethod
-    async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None: ...
+    async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None:
+        """Remove a file or directory in the Sandbox."""
+        ...
+
     @classmethod
     def watch(
         cls,
@@ -77,7 +162,10 @@ class _FileIO(typing.Generic[T]):
         timeout: typing.Optional[int] = None,
     ) -> typing.AsyncIterator[FileWatchEvent]: ...
     async def _close(self) -> None: ...
-    async def close(self) -> None: ...
+    async def close(self) -> None:
+        """Flush the buffer and close the file."""
+        ...
+
     def _check_writable(self) -> None: ...
     def _check_readable(self) -> None: ...
     def _check_closed(self) -> None: ...
@@ -85,22 +173,46 @@ class _FileIO(typing.Generic[T]):
     async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
 
 class __delete_bytes_spec(typing_extensions.Protocol):
-    def __call__(
-        self, /, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-    ) -> None: ...
-    async def aio(
-        self, /, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-    ) -> None: ...
+    def __call__(self, /, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None:
+        """Delete a range of bytes from the file.
+
+        `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+        If either is None, the start or end of the file is used, respectively.
+        """
+        ...
+
+    async def aio(self, /, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None:
+        """Delete a range of bytes from the file.
+
+        `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+        If either is None, the start or end of the file is used, respectively.
+        """
+        ...
 
 delete_bytes: __delete_bytes_spec
 
 class __replace_bytes_spec(typing_extensions.Protocol):
     def __call__(
         self, /, file: FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-    ) -> None: ...
+    ) -> None:
+        """Replace a range of bytes in the file with new data. The length of the data does not
+        have to be the same as the length of the range being replaced.
+
+        `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+        If either is None, the start or end of the file is used, respectively.
+        """
+        ...
+
     async def aio(
         self, /, file: FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
-    ) -> None: ...
+    ) -> None:
+        """Replace a range of bytes in the file with new data. The length of the data does not
+        have to be the same as the length of the range being replaced.
+
+        `start` and `end` are byte offsets. `start` is inclusive, `end` is exclusive.
+        If either is None, the start or end of the file is used, respectively.
+        """
+        ...
 
 replace_bytes: __replace_bytes_spec
 
@@ -109,18 +221,40 @@ SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
 T_INNER = typing.TypeVar("T_INNER", covariant=True)
 
 class FileIO(typing.Generic[T]):
+    """[Alpha] FileIO handle, used in the Sandbox filesystem API.
+
+    The API is designed to mimic Python's io.FileIO.
+
+    Currently this API is in Alpha and is subject to change. File I/O operations
+    may be limited in size to 100 MiB, and the throughput of requests is
+    restricted in the current implementation. For our recommendations on large file transfers
+    see the Sandbox [filesystem access guide](https://modal.com/docs/guide/sandbox-files).
+
+    **Usage**
+
+    ```python notest
+    import modal
+
+    app = modal.App.lookup("my-app", create_if_missing=True)
+
+    sb = modal.Sandbox.create(app=app)
+    f = sb.open("/tmp/foo.txt", "w")
+    f.write("hello")
+    f.close()
+    ```
+    """
+
     _task_id: str
     _file_descriptor: str
     _client: modal.client.Client
-    _watch_output_buffer: list[typing.Optional[bytes]]
+    _watch_output_buffer: list[typing.Union[bytes, None, Exception]]
 
     def __init__(self, client: modal.client.Client, task_id: str) -> None: ...
     def _validate_mode(self, mode: str) -> None: ...
-    def _handle_error(self, error: modal_proto.api_pb2.SystemErrorMessage) -> None: ...
 
     class ___consume_output_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, exec_id: str) -> typing.Iterator[typing.Optional[bytes]]: ...
-        def aio(self, /, exec_id: str) -> typing.AsyncIterator[typing.Optional[bytes]]: ...
+        def __call__(self, /, exec_id: str) -> typing.Iterator[typing.Union[bytes, None, Exception]]: ...
+        def aio(self, /, exec_id: str) -> typing.AsyncIterator[typing.Union[bytes, None, Exception]]: ...
 
     _consume_output: ___consume_output_spec[typing_extensions.Self]
 
@@ -157,7 +291,9 @@ class FileIO(typing.Generic[T]):
         mode: typing.Union[_typeshed.OpenTextMode, _typeshed.OpenBinaryMode],
         client: modal.client.Client,
         task_id: str,
-    ) -> FileIO: ...
+    ) -> FileIO:
+        """Create a new FileIO handle."""
+        ...
 
     class ___make_read_request_spec(typing_extensions.Protocol[SUPERSELF]):
         def __call__(self, /, n: typing.Optional[int]) -> bytes: ...
@@ -166,49 +302,106 @@ class FileIO(typing.Generic[T]):
     _make_read_request: ___make_read_request_spec[typing_extensions.Self]
 
     class __read_spec(typing_extensions.Protocol[T_INNER, SUPERSELF]):
-        def __call__(self, /, n: typing.Optional[int] = None) -> T_INNER: ...
-        async def aio(self, /, n: typing.Optional[int] = None) -> T_INNER: ...
+        def __call__(self, /, n: typing.Optional[int] = None) -> T_INNER:
+            """Read n bytes from the current position, or the entire remaining file if n is None."""
+            ...
+
+        async def aio(self, /, n: typing.Optional[int] = None) -> T_INNER:
+            """Read n bytes from the current position, or the entire remaining file if n is None."""
+            ...
 
     read: __read_spec[T, typing_extensions.Self]
 
     class __readline_spec(typing_extensions.Protocol[T_INNER, SUPERSELF]):
-        def __call__(self, /) -> T_INNER: ...
-        async def aio(self, /) -> T_INNER: ...
+        def __call__(self, /) -> T_INNER:
+            """Read a single line from the current position."""
+            ...
+
+        async def aio(self, /) -> T_INNER:
+            """Read a single line from the current position."""
+            ...
 
     readline: __readline_spec[T, typing_extensions.Self]
 
     class __readlines_spec(typing_extensions.Protocol[T_INNER, SUPERSELF]):
-        def __call__(self, /) -> typing.Sequence[T_INNER]: ...
-        async def aio(self, /) -> typing.Sequence[T_INNER]: ...
+        def __call__(self, /) -> typing.Sequence[T_INNER]:
+            """Read all lines from the current position."""
+            ...
+
+        async def aio(self, /) -> typing.Sequence[T_INNER]:
+            """Read all lines from the current position."""
+            ...
 
     readlines: __readlines_spec[T, typing_extensions.Self]
 
     class __write_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, data: typing.Union[bytes, str]) -> None: ...
-        async def aio(self, /, data: typing.Union[bytes, str]) -> None: ...
+        def __call__(self, /, data: typing.Union[bytes, str]) -> None:
+            """Write data to the current position.
+
+            Writes may not appear until the entire buffer is flushed, which
+            can be done manually with `flush()` or automatically when the file is
+            closed.
+            """
+            ...
+
+        async def aio(self, /, data: typing.Union[bytes, str]) -> None:
+            """Write data to the current position.
+
+            Writes may not appear until the entire buffer is flushed, which
+            can be done manually with `flush()` or automatically when the file is
+            closed.
+            """
+            ...
 
     write: __write_spec[typing_extensions.Self]
 
     class __flush_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /) -> None: ...
-        async def aio(self, /) -> None: ...
+        def __call__(self, /) -> None:
+            """Flush the buffer to disk."""
+            ...
+
+        async def aio(self, /) -> None:
+            """Flush the buffer to disk."""
+            ...
 
     flush: __flush_spec[typing_extensions.Self]
 
     def _get_whence(self, whence: int): ...
 
     class __seek_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /, offset: int, whence: int = 0) -> None: ...
-        async def aio(self, /, offset: int, whence: int = 0) -> None: ...
+        def __call__(self, /, offset: int, whence: int = 0) -> None:
+            """Move to a new position in the file.
+
+            `whence` defaults to 0 (absolute file positioning); other values are 1
+            (relative to the current position) and 2 (relative to the file's end).
+            """
+            ...
+
+        async def aio(self, /, offset: int, whence: int = 0) -> None:
+            """Move to a new position in the file.
+
+            `whence` defaults to 0 (absolute file positioning); other values are 1
+            (relative to the current position) and 2 (relative to the file's end).
+            """
+            ...
 
     seek: __seek_spec[typing_extensions.Self]
 
     @classmethod
-    def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]: ...
+    def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]:
+        """List the contents of the provided directory."""
+        ...
+
     @classmethod
-    def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None: ...
+    def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None:
+        """Create a new directory."""
+        ...
+
     @classmethod
-    def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None: ...
+    def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None:
+        """Remove a file or directory in the Sandbox."""
+        ...
+
     @classmethod
     def watch(
         cls,
@@ -227,8 +420,13 @@ class FileIO(typing.Generic[T]):
     _close: ___close_spec[typing_extensions.Self]
 
     class __close_spec(typing_extensions.Protocol[SUPERSELF]):
-        def __call__(self, /) -> None: ...
-        async def aio(self, /) -> None: ...
+        def __call__(self, /) -> None:
+            """Flush the buffer and close the file."""
+            ...
+
+        async def aio(self, /) -> None:
+            """Flush the buffer and close the file."""
+            ...
 
     close: __close_spec[typing_extensions.Self]
 

modal/file_pattern_matcher.py

@@ -11,6 +11,7 @@ then asking it whether file paths match any of its patterns.
 
 import os
 from abc import abstractmethod
+from functools import cached_property
 from pathlib import Path
 from typing import Callable, Optional, Sequence, Union
 
@@ -46,6 +47,18 @@ class _AbstractPatternMatcher:
 
         return super().__repr__()
 
+    @abstractmethod
+    def can_prune_directories(self) -> bool:
+        """
+        Returns True if this pattern matcher allows safe early directory pruning.
+
+        Directory pruning is safe when matching directories can be skipped entirely
+        without missing any files that should be included.
+
+        An example where pruning is not safe is for inverted patterns, like "!**/*.py".
+        """
+        ...
+
     @abstractmethod
     def __call__(self, path: Path) -> bool: ...
 
@@ -54,6 +67,15 @@ class _CustomPatternMatcher(_AbstractPatternMatcher):
     def __init__(self, predicate: Callable[[Path], bool]):
         self._predicate = predicate
 
+    def can_prune_directories(self) -> bool:
+        """
+        Custom pattern matchers (like negated matchers) cannot safely prune directories.
+
+        Since these are arbitrary predicates, we cannot determine if a directory
+        can be safely skipped without evaluating all files within it.
+        """
+        return False
+
     def __call__(self, path: Path) -> bool:
         return self._predicate(path)
 
@@ -78,11 +100,11 @@ class FilePatternMatcher(_AbstractPatternMatcher):
     ```
     """
 
-    patterns: list[Pattern]
-    _delayed_init: Callable[[], None] = None
+    _file_path: Optional[Union[str, Path]]
+    _pattern_strings: Optional[Sequence[str]]
 
-    def _set_patterns(self, patterns: Sequence[str]) -> None:
-        self.patterns = []
+    def _parse_patterns(self, patterns: Sequence[str]) -> list[Pattern]:
+        parsed_patterns = []
         for pattern in list(patterns):
             pattern = pattern.strip().strip(os.path.sep)
             if not pattern:
@@ -97,7 +119,8 @@ class FilePatternMatcher(_AbstractPatternMatcher):
             # In Python, we can proceed without explicit syntax checking
             new_pattern.cleaned_pattern = pattern
             new_pattern.dirs = pattern.split(os.path.sep)
-            self.patterns.append(new_pattern)
+            parsed_patterns.append(new_pattern)
+        return parsed_patterns
 
     def __init__(self, *pattern: str) -> None:
         """Initialize a new FilePatternMatcher instance.
@@ -108,7 +131,8 @@ class FilePatternMatcher(_AbstractPatternMatcher):
         Raises:
             ValueError: If an illegal exclusion pattern is provided.
         """
-        self._set_patterns(pattern)
+        self._pattern_strings = pattern
+        self._file_path = None
 
     @classmethod
     def from_file(cls, file_path: Union[str, Path]) -> "FilePatternMatcher":
@@ -127,14 +151,10 @@ class FilePatternMatcher(_AbstractPatternMatcher):
         ```
 
         """
-        uninitialized = cls.__new__(cls)
-
-        def _delayed_init():
-            uninitialized._set_patterns(Path(file_path).read_text("utf8").splitlines())
-            uninitialized._delayed_init = None
-
-        uninitialized._delayed_init = _delayed_init
-        return uninitialized
+        instance = cls.__new__(cls)
+        instance._file_path = file_path
+        instance._pattern_strings = None
+        return instance
 
     def _matches(self, file_path: str) -> bool:
         """Check if the file path or any of its parent directories match the patterns.
@@ -173,9 +193,29 @@ class FilePatternMatcher(_AbstractPatternMatcher):
 
         return matched
 
+    @cached_property
+    def patterns(self) -> list[Pattern]:
+        """Get the patterns, loading from file if necessary."""
+        if self._file_path is not None:
+            # Lazy load from file
+            pattern_strings = Path(self._file_path).read_text("utf8").splitlines()
+        else:
+            # Use patterns provided in __init__
+            pattern_strings = list(self._pattern_strings)
+
+        return self._parse_patterns(pattern_strings)
+
+    def can_prune_directories(self) -> bool:
+        """
+        Returns True if this pattern matcher allows safe early directory pruning.
+
+        Directory pruning is safe when matching directories can be skipped entirely
+        without missing any files that should be included. This is for example not
+        safe when we have inverted/negated ignore patterns (e.g. "!**/*.py").
+        """
+        return not any(pattern.exclusion for pattern in self.patterns)
+
     def __call__(self, file_path: Path) -> bool:
-        if self._delayed_init:
-            self._delayed_init()
         return self._matches(str(file_path))