modal 0.62.115__py3-none-any.whl → 0.72.13__py3-none-any.whl

modal/file_io.pyi

@@ -0,0 +1,235 @@
+import _typeshed
+import enum
+import modal.client
+import modal_proto.api_pb2
+import typing
+import typing_extensions
+
+T = typing.TypeVar("T")
+
+async def _delete_bytes(
+    file: _FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None
+) -> None: ...
+async def _replace_bytes(
+    file: _FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
+) -> None: ...
+
+class FileWatchEventType(enum.Enum):
+    Unknown = "Unknown"
+    Access = "Access"
+    Create = "Create"
+    Modify = "Modify"
+    Remove = "Remove"
+
+class FileWatchEvent:
+    paths: list[str]
+    type: FileWatchEventType
+
+    def __init__(self, paths: list[str], type: FileWatchEventType) -> None: ...
+    def __repr__(self): ...
+    def __eq__(self, other): ...
+
+class _FileIO(typing.Generic[T]):
+    _task_id: str
+    _file_descriptor: str
+    _client: modal.client._Client
+    _watch_output_buffer: list[typing.Optional[bytes]]
+
+    def __init__(self, client: modal.client._Client, task_id: str) -> None: ...
+    def _validate_mode(self, mode: str) -> None: ...
+    def _handle_error(self, error: modal_proto.api_pb2.SystemErrorMessage) -> None: ...
+    def _consume_output(self, exec_id: str) -> typing.AsyncIterator[typing.Optional[bytes]]: ...
+    async def _consume_watch_output(self, exec_id: str) -> None: ...
+    async def _parse_watch_output(self, event: bytes) -> typing.Optional[FileWatchEvent]: ...
+    async def _wait(self, exec_id: str) -> bytes: ...
+    def _validate_type(self, data: typing.Union[bytes, str]) -> None: ...
+    async def _open_file(self, path: str, mode: str) -> None: ...
+    @classmethod
+    async def create(
+        cls,
+        path: str,
+        mode: typing.Union[_typeshed.OpenTextMode, _typeshed.OpenBinaryMode],
+        client: modal.client._Client,
+        task_id: str,
+    ) -> _FileIO: ...
+    async def _make_read_request(self, n: typing.Optional[int]) -> bytes: ...
+    async def read(self, n: typing.Optional[int] = None) -> T: ...
+    async def readline(self) -> T: ...
+    async def readlines(self) -> typing.Sequence[T]: ...
+    async def write(self, data: typing.Union[bytes, str]) -> None: ...
+    async def flush(self) -> None: ...
+    def _get_whence(self, whence: int): ...
+    async def seek(self, offset: int, whence: int = 0) -> None: ...
+    @classmethod
+    async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]: ...
+    @classmethod
+    async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None: ...
+    @classmethod
+    async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None: ...
+    @classmethod
+    def watch(
+        cls,
+        path: str,
+        client: modal.client._Client,
+        task_id: str,
+        filter: typing.Optional[list[FileWatchEventType]] = None,
+        recursive: bool = False,
+        timeout: typing.Optional[int] = None,
+    ) -> typing.AsyncIterator[FileWatchEvent]: ...
+    async def _close(self) -> None: ...
+    async def close(self) -> None: ...
+    def _check_writable(self) -> None: ...
+    def _check_readable(self) -> None: ...
+    def _check_closed(self) -> None: ...
+    async def __aenter__(self) -> _FileIO: ...
+    async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
+
+class __delete_bytes_spec(typing_extensions.Protocol):
+    def __call__(self, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None: ...
+    async def aio(self, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None: ...
+
+delete_bytes: __delete_bytes_spec
+
+class __replace_bytes_spec(typing_extensions.Protocol):
+    def __call__(
+        self, file: FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
+    ) -> None: ...
+    async def aio(
+        self, file: FileIO, data: bytes, start: typing.Optional[int] = None, end: typing.Optional[int] = None
+    ) -> None: ...
+
+replace_bytes: __replace_bytes_spec
+
+T_INNER = typing.TypeVar("T_INNER", covariant=True)
+
+class FileIO(typing.Generic[T]):
+    _task_id: str
+    _file_descriptor: str
+    _client: modal.client.Client
+    _watch_output_buffer: list[typing.Optional[bytes]]
+
+    def __init__(self, client: modal.client.Client, task_id: str) -> None: ...
+    def _validate_mode(self, mode: str) -> None: ...
+    def _handle_error(self, error: modal_proto.api_pb2.SystemErrorMessage) -> None: ...
+
+    class ___consume_output_spec(typing_extensions.Protocol):
+        def __call__(self, exec_id: str) -> typing.Iterator[typing.Optional[bytes]]: ...
+        def aio(self, exec_id: str) -> typing.AsyncIterator[typing.Optional[bytes]]: ...
+
+    _consume_output: ___consume_output_spec
+
+    class ___consume_watch_output_spec(typing_extensions.Protocol):
+        def __call__(self, exec_id: str) -> None: ...
+        async def aio(self, exec_id: str) -> None: ...
+
+    _consume_watch_output: ___consume_watch_output_spec
+
+    class ___parse_watch_output_spec(typing_extensions.Protocol):
+        def __call__(self, event: bytes) -> typing.Optional[FileWatchEvent]: ...
+        async def aio(self, event: bytes) -> typing.Optional[FileWatchEvent]: ...
+
+    _parse_watch_output: ___parse_watch_output_spec
+
+    class ___wait_spec(typing_extensions.Protocol):
+        def __call__(self, exec_id: str) -> bytes: ...
+        async def aio(self, exec_id: str) -> bytes: ...
+
+    _wait: ___wait_spec
+
+    def _validate_type(self, data: typing.Union[bytes, str]) -> None: ...
+
+    class ___open_file_spec(typing_extensions.Protocol):
+        def __call__(self, path: str, mode: str) -> None: ...
+        async def aio(self, path: str, mode: str) -> None: ...
+
+    _open_file: ___open_file_spec
+
+    @classmethod
+    def create(
+        cls,
+        path: str,
+        mode: typing.Union[_typeshed.OpenTextMode, _typeshed.OpenBinaryMode],
+        client: modal.client.Client,
+        task_id: str,
+    ) -> FileIO: ...
+
+    class ___make_read_request_spec(typing_extensions.Protocol):
+        def __call__(self, n: typing.Optional[int]) -> bytes: ...
+        async def aio(self, n: typing.Optional[int]) -> bytes: ...
+
+    _make_read_request: ___make_read_request_spec
+
+    class __read_spec(typing_extensions.Protocol[T_INNER]):
+        def __call__(self, n: typing.Optional[int] = None) -> T_INNER: ...
+        async def aio(self, n: typing.Optional[int] = None) -> T_INNER: ...
+
+    read: __read_spec[T]
+
+    class __readline_spec(typing_extensions.Protocol[T_INNER]):
+        def __call__(self) -> T_INNER: ...
+        async def aio(self) -> T_INNER: ...
+
+    readline: __readline_spec[T]
+
+    class __readlines_spec(typing_extensions.Protocol[T_INNER]):
+        def __call__(self) -> typing.Sequence[T_INNER]: ...
+        async def aio(self) -> typing.Sequence[T_INNER]: ...
+
+    readlines: __readlines_spec[T]
+
+    class __write_spec(typing_extensions.Protocol):
+        def __call__(self, data: typing.Union[bytes, str]) -> None: ...
+        async def aio(self, data: typing.Union[bytes, str]) -> None: ...
+
+    write: __write_spec
+
+    class __flush_spec(typing_extensions.Protocol):
+        def __call__(self) -> None: ...
+        async def aio(self) -> None: ...
+
+    flush: __flush_spec
+
+    def _get_whence(self, whence: int): ...
+
+    class __seek_spec(typing_extensions.Protocol):
+        def __call__(self, offset: int, whence: int = 0) -> None: ...
+        async def aio(self, offset: int, whence: int = 0) -> None: ...
+
+    seek: __seek_spec
+
+    @classmethod
+    def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]: ...
+    @classmethod
+    def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None: ...
+    @classmethod
+    def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None: ...
+    @classmethod
+    def watch(
+        cls,
+        path: str,
+        client: modal.client.Client,
+        task_id: str,
+        filter: typing.Optional[list[FileWatchEventType]] = None,
+        recursive: bool = False,
+        timeout: typing.Optional[int] = None,
+    ) -> typing.Iterator[FileWatchEvent]: ...
+
+    class ___close_spec(typing_extensions.Protocol):
+        def __call__(self) -> None: ...
+        async def aio(self) -> None: ...
+
+    _close: ___close_spec
+
+    class __close_spec(typing_extensions.Protocol):
+        def __call__(self) -> None: ...
+        async def aio(self) -> None: ...
+
+    close: __close_spec
+
+    def _check_writable(self) -> None: ...
+    def _check_readable(self) -> None: ...
+    def _check_closed(self) -> None: ...
+    def __enter__(self) -> FileIO: ...
+    async def __aenter__(self) -> FileIO: ...
+    def __exit__(self, exc_type, exc_value, traceback) -> None: ...
+    async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...

modal/file_pattern_matcher.py

@@ -0,0 +1,196 @@
+# Copyright Modal Labs 2024
+"""Pattern matching library ported from https://github.com/moby/patternmatcher.
+
+This is the same pattern-matching logic used by Docker, except it is written in
+Python rather than Go. Also, the original Go library has a couple deprecated
+functions that we don't implement in this port.
+
+The main way to use this library is by constructing a `FilePatternMatcher` object,
+then asking it whether file paths match any of its patterns.
+"""
+
+import os
+from abc import abstractmethod
+from pathlib import Path
+from typing import Callable, Optional, Sequence, Union
+
+from ._utils.pattern_utils import Pattern
+
+
+class _AbstractPatternMatcher:
+    _custom_repr: Optional[str] = None
+
+    def __invert__(self) -> "_AbstractPatternMatcher":
+        """Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
+
+        Usage:
+        ```python
+        from pathlib import Path
+        from modal import FilePatternMatcher
+
+        inverted_matcher = ~FilePatternMatcher("**/*.py")
+
+        assert not inverted_matcher(Path("foo.py"))
+        ```
+        """
+        return _CustomPatternMatcher(lambda path: not self(path))
+
+    def _with_repr(self, custom_repr) -> "_AbstractPatternMatcher":
+        # use to give an instance of a matcher a custom name - useful for visualizing default values in signatures
+        self._custom_repr = custom_repr
+        return self
+
+    def __repr__(self) -> str:
+        if self._custom_repr:
+            return self._custom_repr
+
+        return super().__repr__()
+
+    @abstractmethod
+    def __call__(self, path: Path) -> bool:
+        ...
+
+
+class _CustomPatternMatcher(_AbstractPatternMatcher):
+    def __init__(self, predicate: Callable[[Path], bool]):
+        self._predicate = predicate
+
+    def __call__(self, path: Path) -> bool:
+        return self._predicate(path)
+
+
+class FilePatternMatcher(_AbstractPatternMatcher):
+    """
+    Allows matching file Path objects against a list of patterns.
+
+    **Usage:**
+    ```python
+    from pathlib import Path
+    from modal import FilePatternMatcher
+
+    matcher = FilePatternMatcher("*.py")
+
+    assert matcher(Path("foo.py"))
+
+    # You can also negate the matcher.
+    negated_matcher = ~matcher
+
+    assert not negated_matcher(Path("foo.py"))
+    ```
+    """
+
+    patterns: list[Pattern]
+    _delayed_init: Callable[[], None] = None
+
+    def _set_patterns(self, patterns: Sequence[str]) -> None:
+        self.patterns = []
+        for pattern in list(patterns):
+            pattern = pattern.strip()
+            if not pattern:
+                continue
+            pattern = os.path.normpath(pattern)
+            new_pattern = Pattern()
+            if pattern[0] == "!":
+                if len(pattern) == 1:
+                    raise ValueError('Illegal exclusion pattern: "!"')
+                new_pattern.exclusion = True
+                pattern = pattern[1:]
+            # In Python, we can proceed without explicit syntax checking
+            new_pattern.cleaned_pattern = pattern
+            new_pattern.dirs = pattern.split(os.path.sep)
+            self.patterns.append(new_pattern)
+
+    def __init__(self, *pattern: str) -> None:
+        """Initialize a new FilePatternMatcher instance.
+
+        Args:
+            pattern (str): One or more pattern strings.
+
+        Raises:
+            ValueError: If an illegal exclusion pattern is provided.
+        """
+        self._set_patterns(pattern)
+
+    @classmethod
+    def from_file(cls, file_path: Union[str, Path]) -> "FilePatternMatcher":
+        """Initialize a new FilePatternMatcher instance from a file.
+
+        The patterns in the file will be read lazily when the matcher is first used.
+
+        Args:
+            file_path (Path): The path to the file containing patterns.
+
+        **Usage:**
+        ```python
+        from modal import FilePatternMatcher
+
+        matcher = FilePatternMatcher.from_file("/path/to/ignorefile")
+        ```
+
+        """
+        uninitialized = cls.__new__(cls)
+
+        def _delayed_init():
+            uninitialized._set_patterns(Path(file_path).read_text("utf8").splitlines())
+            uninitialized._delayed_init = None
+
+        uninitialized._delayed_init = _delayed_init
+        return uninitialized
+
+    def _matches(self, file_path: str) -> bool:
+        """Check if the file path or any of its parent directories match the patterns.
+
+        This is equivalent to `MatchesOrParentMatches()` in the original Go
+        library. The reason is that `Matches()` in the original library is
+        deprecated due to buggy behavior.
+        """
+
+        matched = False
+        file_path = os.path.normpath(file_path)
+        if file_path == ".":
+            # Don't let them exclude everything; kind of silly.
+            return False
+        parent_path = os.path.dirname(file_path)
+        if parent_path == "":
+            parent_path = "."
+        parent_path_dirs = parent_path.split(os.path.sep)
+
+        for pattern in self.patterns:
+            # Skip evaluation based on current match status and pattern exclusion
+            if pattern.exclusion != matched:
+                continue
+
+            match = pattern.match(file_path)
+
+            if not match and parent_path != ".":
+                # Check if the pattern matches any of the parent directories
+                for i in range(len(parent_path_dirs)):
+                    dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
+                    if pattern.match(dir_path):
+                        match = True
+                        break
+
+            if match:
+                matched = not pattern.exclusion
+
+        return matched
+
+    def __call__(self, file_path: Path) -> bool:
+        if self._delayed_init:
+            self._delayed_init()
+        return self._matches(str(file_path))
+
+
+# _with_repr allows us to use this matcher as a default value in a function signature
+#  and get a nice repr in the docs and auto-generated type stubs:
+NON_PYTHON_FILES = (~FilePatternMatcher("**/*.py"))._with_repr(f"{__name__}.NON_PYTHON_FILES")
+_NOTHING = (~FilePatternMatcher())._with_repr(f"{__name__}._NOTHING")  # match everything = ignore nothing
+
+
+def _ignore_fn(ignore: Union[Sequence[str], Callable[[Path], bool]]) -> Callable[[Path], bool]:
+    # if a callable is passed, return it
+    # otherwise, treat input as a sequence of patterns and return a callable pattern matcher for those
+    if callable(ignore):
+        return ignore
+
+    return FilePatternMatcher(*ignore)