modal 0.68.17__py3-none-any.whl → 0.68.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +2 -0
- modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
- modal/_utils/shell_utils.py +11 -5
- modal/client.pyi +2 -2
- modal/file_pattern_matcher.py +121 -0
- modal/image.py +88 -8
- modal/image.pyi +20 -4
- modal/mount.py +46 -8
- modal/mount.pyi +19 -4
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/METADATA +1 -1
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/RECORD +16 -15
- modal_version/_version_generated.py +1 -1
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/LICENSE +0 -0
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/WHEEL +0 -0
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/entry_points.txt +0 -0
- {modal-0.68.17.dist-info → modal-0.68.22.dist-info}/top_level.txt +0 -0
modal/__init__.py
CHANGED
@@ -17,6 +17,7 @@ try:
|
|
17
17
|
from .cls import Cls, parameter
|
18
18
|
from .dict import Dict
|
19
19
|
from .exception import Error
|
20
|
+
from .file_pattern_matcher import FilePatternMatcher
|
20
21
|
from .functions import Function
|
21
22
|
from .image import Image
|
22
23
|
from .mount import Mount
|
@@ -48,6 +49,7 @@ __all__ = [
|
|
48
49
|
"Cron",
|
49
50
|
"Dict",
|
50
51
|
"Error",
|
52
|
+
"FilePatternMatcher",
|
51
53
|
"Function",
|
52
54
|
"Image",
|
53
55
|
"Mount",
|
@@ -5,7 +5,7 @@ This is the same pattern-matching logic used by Docker, except it is written in
|
|
5
5
|
Python rather than Go. Also, the original Go library has a couple deprecated
|
6
6
|
functions that we don't implement in this port.
|
7
7
|
|
8
|
-
The main way to use this library is by constructing a `
|
8
|
+
The main way to use this library is by constructing a `FilePatternMatcher` object,
|
9
9
|
then asking it whether file paths match any of its patterns.
|
10
10
|
"""
|
11
11
|
|
@@ -148,75 +148,6 @@ class Pattern:
|
|
148
148
|
return False
|
149
149
|
|
150
150
|
|
151
|
-
class PatternMatcher:
|
152
|
-
"""Allows checking paths against a list of patterns."""
|
153
|
-
|
154
|
-
def __init__(self, patterns: list[str]) -> None:
|
155
|
-
"""Initialize a new PatternMatcher instance.
|
156
|
-
|
157
|
-
Args:
|
158
|
-
patterns (list): A list of pattern strings.
|
159
|
-
|
160
|
-
Raises:
|
161
|
-
ValueError: If an illegal exclusion pattern is provided.
|
162
|
-
"""
|
163
|
-
self.patterns: list[Pattern] = []
|
164
|
-
self.exclusions = False
|
165
|
-
for pattern in patterns:
|
166
|
-
pattern = pattern.strip()
|
167
|
-
if not pattern:
|
168
|
-
continue
|
169
|
-
pattern = os.path.normpath(pattern)
|
170
|
-
new_pattern = Pattern()
|
171
|
-
if pattern[0] == "!":
|
172
|
-
if len(pattern) == 1:
|
173
|
-
raise ValueError('Illegal exclusion pattern: "!"')
|
174
|
-
new_pattern.exclusion = True
|
175
|
-
pattern = pattern[1:]
|
176
|
-
self.exclusions = True
|
177
|
-
# In Python, we can proceed without explicit syntax checking
|
178
|
-
new_pattern.cleaned_pattern = pattern
|
179
|
-
new_pattern.dirs = pattern.split(os.path.sep)
|
180
|
-
self.patterns.append(new_pattern)
|
181
|
-
|
182
|
-
def matches(self, file_path: str) -> bool:
|
183
|
-
"""Check if the file path or any of its parent directories match the patterns.
|
184
|
-
|
185
|
-
This is equivalent to `MatchesOrParentMatches()` in the original Go
|
186
|
-
library. The reason is that `Matches()` in the original library is
|
187
|
-
deprecated due to buggy behavior.
|
188
|
-
"""
|
189
|
-
matched = False
|
190
|
-
file_path = os.path.normpath(file_path)
|
191
|
-
if file_path == ".":
|
192
|
-
# Don't let them exclude everything; kind of silly.
|
193
|
-
return False
|
194
|
-
parent_path = os.path.dirname(file_path)
|
195
|
-
if parent_path == "":
|
196
|
-
parent_path = "."
|
197
|
-
parent_path_dirs = parent_path.split(os.path.sep)
|
198
|
-
|
199
|
-
for pattern in self.patterns:
|
200
|
-
# Skip evaluation based on current match status and pattern exclusion
|
201
|
-
if pattern.exclusion != matched:
|
202
|
-
continue
|
203
|
-
|
204
|
-
match = pattern.match(file_path)
|
205
|
-
|
206
|
-
if not match and parent_path != ".":
|
207
|
-
# Check if the pattern matches any of the parent directories
|
208
|
-
for i in range(len(parent_path_dirs)):
|
209
|
-
dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
|
210
|
-
if pattern.match(dir_path):
|
211
|
-
match = True
|
212
|
-
break
|
213
|
-
|
214
|
-
if match:
|
215
|
-
matched = not pattern.exclusion
|
216
|
-
|
217
|
-
return matched
|
218
|
-
|
219
|
-
|
220
151
|
def read_ignorefile(reader: TextIO) -> list[str]:
|
221
152
|
"""Read an ignore file from a reader and return the list of file patterns to
|
222
153
|
ignore, applying the following rules:
|
modal/_utils/shell_utils.py
CHANGED
@@ -19,14 +19,20 @@ def write_to_fd(fd: int, data: bytes):
|
|
19
19
|
future = loop.create_future()
|
20
20
|
|
21
21
|
def try_write():
|
22
|
+
nonlocal data
|
22
23
|
try:
|
23
24
|
nbytes = os.write(fd, data)
|
24
|
-
|
25
|
-
|
25
|
+
data = data[nbytes:]
|
26
|
+
if not data:
|
27
|
+
loop.remove_writer(fd)
|
28
|
+
future.set_result(None)
|
26
29
|
except OSError as e:
|
27
|
-
if e.errno
|
28
|
-
|
29
|
-
|
30
|
+
if e.errno == errno.EAGAIN:
|
31
|
+
# Wait for the next write notification
|
32
|
+
return
|
33
|
+
# Fail if it's not EAGAIN
|
34
|
+
loop.remove_writer(fd)
|
35
|
+
future.set_exception(e)
|
30
36
|
|
31
37
|
loop.add_writer(fd, try_write)
|
32
38
|
return future
|
modal/client.pyi
CHANGED
@@ -26,7 +26,7 @@ class _Client:
|
|
26
26
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
27
27
|
|
28
28
|
def __init__(
|
29
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
29
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.22"
|
30
30
|
): ...
|
31
31
|
def is_closed(self) -> bool: ...
|
32
32
|
@property
|
@@ -81,7 +81,7 @@ class Client:
|
|
81
81
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
82
82
|
|
83
83
|
def __init__(
|
84
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
84
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.22"
|
85
85
|
): ...
|
86
86
|
def is_closed(self) -> bool: ...
|
87
87
|
@property
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# Copyright Modal Labs 2024
|
2
|
+
"""Pattern matching library ported from https://github.com/moby/patternmatcher.
|
3
|
+
|
4
|
+
This is the same pattern-matching logic used by Docker, except it is written in
|
5
|
+
Python rather than Go. Also, the original Go library has a couple deprecated
|
6
|
+
functions that we don't implement in this port.
|
7
|
+
|
8
|
+
The main way to use this library is by constructing a `FilePatternMatcher` object,
|
9
|
+
then asking it whether file paths match any of its patterns.
|
10
|
+
"""
|
11
|
+
|
12
|
+
import os
|
13
|
+
from pathlib import Path
|
14
|
+
from typing import Callable
|
15
|
+
|
16
|
+
from ._utils.pattern_utils import Pattern
|
17
|
+
|
18
|
+
|
19
|
+
class FilePatternMatcher:
|
20
|
+
"""Allows matching file paths against a list of patterns."""
|
21
|
+
|
22
|
+
def __init__(self, *pattern: str) -> None:
|
23
|
+
"""Initialize a new FilePatternMatcher instance.
|
24
|
+
|
25
|
+
Args:
|
26
|
+
pattern (str): One or more pattern strings.
|
27
|
+
|
28
|
+
Raises:
|
29
|
+
ValueError: If an illegal exclusion pattern is provided.
|
30
|
+
"""
|
31
|
+
self.patterns: list[Pattern] = []
|
32
|
+
self.exclusions = False
|
33
|
+
for p in list(pattern):
|
34
|
+
p = p.strip()
|
35
|
+
if not p:
|
36
|
+
continue
|
37
|
+
p = os.path.normpath(p)
|
38
|
+
new_pattern = Pattern()
|
39
|
+
if p[0] == "!":
|
40
|
+
if len(p) == 1:
|
41
|
+
raise ValueError('Illegal exclusion pattern: "!"')
|
42
|
+
new_pattern.exclusion = True
|
43
|
+
p = p[1:]
|
44
|
+
self.exclusions = True
|
45
|
+
# In Python, we can proceed without explicit syntax checking
|
46
|
+
new_pattern.cleaned_pattern = p
|
47
|
+
new_pattern.dirs = p.split(os.path.sep)
|
48
|
+
self.patterns.append(new_pattern)
|
49
|
+
|
50
|
+
def _matches(self, file_path: str) -> bool:
|
51
|
+
"""Check if the file path or any of its parent directories match the patterns.
|
52
|
+
|
53
|
+
This is equivalent to `MatchesOrParentMatches()` in the original Go
|
54
|
+
library. The reason is that `Matches()` in the original library is
|
55
|
+
deprecated due to buggy behavior.
|
56
|
+
"""
|
57
|
+
matched = False
|
58
|
+
file_path = os.path.normpath(file_path)
|
59
|
+
if file_path == ".":
|
60
|
+
# Don't let them exclude everything; kind of silly.
|
61
|
+
return False
|
62
|
+
parent_path = os.path.dirname(file_path)
|
63
|
+
if parent_path == "":
|
64
|
+
parent_path = "."
|
65
|
+
parent_path_dirs = parent_path.split(os.path.sep)
|
66
|
+
|
67
|
+
for pattern in self.patterns:
|
68
|
+
# Skip evaluation based on current match status and pattern exclusion
|
69
|
+
if pattern.exclusion != matched:
|
70
|
+
continue
|
71
|
+
|
72
|
+
match = pattern.match(file_path)
|
73
|
+
|
74
|
+
if not match and parent_path != ".":
|
75
|
+
# Check if the pattern matches any of the parent directories
|
76
|
+
for i in range(len(parent_path_dirs)):
|
77
|
+
dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
|
78
|
+
if pattern.match(dir_path):
|
79
|
+
match = True
|
80
|
+
break
|
81
|
+
|
82
|
+
if match:
|
83
|
+
matched = not pattern.exclusion
|
84
|
+
|
85
|
+
return matched
|
86
|
+
|
87
|
+
def __call__(self, file_path: Path) -> bool:
|
88
|
+
"""Check if the path matches any of the patterns.
|
89
|
+
|
90
|
+
Args:
|
91
|
+
file_path (Path): The path to check.
|
92
|
+
|
93
|
+
Returns:
|
94
|
+
True if the path matches any of the patterns.
|
95
|
+
|
96
|
+
Usage:
|
97
|
+
```python
|
98
|
+
from pathlib import Path
|
99
|
+
from modal import FilePatternMatcher
|
100
|
+
|
101
|
+
matcher = FilePatternMatcher("*.py")
|
102
|
+
|
103
|
+
assert matcher(Path("foo.py"))
|
104
|
+
```
|
105
|
+
"""
|
106
|
+
return self._matches(str(file_path))
|
107
|
+
|
108
|
+
def __invert__(self) -> Callable[[Path], bool]:
|
109
|
+
"""Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
|
110
|
+
|
111
|
+
Usage:
|
112
|
+
```python
|
113
|
+
from pathlib import Path
|
114
|
+
from modal import FilePatternMatcher
|
115
|
+
|
116
|
+
inverted_matcher = ~FilePatternMatcher("**/*.py")
|
117
|
+
|
118
|
+
assert not inverted_matcher(Path("foo.py"))
|
119
|
+
```
|
120
|
+
"""
|
121
|
+
return lambda path: not self(path)
|
modal/image.py
CHANGED
@@ -37,6 +37,7 @@ from .cloud_bucket_mount import _CloudBucketMount
|
|
37
37
|
from .config import config, logger, user_config_path
|
38
38
|
from .environments import _get_environment_cached
|
39
39
|
from .exception import InvalidError, NotFoundError, RemoteError, VersionError, deprecation_error, deprecation_warning
|
40
|
+
from .file_pattern_matcher import FilePatternMatcher
|
40
41
|
from .gpu import GPU_T, parse_gpu_config
|
41
42
|
from .mount import _Mount, python_standalone_mount_name
|
42
43
|
from .network_file_system import _NetworkFileSystem
|
@@ -638,7 +639,17 @@ class _Image(_Object, type_prefix="im"):
|
|
638
639
|
mount = _Mount.from_local_file(local_path, remote_path)
|
639
640
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
640
641
|
|
641
|
-
def add_local_dir(
|
642
|
+
def add_local_dir(
|
643
|
+
self,
|
644
|
+
local_path: Union[str, Path],
|
645
|
+
remote_path: str,
|
646
|
+
*,
|
647
|
+
copy: bool = False,
|
648
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
649
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
650
|
+
# Which follows dockerignore syntax.
|
651
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
652
|
+
) -> "_Image":
|
642
653
|
"""Adds a local directory's content to the image at `remote_path` within the container
|
643
654
|
|
644
655
|
By default (`copy=False`), the files are added to containers on startup and are not built into the actual Image,
|
@@ -650,12 +661,44 @@ class _Image(_Object, type_prefix="im"):
|
|
650
661
|
copy=True can slow down iteration since it requires a rebuild of the Image and any subsequent
|
651
662
|
build steps whenever the included files change, but it is required if you want to run additional
|
652
663
|
build steps after this one.
|
664
|
+
|
665
|
+
**Usage:**
|
666
|
+
|
667
|
+
```python
|
668
|
+
from modal import FilePatternMatcher
|
669
|
+
|
670
|
+
image = modal.Image.debian_slim().add_local_dir(
|
671
|
+
"~/assets",
|
672
|
+
remote_path="/assets",
|
673
|
+
ignore=["*.venv"],
|
674
|
+
)
|
675
|
+
|
676
|
+
image = modal.Image.debian_slim().add_local_dir(
|
677
|
+
"~/assets",
|
678
|
+
remote_path="/assets",
|
679
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
680
|
+
)
|
681
|
+
|
682
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
683
|
+
"~/assets",
|
684
|
+
remote_path="/assets",
|
685
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
686
|
+
)
|
687
|
+
|
688
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
689
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
690
|
+
"~/assets",
|
691
|
+
remote_path="/assets",
|
692
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
693
|
+
)
|
694
|
+
```
|
653
695
|
"""
|
654
696
|
if not PurePosixPath(remote_path).is_absolute():
|
655
697
|
# TODO(elias): implement relative to absolute resolution using image workdir metadata
|
656
698
|
# + make default remote_path="./"
|
657
699
|
raise InvalidError("image.add_local_dir() currently only supports absolute remote_path values")
|
658
|
-
|
700
|
+
|
701
|
+
mount = _Mount._add_local_dir(Path(local_path), Path(remote_path), ignore)
|
659
702
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
660
703
|
|
661
704
|
def copy_local_file(self, local_path: Union[str, Path], remote_path: Union[str, Path] = "./") -> "_Image":
|
@@ -697,19 +740,56 @@ class _Image(_Object, type_prefix="im"):
|
|
697
740
|
the destination directory.
|
698
741
|
"""
|
699
742
|
|
700
|
-
|
701
|
-
return filename.endswith(".py")
|
702
|
-
|
703
|
-
mount = _Mount.from_local_python_packages(*modules, condition=only_py_files)
|
743
|
+
mount = _Mount.from_local_python_packages(*modules, ignore=~FilePatternMatcher("**/*.py"))
|
704
744
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
705
745
|
|
706
|
-
def copy_local_dir(
|
746
|
+
def copy_local_dir(
|
747
|
+
self,
|
748
|
+
local_path: Union[str, Path],
|
749
|
+
remote_path: Union[str, Path] = ".",
|
750
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
751
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
752
|
+
# Which follows dockerignore syntax.
|
753
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
754
|
+
) -> "_Image":
|
707
755
|
"""Copy a directory into the image as a part of building the image.
|
708
756
|
|
709
757
|
This works in a similar way to [`COPY`](https://docs.docker.com/engine/reference/builder/#copy)
|
710
758
|
works in a `Dockerfile`.
|
759
|
+
|
760
|
+
**Usage:**
|
761
|
+
|
762
|
+
```python
|
763
|
+
from modal import FilePatternMatcher
|
764
|
+
|
765
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
766
|
+
"~/assets",
|
767
|
+
remote_path="/assets",
|
768
|
+
ignore=["**/*.venv"],
|
769
|
+
)
|
770
|
+
|
771
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
772
|
+
"~/assets",
|
773
|
+
remote_path="/assets",
|
774
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
775
|
+
)
|
776
|
+
|
777
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
778
|
+
"~/assets",
|
779
|
+
remote_path="/assets",
|
780
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
781
|
+
)
|
782
|
+
|
783
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
784
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
785
|
+
"~/assets",
|
786
|
+
remote_path="/assets",
|
787
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
788
|
+
)
|
789
|
+
```
|
711
790
|
"""
|
712
|
-
|
791
|
+
|
792
|
+
mount = _Mount._add_local_dir(Path(local_path), Path("/"), ignore)
|
713
793
|
|
714
794
|
def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
|
715
795
|
return DockerfileSpec(commands=["FROM base", f"COPY . {remote_path}"], context_files={})
|
modal/image.pyi
CHANGED
@@ -110,14 +110,22 @@ class _Image(modal.object._Object):
|
|
110
110
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
111
111
|
) -> _Image: ...
|
112
112
|
def add_local_dir(
|
113
|
-
self,
|
113
|
+
self,
|
114
|
+
local_path: typing.Union[str, pathlib.Path],
|
115
|
+
remote_path: str,
|
116
|
+
*,
|
117
|
+
copy: bool = False,
|
118
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
114
119
|
) -> _Image: ...
|
115
120
|
def copy_local_file(
|
116
121
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
117
122
|
) -> _Image: ...
|
118
123
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> _Image: ...
|
119
124
|
def copy_local_dir(
|
120
|
-
self,
|
125
|
+
self,
|
126
|
+
local_path: typing.Union[str, pathlib.Path],
|
127
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
128
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
121
129
|
) -> _Image: ...
|
122
130
|
def pip_install(
|
123
131
|
self,
|
@@ -367,14 +375,22 @@ class Image(modal.object.Object):
|
|
367
375
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
368
376
|
) -> Image: ...
|
369
377
|
def add_local_dir(
|
370
|
-
self,
|
378
|
+
self,
|
379
|
+
local_path: typing.Union[str, pathlib.Path],
|
380
|
+
remote_path: str,
|
381
|
+
*,
|
382
|
+
copy: bool = False,
|
383
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
371
384
|
) -> Image: ...
|
372
385
|
def copy_local_file(
|
373
386
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
374
387
|
) -> Image: ...
|
375
388
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> Image: ...
|
376
389
|
def copy_local_dir(
|
377
|
-
self,
|
390
|
+
self,
|
391
|
+
local_path: typing.Union[str, pathlib.Path],
|
392
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
393
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
378
394
|
) -> Image: ...
|
379
395
|
def pip_install(
|
380
396
|
self,
|
modal/mount.py
CHANGED
@@ -11,7 +11,7 @@ import time
|
|
11
11
|
import typing
|
12
12
|
from collections.abc import AsyncGenerator
|
13
13
|
from pathlib import Path, PurePosixPath
|
14
|
-
from typing import Callable, Optional, Union
|
14
|
+
from typing import Callable, Optional, Sequence, Union
|
15
15
|
|
16
16
|
from google.protobuf.message import Message
|
17
17
|
|
@@ -27,7 +27,8 @@ from ._utils.name_utils import check_object_name
|
|
27
27
|
from ._utils.package_utils import get_module_mount_info
|
28
28
|
from .client import _Client
|
29
29
|
from .config import config, logger
|
30
|
-
from .exception import ModuleNotMountable
|
30
|
+
from .exception import InvalidError, ModuleNotMountable
|
31
|
+
from .file_pattern_matcher import FilePatternMatcher
|
31
32
|
from .object import _get_environment_name, _Object
|
32
33
|
|
33
34
|
ROOT_DIR: PurePosixPath = PurePosixPath("/root")
|
@@ -122,7 +123,7 @@ class _MountFile(_MountEntry):
|
|
122
123
|
class _MountDir(_MountEntry):
|
123
124
|
local_dir: Path
|
124
125
|
remote_path: PurePosixPath
|
125
|
-
|
126
|
+
ignore: Callable[[Path], bool]
|
126
127
|
recursive: bool
|
127
128
|
|
128
129
|
def description(self):
|
@@ -143,7 +144,7 @@ class _MountDir(_MountEntry):
|
|
143
144
|
gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
|
144
145
|
|
145
146
|
for local_filename in gen:
|
146
|
-
if self.
|
147
|
+
if not self.ignore(Path(local_filename)):
|
147
148
|
local_relpath = Path(local_filename).expanduser().absolute().relative_to(local_dir)
|
148
149
|
mount_path = self.remote_path / local_relpath.as_posix()
|
149
150
|
yield local_filename, mount_path
|
@@ -182,6 +183,10 @@ def module_mount_condition(module_base: Path):
|
|
182
183
|
return condition
|
183
184
|
|
184
185
|
|
186
|
+
def module_mount_ignore_condition(module_base: Path):
|
187
|
+
return lambda f: not module_mount_condition(module_base)(str(f))
|
188
|
+
|
189
|
+
|
185
190
|
@dataclasses.dataclass
|
186
191
|
class _MountedPythonModule(_MountEntry):
|
187
192
|
# the purpose of this is to keep printable information about which Python package
|
@@ -190,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
|
|
190
195
|
|
191
196
|
module_name: str
|
192
197
|
remote_dir: Union[PurePosixPath, str] = ROOT_DIR.as_posix() # cast needed here for type stub generation...
|
193
|
-
|
198
|
+
ignore: Optional[Callable[[Path], bool]] = None
|
194
199
|
|
195
200
|
def description(self) -> str:
|
196
201
|
return f"PythonPackage:{self.module_name}"
|
@@ -206,7 +211,7 @@ class _MountedPythonModule(_MountEntry):
|
|
206
211
|
_MountDir(
|
207
212
|
base_path,
|
208
213
|
remote_path=remote_dir,
|
209
|
-
|
214
|
+
ignore=self.ignore or module_mount_ignore_condition(base_path),
|
210
215
|
recursive=True,
|
211
216
|
)
|
212
217
|
)
|
@@ -313,6 +318,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
313
318
|
# we can't rely on it to be set. Let's clean this up later.
|
314
319
|
return getattr(self, "_is_local", False)
|
315
320
|
|
321
|
+
@staticmethod
|
322
|
+
def _add_local_dir(
|
323
|
+
local_path: Path,
|
324
|
+
remote_path: Path,
|
325
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
326
|
+
):
|
327
|
+
if isinstance(ignore, list):
|
328
|
+
ignore = FilePatternMatcher(*ignore)
|
329
|
+
|
330
|
+
return _Mount._new()._extend(
|
331
|
+
_MountDir(
|
332
|
+
local_dir=local_path,
|
333
|
+
ignore=ignore,
|
334
|
+
remote_path=remote_path,
|
335
|
+
recursive=True,
|
336
|
+
),
|
337
|
+
)
|
338
|
+
|
316
339
|
def add_local_dir(
|
317
340
|
self,
|
318
341
|
local_path: Union[str, Path],
|
@@ -339,10 +362,13 @@ class _Mount(_Object, type_prefix="mo"):
|
|
339
362
|
|
340
363
|
condition = include_all
|
341
364
|
|
365
|
+
def converted_condition(path: Path) -> bool:
|
366
|
+
return not condition(str(path))
|
367
|
+
|
342
368
|
return self._extend(
|
343
369
|
_MountDir(
|
344
370
|
local_dir=local_path,
|
345
|
-
|
371
|
+
ignore=converted_condition,
|
346
372
|
remote_path=remote_path,
|
347
373
|
recursive=recursive,
|
348
374
|
),
|
@@ -551,6 +577,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
551
577
|
# Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
|
552
578
|
# Defaults to including all files.
|
553
579
|
condition: Optional[Callable[[str], bool]] = None,
|
580
|
+
ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
|
554
581
|
) -> "_Mount":
|
555
582
|
"""
|
556
583
|
Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
|
@@ -575,13 +602,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
575
602
|
|
576
603
|
# Don't re-run inside container.
|
577
604
|
|
605
|
+
if condition is not None:
|
606
|
+
if ignore is not None:
|
607
|
+
raise InvalidError("Cannot specify both `ignore` and `condition`")
|
608
|
+
|
609
|
+
def converted_condition(path: Path) -> bool:
|
610
|
+
return not condition(str(path))
|
611
|
+
|
612
|
+
ignore = converted_condition
|
613
|
+
elif isinstance(ignore, list):
|
614
|
+
ignore = FilePatternMatcher(*ignore)
|
615
|
+
|
578
616
|
mount = _Mount._new()
|
579
617
|
from ._runtime.execution_context import is_local
|
580
618
|
|
581
619
|
if not is_local():
|
582
620
|
return mount # empty/non-mountable mount in case it's used from within a container
|
583
621
|
for module_name in module_names:
|
584
|
-
mount = mount._extend(_MountedPythonModule(module_name, remote_dir,
|
622
|
+
mount = mount._extend(_MountedPythonModule(module_name, remote_dir, ignore))
|
585
623
|
return mount
|
586
624
|
|
587
625
|
@staticmethod
|
modal/mount.pyi
CHANGED
@@ -35,7 +35,7 @@ class _MountFile(_MountEntry):
|
|
35
35
|
class _MountDir(_MountEntry):
|
36
36
|
local_dir: pathlib.Path
|
37
37
|
remote_path: pathlib.PurePosixPath
|
38
|
-
|
38
|
+
ignore: typing.Callable[[pathlib.Path], bool]
|
39
39
|
recursive: bool
|
40
40
|
|
41
41
|
def description(self): ...
|
@@ -46,18 +46,19 @@ class _MountDir(_MountEntry):
|
|
46
46
|
self,
|
47
47
|
local_dir: pathlib.Path,
|
48
48
|
remote_path: pathlib.PurePosixPath,
|
49
|
-
|
49
|
+
ignore: typing.Callable[[pathlib.Path], bool],
|
50
50
|
recursive: bool,
|
51
51
|
) -> None: ...
|
52
52
|
def __repr__(self): ...
|
53
53
|
def __eq__(self, other): ...
|
54
54
|
|
55
55
|
def module_mount_condition(module_base: pathlib.Path): ...
|
56
|
+
def module_mount_ignore_condition(module_base: pathlib.Path): ...
|
56
57
|
|
57
58
|
class _MountedPythonModule(_MountEntry):
|
58
59
|
module_name: str
|
59
60
|
remote_dir: typing.Union[pathlib.PurePosixPath, str]
|
60
|
-
|
61
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]]
|
61
62
|
|
62
63
|
def description(self) -> str: ...
|
63
64
|
def _proxy_entries(self) -> list[_MountEntry]: ...
|
@@ -68,7 +69,7 @@ class _MountedPythonModule(_MountEntry):
|
|
68
69
|
self,
|
69
70
|
module_name: str,
|
70
71
|
remote_dir: typing.Union[pathlib.PurePosixPath, str] = "/root",
|
71
|
-
|
72
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]] = None,
|
72
73
|
) -> None: ...
|
73
74
|
def __repr__(self): ...
|
74
75
|
def __eq__(self, other): ...
|
@@ -90,6 +91,12 @@ class _Mount(modal.object._Object):
|
|
90
91
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
91
92
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
92
93
|
def is_local(self) -> bool: ...
|
94
|
+
@staticmethod
|
95
|
+
def _add_local_dir(
|
96
|
+
local_path: pathlib.Path,
|
97
|
+
remote_path: pathlib.Path,
|
98
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
99
|
+
): ...
|
93
100
|
def add_local_dir(
|
94
101
|
self,
|
95
102
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -129,6 +136,7 @@ class _Mount(modal.object._Object):
|
|
129
136
|
*module_names: str,
|
130
137
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
131
138
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
139
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
132
140
|
) -> _Mount: ...
|
133
141
|
@staticmethod
|
134
142
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> _Mount: ...
|
@@ -165,6 +173,12 @@ class Mount(modal.object.Object):
|
|
165
173
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
166
174
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
167
175
|
def is_local(self) -> bool: ...
|
176
|
+
@staticmethod
|
177
|
+
def _add_local_dir(
|
178
|
+
local_path: pathlib.Path,
|
179
|
+
remote_path: pathlib.Path,
|
180
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
181
|
+
): ...
|
168
182
|
def add_local_dir(
|
169
183
|
self,
|
170
184
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -214,6 +228,7 @@ class Mount(modal.object.Object):
|
|
214
228
|
*module_names: str,
|
215
229
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
216
230
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
231
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
217
232
|
) -> Mount: ...
|
218
233
|
@staticmethod
|
219
234
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> Mount: ...
|
@@ -1,4 +1,4 @@
|
|
1
|
-
modal/__init__.py,sha256=
|
1
|
+
modal/__init__.py,sha256=3NJLLHb0TRc2tc68kf8NHzORx38GbtbZvPEWDWrQ6N4,2234
|
2
2
|
modal/__main__.py,sha256=scYhGFqh8OJcVDo-VOxIT6CCwxOgzgflYWMnIZiMRqE,2871
|
3
3
|
modal/_clustered_functions.py,sha256=kTf-9YBXY88NutC1akI-gCbvf01RhMPCw-zoOI_YIUE,2700
|
4
4
|
modal/_clustered_functions.pyi,sha256=vllkegc99A0jrUOWa8mdlSbdp6uz36TsHhGxysAOpaQ,771
|
@@ -19,7 +19,7 @@ modal/app.py,sha256=EJ7FUN6rWnSwLJoYJh8nmKg_t-8hdN8_rt0OrkP7JvQ,46084
|
|
19
19
|
modal/app.pyi,sha256=BE5SlR5tRECuc6-e2lUuOknDdov3zxgZ4N0AsLb5ZVQ,25270
|
20
20
|
modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
|
21
21
|
modal/client.py,sha256=JAnd4-GCN093BwkvOFAK5a6iy5ycxofjpUncMxlrIMw,15253
|
22
|
-
modal/client.pyi,sha256=
|
22
|
+
modal/client.pyi,sha256=dfEnFo8-UHPgN1obEZZNljgyXnRbGEjXdZld0uZtxL0,7280
|
23
23
|
modal/cloud_bucket_mount.py,sha256=G7T7jWLD0QkmrfKR75mSTwdUZ2xNfj7pkVqb4ipmxmI,5735
|
24
24
|
modal/cloud_bucket_mount.pyi,sha256=CEi7vrH3kDUF4LAy4qP6tfImy2UJuFRcRbsgRNM1wo8,1403
|
25
25
|
modal/cls.py,sha256=ONnrfZ2vPcaY2JuKypPiBA9eTiyg8Qfg-Ull40nn9zs,30956
|
@@ -35,15 +35,16 @@ modal/exception.py,sha256=dRK789TD1HaB63kHhu1yZuvS2vP_Vua3iLMBtA6dgqk,7128
|
|
35
35
|
modal/experimental.py,sha256=jFuNbwrNHos47viMB9q-cHJSvf2RDxDdoEcss9plaZE,2302
|
36
36
|
modal/file_io.py,sha256=q8s872qf6Ntdw7dPogDlpYbixxGkwCA0BlQn2UUoVhY,14637
|
37
37
|
modal/file_io.pyi,sha256=pfkmJiaBpMCZReE6-KCjYOzB1dVtyYDYokJoYX8ARK4,6932
|
38
|
+
modal/file_pattern_matcher.py,sha256=vX6MjWRGdonE4I8QPdjFUnz6moBjSzvgD6417BNQrW4,4021
|
38
39
|
modal/functions.py,sha256=IIdHw0FNOdoMksG1b2zvkn8f-xskhJu07ZvHMey9iq4,67667
|
39
40
|
modal/functions.pyi,sha256=EYH4w4VgQtdbEWLGarnU5QtYVfuM2_tnovKFEbYyg2c,25068
|
40
41
|
modal/gpu.py,sha256=r4rL6uH3UJIQthzYvfWauXNyh01WqCPtKZCmmSX1fd4,6881
|
41
|
-
modal/image.py,sha256=
|
42
|
-
modal/image.pyi,sha256=
|
42
|
+
modal/image.py,sha256=xQAC1gWOG0L77QfVfAbHfLwfVkvMYi2sy0V_Ah7GWPg,82253
|
43
|
+
modal/image.pyi,sha256=8R8Ac9eZ83ocM_1qrFUvH3rCbI5zRnq-Eq0xaAQq4nI,25105
|
43
44
|
modal/io_streams.py,sha256=QkQiizKRzd5bnbKQsap31LJgBYlAnj4-XkV_50xPYX0,15079
|
44
45
|
modal/io_streams.pyi,sha256=bCCVSxkMcosYd8O3PQDDwJw7TQ8JEcnYonLJ5t27TQs,4804
|
45
|
-
modal/mount.py,sha256=
|
46
|
-
modal/mount.pyi,sha256=
|
46
|
+
modal/mount.py,sha256=QKvrgpS_FMqrGdoyVZWeWnkNpQeDSLpuiwZFSGgRp_Y,29017
|
47
|
+
modal/mount.pyi,sha256=a0WAFmT7kZvoq_ZAu6R6fwxiEUR6QSmeC_avUpJKGWM,10495
|
47
48
|
modal/network_file_system.py,sha256=kwwQLCJVO086FTiAWSF_jz9BkqijZLpSbEYXpFvS0Ik,14600
|
48
49
|
modal/network_file_system.pyi,sha256=8mHKXuRkxHPazF6ljIW7g4M5aVqLSl6eKUPLgDCug5c,7901
|
49
50
|
modal/object.py,sha256=HZs3N59C6JxlMuPQWJYvrWV1FEEkH9txUovVDorVUbs,9763
|
@@ -95,9 +96,9 @@ modal/_utils/logger.py,sha256=ePzdudrtx9jJCjuO6-bcL_kwUJfi4AwloUmIiNtqkY0,1330
|
|
95
96
|
modal/_utils/mount_utils.py,sha256=J-FRZbPQv1i__Tob-FIpbB1oXWpFLAwZiB4OCiJpFG0,3206
|
96
97
|
modal/_utils/name_utils.py,sha256=TW1iyJedvDNPEJ5UVp93u8xuD5J2gQL_CUt1mgov_aI,1939
|
97
98
|
modal/_utils/package_utils.py,sha256=LcL2olGN4xaUzu2Tbv-C-Ft9Qp6bsLxEfETOAVd-mjU,2073
|
98
|
-
modal/_utils/
|
99
|
+
modal/_utils/pattern_utils.py,sha256=ZUffaECfe2iYBhH6cvCB-0-UWhmEBTZEl_TwG_So3ag,6714
|
99
100
|
modal/_utils/rand_pb_testing.py,sha256=NYM8W6HF-6_bzxJCAj4ITvItZYrclacEZlBhTptOT_Q,3845
|
100
|
-
modal/_utils/shell_utils.py,sha256=
|
101
|
+
modal/_utils/shell_utils.py,sha256=hWHzv730Br2Xyj6cGPiMZ-198Z3RZuOu3pDXhFSZ22c,2157
|
101
102
|
modal/_vendor/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
|
102
103
|
modal/_vendor/a2wsgi_wsgi.py,sha256=Q1AsjpV_Q_vzQsz_cSqmP9jWzsGsB-ARFU6vpQYml8k,21878
|
103
104
|
modal/_vendor/cloudpickle.py,sha256=Loq12qo7PBNbE4LFVEW6BdMMwY10MG94EOW1SCpcnQ0,55217
|
@@ -162,10 +163,10 @@ modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0y
|
|
162
163
|
modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
163
164
|
modal_version/__init__.py,sha256=RT6zPoOdFO99u5Wcxxaoir4ZCuPTbQ22cvzFAXl3vUY,470
|
164
165
|
modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
|
165
|
-
modal_version/_version_generated.py,sha256=
|
166
|
-
modal-0.68.
|
167
|
-
modal-0.68.
|
168
|
-
modal-0.68.
|
169
|
-
modal-0.68.
|
170
|
-
modal-0.68.
|
171
|
-
modal-0.68.
|
166
|
+
modal_version/_version_generated.py,sha256=wLHIwO28sQtEk0ciwX4jSNkWaBSvrjwFXwTKHNJnrk8,149
|
167
|
+
modal-0.68.22.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
|
168
|
+
modal-0.68.22.dist-info/METADATA,sha256=lxET0EL_6taaugt-THklYA5DDoV-yEiaulTb6-hv33I,2329
|
169
|
+
modal-0.68.22.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
170
|
+
modal-0.68.22.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
|
171
|
+
modal-0.68.22.dist-info/top_level.txt,sha256=1nvYbOSIKcmU50fNrpnQnrrOpj269ei3LzgB6j9xGqg,64
|
172
|
+
modal-0.68.22.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|