modal 0.68.11__py3-none-any.whl → 0.68.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +2 -0
- modal/_ipython.py +3 -13
- modal/_runtime/asgi.py +4 -0
- modal/_runtime/user_code_imports.py +13 -18
- modal/_utils/blob_utils.py +27 -92
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/hash_utils.py +38 -9
- modal/_utils/http_utils.py +19 -10
- modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
- modal/_utils/shell_utils.py +11 -5
- modal/cli/run.py +25 -5
- modal/client.py +1 -1
- modal/client.pyi +2 -2
- modal/file_pattern_matcher.py +121 -0
- modal/functions.pyi +6 -6
- modal/image.py +88 -8
- modal/image.pyi +20 -4
- modal/mount.py +49 -9
- modal/mount.pyi +19 -4
- modal/network_file_system.py +4 -1
- modal/partial_function.py +8 -0
- modal/partial_function.pyi +8 -0
- modal/volume.py +5 -1
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/METADATA +1 -1
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/RECORD +33 -31
- modal_proto/api.proto +8 -0
- modal_proto/api_pb2.py +781 -745
- modal_proto/api_pb2.pyi +65 -3
- modal_version/_version_generated.py +1 -1
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/LICENSE +0 -0
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/WHEEL +0 -0
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/entry_points.txt +0 -0
- {modal-0.68.11.dist-info → modal-0.68.24.dist-info}/top_level.txt +0 -0
modal/cli/run.py
CHANGED
@@ -133,6 +133,18 @@ def _get_clean_app_description(func_ref: str) -> str:
|
|
133
133
|
return " ".join(sys.argv)
|
134
134
|
|
135
135
|
|
136
|
+
def _write_local_result(result_path: str, res: Any):
|
137
|
+
if isinstance(res, str):
|
138
|
+
mode = "wt"
|
139
|
+
elif isinstance(res, bytes):
|
140
|
+
mode = "wb"
|
141
|
+
else:
|
142
|
+
res_type = type(res).__name__
|
143
|
+
raise InvalidError(f"Function must return str or bytes when using `--write-result`; got {res_type}.")
|
144
|
+
with open(result_path, mode) as fid:
|
145
|
+
fid.write(res)
|
146
|
+
|
147
|
+
|
136
148
|
def _get_click_command_for_function(app: App, function_tag):
|
137
149
|
function = app.registered_functions.get(function_tag)
|
138
150
|
if not function or (isinstance(function, Function) and function.info.user_cls is not None):
|
@@ -177,7 +189,7 @@ def _get_click_command_for_function(app: App, function_tag):
|
|
177
189
|
interactive=ctx.obj["interactive"],
|
178
190
|
):
|
179
191
|
if cls is None:
|
180
|
-
function.remote(**kwargs)
|
192
|
+
res = function.remote(**kwargs)
|
181
193
|
else:
|
182
194
|
# unpool class and method arguments
|
183
195
|
# TODO(erikbern): this code is a bit hacky
|
@@ -186,7 +198,10 @@ def _get_click_command_for_function(app: App, function_tag):
|
|
186
198
|
|
187
199
|
instance = cls(**cls_kwargs)
|
188
200
|
method: Function = getattr(instance, method_name)
|
189
|
-
method.remote(**fun_kwargs)
|
201
|
+
res = method.remote(**fun_kwargs)
|
202
|
+
|
203
|
+
if result_path := ctx.obj["result_path"]:
|
204
|
+
_write_local_result(result_path, res)
|
190
205
|
|
191
206
|
with_click_options = _add_click_options(f, signature)
|
192
207
|
return click.command(with_click_options)
|
@@ -214,12 +229,15 @@ def _get_click_command_for_local_entrypoint(app: App, entrypoint: LocalEntrypoin
|
|
214
229
|
):
|
215
230
|
try:
|
216
231
|
if isasync:
|
217
|
-
asyncio.run(func(*args, **kwargs))
|
232
|
+
res = asyncio.run(func(*args, **kwargs))
|
218
233
|
else:
|
219
|
-
func(*args, **kwargs)
|
234
|
+
res = func(*args, **kwargs)
|
220
235
|
except Exception as exc:
|
221
236
|
raise _CliUserExecutionError(inspect.getsourcefile(func)) from exc
|
222
237
|
|
238
|
+
if result_path := ctx.obj["result_path"]:
|
239
|
+
_write_local_result(result_path, res)
|
240
|
+
|
223
241
|
with_click_options = _add_click_options(f, _get_signature(func))
|
224
242
|
return click.command(with_click_options)
|
225
243
|
|
@@ -248,12 +266,13 @@ class RunGroup(click.Group):
|
|
248
266
|
cls=RunGroup,
|
249
267
|
subcommand_metavar="FUNC_REF",
|
250
268
|
)
|
269
|
+
@click.option("-w", "--write-result", help="Write return value (which must be str or bytes) to this local path.")
|
251
270
|
@click.option("-q", "--quiet", is_flag=True, help="Don't show Modal progress indicators.")
|
252
271
|
@click.option("-d", "--detach", is_flag=True, help="Don't stop the app if the local process dies or disconnects.")
|
253
272
|
@click.option("-i", "--interactive", is_flag=True, help="Run the app in interactive mode.")
|
254
273
|
@click.option("-e", "--env", help=ENV_OPTION_HELP, default=None)
|
255
274
|
@click.pass_context
|
256
|
-
def run(ctx, detach, quiet, interactive, env):
|
275
|
+
def run(ctx, write_result, detach, quiet, interactive, env):
|
257
276
|
"""Run a Modal function or local entrypoint.
|
258
277
|
|
259
278
|
`FUNC_REF` should be of the format `{file or module}::{function name}`.
|
@@ -284,6 +303,7 @@ def run(ctx, detach, quiet, interactive, env):
|
|
284
303
|
```
|
285
304
|
"""
|
286
305
|
ctx.ensure_object(dict)
|
306
|
+
ctx.obj["result_path"] = write_result
|
287
307
|
ctx.obj["detach"] = detach # if subcommand would be a click command...
|
288
308
|
ctx.obj["show_progress"] = False if quiet else True
|
289
309
|
ctx.obj["interactive"] = interactive
|
modal/client.py
CHANGED
@@ -236,7 +236,7 @@ class _Client:
|
|
236
236
|
Check whether can the client can connect to this server with these credentials; raise if not.
|
237
237
|
"""
|
238
238
|
async with cls(server_url, api_pb2.CLIENT_TYPE_CLIENT, credentials) as client:
|
239
|
-
client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
|
239
|
+
await client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
|
240
240
|
|
241
241
|
@classmethod
|
242
242
|
def set_env_client(cls, client: Optional["_Client"]):
|
modal/client.pyi
CHANGED
@@ -26,7 +26,7 @@ class _Client:
|
|
26
26
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
27
27
|
|
28
28
|
def __init__(
|
29
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
29
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.24"
|
30
30
|
): ...
|
31
31
|
def is_closed(self) -> bool: ...
|
32
32
|
@property
|
@@ -81,7 +81,7 @@ class Client:
|
|
81
81
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
82
82
|
|
83
83
|
def __init__(
|
84
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
84
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.24"
|
85
85
|
): ...
|
86
86
|
def is_closed(self) -> bool: ...
|
87
87
|
@property
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# Copyright Modal Labs 2024
|
2
|
+
"""Pattern matching library ported from https://github.com/moby/patternmatcher.
|
3
|
+
|
4
|
+
This is the same pattern-matching logic used by Docker, except it is written in
|
5
|
+
Python rather than Go. Also, the original Go library has a couple deprecated
|
6
|
+
functions that we don't implement in this port.
|
7
|
+
|
8
|
+
The main way to use this library is by constructing a `FilePatternMatcher` object,
|
9
|
+
then asking it whether file paths match any of its patterns.
|
10
|
+
"""
|
11
|
+
|
12
|
+
import os
|
13
|
+
from pathlib import Path
|
14
|
+
from typing import Callable
|
15
|
+
|
16
|
+
from ._utils.pattern_utils import Pattern
|
17
|
+
|
18
|
+
|
19
|
+
class FilePatternMatcher:
|
20
|
+
"""Allows matching file paths against a list of patterns."""
|
21
|
+
|
22
|
+
def __init__(self, *pattern: str) -> None:
|
23
|
+
"""Initialize a new FilePatternMatcher instance.
|
24
|
+
|
25
|
+
Args:
|
26
|
+
pattern (str): One or more pattern strings.
|
27
|
+
|
28
|
+
Raises:
|
29
|
+
ValueError: If an illegal exclusion pattern is provided.
|
30
|
+
"""
|
31
|
+
self.patterns: list[Pattern] = []
|
32
|
+
self.exclusions = False
|
33
|
+
for p in list(pattern):
|
34
|
+
p = p.strip()
|
35
|
+
if not p:
|
36
|
+
continue
|
37
|
+
p = os.path.normpath(p)
|
38
|
+
new_pattern = Pattern()
|
39
|
+
if p[0] == "!":
|
40
|
+
if len(p) == 1:
|
41
|
+
raise ValueError('Illegal exclusion pattern: "!"')
|
42
|
+
new_pattern.exclusion = True
|
43
|
+
p = p[1:]
|
44
|
+
self.exclusions = True
|
45
|
+
# In Python, we can proceed without explicit syntax checking
|
46
|
+
new_pattern.cleaned_pattern = p
|
47
|
+
new_pattern.dirs = p.split(os.path.sep)
|
48
|
+
self.patterns.append(new_pattern)
|
49
|
+
|
50
|
+
def _matches(self, file_path: str) -> bool:
|
51
|
+
"""Check if the file path or any of its parent directories match the patterns.
|
52
|
+
|
53
|
+
This is equivalent to `MatchesOrParentMatches()` in the original Go
|
54
|
+
library. The reason is that `Matches()` in the original library is
|
55
|
+
deprecated due to buggy behavior.
|
56
|
+
"""
|
57
|
+
matched = False
|
58
|
+
file_path = os.path.normpath(file_path)
|
59
|
+
if file_path == ".":
|
60
|
+
# Don't let them exclude everything; kind of silly.
|
61
|
+
return False
|
62
|
+
parent_path = os.path.dirname(file_path)
|
63
|
+
if parent_path == "":
|
64
|
+
parent_path = "."
|
65
|
+
parent_path_dirs = parent_path.split(os.path.sep)
|
66
|
+
|
67
|
+
for pattern in self.patterns:
|
68
|
+
# Skip evaluation based on current match status and pattern exclusion
|
69
|
+
if pattern.exclusion != matched:
|
70
|
+
continue
|
71
|
+
|
72
|
+
match = pattern.match(file_path)
|
73
|
+
|
74
|
+
if not match and parent_path != ".":
|
75
|
+
# Check if the pattern matches any of the parent directories
|
76
|
+
for i in range(len(parent_path_dirs)):
|
77
|
+
dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
|
78
|
+
if pattern.match(dir_path):
|
79
|
+
match = True
|
80
|
+
break
|
81
|
+
|
82
|
+
if match:
|
83
|
+
matched = not pattern.exclusion
|
84
|
+
|
85
|
+
return matched
|
86
|
+
|
87
|
+
def __call__(self, file_path: Path) -> bool:
|
88
|
+
"""Check if the path matches any of the patterns.
|
89
|
+
|
90
|
+
Args:
|
91
|
+
file_path (Path): The path to check.
|
92
|
+
|
93
|
+
Returns:
|
94
|
+
True if the path matches any of the patterns.
|
95
|
+
|
96
|
+
Usage:
|
97
|
+
```python
|
98
|
+
from pathlib import Path
|
99
|
+
from modal import FilePatternMatcher
|
100
|
+
|
101
|
+
matcher = FilePatternMatcher("*.py")
|
102
|
+
|
103
|
+
assert matcher(Path("foo.py"))
|
104
|
+
```
|
105
|
+
"""
|
106
|
+
return self._matches(str(file_path))
|
107
|
+
|
108
|
+
def __invert__(self) -> Callable[[Path], bool]:
|
109
|
+
"""Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
|
110
|
+
|
111
|
+
Usage:
|
112
|
+
```python
|
113
|
+
from pathlib import Path
|
114
|
+
from modal import FilePatternMatcher
|
115
|
+
|
116
|
+
inverted_matcher = ~FilePatternMatcher("**/*.py")
|
117
|
+
|
118
|
+
assert not inverted_matcher(Path("foo.py"))
|
119
|
+
```
|
120
|
+
"""
|
121
|
+
return lambda path: not self(path)
|
modal/functions.pyi
CHANGED
@@ -456,11 +456,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
456
456
|
|
457
457
|
_call_generator_nowait: ___call_generator_nowait_spec
|
458
458
|
|
459
|
-
class __remote_spec(typing_extensions.Protocol[
|
459
|
+
class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
460
460
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
461
461
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
462
462
|
|
463
|
-
remote: __remote_spec[
|
463
|
+
remote: __remote_spec[ReturnType, P]
|
464
464
|
|
465
465
|
class __remote_gen_spec(typing_extensions.Protocol):
|
466
466
|
def __call__(self, *args, **kwargs) -> typing.Generator[typing.Any, None, None]: ...
|
@@ -473,17 +473,17 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
473
473
|
def _get_obj(self) -> typing.Optional[modal.cls.Obj]: ...
|
474
474
|
def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
|
475
475
|
|
476
|
-
class ___experimental_spawn_spec(typing_extensions.Protocol[
|
476
|
+
class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
477
477
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
478
478
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
479
479
|
|
480
|
-
_experimental_spawn: ___experimental_spawn_spec[
|
480
|
+
_experimental_spawn: ___experimental_spawn_spec[ReturnType, P]
|
481
481
|
|
482
|
-
class __spawn_spec(typing_extensions.Protocol[
|
482
|
+
class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
483
483
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
484
484
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
485
485
|
|
486
|
-
spawn: __spawn_spec[
|
486
|
+
spawn: __spawn_spec[ReturnType, P]
|
487
487
|
|
488
488
|
def get_raw_f(self) -> typing.Callable[..., typing.Any]: ...
|
489
489
|
|
modal/image.py
CHANGED
@@ -37,6 +37,7 @@ from .cloud_bucket_mount import _CloudBucketMount
|
|
37
37
|
from .config import config, logger, user_config_path
|
38
38
|
from .environments import _get_environment_cached
|
39
39
|
from .exception import InvalidError, NotFoundError, RemoteError, VersionError, deprecation_error, deprecation_warning
|
40
|
+
from .file_pattern_matcher import FilePatternMatcher
|
40
41
|
from .gpu import GPU_T, parse_gpu_config
|
41
42
|
from .mount import _Mount, python_standalone_mount_name
|
42
43
|
from .network_file_system import _NetworkFileSystem
|
@@ -638,7 +639,17 @@ class _Image(_Object, type_prefix="im"):
|
|
638
639
|
mount = _Mount.from_local_file(local_path, remote_path)
|
639
640
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
640
641
|
|
641
|
-
def add_local_dir(
|
642
|
+
def add_local_dir(
|
643
|
+
self,
|
644
|
+
local_path: Union[str, Path],
|
645
|
+
remote_path: str,
|
646
|
+
*,
|
647
|
+
copy: bool = False,
|
648
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
649
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
650
|
+
# Which follows dockerignore syntax.
|
651
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
652
|
+
) -> "_Image":
|
642
653
|
"""Adds a local directory's content to the image at `remote_path` within the container
|
643
654
|
|
644
655
|
By default (`copy=False`), the files are added to containers on startup and are not built into the actual Image,
|
@@ -650,12 +661,44 @@ class _Image(_Object, type_prefix="im"):
|
|
650
661
|
copy=True can slow down iteration since it requires a rebuild of the Image and any subsequent
|
651
662
|
build steps whenever the included files change, but it is required if you want to run additional
|
652
663
|
build steps after this one.
|
664
|
+
|
665
|
+
**Usage:**
|
666
|
+
|
667
|
+
```python
|
668
|
+
from modal import FilePatternMatcher
|
669
|
+
|
670
|
+
image = modal.Image.debian_slim().add_local_dir(
|
671
|
+
"~/assets",
|
672
|
+
remote_path="/assets",
|
673
|
+
ignore=["*.venv"],
|
674
|
+
)
|
675
|
+
|
676
|
+
image = modal.Image.debian_slim().add_local_dir(
|
677
|
+
"~/assets",
|
678
|
+
remote_path="/assets",
|
679
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
680
|
+
)
|
681
|
+
|
682
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
683
|
+
"~/assets",
|
684
|
+
remote_path="/assets",
|
685
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
686
|
+
)
|
687
|
+
|
688
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
689
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
690
|
+
"~/assets",
|
691
|
+
remote_path="/assets",
|
692
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
693
|
+
)
|
694
|
+
```
|
653
695
|
"""
|
654
696
|
if not PurePosixPath(remote_path).is_absolute():
|
655
697
|
# TODO(elias): implement relative to absolute resolution using image workdir metadata
|
656
698
|
# + make default remote_path="./"
|
657
699
|
raise InvalidError("image.add_local_dir() currently only supports absolute remote_path values")
|
658
|
-
|
700
|
+
|
701
|
+
mount = _Mount._add_local_dir(Path(local_path), Path(remote_path), ignore)
|
659
702
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
660
703
|
|
661
704
|
def copy_local_file(self, local_path: Union[str, Path], remote_path: Union[str, Path] = "./") -> "_Image":
|
@@ -697,19 +740,56 @@ class _Image(_Object, type_prefix="im"):
|
|
697
740
|
the destination directory.
|
698
741
|
"""
|
699
742
|
|
700
|
-
|
701
|
-
return filename.endswith(".py")
|
702
|
-
|
703
|
-
mount = _Mount.from_local_python_packages(*modules, condition=only_py_files)
|
743
|
+
mount = _Mount.from_local_python_packages(*modules, ignore=~FilePatternMatcher("**/*.py"))
|
704
744
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
705
745
|
|
706
|
-
def copy_local_dir(
|
746
|
+
def copy_local_dir(
|
747
|
+
self,
|
748
|
+
local_path: Union[str, Path],
|
749
|
+
remote_path: Union[str, Path] = ".",
|
750
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
751
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
752
|
+
# Which follows dockerignore syntax.
|
753
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
754
|
+
) -> "_Image":
|
707
755
|
"""Copy a directory into the image as a part of building the image.
|
708
756
|
|
709
757
|
This works in a similar way to [`COPY`](https://docs.docker.com/engine/reference/builder/#copy)
|
710
758
|
works in a `Dockerfile`.
|
759
|
+
|
760
|
+
**Usage:**
|
761
|
+
|
762
|
+
```python
|
763
|
+
from modal import FilePatternMatcher
|
764
|
+
|
765
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
766
|
+
"~/assets",
|
767
|
+
remote_path="/assets",
|
768
|
+
ignore=["**/*.venv"],
|
769
|
+
)
|
770
|
+
|
771
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
772
|
+
"~/assets",
|
773
|
+
remote_path="/assets",
|
774
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
775
|
+
)
|
776
|
+
|
777
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
778
|
+
"~/assets",
|
779
|
+
remote_path="/assets",
|
780
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
781
|
+
)
|
782
|
+
|
783
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
784
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
785
|
+
"~/assets",
|
786
|
+
remote_path="/assets",
|
787
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
788
|
+
)
|
789
|
+
```
|
711
790
|
"""
|
712
|
-
|
791
|
+
|
792
|
+
mount = _Mount._add_local_dir(Path(local_path), Path("/"), ignore)
|
713
793
|
|
714
794
|
def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
|
715
795
|
return DockerfileSpec(commands=["FROM base", f"COPY . {remote_path}"], context_files={})
|
modal/image.pyi
CHANGED
@@ -110,14 +110,22 @@ class _Image(modal.object._Object):
|
|
110
110
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
111
111
|
) -> _Image: ...
|
112
112
|
def add_local_dir(
|
113
|
-
self,
|
113
|
+
self,
|
114
|
+
local_path: typing.Union[str, pathlib.Path],
|
115
|
+
remote_path: str,
|
116
|
+
*,
|
117
|
+
copy: bool = False,
|
118
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
114
119
|
) -> _Image: ...
|
115
120
|
def copy_local_file(
|
116
121
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
117
122
|
) -> _Image: ...
|
118
123
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> _Image: ...
|
119
124
|
def copy_local_dir(
|
120
|
-
self,
|
125
|
+
self,
|
126
|
+
local_path: typing.Union[str, pathlib.Path],
|
127
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
128
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
121
129
|
) -> _Image: ...
|
122
130
|
def pip_install(
|
123
131
|
self,
|
@@ -367,14 +375,22 @@ class Image(modal.object.Object):
|
|
367
375
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
368
376
|
) -> Image: ...
|
369
377
|
def add_local_dir(
|
370
|
-
self,
|
378
|
+
self,
|
379
|
+
local_path: typing.Union[str, pathlib.Path],
|
380
|
+
remote_path: str,
|
381
|
+
*,
|
382
|
+
copy: bool = False,
|
383
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
371
384
|
) -> Image: ...
|
372
385
|
def copy_local_file(
|
373
386
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
374
387
|
) -> Image: ...
|
375
388
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> Image: ...
|
376
389
|
def copy_local_dir(
|
377
|
-
self,
|
390
|
+
self,
|
391
|
+
local_path: typing.Union[str, pathlib.Path],
|
392
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
393
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
378
394
|
) -> Image: ...
|
379
395
|
def pip_install(
|
380
396
|
self,
|
modal/mount.py
CHANGED
@@ -11,7 +11,7 @@ import time
|
|
11
11
|
import typing
|
12
12
|
from collections.abc import AsyncGenerator
|
13
13
|
from pathlib import Path, PurePosixPath
|
14
|
-
from typing import Callable, Optional, Union
|
14
|
+
from typing import Callable, Optional, Sequence, Union
|
15
15
|
|
16
16
|
from google.protobuf.message import Message
|
17
17
|
|
@@ -27,7 +27,8 @@ from ._utils.name_utils import check_object_name
|
|
27
27
|
from ._utils.package_utils import get_module_mount_info
|
28
28
|
from .client import _Client
|
29
29
|
from .config import config, logger
|
30
|
-
from .exception import ModuleNotMountable
|
30
|
+
from .exception import InvalidError, ModuleNotMountable
|
31
|
+
from .file_pattern_matcher import FilePatternMatcher
|
31
32
|
from .object import _get_environment_name, _Object
|
32
33
|
|
33
34
|
ROOT_DIR: PurePosixPath = PurePosixPath("/root")
|
@@ -122,7 +123,7 @@ class _MountFile(_MountEntry):
|
|
122
123
|
class _MountDir(_MountEntry):
|
123
124
|
local_dir: Path
|
124
125
|
remote_path: PurePosixPath
|
125
|
-
|
126
|
+
ignore: Callable[[Path], bool]
|
126
127
|
recursive: bool
|
127
128
|
|
128
129
|
def description(self):
|
@@ -143,7 +144,7 @@ class _MountDir(_MountEntry):
|
|
143
144
|
gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
|
144
145
|
|
145
146
|
for local_filename in gen:
|
146
|
-
if self.
|
147
|
+
if not self.ignore(Path(local_filename)):
|
147
148
|
local_relpath = Path(local_filename).expanduser().absolute().relative_to(local_dir)
|
148
149
|
mount_path = self.remote_path / local_relpath.as_posix()
|
149
150
|
yield local_filename, mount_path
|
@@ -182,6 +183,10 @@ def module_mount_condition(module_base: Path):
|
|
182
183
|
return condition
|
183
184
|
|
184
185
|
|
186
|
+
def module_mount_ignore_condition(module_base: Path):
|
187
|
+
return lambda f: not module_mount_condition(module_base)(str(f))
|
188
|
+
|
189
|
+
|
185
190
|
@dataclasses.dataclass
|
186
191
|
class _MountedPythonModule(_MountEntry):
|
187
192
|
# the purpose of this is to keep printable information about which Python package
|
@@ -190,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
|
|
190
195
|
|
191
196
|
module_name: str
|
192
197
|
remote_dir: Union[PurePosixPath, str] = ROOT_DIR.as_posix() # cast needed here for type stub generation...
|
193
|
-
|
198
|
+
ignore: Optional[Callable[[Path], bool]] = None
|
194
199
|
|
195
200
|
def description(self) -> str:
|
196
201
|
return f"PythonPackage:{self.module_name}"
|
@@ -206,7 +211,7 @@ class _MountedPythonModule(_MountEntry):
|
|
206
211
|
_MountDir(
|
207
212
|
base_path,
|
208
213
|
remote_path=remote_dir,
|
209
|
-
|
214
|
+
ignore=self.ignore or module_mount_ignore_condition(base_path),
|
210
215
|
recursive=True,
|
211
216
|
)
|
212
217
|
)
|
@@ -313,6 +318,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
313
318
|
# we can't rely on it to be set. Let's clean this up later.
|
314
319
|
return getattr(self, "_is_local", False)
|
315
320
|
|
321
|
+
@staticmethod
|
322
|
+
def _add_local_dir(
|
323
|
+
local_path: Path,
|
324
|
+
remote_path: Path,
|
325
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
326
|
+
):
|
327
|
+
if isinstance(ignore, list):
|
328
|
+
ignore = FilePatternMatcher(*ignore)
|
329
|
+
|
330
|
+
return _Mount._new()._extend(
|
331
|
+
_MountDir(
|
332
|
+
local_dir=local_path,
|
333
|
+
ignore=ignore,
|
334
|
+
remote_path=remote_path,
|
335
|
+
recursive=True,
|
336
|
+
),
|
337
|
+
)
|
338
|
+
|
316
339
|
def add_local_dir(
|
317
340
|
self,
|
318
341
|
local_path: Union[str, Path],
|
@@ -339,10 +362,13 @@ class _Mount(_Object, type_prefix="mo"):
|
|
339
362
|
|
340
363
|
condition = include_all
|
341
364
|
|
365
|
+
def converted_condition(path: Path) -> bool:
|
366
|
+
return not condition(str(path))
|
367
|
+
|
342
368
|
return self._extend(
|
343
369
|
_MountDir(
|
344
370
|
local_dir=local_path,
|
345
|
-
|
371
|
+
ignore=converted_condition,
|
346
372
|
remote_path=remote_path,
|
347
373
|
recursive=recursive,
|
348
374
|
),
|
@@ -483,7 +509,9 @@ class _Mount(_Object, type_prefix="mo"):
|
|
483
509
|
logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
|
484
510
|
async with blob_upload_concurrency:
|
485
511
|
with file_spec.source() as fp:
|
486
|
-
blob_id = await blob_upload_file(
|
512
|
+
blob_id = await blob_upload_file(
|
513
|
+
fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
|
514
|
+
)
|
487
515
|
logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
|
488
516
|
request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
|
489
517
|
else:
|
@@ -549,6 +577,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
549
577
|
# Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
|
550
578
|
# Defaults to including all files.
|
551
579
|
condition: Optional[Callable[[str], bool]] = None,
|
580
|
+
ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
|
552
581
|
) -> "_Mount":
|
553
582
|
"""
|
554
583
|
Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
|
@@ -573,13 +602,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
573
602
|
|
574
603
|
# Don't re-run inside container.
|
575
604
|
|
605
|
+
if condition is not None:
|
606
|
+
if ignore is not None:
|
607
|
+
raise InvalidError("Cannot specify both `ignore` and `condition`")
|
608
|
+
|
609
|
+
def converted_condition(path: Path) -> bool:
|
610
|
+
return not condition(str(path))
|
611
|
+
|
612
|
+
ignore = converted_condition
|
613
|
+
elif isinstance(ignore, list):
|
614
|
+
ignore = FilePatternMatcher(*ignore)
|
615
|
+
|
576
616
|
mount = _Mount._new()
|
577
617
|
from ._runtime.execution_context import is_local
|
578
618
|
|
579
619
|
if not is_local():
|
580
620
|
return mount # empty/non-mountable mount in case it's used from within a container
|
581
621
|
for module_name in module_names:
|
582
|
-
mount = mount._extend(_MountedPythonModule(module_name, remote_dir,
|
622
|
+
mount = mount._extend(_MountedPythonModule(module_name, remote_dir, ignore))
|
583
623
|
return mount
|
584
624
|
|
585
625
|
@staticmethod
|