modal 0.67.43__py3-none-any.whl → 0.68.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +2 -0
- modal/_container_entrypoint.py +4 -1
- modal/_ipython.py +3 -13
- modal/_runtime/asgi.py +4 -0
- modal/_runtime/container_io_manager.py +3 -0
- modal/_runtime/user_code_imports.py +17 -20
- modal/_traceback.py +16 -2
- modal/_utils/blob_utils.py +27 -92
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/function_utils.py +5 -1
- modal/_utils/grpc_testing.py +6 -2
- modal/_utils/hash_utils.py +51 -10
- modal/_utils/http_utils.py +19 -10
- modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
- modal/_utils/shell_utils.py +11 -5
- modal/cli/_traceback.py +11 -4
- modal/cli/run.py +25 -12
- modal/client.py +6 -37
- modal/client.pyi +2 -6
- modal/cls.py +132 -62
- modal/cls.pyi +13 -7
- modal/exception.py +20 -0
- modal/file_io.py +380 -0
- modal/file_io.pyi +185 -0
- modal/file_pattern_matcher.py +121 -0
- modal/functions.py +33 -11
- modal/functions.pyi +11 -9
- modal/image.py +88 -8
- modal/image.pyi +20 -4
- modal/mount.py +49 -9
- modal/mount.pyi +19 -4
- modal/network_file_system.py +4 -1
- modal/object.py +4 -2
- modal/partial_function.py +22 -10
- modal/partial_function.pyi +10 -2
- modal/runner.py +5 -4
- modal/runner.pyi +2 -1
- modal/sandbox.py +40 -0
- modal/sandbox.pyi +18 -0
- modal/volume.py +5 -1
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/METADATA +2 -2
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/RECORD +52 -48
- modal_docs/gen_reference_docs.py +1 -0
- modal_proto/api.proto +33 -1
- modal_proto/api_pb2.py +813 -737
- modal_proto/api_pb2.pyi +160 -13
- modal_version/__init__.py +1 -1
- modal_version/_version_generated.py +1 -1
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/LICENSE +0 -0
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/WHEEL +0 -0
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/entry_points.txt +0 -0
- {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/top_level.txt +0 -0
modal/functions.py
CHANGED
@@ -32,6 +32,7 @@ from ._resolver import Resolver
|
|
32
32
|
from ._resources import convert_fn_config_to_resources_config
|
33
33
|
from ._runtime.execution_context import current_input_id, is_local
|
34
34
|
from ._serialization import serialize, serialize_proto_params
|
35
|
+
from ._traceback import print_server_warnings
|
35
36
|
from ._utils.async_utils import (
|
36
37
|
TaskContext,
|
37
38
|
async_merge,
|
@@ -431,7 +432,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
431
432
|
image: _Image,
|
432
433
|
secrets: Sequence[_Secret] = (),
|
433
434
|
schedule: Optional[Schedule] = None,
|
434
|
-
is_generator=False,
|
435
|
+
is_generator: bool = False,
|
435
436
|
gpu: Union[GPU_T, list[GPU_T]] = None,
|
436
437
|
# TODO: maybe break this out into a separate decorator for notebooks.
|
437
438
|
mounts: Collection[_Mount] = (),
|
@@ -627,7 +628,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
627
628
|
raise InvalidError(f"Expected modal.Image object. Got {type(image)}.")
|
628
629
|
|
629
630
|
method_definitions: Optional[dict[str, api_pb2.MethodDefinition]] = None
|
630
|
-
|
631
|
+
|
631
632
|
if info.user_cls:
|
632
633
|
method_definitions = {}
|
633
634
|
partial_functions = _find_partial_methods_for_user_cls(info.user_cls, _PartialFunctionFlags.FUNCTION)
|
@@ -1061,6 +1062,8 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1061
1062
|
else:
|
1062
1063
|
raise
|
1063
1064
|
|
1065
|
+
print_server_warnings(response.server_warnings)
|
1066
|
+
|
1064
1067
|
self._hydrate(response.function_id, resolver.client, response.handle_metadata)
|
1065
1068
|
|
1066
1069
|
rep = f"Ref({app_name})"
|
@@ -1188,9 +1191,16 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1188
1191
|
return self._web_url
|
1189
1192
|
|
1190
1193
|
@property
|
1191
|
-
def is_generator(self) -> bool:
|
1194
|
+
async def is_generator(self) -> bool:
|
1192
1195
|
"""mdmd:hidden"""
|
1193
|
-
|
1196
|
+
# hacky: kind of like @live_method, but not hydrating if we have the value already from local source
|
1197
|
+
if self._is_generator is not None:
|
1198
|
+
# this is set if the function or class is local
|
1199
|
+
return self._is_generator
|
1200
|
+
|
1201
|
+
# not set - this is a from_name lookup - hydrate
|
1202
|
+
await self.resolve()
|
1203
|
+
assert self._is_generator is not None # should be set now
|
1194
1204
|
return self._is_generator
|
1195
1205
|
|
1196
1206
|
@property
|
@@ -1272,6 +1282,10 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1272
1282
|
|
1273
1283
|
@synchronizer.no_io_translation
|
1274
1284
|
async def _call_generator_nowait(self, args, kwargs):
|
1285
|
+
deprecation_warning(
|
1286
|
+
(2024, 12, 11),
|
1287
|
+
"Calling spawn on a generator function is deprecated and will soon raise an exception.",
|
1288
|
+
)
|
1275
1289
|
return await _Invocation.create(
|
1276
1290
|
self,
|
1277
1291
|
args,
|
@@ -1311,6 +1325,9 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1311
1325
|
async for item in self._call_generator(args, kwargs): # type: ignore
|
1312
1326
|
yield item
|
1313
1327
|
|
1328
|
+
def _is_local(self):
|
1329
|
+
return self._info is not None
|
1330
|
+
|
1314
1331
|
def _get_info(self) -> FunctionInfo:
|
1315
1332
|
if not self._info:
|
1316
1333
|
raise ExecutionError("Can't get info for a function that isn't locally defined")
|
@@ -1335,19 +1352,24 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1335
1352
|
"""
|
1336
1353
|
# TODO(erikbern): it would be nice to remove the nowrap thing, but right now that would cause
|
1337
1354
|
# "user code" to run on the synchronicity thread, which seems bad
|
1355
|
+
if not self._is_local():
|
1356
|
+
msg = (
|
1357
|
+
"The definition for this function is missing here so it is not possible to invoke it locally. "
|
1358
|
+
"If this function was retrieved via `Function.lookup` you need to use `.remote()`."
|
1359
|
+
)
|
1360
|
+
raise ExecutionError(msg)
|
1361
|
+
|
1338
1362
|
info = self._get_info()
|
1363
|
+
if not info.raw_f:
|
1364
|
+
# Here if calling .local on a service function itself which should never happen
|
1365
|
+
# TODO: check if we end up here in a container for a serialized function?
|
1366
|
+
raise ExecutionError("Can't call .local on service function")
|
1339
1367
|
|
1340
1368
|
if is_local() and self.spec.volumes or self.spec.network_file_systems:
|
1341
1369
|
warnings.warn(
|
1342
1370
|
f"The {info.function_name} function is executing locally "
|
1343
1371
|
+ "and will not have access to the mounted Volume or NetworkFileSystem data"
|
1344
1372
|
)
|
1345
|
-
if not info or not info.raw_f:
|
1346
|
-
msg = (
|
1347
|
-
"The definition for this function is missing so it is not possible to invoke it locally. "
|
1348
|
-
"If this function was retrieved via `Function.lookup` you need to use `.remote()`."
|
1349
|
-
)
|
1350
|
-
raise ExecutionError(msg)
|
1351
1373
|
|
1352
1374
|
obj: Optional["modal.cls._Obj"] = self._get_obj()
|
1353
1375
|
|
@@ -1357,9 +1379,9 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
|
|
1357
1379
|
else:
|
1358
1380
|
# This is a method on a class, so bind the self to the function
|
1359
1381
|
user_cls_instance = obj._cached_user_cls_instance()
|
1360
|
-
|
1361
1382
|
fun = info.raw_f.__get__(user_cls_instance)
|
1362
1383
|
|
1384
|
+
# TODO: replace implicit local enter/exit with a context manager
|
1363
1385
|
if is_async(info.raw_f):
|
1364
1386
|
# We want to run __aenter__ and fun in the same coroutine
|
1365
1387
|
async def coro():
|
modal/functions.pyi
CHANGED
@@ -157,7 +157,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
|
|
157
157
|
image: modal.image._Image,
|
158
158
|
secrets: collections.abc.Sequence[modal.secret._Secret] = (),
|
159
159
|
schedule: typing.Optional[modal.schedule.Schedule] = None,
|
160
|
-
is_generator=False,
|
160
|
+
is_generator: bool = False,
|
161
161
|
gpu: typing.Union[
|
162
162
|
None, bool, str, modal.gpu._GPUConfig, list[typing.Union[None, bool, str, modal.gpu._GPUConfig]]
|
163
163
|
] = None,
|
@@ -234,7 +234,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
|
|
234
234
|
@property
|
235
235
|
async def web_url(self) -> str: ...
|
236
236
|
@property
|
237
|
-
def is_generator(self) -> bool: ...
|
237
|
+
async def is_generator(self) -> bool: ...
|
238
238
|
@property
|
239
239
|
def cluster_size(self) -> int: ...
|
240
240
|
def _map(
|
@@ -246,6 +246,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
|
|
246
246
|
async def _call_generator_nowait(self, args, kwargs): ...
|
247
247
|
async def remote(self, *args: P.args, **kwargs: P.kwargs) -> ReturnType: ...
|
248
248
|
def remote_gen(self, *args, **kwargs) -> collections.abc.AsyncGenerator[typing.Any, None]: ...
|
249
|
+
def _is_local(self): ...
|
249
250
|
def _get_info(self) -> modal._utils.function_utils.FunctionInfo: ...
|
250
251
|
def _get_obj(self) -> typing.Optional[modal.cls._Obj]: ...
|
251
252
|
def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
|
@@ -325,7 +326,7 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
325
326
|
image: modal.image.Image,
|
326
327
|
secrets: collections.abc.Sequence[modal.secret.Secret] = (),
|
327
328
|
schedule: typing.Optional[modal.schedule.Schedule] = None,
|
328
|
-
is_generator=False,
|
329
|
+
is_generator: bool = False,
|
329
330
|
gpu: typing.Union[
|
330
331
|
None, bool, str, modal.gpu._GPUConfig, list[typing.Union[None, bool, str, modal.gpu._GPUConfig]]
|
331
332
|
] = None,
|
@@ -455,11 +456,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
455
456
|
|
456
457
|
_call_generator_nowait: ___call_generator_nowait_spec
|
457
458
|
|
458
|
-
class __remote_spec(typing_extensions.Protocol[
|
459
|
+
class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
459
460
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
460
461
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
461
462
|
|
462
|
-
remote: __remote_spec[
|
463
|
+
remote: __remote_spec[ReturnType, P]
|
463
464
|
|
464
465
|
class __remote_gen_spec(typing_extensions.Protocol):
|
465
466
|
def __call__(self, *args, **kwargs) -> typing.Generator[typing.Any, None, None]: ...
|
@@ -467,21 +468,22 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
467
468
|
|
468
469
|
remote_gen: __remote_gen_spec
|
469
470
|
|
471
|
+
def _is_local(self): ...
|
470
472
|
def _get_info(self) -> modal._utils.function_utils.FunctionInfo: ...
|
471
473
|
def _get_obj(self) -> typing.Optional[modal.cls.Obj]: ...
|
472
474
|
def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
|
473
475
|
|
474
|
-
class ___experimental_spawn_spec(typing_extensions.Protocol[
|
476
|
+
class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
475
477
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
476
478
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
477
479
|
|
478
|
-
_experimental_spawn: ___experimental_spawn_spec[
|
480
|
+
_experimental_spawn: ___experimental_spawn_spec[ReturnType, P]
|
479
481
|
|
480
|
-
class __spawn_spec(typing_extensions.Protocol[
|
482
|
+
class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
|
481
483
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
482
484
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
483
485
|
|
484
|
-
spawn: __spawn_spec[
|
486
|
+
spawn: __spawn_spec[ReturnType, P]
|
485
487
|
|
486
488
|
def get_raw_f(self) -> typing.Callable[..., typing.Any]: ...
|
487
489
|
|
modal/image.py
CHANGED
@@ -37,6 +37,7 @@ from .cloud_bucket_mount import _CloudBucketMount
|
|
37
37
|
from .config import config, logger, user_config_path
|
38
38
|
from .environments import _get_environment_cached
|
39
39
|
from .exception import InvalidError, NotFoundError, RemoteError, VersionError, deprecation_error, deprecation_warning
|
40
|
+
from .file_pattern_matcher import FilePatternMatcher
|
40
41
|
from .gpu import GPU_T, parse_gpu_config
|
41
42
|
from .mount import _Mount, python_standalone_mount_name
|
42
43
|
from .network_file_system import _NetworkFileSystem
|
@@ -638,7 +639,17 @@ class _Image(_Object, type_prefix="im"):
|
|
638
639
|
mount = _Mount.from_local_file(local_path, remote_path)
|
639
640
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
640
641
|
|
641
|
-
def add_local_dir(
|
642
|
+
def add_local_dir(
|
643
|
+
self,
|
644
|
+
local_path: Union[str, Path],
|
645
|
+
remote_path: str,
|
646
|
+
*,
|
647
|
+
copy: bool = False,
|
648
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
649
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
650
|
+
# Which follows dockerignore syntax.
|
651
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
652
|
+
) -> "_Image":
|
642
653
|
"""Adds a local directory's content to the image at `remote_path` within the container
|
643
654
|
|
644
655
|
By default (`copy=False`), the files are added to containers on startup and are not built into the actual Image,
|
@@ -650,12 +661,44 @@ class _Image(_Object, type_prefix="im"):
|
|
650
661
|
copy=True can slow down iteration since it requires a rebuild of the Image and any subsequent
|
651
662
|
build steps whenever the included files change, but it is required if you want to run additional
|
652
663
|
build steps after this one.
|
664
|
+
|
665
|
+
**Usage:**
|
666
|
+
|
667
|
+
```python
|
668
|
+
from modal import FilePatternMatcher
|
669
|
+
|
670
|
+
image = modal.Image.debian_slim().add_local_dir(
|
671
|
+
"~/assets",
|
672
|
+
remote_path="/assets",
|
673
|
+
ignore=["*.venv"],
|
674
|
+
)
|
675
|
+
|
676
|
+
image = modal.Image.debian_slim().add_local_dir(
|
677
|
+
"~/assets",
|
678
|
+
remote_path="/assets",
|
679
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
680
|
+
)
|
681
|
+
|
682
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
683
|
+
"~/assets",
|
684
|
+
remote_path="/assets",
|
685
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
686
|
+
)
|
687
|
+
|
688
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
689
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
690
|
+
"~/assets",
|
691
|
+
remote_path="/assets",
|
692
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
693
|
+
)
|
694
|
+
```
|
653
695
|
"""
|
654
696
|
if not PurePosixPath(remote_path).is_absolute():
|
655
697
|
# TODO(elias): implement relative to absolute resolution using image workdir metadata
|
656
698
|
# + make default remote_path="./"
|
657
699
|
raise InvalidError("image.add_local_dir() currently only supports absolute remote_path values")
|
658
|
-
|
700
|
+
|
701
|
+
mount = _Mount._add_local_dir(Path(local_path), Path(remote_path), ignore)
|
659
702
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
660
703
|
|
661
704
|
def copy_local_file(self, local_path: Union[str, Path], remote_path: Union[str, Path] = "./") -> "_Image":
|
@@ -697,19 +740,56 @@ class _Image(_Object, type_prefix="im"):
|
|
697
740
|
the destination directory.
|
698
741
|
"""
|
699
742
|
|
700
|
-
|
701
|
-
return filename.endswith(".py")
|
702
|
-
|
703
|
-
mount = _Mount.from_local_python_packages(*modules, condition=only_py_files)
|
743
|
+
mount = _Mount.from_local_python_packages(*modules, ignore=~FilePatternMatcher("**/*.py"))
|
704
744
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
705
745
|
|
706
|
-
def copy_local_dir(
|
746
|
+
def copy_local_dir(
|
747
|
+
self,
|
748
|
+
local_path: Union[str, Path],
|
749
|
+
remote_path: Union[str, Path] = ".",
|
750
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
751
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
752
|
+
# Which follows dockerignore syntax.
|
753
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
754
|
+
) -> "_Image":
|
707
755
|
"""Copy a directory into the image as a part of building the image.
|
708
756
|
|
709
757
|
This works in a similar way to [`COPY`](https://docs.docker.com/engine/reference/builder/#copy)
|
710
758
|
works in a `Dockerfile`.
|
759
|
+
|
760
|
+
**Usage:**
|
761
|
+
|
762
|
+
```python
|
763
|
+
from modal import FilePatternMatcher
|
764
|
+
|
765
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
766
|
+
"~/assets",
|
767
|
+
remote_path="/assets",
|
768
|
+
ignore=["**/*.venv"],
|
769
|
+
)
|
770
|
+
|
771
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
772
|
+
"~/assets",
|
773
|
+
remote_path="/assets",
|
774
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
775
|
+
)
|
776
|
+
|
777
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
778
|
+
"~/assets",
|
779
|
+
remote_path="/assets",
|
780
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
781
|
+
)
|
782
|
+
|
783
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
784
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
785
|
+
"~/assets",
|
786
|
+
remote_path="/assets",
|
787
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
788
|
+
)
|
789
|
+
```
|
711
790
|
"""
|
712
|
-
|
791
|
+
|
792
|
+
mount = _Mount._add_local_dir(Path(local_path), Path("/"), ignore)
|
713
793
|
|
714
794
|
def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
|
715
795
|
return DockerfileSpec(commands=["FROM base", f"COPY . {remote_path}"], context_files={})
|
modal/image.pyi
CHANGED
@@ -110,14 +110,22 @@ class _Image(modal.object._Object):
|
|
110
110
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
111
111
|
) -> _Image: ...
|
112
112
|
def add_local_dir(
|
113
|
-
self,
|
113
|
+
self,
|
114
|
+
local_path: typing.Union[str, pathlib.Path],
|
115
|
+
remote_path: str,
|
116
|
+
*,
|
117
|
+
copy: bool = False,
|
118
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
114
119
|
) -> _Image: ...
|
115
120
|
def copy_local_file(
|
116
121
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
117
122
|
) -> _Image: ...
|
118
123
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> _Image: ...
|
119
124
|
def copy_local_dir(
|
120
|
-
self,
|
125
|
+
self,
|
126
|
+
local_path: typing.Union[str, pathlib.Path],
|
127
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
128
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
121
129
|
) -> _Image: ...
|
122
130
|
def pip_install(
|
123
131
|
self,
|
@@ -367,14 +375,22 @@ class Image(modal.object.Object):
|
|
367
375
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
368
376
|
) -> Image: ...
|
369
377
|
def add_local_dir(
|
370
|
-
self,
|
378
|
+
self,
|
379
|
+
local_path: typing.Union[str, pathlib.Path],
|
380
|
+
remote_path: str,
|
381
|
+
*,
|
382
|
+
copy: bool = False,
|
383
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
371
384
|
) -> Image: ...
|
372
385
|
def copy_local_file(
|
373
386
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
374
387
|
) -> Image: ...
|
375
388
|
def add_local_python_source(self, *module_names: str, copy: bool = False) -> Image: ...
|
376
389
|
def copy_local_dir(
|
377
|
-
self,
|
390
|
+
self,
|
391
|
+
local_path: typing.Union[str, pathlib.Path],
|
392
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
393
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
378
394
|
) -> Image: ...
|
379
395
|
def pip_install(
|
380
396
|
self,
|
modal/mount.py
CHANGED
@@ -11,7 +11,7 @@ import time
|
|
11
11
|
import typing
|
12
12
|
from collections.abc import AsyncGenerator
|
13
13
|
from pathlib import Path, PurePosixPath
|
14
|
-
from typing import Callable, Optional, Union
|
14
|
+
from typing import Callable, Optional, Sequence, Union
|
15
15
|
|
16
16
|
from google.protobuf.message import Message
|
17
17
|
|
@@ -27,7 +27,8 @@ from ._utils.name_utils import check_object_name
|
|
27
27
|
from ._utils.package_utils import get_module_mount_info
|
28
28
|
from .client import _Client
|
29
29
|
from .config import config, logger
|
30
|
-
from .exception import ModuleNotMountable
|
30
|
+
from .exception import InvalidError, ModuleNotMountable
|
31
|
+
from .file_pattern_matcher import FilePatternMatcher
|
31
32
|
from .object import _get_environment_name, _Object
|
32
33
|
|
33
34
|
ROOT_DIR: PurePosixPath = PurePosixPath("/root")
|
@@ -122,7 +123,7 @@ class _MountFile(_MountEntry):
|
|
122
123
|
class _MountDir(_MountEntry):
|
123
124
|
local_dir: Path
|
124
125
|
remote_path: PurePosixPath
|
125
|
-
|
126
|
+
ignore: Callable[[Path], bool]
|
126
127
|
recursive: bool
|
127
128
|
|
128
129
|
def description(self):
|
@@ -143,7 +144,7 @@ class _MountDir(_MountEntry):
|
|
143
144
|
gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
|
144
145
|
|
145
146
|
for local_filename in gen:
|
146
|
-
if self.
|
147
|
+
if not self.ignore(Path(local_filename)):
|
147
148
|
local_relpath = Path(local_filename).expanduser().absolute().relative_to(local_dir)
|
148
149
|
mount_path = self.remote_path / local_relpath.as_posix()
|
149
150
|
yield local_filename, mount_path
|
@@ -182,6 +183,10 @@ def module_mount_condition(module_base: Path):
|
|
182
183
|
return condition
|
183
184
|
|
184
185
|
|
186
|
+
def module_mount_ignore_condition(module_base: Path):
|
187
|
+
return lambda f: not module_mount_condition(module_base)(str(f))
|
188
|
+
|
189
|
+
|
185
190
|
@dataclasses.dataclass
|
186
191
|
class _MountedPythonModule(_MountEntry):
|
187
192
|
# the purpose of this is to keep printable information about which Python package
|
@@ -190,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
|
|
190
195
|
|
191
196
|
module_name: str
|
192
197
|
remote_dir: Union[PurePosixPath, str] = ROOT_DIR.as_posix() # cast needed here for type stub generation...
|
193
|
-
|
198
|
+
ignore: Optional[Callable[[Path], bool]] = None
|
194
199
|
|
195
200
|
def description(self) -> str:
|
196
201
|
return f"PythonPackage:{self.module_name}"
|
@@ -206,7 +211,7 @@ class _MountedPythonModule(_MountEntry):
|
|
206
211
|
_MountDir(
|
207
212
|
base_path,
|
208
213
|
remote_path=remote_dir,
|
209
|
-
|
214
|
+
ignore=self.ignore or module_mount_ignore_condition(base_path),
|
210
215
|
recursive=True,
|
211
216
|
)
|
212
217
|
)
|
@@ -313,6 +318,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
313
318
|
# we can't rely on it to be set. Let's clean this up later.
|
314
319
|
return getattr(self, "_is_local", False)
|
315
320
|
|
321
|
+
@staticmethod
|
322
|
+
def _add_local_dir(
|
323
|
+
local_path: Path,
|
324
|
+
remote_path: Path,
|
325
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
326
|
+
):
|
327
|
+
if isinstance(ignore, list):
|
328
|
+
ignore = FilePatternMatcher(*ignore)
|
329
|
+
|
330
|
+
return _Mount._new()._extend(
|
331
|
+
_MountDir(
|
332
|
+
local_dir=local_path,
|
333
|
+
ignore=ignore,
|
334
|
+
remote_path=remote_path,
|
335
|
+
recursive=True,
|
336
|
+
),
|
337
|
+
)
|
338
|
+
|
316
339
|
def add_local_dir(
|
317
340
|
self,
|
318
341
|
local_path: Union[str, Path],
|
@@ -339,10 +362,13 @@ class _Mount(_Object, type_prefix="mo"):
|
|
339
362
|
|
340
363
|
condition = include_all
|
341
364
|
|
365
|
+
def converted_condition(path: Path) -> bool:
|
366
|
+
return not condition(str(path))
|
367
|
+
|
342
368
|
return self._extend(
|
343
369
|
_MountDir(
|
344
370
|
local_dir=local_path,
|
345
|
-
|
371
|
+
ignore=converted_condition,
|
346
372
|
remote_path=remote_path,
|
347
373
|
recursive=recursive,
|
348
374
|
),
|
@@ -483,7 +509,9 @@ class _Mount(_Object, type_prefix="mo"):
|
|
483
509
|
logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
|
484
510
|
async with blob_upload_concurrency:
|
485
511
|
with file_spec.source() as fp:
|
486
|
-
blob_id = await blob_upload_file(
|
512
|
+
blob_id = await blob_upload_file(
|
513
|
+
fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
|
514
|
+
)
|
487
515
|
logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
|
488
516
|
request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
|
489
517
|
else:
|
@@ -549,6 +577,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
549
577
|
# Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
|
550
578
|
# Defaults to including all files.
|
551
579
|
condition: Optional[Callable[[str], bool]] = None,
|
580
|
+
ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
|
552
581
|
) -> "_Mount":
|
553
582
|
"""
|
554
583
|
Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
|
@@ -573,13 +602,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
573
602
|
|
574
603
|
# Don't re-run inside container.
|
575
604
|
|
605
|
+
if condition is not None:
|
606
|
+
if ignore is not None:
|
607
|
+
raise InvalidError("Cannot specify both `ignore` and `condition`")
|
608
|
+
|
609
|
+
def converted_condition(path: Path) -> bool:
|
610
|
+
return not condition(str(path))
|
611
|
+
|
612
|
+
ignore = converted_condition
|
613
|
+
elif isinstance(ignore, list):
|
614
|
+
ignore = FilePatternMatcher(*ignore)
|
615
|
+
|
576
616
|
mount = _Mount._new()
|
577
617
|
from ._runtime.execution_context import is_local
|
578
618
|
|
579
619
|
if not is_local():
|
580
620
|
return mount # empty/non-mountable mount in case it's used from within a container
|
581
621
|
for module_name in module_names:
|
582
|
-
mount = mount._extend(_MountedPythonModule(module_name, remote_dir,
|
622
|
+
mount = mount._extend(_MountedPythonModule(module_name, remote_dir, ignore))
|
583
623
|
return mount
|
584
624
|
|
585
625
|
@staticmethod
|
modal/mount.pyi
CHANGED
@@ -35,7 +35,7 @@ class _MountFile(_MountEntry):
|
|
35
35
|
class _MountDir(_MountEntry):
|
36
36
|
local_dir: pathlib.Path
|
37
37
|
remote_path: pathlib.PurePosixPath
|
38
|
-
|
38
|
+
ignore: typing.Callable[[pathlib.Path], bool]
|
39
39
|
recursive: bool
|
40
40
|
|
41
41
|
def description(self): ...
|
@@ -46,18 +46,19 @@ class _MountDir(_MountEntry):
|
|
46
46
|
self,
|
47
47
|
local_dir: pathlib.Path,
|
48
48
|
remote_path: pathlib.PurePosixPath,
|
49
|
-
|
49
|
+
ignore: typing.Callable[[pathlib.Path], bool],
|
50
50
|
recursive: bool,
|
51
51
|
) -> None: ...
|
52
52
|
def __repr__(self): ...
|
53
53
|
def __eq__(self, other): ...
|
54
54
|
|
55
55
|
def module_mount_condition(module_base: pathlib.Path): ...
|
56
|
+
def module_mount_ignore_condition(module_base: pathlib.Path): ...
|
56
57
|
|
57
58
|
class _MountedPythonModule(_MountEntry):
|
58
59
|
module_name: str
|
59
60
|
remote_dir: typing.Union[pathlib.PurePosixPath, str]
|
60
|
-
|
61
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]]
|
61
62
|
|
62
63
|
def description(self) -> str: ...
|
63
64
|
def _proxy_entries(self) -> list[_MountEntry]: ...
|
@@ -68,7 +69,7 @@ class _MountedPythonModule(_MountEntry):
|
|
68
69
|
self,
|
69
70
|
module_name: str,
|
70
71
|
remote_dir: typing.Union[pathlib.PurePosixPath, str] = "/root",
|
71
|
-
|
72
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]] = None,
|
72
73
|
) -> None: ...
|
73
74
|
def __repr__(self): ...
|
74
75
|
def __eq__(self, other): ...
|
@@ -90,6 +91,12 @@ class _Mount(modal.object._Object):
|
|
90
91
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
91
92
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
92
93
|
def is_local(self) -> bool: ...
|
94
|
+
@staticmethod
|
95
|
+
def _add_local_dir(
|
96
|
+
local_path: pathlib.Path,
|
97
|
+
remote_path: pathlib.Path,
|
98
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
99
|
+
): ...
|
93
100
|
def add_local_dir(
|
94
101
|
self,
|
95
102
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -129,6 +136,7 @@ class _Mount(modal.object._Object):
|
|
129
136
|
*module_names: str,
|
130
137
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
131
138
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
139
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
132
140
|
) -> _Mount: ...
|
133
141
|
@staticmethod
|
134
142
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> _Mount: ...
|
@@ -165,6 +173,12 @@ class Mount(modal.object.Object):
|
|
165
173
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
166
174
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
167
175
|
def is_local(self) -> bool: ...
|
176
|
+
@staticmethod
|
177
|
+
def _add_local_dir(
|
178
|
+
local_path: pathlib.Path,
|
179
|
+
remote_path: pathlib.Path,
|
180
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
181
|
+
): ...
|
168
182
|
def add_local_dir(
|
169
183
|
self,
|
170
184
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -214,6 +228,7 @@ class Mount(modal.object.Object):
|
|
214
228
|
*module_names: str,
|
215
229
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
216
230
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
231
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
217
232
|
) -> Mount: ...
|
218
233
|
@staticmethod
|
219
234
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> Mount: ...
|
modal/network_file_system.py
CHANGED
@@ -245,7 +245,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
245
245
|
if data_size > LARGE_FILE_LIMIT:
|
246
246
|
progress_task_id = progress_cb(name=remote_path, size=data_size)
|
247
247
|
blob_id = await blob_upload_file(
|
248
|
-
fp,
|
248
|
+
fp,
|
249
|
+
self._client.stub,
|
250
|
+
progress_report_cb=functools.partial(progress_cb, progress_task_id),
|
251
|
+
sha256_hex=sha_hash,
|
249
252
|
)
|
250
253
|
req = api_pb2.SharedVolumePutFileRequest(
|
251
254
|
shared_volume_id=self.object_id,
|
modal/object.py
CHANGED
@@ -96,7 +96,9 @@ class _Object:
|
|
96
96
|
|
97
97
|
def _initialize_from_other(self, other):
|
98
98
|
# default implementation, can be overriden in subclasses
|
99
|
-
|
99
|
+
self._object_id = other._object_id
|
100
|
+
self._is_hydrated = other._is_hydrated
|
101
|
+
self._client = other._client
|
100
102
|
|
101
103
|
def _hydrate(self, object_id: str, client: _Client, metadata: Optional[Message]):
|
102
104
|
assert isinstance(object_id, str)
|
@@ -139,7 +141,7 @@ class _Object:
|
|
139
141
|
|
140
142
|
# Object to clone must already be hydrated, otherwise from_loader is more suitable.
|
141
143
|
self._validate_is_hydrated()
|
142
|
-
obj =
|
144
|
+
obj = type(self).__new__(type(self))
|
143
145
|
obj._initialize_from_other(self)
|
144
146
|
return obj
|
145
147
|
|