modal 0.68.11__py3-none-any.whl → 0.68.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +2 -0
- modal/_ipython.py +3 -13
- modal/_runtime/asgi.py +4 -0
- modal/_runtime/user_code_imports.py +13 -18
- modal/_utils/blob_utils.py +27 -92
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/deprecation.py +44 -0
- modal/_utils/hash_utils.py +38 -9
- modal/_utils/http_utils.py +19 -10
- modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
- modal/_utils/shell_utils.py +11 -5
- modal/app.py +11 -31
- modal/app.pyi +3 -4
- modal/cli/app.py +1 -1
- modal/cli/run.py +25 -5
- modal/client.py +1 -1
- modal/client.pyi +2 -2
- modal/config.py +2 -1
- modal/container_process.py +2 -1
- modal/dict.py +2 -1
- modal/exception.py +0 -54
- modal/file_io.py +54 -7
- modal/file_io.pyi +18 -8
- modal/file_pattern_matcher.py +154 -0
- modal/functions.py +2 -8
- modal/functions.pyi +5 -1
- modal/image.py +106 -10
- modal/image.pyi +36 -6
- modal/mount.py +49 -9
- modal/mount.pyi +19 -4
- modal/network_file_system.py +6 -2
- modal/partial_function.py +10 -1
- modal/partial_function.pyi +8 -0
- modal/queue.py +2 -1
- modal/runner.py +2 -7
- modal/sandbox.py +23 -13
- modal/sandbox.pyi +21 -0
- modal/serving.py +1 -1
- modal/volume.py +7 -2
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/METADATA +1 -1
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/RECORD +49 -46
- modal_proto/api.proto +8 -0
- modal_proto/api_pb2.py +781 -745
- modal_proto/api_pb2.pyi +65 -3
- modal_version/_version_generated.py +1 -1
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/LICENSE +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/WHEEL +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/entry_points.txt +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/top_level.txt +0 -0
modal/functions.py
CHANGED
@@ -41,6 +41,7 @@ from ._utils.async_utils import (
|
|
41
41
|
synchronizer,
|
42
42
|
warn_if_generator_is_not_consumed,
|
43
43
|
)
|
44
|
+
from ._utils.deprecation import deprecation_warning
|
44
45
|
from ._utils.function_utils import (
|
45
46
|
ATTEMPT_TIMEOUT_GRACE_PERIOD,
|
46
47
|
OUTPUTS_TIMEOUT,
|
@@ -58,14 +59,7 @@ from .call_graph import InputInfo, _reconstruct_call_graph
|
|
58
59
|
from .client import _Client
|
59
60
|
from .cloud_bucket_mount import _CloudBucketMount, cloud_bucket_mounts_to_proto
|
60
61
|
from .config import config
|
61
|
-
from .exception import
|
62
|
-
ExecutionError,
|
63
|
-
FunctionTimeoutError,
|
64
|
-
InvalidError,
|
65
|
-
NotFoundError,
|
66
|
-
OutputExpiredError,
|
67
|
-
deprecation_warning,
|
68
|
-
)
|
62
|
+
from .exception import ExecutionError, FunctionTimeoutError, InvalidError, NotFoundError, OutputExpiredError
|
69
63
|
from .gpu import GPU_T, parse_gpu_config
|
70
64
|
from .image import _Image
|
71
65
|
from .mount import _get_client_mount, _Mount, get_auto_mounts
|
modal/functions.pyi
CHANGED
@@ -448,7 +448,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
448
448
|
|
449
449
|
_call_function_nowait: ___call_function_nowait_spec
|
450
450
|
|
451
|
-
|
451
|
+
class ___call_generator_spec(typing_extensions.Protocol):
|
452
|
+
def __call__(self, args, kwargs): ...
|
453
|
+
def aio(self, args, kwargs): ...
|
454
|
+
|
455
|
+
_call_generator: ___call_generator_spec
|
452
456
|
|
453
457
|
class ___call_generator_nowait_spec(typing_extensions.Protocol):
|
454
458
|
def __call__(self, args, kwargs): ...
|
modal/image.py
CHANGED
@@ -30,13 +30,15 @@ from ._resolver import Resolver
|
|
30
30
|
from ._serialization import serialize
|
31
31
|
from ._utils.async_utils import synchronize_api
|
32
32
|
from ._utils.blob_utils import MAX_OBJECT_SIZE_BYTES
|
33
|
+
from ._utils.deprecation import deprecation_error, deprecation_warning
|
33
34
|
from ._utils.function_utils import FunctionInfo
|
34
35
|
from ._utils.grpc_utils import RETRYABLE_GRPC_STATUS_CODES, retry_transient_errors
|
35
36
|
from .client import _Client
|
36
37
|
from .cloud_bucket_mount import _CloudBucketMount
|
37
38
|
from .config import config, logger, user_config_path
|
38
39
|
from .environments import _get_environment_cached
|
39
|
-
from .exception import InvalidError, NotFoundError, RemoteError, VersionError
|
40
|
+
from .exception import InvalidError, NotFoundError, RemoteError, VersionError
|
41
|
+
from .file_pattern_matcher import NON_PYTHON_FILES
|
40
42
|
from .gpu import GPU_T, parse_gpu_config
|
41
43
|
from .mount import _Mount, python_standalone_mount_name
|
42
44
|
from .network_file_system import _NetworkFileSystem
|
@@ -638,7 +640,17 @@ class _Image(_Object, type_prefix="im"):
|
|
638
640
|
mount = _Mount.from_local_file(local_path, remote_path)
|
639
641
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
640
642
|
|
641
|
-
def add_local_dir(
|
643
|
+
def add_local_dir(
|
644
|
+
self,
|
645
|
+
local_path: Union[str, Path],
|
646
|
+
remote_path: str,
|
647
|
+
*,
|
648
|
+
copy: bool = False,
|
649
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
650
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
651
|
+
# Which follows dockerignore syntax.
|
652
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
653
|
+
) -> "_Image":
|
642
654
|
"""Adds a local directory's content to the image at `remote_path` within the container
|
643
655
|
|
644
656
|
By default (`copy=False`), the files are added to containers on startup and are not built into the actual Image,
|
@@ -650,12 +662,44 @@ class _Image(_Object, type_prefix="im"):
|
|
650
662
|
copy=True can slow down iteration since it requires a rebuild of the Image and any subsequent
|
651
663
|
build steps whenever the included files change, but it is required if you want to run additional
|
652
664
|
build steps after this one.
|
665
|
+
|
666
|
+
**Usage:**
|
667
|
+
|
668
|
+
```python
|
669
|
+
from modal import FilePatternMatcher
|
670
|
+
|
671
|
+
image = modal.Image.debian_slim().add_local_dir(
|
672
|
+
"~/assets",
|
673
|
+
remote_path="/assets",
|
674
|
+
ignore=["*.venv"],
|
675
|
+
)
|
676
|
+
|
677
|
+
image = modal.Image.debian_slim().add_local_dir(
|
678
|
+
"~/assets",
|
679
|
+
remote_path="/assets",
|
680
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
681
|
+
)
|
682
|
+
|
683
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
684
|
+
"~/assets",
|
685
|
+
remote_path="/assets",
|
686
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
687
|
+
)
|
688
|
+
|
689
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
690
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
691
|
+
"~/assets",
|
692
|
+
remote_path="/assets",
|
693
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
694
|
+
)
|
695
|
+
```
|
653
696
|
"""
|
654
697
|
if not PurePosixPath(remote_path).is_absolute():
|
655
698
|
# TODO(elias): implement relative to absolute resolution using image workdir metadata
|
656
699
|
# + make default remote_path="./"
|
657
700
|
raise InvalidError("image.add_local_dir() currently only supports absolute remote_path values")
|
658
|
-
|
701
|
+
|
702
|
+
mount = _Mount._add_local_dir(Path(local_path), Path(remote_path), ignore)
|
659
703
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
660
704
|
|
661
705
|
def copy_local_file(self, local_path: Union[str, Path], remote_path: Union[str, Path] = "./") -> "_Image":
|
@@ -677,7 +721,9 @@ class _Image(_Object, type_prefix="im"):
|
|
677
721
|
context_mount=mount,
|
678
722
|
)
|
679
723
|
|
680
|
-
def add_local_python_source(
|
724
|
+
def add_local_python_source(
|
725
|
+
self, *modules: str, copy: bool = False, ignore: Union[Sequence[str], Callable[[Path], bool]] = NON_PYTHON_FILES
|
726
|
+
) -> "_Image":
|
681
727
|
"""Adds locally available Python packages/modules to containers
|
682
728
|
|
683
729
|
Adds all files from the specified Python package or module to containers running the Image.
|
@@ -695,21 +741,71 @@ class _Image(_Object, type_prefix="im"):
|
|
695
741
|
**Note:** This excludes all dot-prefixed subdirectories or files and all `.pyc`/`__pycache__` files.
|
696
742
|
To add full directories with finer control, use `.add_local_dir()` instead and specify `/root` as
|
697
743
|
the destination directory.
|
698
|
-
"""
|
699
744
|
|
700
|
-
|
701
|
-
|
745
|
+
By default only includes `.py`-files in the source modules. Set the `ignore` argument to a list of patterns
|
746
|
+
or a callable to override this behavior, e.g.:
|
747
|
+
|
748
|
+
```py
|
749
|
+
# includes everything except data.json
|
750
|
+
modal.Image.debian_slim().add_local_python_source("mymodule", ignore=["data.json"])
|
702
751
|
|
703
|
-
|
752
|
+
# exclude large files
|
753
|
+
modal.Image.debian_slim().add_local_python_source(
|
754
|
+
"mymodule",
|
755
|
+
ignore=lambda p: p.stat().st_size > 1e9
|
756
|
+
)
|
757
|
+
```
|
758
|
+
"""
|
759
|
+
mount = _Mount.from_local_python_packages(*modules, ignore=ignore)
|
704
760
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
705
761
|
|
706
|
-
def copy_local_dir(
|
762
|
+
def copy_local_dir(
|
763
|
+
self,
|
764
|
+
local_path: Union[str, Path],
|
765
|
+
remote_path: Union[str, Path] = ".",
|
766
|
+
# Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
|
767
|
+
# Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
|
768
|
+
# Which follows dockerignore syntax.
|
769
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
770
|
+
) -> "_Image":
|
707
771
|
"""Copy a directory into the image as a part of building the image.
|
708
772
|
|
709
773
|
This works in a similar way to [`COPY`](https://docs.docker.com/engine/reference/builder/#copy)
|
710
774
|
works in a `Dockerfile`.
|
775
|
+
|
776
|
+
**Usage:**
|
777
|
+
|
778
|
+
```python
|
779
|
+
from modal import FilePatternMatcher
|
780
|
+
|
781
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
782
|
+
"~/assets",
|
783
|
+
remote_path="/assets",
|
784
|
+
ignore=["**/*.venv"],
|
785
|
+
)
|
786
|
+
|
787
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
788
|
+
"~/assets",
|
789
|
+
remote_path="/assets",
|
790
|
+
ignore=lambda p: p.is_relative_to(".venv"),
|
791
|
+
)
|
792
|
+
|
793
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
794
|
+
"~/assets",
|
795
|
+
remote_path="/assets",
|
796
|
+
ignore=FilePatternMatcher("**/*.txt"),
|
797
|
+
)
|
798
|
+
|
799
|
+
# When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
|
800
|
+
image = modal.Image.debian_slim().copy_local_dir(
|
801
|
+
"~/assets",
|
802
|
+
remote_path="/assets",
|
803
|
+
ignore=~FilePatternMatcher("**/*.py"),
|
804
|
+
)
|
805
|
+
```
|
711
806
|
"""
|
712
|
-
|
807
|
+
|
808
|
+
mount = _Mount._add_local_dir(Path(local_path), Path("/"), ignore)
|
713
809
|
|
714
810
|
def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
|
715
811
|
return DockerfileSpec(commands=["FROM base", f"COPY . {remote_path}"], context_files={})
|
modal/image.pyi
CHANGED
@@ -110,14 +110,29 @@ class _Image(modal.object._Object):
|
|
110
110
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
111
111
|
) -> _Image: ...
|
112
112
|
def add_local_dir(
|
113
|
-
self,
|
113
|
+
self,
|
114
|
+
local_path: typing.Union[str, pathlib.Path],
|
115
|
+
remote_path: str,
|
116
|
+
*,
|
117
|
+
copy: bool = False,
|
118
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
114
119
|
) -> _Image: ...
|
115
120
|
def copy_local_file(
|
116
121
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
117
122
|
) -> _Image: ...
|
118
|
-
def add_local_python_source(
|
123
|
+
def add_local_python_source(
|
124
|
+
self,
|
125
|
+
*module_names: str,
|
126
|
+
copy: bool = False,
|
127
|
+
ignore: typing.Union[
|
128
|
+
collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]
|
129
|
+
] = modal.file_pattern_matcher.NON_PYTHON_FILES,
|
130
|
+
) -> _Image: ...
|
119
131
|
def copy_local_dir(
|
120
|
-
self,
|
132
|
+
self,
|
133
|
+
local_path: typing.Union[str, pathlib.Path],
|
134
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
135
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
121
136
|
) -> _Image: ...
|
122
137
|
def pip_install(
|
123
138
|
self,
|
@@ -367,14 +382,29 @@ class Image(modal.object.Object):
|
|
367
382
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
|
368
383
|
) -> Image: ...
|
369
384
|
def add_local_dir(
|
370
|
-
self,
|
385
|
+
self,
|
386
|
+
local_path: typing.Union[str, pathlib.Path],
|
387
|
+
remote_path: str,
|
388
|
+
*,
|
389
|
+
copy: bool = False,
|
390
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
371
391
|
) -> Image: ...
|
372
392
|
def copy_local_file(
|
373
393
|
self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
|
374
394
|
) -> Image: ...
|
375
|
-
def add_local_python_source(
|
395
|
+
def add_local_python_source(
|
396
|
+
self,
|
397
|
+
*module_names: str,
|
398
|
+
copy: bool = False,
|
399
|
+
ignore: typing.Union[
|
400
|
+
collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]
|
401
|
+
] = modal.file_pattern_matcher.NON_PYTHON_FILES,
|
402
|
+
) -> Image: ...
|
376
403
|
def copy_local_dir(
|
377
|
-
self,
|
404
|
+
self,
|
405
|
+
local_path: typing.Union[str, pathlib.Path],
|
406
|
+
remote_path: typing.Union[str, pathlib.Path] = ".",
|
407
|
+
ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
378
408
|
) -> Image: ...
|
379
409
|
def pip_install(
|
380
410
|
self,
|
modal/mount.py
CHANGED
@@ -11,7 +11,7 @@ import time
|
|
11
11
|
import typing
|
12
12
|
from collections.abc import AsyncGenerator
|
13
13
|
from pathlib import Path, PurePosixPath
|
14
|
-
from typing import Callable, Optional, Union
|
14
|
+
from typing import Callable, Optional, Sequence, Union
|
15
15
|
|
16
16
|
from google.protobuf.message import Message
|
17
17
|
|
@@ -27,7 +27,8 @@ from ._utils.name_utils import check_object_name
|
|
27
27
|
from ._utils.package_utils import get_module_mount_info
|
28
28
|
from .client import _Client
|
29
29
|
from .config import config, logger
|
30
|
-
from .exception import ModuleNotMountable
|
30
|
+
from .exception import InvalidError, ModuleNotMountable
|
31
|
+
from .file_pattern_matcher import FilePatternMatcher
|
31
32
|
from .object import _get_environment_name, _Object
|
32
33
|
|
33
34
|
ROOT_DIR: PurePosixPath = PurePosixPath("/root")
|
@@ -122,7 +123,7 @@ class _MountFile(_MountEntry):
|
|
122
123
|
class _MountDir(_MountEntry):
|
123
124
|
local_dir: Path
|
124
125
|
remote_path: PurePosixPath
|
125
|
-
|
126
|
+
ignore: Callable[[Path], bool]
|
126
127
|
recursive: bool
|
127
128
|
|
128
129
|
def description(self):
|
@@ -143,7 +144,7 @@ class _MountDir(_MountEntry):
|
|
143
144
|
gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
|
144
145
|
|
145
146
|
for local_filename in gen:
|
146
|
-
if self.
|
147
|
+
if not self.ignore(Path(local_filename)):
|
147
148
|
local_relpath = Path(local_filename).expanduser().absolute().relative_to(local_dir)
|
148
149
|
mount_path = self.remote_path / local_relpath.as_posix()
|
149
150
|
yield local_filename, mount_path
|
@@ -182,6 +183,10 @@ def module_mount_condition(module_base: Path):
|
|
182
183
|
return condition
|
183
184
|
|
184
185
|
|
186
|
+
def module_mount_ignore_condition(module_base: Path):
|
187
|
+
return lambda f: not module_mount_condition(module_base)(str(f))
|
188
|
+
|
189
|
+
|
185
190
|
@dataclasses.dataclass
|
186
191
|
class _MountedPythonModule(_MountEntry):
|
187
192
|
# the purpose of this is to keep printable information about which Python package
|
@@ -190,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
|
|
190
195
|
|
191
196
|
module_name: str
|
192
197
|
remote_dir: Union[PurePosixPath, str] = ROOT_DIR.as_posix() # cast needed here for type stub generation...
|
193
|
-
|
198
|
+
ignore: Optional[Callable[[Path], bool]] = None
|
194
199
|
|
195
200
|
def description(self) -> str:
|
196
201
|
return f"PythonPackage:{self.module_name}"
|
@@ -206,7 +211,7 @@ class _MountedPythonModule(_MountEntry):
|
|
206
211
|
_MountDir(
|
207
212
|
base_path,
|
208
213
|
remote_path=remote_dir,
|
209
|
-
|
214
|
+
ignore=self.ignore or module_mount_ignore_condition(base_path),
|
210
215
|
recursive=True,
|
211
216
|
)
|
212
217
|
)
|
@@ -313,6 +318,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
313
318
|
# we can't rely on it to be set. Let's clean this up later.
|
314
319
|
return getattr(self, "_is_local", False)
|
315
320
|
|
321
|
+
@staticmethod
|
322
|
+
def _add_local_dir(
|
323
|
+
local_path: Path,
|
324
|
+
remote_path: Path,
|
325
|
+
ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
|
326
|
+
):
|
327
|
+
if isinstance(ignore, list):
|
328
|
+
ignore = FilePatternMatcher(*ignore)
|
329
|
+
|
330
|
+
return _Mount._new()._extend(
|
331
|
+
_MountDir(
|
332
|
+
local_dir=local_path,
|
333
|
+
ignore=ignore,
|
334
|
+
remote_path=remote_path,
|
335
|
+
recursive=True,
|
336
|
+
),
|
337
|
+
)
|
338
|
+
|
316
339
|
def add_local_dir(
|
317
340
|
self,
|
318
341
|
local_path: Union[str, Path],
|
@@ -339,10 +362,13 @@ class _Mount(_Object, type_prefix="mo"):
|
|
339
362
|
|
340
363
|
condition = include_all
|
341
364
|
|
365
|
+
def converted_condition(path: Path) -> bool:
|
366
|
+
return not condition(str(path))
|
367
|
+
|
342
368
|
return self._extend(
|
343
369
|
_MountDir(
|
344
370
|
local_dir=local_path,
|
345
|
-
|
371
|
+
ignore=converted_condition,
|
346
372
|
remote_path=remote_path,
|
347
373
|
recursive=recursive,
|
348
374
|
),
|
@@ -483,7 +509,9 @@ class _Mount(_Object, type_prefix="mo"):
|
|
483
509
|
logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
|
484
510
|
async with blob_upload_concurrency:
|
485
511
|
with file_spec.source() as fp:
|
486
|
-
blob_id = await blob_upload_file(
|
512
|
+
blob_id = await blob_upload_file(
|
513
|
+
fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
|
514
|
+
)
|
487
515
|
logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
|
488
516
|
request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
|
489
517
|
else:
|
@@ -549,6 +577,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
549
577
|
# Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
|
550
578
|
# Defaults to including all files.
|
551
579
|
condition: Optional[Callable[[str], bool]] = None,
|
580
|
+
ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
|
552
581
|
) -> "_Mount":
|
553
582
|
"""
|
554
583
|
Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
|
@@ -573,13 +602,24 @@ class _Mount(_Object, type_prefix="mo"):
|
|
573
602
|
|
574
603
|
# Don't re-run inside container.
|
575
604
|
|
605
|
+
if condition is not None:
|
606
|
+
if ignore is not None:
|
607
|
+
raise InvalidError("Cannot specify both `ignore` and `condition`")
|
608
|
+
|
609
|
+
def converted_condition(path: Path) -> bool:
|
610
|
+
return not condition(str(path))
|
611
|
+
|
612
|
+
ignore = converted_condition
|
613
|
+
elif isinstance(ignore, list):
|
614
|
+
ignore = FilePatternMatcher(*ignore)
|
615
|
+
|
576
616
|
mount = _Mount._new()
|
577
617
|
from ._runtime.execution_context import is_local
|
578
618
|
|
579
619
|
if not is_local():
|
580
620
|
return mount # empty/non-mountable mount in case it's used from within a container
|
581
621
|
for module_name in module_names:
|
582
|
-
mount = mount._extend(_MountedPythonModule(module_name, remote_dir,
|
622
|
+
mount = mount._extend(_MountedPythonModule(module_name, remote_dir, ignore))
|
583
623
|
return mount
|
584
624
|
|
585
625
|
@staticmethod
|
modal/mount.pyi
CHANGED
@@ -35,7 +35,7 @@ class _MountFile(_MountEntry):
|
|
35
35
|
class _MountDir(_MountEntry):
|
36
36
|
local_dir: pathlib.Path
|
37
37
|
remote_path: pathlib.PurePosixPath
|
38
|
-
|
38
|
+
ignore: typing.Callable[[pathlib.Path], bool]
|
39
39
|
recursive: bool
|
40
40
|
|
41
41
|
def description(self): ...
|
@@ -46,18 +46,19 @@ class _MountDir(_MountEntry):
|
|
46
46
|
self,
|
47
47
|
local_dir: pathlib.Path,
|
48
48
|
remote_path: pathlib.PurePosixPath,
|
49
|
-
|
49
|
+
ignore: typing.Callable[[pathlib.Path], bool],
|
50
50
|
recursive: bool,
|
51
51
|
) -> None: ...
|
52
52
|
def __repr__(self): ...
|
53
53
|
def __eq__(self, other): ...
|
54
54
|
|
55
55
|
def module_mount_condition(module_base: pathlib.Path): ...
|
56
|
+
def module_mount_ignore_condition(module_base: pathlib.Path): ...
|
56
57
|
|
57
58
|
class _MountedPythonModule(_MountEntry):
|
58
59
|
module_name: str
|
59
60
|
remote_dir: typing.Union[pathlib.PurePosixPath, str]
|
60
|
-
|
61
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]]
|
61
62
|
|
62
63
|
def description(self) -> str: ...
|
63
64
|
def _proxy_entries(self) -> list[_MountEntry]: ...
|
@@ -68,7 +69,7 @@ class _MountedPythonModule(_MountEntry):
|
|
68
69
|
self,
|
69
70
|
module_name: str,
|
70
71
|
remote_dir: typing.Union[pathlib.PurePosixPath, str] = "/root",
|
71
|
-
|
72
|
+
ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]] = None,
|
72
73
|
) -> None: ...
|
73
74
|
def __repr__(self): ...
|
74
75
|
def __eq__(self, other): ...
|
@@ -90,6 +91,12 @@ class _Mount(modal.object._Object):
|
|
90
91
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
91
92
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
92
93
|
def is_local(self) -> bool: ...
|
94
|
+
@staticmethod
|
95
|
+
def _add_local_dir(
|
96
|
+
local_path: pathlib.Path,
|
97
|
+
remote_path: pathlib.Path,
|
98
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
99
|
+
): ...
|
93
100
|
def add_local_dir(
|
94
101
|
self,
|
95
102
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -129,6 +136,7 @@ class _Mount(modal.object._Object):
|
|
129
136
|
*module_names: str,
|
130
137
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
131
138
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
139
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
132
140
|
) -> _Mount: ...
|
133
141
|
@staticmethod
|
134
142
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> _Mount: ...
|
@@ -165,6 +173,12 @@ class Mount(modal.object.Object):
|
|
165
173
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
166
174
|
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
167
175
|
def is_local(self) -> bool: ...
|
176
|
+
@staticmethod
|
177
|
+
def _add_local_dir(
|
178
|
+
local_path: pathlib.Path,
|
179
|
+
remote_path: pathlib.Path,
|
180
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
|
181
|
+
): ...
|
168
182
|
def add_local_dir(
|
169
183
|
self,
|
170
184
|
local_path: typing.Union[str, pathlib.Path],
|
@@ -214,6 +228,7 @@ class Mount(modal.object.Object):
|
|
214
228
|
*module_names: str,
|
215
229
|
remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
|
216
230
|
condition: typing.Optional[typing.Callable[[str], bool]] = None,
|
231
|
+
ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
|
217
232
|
) -> Mount: ...
|
218
233
|
@staticmethod
|
219
234
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> Mount: ...
|
modal/network_file_system.py
CHANGED
@@ -15,11 +15,12 @@ from modal_proto import api_pb2
|
|
15
15
|
from ._resolver import Resolver
|
16
16
|
from ._utils.async_utils import TaskContext, aclosing, async_map, sync_or_async_iter, synchronize_api
|
17
17
|
from ._utils.blob_utils import LARGE_FILE_LIMIT, blob_iter, blob_upload_file
|
18
|
+
from ._utils.deprecation import deprecation_error
|
18
19
|
from ._utils.grpc_utils import retry_transient_errors
|
19
20
|
from ._utils.hash_utils import get_sha256_hex
|
20
21
|
from ._utils.name_utils import check_object_name
|
21
22
|
from .client import _Client
|
22
|
-
from .exception import InvalidError
|
23
|
+
from .exception import InvalidError
|
23
24
|
from .object import (
|
24
25
|
EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
25
26
|
_get_environment_name,
|
@@ -245,7 +246,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
245
246
|
if data_size > LARGE_FILE_LIMIT:
|
246
247
|
progress_task_id = progress_cb(name=remote_path, size=data_size)
|
247
248
|
blob_id = await blob_upload_file(
|
248
|
-
fp,
|
249
|
+
fp,
|
250
|
+
self._client.stub,
|
251
|
+
progress_report_cb=functools.partial(progress_cb, progress_task_id),
|
252
|
+
sha256_hex=sha_hash,
|
249
253
|
)
|
250
254
|
req = api_pb2.SharedVolumePutFileRequest(
|
251
255
|
shared_volume_id=self.object_id,
|
modal/partial_function.py
CHANGED
@@ -15,9 +15,10 @@ import typing_extensions
|
|
15
15
|
from modal_proto import api_pb2
|
16
16
|
|
17
17
|
from ._utils.async_utils import synchronize_api, synchronizer
|
18
|
+
from ._utils.deprecation import deprecation_error, deprecation_warning
|
18
19
|
from ._utils.function_utils import callable_has_non_self_non_default_params, callable_has_non_self_params
|
19
20
|
from .config import logger
|
20
|
-
from .exception import InvalidError
|
21
|
+
from .exception import InvalidError
|
21
22
|
from .functions import _Function
|
22
23
|
|
23
24
|
MAX_MAX_BATCH_SIZE = 1000
|
@@ -275,6 +276,7 @@ def _web_endpoint(
|
|
275
276
|
custom_domains: Optional[
|
276
277
|
Iterable[str]
|
277
278
|
] = None, # Create an endpoint using a custom domain fully-qualified domain name (FQDN).
|
279
|
+
requires_proxy_auth: bool = False, # Require Proxy-Authorization HTTP Headers on requests to the endpoint
|
278
280
|
wait_for_response: bool = True, # DEPRECATED: this must always be True now
|
279
281
|
) -> Callable[[Callable[P, ReturnType]], _PartialFunction[P, ReturnType, ReturnType]]:
|
280
282
|
"""Register a basic web endpoint with this application.
|
@@ -325,6 +327,7 @@ def _web_endpoint(
|
|
325
327
|
requested_suffix=label,
|
326
328
|
async_mode=api_pb2.WEBHOOK_ASYNC_MODE_AUTO,
|
327
329
|
custom_domains=_parse_custom_domains(custom_domains),
|
330
|
+
requires_proxy_auth=requires_proxy_auth,
|
328
331
|
),
|
329
332
|
)
|
330
333
|
|
@@ -336,6 +339,7 @@ def _asgi_app(
|
|
336
339
|
*,
|
337
340
|
label: Optional[str] = None, # Label for created endpoint. Final subdomain will be <workspace>--<label>.modal.run.
|
338
341
|
custom_domains: Optional[Iterable[str]] = None, # Deploy this endpoint on a custom domain.
|
342
|
+
requires_proxy_auth: bool = False, # Require Proxy-Authorization HTTP Headers on requests to the endpoint
|
339
343
|
wait_for_response: bool = True, # DEPRECATED: this must always be True now
|
340
344
|
) -> Callable[[Callable[..., Any]], _PartialFunction]:
|
341
345
|
"""Decorator for registering an ASGI app with a Modal function.
|
@@ -399,6 +403,7 @@ def _asgi_app(
|
|
399
403
|
requested_suffix=label,
|
400
404
|
async_mode=api_pb2.WEBHOOK_ASYNC_MODE_AUTO,
|
401
405
|
custom_domains=_parse_custom_domains(custom_domains),
|
406
|
+
requires_proxy_auth=requires_proxy_auth,
|
402
407
|
),
|
403
408
|
)
|
404
409
|
|
@@ -410,6 +415,7 @@ def _wsgi_app(
|
|
410
415
|
*,
|
411
416
|
label: Optional[str] = None, # Label for created endpoint. Final subdomain will be <workspace>--<label>.modal.run.
|
412
417
|
custom_domains: Optional[Iterable[str]] = None, # Deploy this endpoint on a custom domain.
|
418
|
+
requires_proxy_auth: bool = False, # Require Proxy-Authorization HTTP Headers on requests to the endpoint
|
413
419
|
wait_for_response: bool = True, # DEPRECATED: this must always be True now
|
414
420
|
) -> Callable[[Callable[..., Any]], _PartialFunction]:
|
415
421
|
"""Decorator for registering a WSGI app with a Modal function.
|
@@ -473,6 +479,7 @@ def _wsgi_app(
|
|
473
479
|
requested_suffix=label,
|
474
480
|
async_mode=api_pb2.WEBHOOK_ASYNC_MODE_AUTO,
|
475
481
|
custom_domains=_parse_custom_domains(custom_domains),
|
482
|
+
requires_proxy_auth=requires_proxy_auth,
|
476
483
|
),
|
477
484
|
)
|
478
485
|
|
@@ -485,6 +492,7 @@ def _web_server(
|
|
485
492
|
startup_timeout: float = 5.0, # Maximum number of seconds to wait for the web server to start.
|
486
493
|
label: Optional[str] = None, # Label for created endpoint. Final subdomain will be <workspace>--<label>.modal.run.
|
487
494
|
custom_domains: Optional[Iterable[str]] = None, # Deploy this endpoint on a custom domain.
|
495
|
+
requires_proxy_auth: bool = False, # Require Proxy-Authorization HTTP Headers on requests to the endpoint
|
488
496
|
) -> Callable[[Callable[..., Any]], _PartialFunction]:
|
489
497
|
"""Decorator that registers an HTTP web server inside the container.
|
490
498
|
|
@@ -528,6 +536,7 @@ def _web_server(
|
|
528
536
|
custom_domains=_parse_custom_domains(custom_domains),
|
529
537
|
web_server_port=port,
|
530
538
|
web_server_startup_timeout=startup_timeout,
|
539
|
+
requires_proxy_auth=requires_proxy_auth,
|
531
540
|
),
|
532
541
|
)
|
533
542
|
|
modal/partial_function.pyi
CHANGED
@@ -118,6 +118,7 @@ def _web_endpoint(
|
|
118
118
|
label: typing.Optional[str] = None,
|
119
119
|
docs: bool = False,
|
120
120
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
121
|
+
requires_proxy_auth: bool = False,
|
121
122
|
wait_for_response: bool = True,
|
122
123
|
) -> typing.Callable[[typing.Callable[P, ReturnType]], _PartialFunction[P, ReturnType, ReturnType]]: ...
|
123
124
|
def _asgi_app(
|
@@ -125,6 +126,7 @@ def _asgi_app(
|
|
125
126
|
*,
|
126
127
|
label: typing.Optional[str] = None,
|
127
128
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
129
|
+
requires_proxy_auth: bool = False,
|
128
130
|
wait_for_response: bool = True,
|
129
131
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], _PartialFunction]: ...
|
130
132
|
def _wsgi_app(
|
@@ -132,6 +134,7 @@ def _wsgi_app(
|
|
132
134
|
*,
|
133
135
|
label: typing.Optional[str] = None,
|
134
136
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
137
|
+
requires_proxy_auth: bool = False,
|
135
138
|
wait_for_response: bool = True,
|
136
139
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], _PartialFunction]: ...
|
137
140
|
def _web_server(
|
@@ -140,6 +143,7 @@ def _web_server(
|
|
140
143
|
startup_timeout: float = 5.0,
|
141
144
|
label: typing.Optional[str] = None,
|
142
145
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
146
|
+
requires_proxy_auth: bool = False,
|
143
147
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], _PartialFunction]: ...
|
144
148
|
def _disallow_wrapping_method(f: _PartialFunction, wrapper: str) -> None: ...
|
145
149
|
def _build(
|
@@ -178,6 +182,7 @@ def web_endpoint(
|
|
178
182
|
label: typing.Optional[str] = None,
|
179
183
|
docs: bool = False,
|
180
184
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
185
|
+
requires_proxy_auth: bool = False,
|
181
186
|
wait_for_response: bool = True,
|
182
187
|
) -> typing.Callable[[typing.Callable[P, ReturnType]], PartialFunction[P, ReturnType, ReturnType]]: ...
|
183
188
|
def asgi_app(
|
@@ -185,6 +190,7 @@ def asgi_app(
|
|
185
190
|
*,
|
186
191
|
label: typing.Optional[str] = None,
|
187
192
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
193
|
+
requires_proxy_auth: bool = False,
|
188
194
|
wait_for_response: bool = True,
|
189
195
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], PartialFunction]: ...
|
190
196
|
def wsgi_app(
|
@@ -192,6 +198,7 @@ def wsgi_app(
|
|
192
198
|
*,
|
193
199
|
label: typing.Optional[str] = None,
|
194
200
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
201
|
+
requires_proxy_auth: bool = False,
|
195
202
|
wait_for_response: bool = True,
|
196
203
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], PartialFunction]: ...
|
197
204
|
def web_server(
|
@@ -200,6 +207,7 @@ def web_server(
|
|
200
207
|
startup_timeout: float = 5.0,
|
201
208
|
label: typing.Optional[str] = None,
|
202
209
|
custom_domains: typing.Optional[collections.abc.Iterable[str]] = None,
|
210
|
+
requires_proxy_auth: bool = False,
|
203
211
|
) -> typing.Callable[[typing.Callable[..., typing.Any]], PartialFunction]: ...
|
204
212
|
def build(
|
205
213
|
_warn_parentheses_missing=None, *, force: bool = False, timeout: int = 86400
|