modal 0.67.43__py3-none-any.whl → 0.68.24__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. modal/__init__.py +2 -0
  2. modal/_container_entrypoint.py +4 -1
  3. modal/_ipython.py +3 -13
  4. modal/_runtime/asgi.py +4 -0
  5. modal/_runtime/container_io_manager.py +3 -0
  6. modal/_runtime/user_code_imports.py +17 -20
  7. modal/_traceback.py +16 -2
  8. modal/_utils/blob_utils.py +27 -92
  9. modal/_utils/bytes_io_segment_payload.py +97 -0
  10. modal/_utils/function_utils.py +5 -1
  11. modal/_utils/grpc_testing.py +6 -2
  12. modal/_utils/hash_utils.py +51 -10
  13. modal/_utils/http_utils.py +19 -10
  14. modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
  15. modal/_utils/shell_utils.py +11 -5
  16. modal/cli/_traceback.py +11 -4
  17. modal/cli/run.py +25 -12
  18. modal/client.py +6 -37
  19. modal/client.pyi +2 -6
  20. modal/cls.py +132 -62
  21. modal/cls.pyi +13 -7
  22. modal/exception.py +20 -0
  23. modal/file_io.py +380 -0
  24. modal/file_io.pyi +185 -0
  25. modal/file_pattern_matcher.py +121 -0
  26. modal/functions.py +33 -11
  27. modal/functions.pyi +11 -9
  28. modal/image.py +88 -8
  29. modal/image.pyi +20 -4
  30. modal/mount.py +49 -9
  31. modal/mount.pyi +19 -4
  32. modal/network_file_system.py +4 -1
  33. modal/object.py +4 -2
  34. modal/partial_function.py +22 -10
  35. modal/partial_function.pyi +10 -2
  36. modal/runner.py +5 -4
  37. modal/runner.pyi +2 -1
  38. modal/sandbox.py +40 -0
  39. modal/sandbox.pyi +18 -0
  40. modal/volume.py +5 -1
  41. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/METADATA +2 -2
  42. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/RECORD +52 -48
  43. modal_docs/gen_reference_docs.py +1 -0
  44. modal_proto/api.proto +33 -1
  45. modal_proto/api_pb2.py +813 -737
  46. modal_proto/api_pb2.pyi +160 -13
  47. modal_version/__init__.py +1 -1
  48. modal_version/_version_generated.py +1 -1
  49. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/LICENSE +0 -0
  50. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/WHEEL +0 -0
  51. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/entry_points.txt +0 -0
  52. {modal-0.67.43.dist-info → modal-0.68.24.dist-info}/top_level.txt +0 -0
modal/functions.py CHANGED
@@ -32,6 +32,7 @@ from ._resolver import Resolver
32
32
  from ._resources import convert_fn_config_to_resources_config
33
33
  from ._runtime.execution_context import current_input_id, is_local
34
34
  from ._serialization import serialize, serialize_proto_params
35
+ from ._traceback import print_server_warnings
35
36
  from ._utils.async_utils import (
36
37
  TaskContext,
37
38
  async_merge,
@@ -431,7 +432,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
431
432
  image: _Image,
432
433
  secrets: Sequence[_Secret] = (),
433
434
  schedule: Optional[Schedule] = None,
434
- is_generator=False,
435
+ is_generator: bool = False,
435
436
  gpu: Union[GPU_T, list[GPU_T]] = None,
436
437
  # TODO: maybe break this out into a separate decorator for notebooks.
437
438
  mounts: Collection[_Mount] = (),
@@ -627,7 +628,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
627
628
  raise InvalidError(f"Expected modal.Image object. Got {type(image)}.")
628
629
 
629
630
  method_definitions: Optional[dict[str, api_pb2.MethodDefinition]] = None
630
- partial_functions: dict[str, "modal.partial_function._PartialFunction"] = {}
631
+
631
632
  if info.user_cls:
632
633
  method_definitions = {}
633
634
  partial_functions = _find_partial_methods_for_user_cls(info.user_cls, _PartialFunctionFlags.FUNCTION)
@@ -1061,6 +1062,8 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1061
1062
  else:
1062
1063
  raise
1063
1064
 
1065
+ print_server_warnings(response.server_warnings)
1066
+
1064
1067
  self._hydrate(response.function_id, resolver.client, response.handle_metadata)
1065
1068
 
1066
1069
  rep = f"Ref({app_name})"
@@ -1188,9 +1191,16 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1188
1191
  return self._web_url
1189
1192
 
1190
1193
  @property
1191
- def is_generator(self) -> bool:
1194
+ async def is_generator(self) -> bool:
1192
1195
  """mdmd:hidden"""
1193
- assert self._is_generator is not None
1196
+ # hacky: kind of like @live_method, but not hydrating if we have the value already from local source
1197
+ if self._is_generator is not None:
1198
+ # this is set if the function or class is local
1199
+ return self._is_generator
1200
+
1201
+ # not set - this is a from_name lookup - hydrate
1202
+ await self.resolve()
1203
+ assert self._is_generator is not None # should be set now
1194
1204
  return self._is_generator
1195
1205
 
1196
1206
  @property
@@ -1272,6 +1282,10 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1272
1282
 
1273
1283
  @synchronizer.no_io_translation
1274
1284
  async def _call_generator_nowait(self, args, kwargs):
1285
+ deprecation_warning(
1286
+ (2024, 12, 11),
1287
+ "Calling spawn on a generator function is deprecated and will soon raise an exception.",
1288
+ )
1275
1289
  return await _Invocation.create(
1276
1290
  self,
1277
1291
  args,
@@ -1311,6 +1325,9 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1311
1325
  async for item in self._call_generator(args, kwargs): # type: ignore
1312
1326
  yield item
1313
1327
 
1328
+ def _is_local(self):
1329
+ return self._info is not None
1330
+
1314
1331
  def _get_info(self) -> FunctionInfo:
1315
1332
  if not self._info:
1316
1333
  raise ExecutionError("Can't get info for a function that isn't locally defined")
@@ -1335,19 +1352,24 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1335
1352
  """
1336
1353
  # TODO(erikbern): it would be nice to remove the nowrap thing, but right now that would cause
1337
1354
  # "user code" to run on the synchronicity thread, which seems bad
1355
+ if not self._is_local():
1356
+ msg = (
1357
+ "The definition for this function is missing here so it is not possible to invoke it locally. "
1358
+ "If this function was retrieved via `Function.lookup` you need to use `.remote()`."
1359
+ )
1360
+ raise ExecutionError(msg)
1361
+
1338
1362
  info = self._get_info()
1363
+ if not info.raw_f:
1364
+ # Here if calling .local on a service function itself which should never happen
1365
+ # TODO: check if we end up here in a container for a serialized function?
1366
+ raise ExecutionError("Can't call .local on service function")
1339
1367
 
1340
1368
  if is_local() and self.spec.volumes or self.spec.network_file_systems:
1341
1369
  warnings.warn(
1342
1370
  f"The {info.function_name} function is executing locally "
1343
1371
  + "and will not have access to the mounted Volume or NetworkFileSystem data"
1344
1372
  )
1345
- if not info or not info.raw_f:
1346
- msg = (
1347
- "The definition for this function is missing so it is not possible to invoke it locally. "
1348
- "If this function was retrieved via `Function.lookup` you need to use `.remote()`."
1349
- )
1350
- raise ExecutionError(msg)
1351
1373
 
1352
1374
  obj: Optional["modal.cls._Obj"] = self._get_obj()
1353
1375
 
@@ -1357,9 +1379,9 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], _Object, type
1357
1379
  else:
1358
1380
  # This is a method on a class, so bind the self to the function
1359
1381
  user_cls_instance = obj._cached_user_cls_instance()
1360
-
1361
1382
  fun = info.raw_f.__get__(user_cls_instance)
1362
1383
 
1384
+ # TODO: replace implicit local enter/exit with a context manager
1363
1385
  if is_async(info.raw_f):
1364
1386
  # We want to run __aenter__ and fun in the same coroutine
1365
1387
  async def coro():
modal/functions.pyi CHANGED
@@ -157,7 +157,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
157
157
  image: modal.image._Image,
158
158
  secrets: collections.abc.Sequence[modal.secret._Secret] = (),
159
159
  schedule: typing.Optional[modal.schedule.Schedule] = None,
160
- is_generator=False,
160
+ is_generator: bool = False,
161
161
  gpu: typing.Union[
162
162
  None, bool, str, modal.gpu._GPUConfig, list[typing.Union[None, bool, str, modal.gpu._GPUConfig]]
163
163
  ] = None,
@@ -234,7 +234,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
234
234
  @property
235
235
  async def web_url(self) -> str: ...
236
236
  @property
237
- def is_generator(self) -> bool: ...
237
+ async def is_generator(self) -> bool: ...
238
238
  @property
239
239
  def cluster_size(self) -> int: ...
240
240
  def _map(
@@ -246,6 +246,7 @@ class _Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.
246
246
  async def _call_generator_nowait(self, args, kwargs): ...
247
247
  async def remote(self, *args: P.args, **kwargs: P.kwargs) -> ReturnType: ...
248
248
  def remote_gen(self, *args, **kwargs) -> collections.abc.AsyncGenerator[typing.Any, None]: ...
249
+ def _is_local(self): ...
249
250
  def _get_info(self) -> modal._utils.function_utils.FunctionInfo: ...
250
251
  def _get_obj(self) -> typing.Optional[modal.cls._Obj]: ...
251
252
  def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
@@ -325,7 +326,7 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
325
326
  image: modal.image.Image,
326
327
  secrets: collections.abc.Sequence[modal.secret.Secret] = (),
327
328
  schedule: typing.Optional[modal.schedule.Schedule] = None,
328
- is_generator=False,
329
+ is_generator: bool = False,
329
330
  gpu: typing.Union[
330
331
  None, bool, str, modal.gpu._GPUConfig, list[typing.Union[None, bool, str, modal.gpu._GPUConfig]]
331
332
  ] = None,
@@ -455,11 +456,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
455
456
 
456
457
  _call_generator_nowait: ___call_generator_nowait_spec
457
458
 
458
- class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
459
+ class __remote_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
459
460
  def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
460
461
  async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
461
462
 
462
- remote: __remote_spec[P, ReturnType]
463
+ remote: __remote_spec[ReturnType, P]
463
464
 
464
465
  class __remote_gen_spec(typing_extensions.Protocol):
465
466
  def __call__(self, *args, **kwargs) -> typing.Generator[typing.Any, None, None]: ...
@@ -467,21 +468,22 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
467
468
 
468
469
  remote_gen: __remote_gen_spec
469
470
 
471
+ def _is_local(self): ...
470
472
  def _get_info(self) -> modal._utils.function_utils.FunctionInfo: ...
471
473
  def _get_obj(self) -> typing.Optional[modal.cls.Obj]: ...
472
474
  def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
473
475
 
474
- class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
476
+ class ___experimental_spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
475
477
  def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
476
478
  async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
477
479
 
478
- _experimental_spawn: ___experimental_spawn_spec[P, ReturnType]
480
+ _experimental_spawn: ___experimental_spawn_spec[ReturnType, P]
479
481
 
480
- class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
482
+ class __spawn_spec(typing_extensions.Protocol[ReturnType_INNER, P_INNER]):
481
483
  def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
482
484
  async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
483
485
 
484
- spawn: __spawn_spec[P, ReturnType]
486
+ spawn: __spawn_spec[ReturnType, P]
485
487
 
486
488
  def get_raw_f(self) -> typing.Callable[..., typing.Any]: ...
487
489
 
modal/image.py CHANGED
@@ -37,6 +37,7 @@ from .cloud_bucket_mount import _CloudBucketMount
37
37
  from .config import config, logger, user_config_path
38
38
  from .environments import _get_environment_cached
39
39
  from .exception import InvalidError, NotFoundError, RemoteError, VersionError, deprecation_error, deprecation_warning
40
+ from .file_pattern_matcher import FilePatternMatcher
40
41
  from .gpu import GPU_T, parse_gpu_config
41
42
  from .mount import _Mount, python_standalone_mount_name
42
43
  from .network_file_system import _NetworkFileSystem
@@ -638,7 +639,17 @@ class _Image(_Object, type_prefix="im"):
638
639
  mount = _Mount.from_local_file(local_path, remote_path)
639
640
  return self._add_mount_layer_or_copy(mount, copy=copy)
640
641
 
641
- def add_local_dir(self, local_path: Union[str, Path], remote_path: str, *, copy: bool = False) -> "_Image":
642
+ def add_local_dir(
643
+ self,
644
+ local_path: Union[str, Path],
645
+ remote_path: str,
646
+ *,
647
+ copy: bool = False,
648
+ # Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
649
+ # Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
650
+ # Which follows dockerignore syntax.
651
+ ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
652
+ ) -> "_Image":
642
653
  """Adds a local directory's content to the image at `remote_path` within the container
643
654
 
644
655
  By default (`copy=False`), the files are added to containers on startup and are not built into the actual Image,
@@ -650,12 +661,44 @@ class _Image(_Object, type_prefix="im"):
650
661
  copy=True can slow down iteration since it requires a rebuild of the Image and any subsequent
651
662
  build steps whenever the included files change, but it is required if you want to run additional
652
663
  build steps after this one.
664
+
665
+ **Usage:**
666
+
667
+ ```python
668
+ from modal import FilePatternMatcher
669
+
670
+ image = modal.Image.debian_slim().add_local_dir(
671
+ "~/assets",
672
+ remote_path="/assets",
673
+ ignore=["*.venv"],
674
+ )
675
+
676
+ image = modal.Image.debian_slim().add_local_dir(
677
+ "~/assets",
678
+ remote_path="/assets",
679
+ ignore=lambda p: p.is_relative_to(".venv"),
680
+ )
681
+
682
+ image = modal.Image.debian_slim().copy_local_dir(
683
+ "~/assets",
684
+ remote_path="/assets",
685
+ ignore=FilePatternMatcher("**/*.txt"),
686
+ )
687
+
688
+ # When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
689
+ image = modal.Image.debian_slim().copy_local_dir(
690
+ "~/assets",
691
+ remote_path="/assets",
692
+ ignore=~FilePatternMatcher("**/*.py"),
693
+ )
694
+ ```
653
695
  """
654
696
  if not PurePosixPath(remote_path).is_absolute():
655
697
  # TODO(elias): implement relative to absolute resolution using image workdir metadata
656
698
  # + make default remote_path="./"
657
699
  raise InvalidError("image.add_local_dir() currently only supports absolute remote_path values")
658
- mount = _Mount.from_local_dir(local_path, remote_path=remote_path)
700
+
701
+ mount = _Mount._add_local_dir(Path(local_path), Path(remote_path), ignore)
659
702
  return self._add_mount_layer_or_copy(mount, copy=copy)
660
703
 
661
704
  def copy_local_file(self, local_path: Union[str, Path], remote_path: Union[str, Path] = "./") -> "_Image":
@@ -697,19 +740,56 @@ class _Image(_Object, type_prefix="im"):
697
740
  the destination directory.
698
741
  """
699
742
 
700
- def only_py_files(filename):
701
- return filename.endswith(".py")
702
-
703
- mount = _Mount.from_local_python_packages(*modules, condition=only_py_files)
743
+ mount = _Mount.from_local_python_packages(*modules, ignore=~FilePatternMatcher("**/*.py"))
704
744
  return self._add_mount_layer_or_copy(mount, copy=copy)
705
745
 
706
- def copy_local_dir(self, local_path: Union[str, Path], remote_path: Union[str, Path] = ".") -> "_Image":
746
+ def copy_local_dir(
747
+ self,
748
+ local_path: Union[str, Path],
749
+ remote_path: Union[str, Path] = ".",
750
+ # Predicate filter function for file exclusion, which should accept a filepath and return `True` for exclusion.
751
+ # Defaults to excluding no files. If a Sequence is provided, it will be converted to a FilePatternMatcher.
752
+ # Which follows dockerignore syntax.
753
+ ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
754
+ ) -> "_Image":
707
755
  """Copy a directory into the image as a part of building the image.
708
756
 
709
757
  This works in a similar way to [`COPY`](https://docs.docker.com/engine/reference/builder/#copy)
710
758
  works in a `Dockerfile`.
759
+
760
+ **Usage:**
761
+
762
+ ```python
763
+ from modal import FilePatternMatcher
764
+
765
+ image = modal.Image.debian_slim().copy_local_dir(
766
+ "~/assets",
767
+ remote_path="/assets",
768
+ ignore=["**/*.venv"],
769
+ )
770
+
771
+ image = modal.Image.debian_slim().copy_local_dir(
772
+ "~/assets",
773
+ remote_path="/assets",
774
+ ignore=lambda p: p.is_relative_to(".venv"),
775
+ )
776
+
777
+ image = modal.Image.debian_slim().copy_local_dir(
778
+ "~/assets",
779
+ remote_path="/assets",
780
+ ignore=FilePatternMatcher("**/*.txt"),
781
+ )
782
+
783
+ # When including files is simpler than excluding them, you can use the `~` operator to invert the matcher.
784
+ image = modal.Image.debian_slim().copy_local_dir(
785
+ "~/assets",
786
+ remote_path="/assets",
787
+ ignore=~FilePatternMatcher("**/*.py"),
788
+ )
789
+ ```
711
790
  """
712
- mount = _Mount.from_local_dir(local_path, remote_path="/")
791
+
792
+ mount = _Mount._add_local_dir(Path(local_path), Path("/"), ignore)
713
793
 
714
794
  def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
715
795
  return DockerfileSpec(commands=["FROM base", f"COPY . {remote_path}"], context_files={})
modal/image.pyi CHANGED
@@ -110,14 +110,22 @@ class _Image(modal.object._Object):
110
110
  self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
111
111
  ) -> _Image: ...
112
112
  def add_local_dir(
113
- self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
113
+ self,
114
+ local_path: typing.Union[str, pathlib.Path],
115
+ remote_path: str,
116
+ *,
117
+ copy: bool = False,
118
+ ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
114
119
  ) -> _Image: ...
115
120
  def copy_local_file(
116
121
  self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
117
122
  ) -> _Image: ...
118
123
  def add_local_python_source(self, *module_names: str, copy: bool = False) -> _Image: ...
119
124
  def copy_local_dir(
120
- self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "."
125
+ self,
126
+ local_path: typing.Union[str, pathlib.Path],
127
+ remote_path: typing.Union[str, pathlib.Path] = ".",
128
+ ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
121
129
  ) -> _Image: ...
122
130
  def pip_install(
123
131
  self,
@@ -367,14 +375,22 @@ class Image(modal.object.Object):
367
375
  self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
368
376
  ) -> Image: ...
369
377
  def add_local_dir(
370
- self, local_path: typing.Union[str, pathlib.Path], remote_path: str, *, copy: bool = False
378
+ self,
379
+ local_path: typing.Union[str, pathlib.Path],
380
+ remote_path: str,
381
+ *,
382
+ copy: bool = False,
383
+ ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
371
384
  ) -> Image: ...
372
385
  def copy_local_file(
373
386
  self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "./"
374
387
  ) -> Image: ...
375
388
  def add_local_python_source(self, *module_names: str, copy: bool = False) -> Image: ...
376
389
  def copy_local_dir(
377
- self, local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.Path] = "."
390
+ self,
391
+ local_path: typing.Union[str, pathlib.Path],
392
+ remote_path: typing.Union[str, pathlib.Path] = ".",
393
+ ignore: typing.Union[collections.abc.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
378
394
  ) -> Image: ...
379
395
  def pip_install(
380
396
  self,
modal/mount.py CHANGED
@@ -11,7 +11,7 @@ import time
11
11
  import typing
12
12
  from collections.abc import AsyncGenerator
13
13
  from pathlib import Path, PurePosixPath
14
- from typing import Callable, Optional, Union
14
+ from typing import Callable, Optional, Sequence, Union
15
15
 
16
16
  from google.protobuf.message import Message
17
17
 
@@ -27,7 +27,8 @@ from ._utils.name_utils import check_object_name
27
27
  from ._utils.package_utils import get_module_mount_info
28
28
  from .client import _Client
29
29
  from .config import config, logger
30
- from .exception import ModuleNotMountable
30
+ from .exception import InvalidError, ModuleNotMountable
31
+ from .file_pattern_matcher import FilePatternMatcher
31
32
  from .object import _get_environment_name, _Object
32
33
 
33
34
  ROOT_DIR: PurePosixPath = PurePosixPath("/root")
@@ -122,7 +123,7 @@ class _MountFile(_MountEntry):
122
123
  class _MountDir(_MountEntry):
123
124
  local_dir: Path
124
125
  remote_path: PurePosixPath
125
- condition: Callable[[str], bool]
126
+ ignore: Callable[[Path], bool]
126
127
  recursive: bool
127
128
 
128
129
  def description(self):
@@ -143,7 +144,7 @@ class _MountDir(_MountEntry):
143
144
  gen = (dir_entry.path for dir_entry in os.scandir(local_dir) if dir_entry.is_file())
144
145
 
145
146
  for local_filename in gen:
146
- if self.condition(local_filename):
147
+ if not self.ignore(Path(local_filename)):
147
148
  local_relpath = Path(local_filename).expanduser().absolute().relative_to(local_dir)
148
149
  mount_path = self.remote_path / local_relpath.as_posix()
149
150
  yield local_filename, mount_path
@@ -182,6 +183,10 @@ def module_mount_condition(module_base: Path):
182
183
  return condition
183
184
 
184
185
 
186
+ def module_mount_ignore_condition(module_base: Path):
187
+ return lambda f: not module_mount_condition(module_base)(str(f))
188
+
189
+
185
190
  @dataclasses.dataclass
186
191
  class _MountedPythonModule(_MountEntry):
187
192
  # the purpose of this is to keep printable information about which Python package
@@ -190,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
190
195
 
191
196
  module_name: str
192
197
  remote_dir: Union[PurePosixPath, str] = ROOT_DIR.as_posix() # cast needed here for type stub generation...
193
- condition: typing.Optional[typing.Callable[[str], bool]] = None
198
+ ignore: Optional[Callable[[Path], bool]] = None
194
199
 
195
200
  def description(self) -> str:
196
201
  return f"PythonPackage:{self.module_name}"
@@ -206,7 +211,7 @@ class _MountedPythonModule(_MountEntry):
206
211
  _MountDir(
207
212
  base_path,
208
213
  remote_path=remote_dir,
209
- condition=self.condition or module_mount_condition(base_path),
214
+ ignore=self.ignore or module_mount_ignore_condition(base_path),
210
215
  recursive=True,
211
216
  )
212
217
  )
@@ -313,6 +318,24 @@ class _Mount(_Object, type_prefix="mo"):
313
318
  # we can't rely on it to be set. Let's clean this up later.
314
319
  return getattr(self, "_is_local", False)
315
320
 
321
+ @staticmethod
322
+ def _add_local_dir(
323
+ local_path: Path,
324
+ remote_path: Path,
325
+ ignore: Union[Sequence[str], Callable[[Path], bool]] = [],
326
+ ):
327
+ if isinstance(ignore, list):
328
+ ignore = FilePatternMatcher(*ignore)
329
+
330
+ return _Mount._new()._extend(
331
+ _MountDir(
332
+ local_dir=local_path,
333
+ ignore=ignore,
334
+ remote_path=remote_path,
335
+ recursive=True,
336
+ ),
337
+ )
338
+
316
339
  def add_local_dir(
317
340
  self,
318
341
  local_path: Union[str, Path],
@@ -339,10 +362,13 @@ class _Mount(_Object, type_prefix="mo"):
339
362
 
340
363
  condition = include_all
341
364
 
365
+ def converted_condition(path: Path) -> bool:
366
+ return not condition(str(path))
367
+
342
368
  return self._extend(
343
369
  _MountDir(
344
370
  local_dir=local_path,
345
- condition=condition,
371
+ ignore=converted_condition,
346
372
  remote_path=remote_path,
347
373
  recursive=recursive,
348
374
  ),
@@ -483,7 +509,9 @@ class _Mount(_Object, type_prefix="mo"):
483
509
  logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
484
510
  async with blob_upload_concurrency:
485
511
  with file_spec.source() as fp:
486
- blob_id = await blob_upload_file(fp, resolver.client.stub)
512
+ blob_id = await blob_upload_file(
513
+ fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
514
+ )
487
515
  logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
488
516
  request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
489
517
  else:
@@ -549,6 +577,7 @@ class _Mount(_Object, type_prefix="mo"):
549
577
  # Predicate filter function for file selection, which should accept a filepath and return `True` for inclusion.
550
578
  # Defaults to including all files.
551
579
  condition: Optional[Callable[[str], bool]] = None,
580
+ ignore: Optional[Union[Sequence[str], Callable[[Path], bool]]] = None,
552
581
  ) -> "_Mount":
553
582
  """
554
583
  Returns a `modal.Mount` that makes local modules listed in `module_names` available inside the container.
@@ -573,13 +602,24 @@ class _Mount(_Object, type_prefix="mo"):
573
602
 
574
603
  # Don't re-run inside container.
575
604
 
605
+ if condition is not None:
606
+ if ignore is not None:
607
+ raise InvalidError("Cannot specify both `ignore` and `condition`")
608
+
609
+ def converted_condition(path: Path) -> bool:
610
+ return not condition(str(path))
611
+
612
+ ignore = converted_condition
613
+ elif isinstance(ignore, list):
614
+ ignore = FilePatternMatcher(*ignore)
615
+
576
616
  mount = _Mount._new()
577
617
  from ._runtime.execution_context import is_local
578
618
 
579
619
  if not is_local():
580
620
  return mount # empty/non-mountable mount in case it's used from within a container
581
621
  for module_name in module_names:
582
- mount = mount._extend(_MountedPythonModule(module_name, remote_dir, condition))
622
+ mount = mount._extend(_MountedPythonModule(module_name, remote_dir, ignore))
583
623
  return mount
584
624
 
585
625
  @staticmethod
modal/mount.pyi CHANGED
@@ -35,7 +35,7 @@ class _MountFile(_MountEntry):
35
35
  class _MountDir(_MountEntry):
36
36
  local_dir: pathlib.Path
37
37
  remote_path: pathlib.PurePosixPath
38
- condition: typing.Callable[[str], bool]
38
+ ignore: typing.Callable[[pathlib.Path], bool]
39
39
  recursive: bool
40
40
 
41
41
  def description(self): ...
@@ -46,18 +46,19 @@ class _MountDir(_MountEntry):
46
46
  self,
47
47
  local_dir: pathlib.Path,
48
48
  remote_path: pathlib.PurePosixPath,
49
- condition: typing.Callable[[str], bool],
49
+ ignore: typing.Callable[[pathlib.Path], bool],
50
50
  recursive: bool,
51
51
  ) -> None: ...
52
52
  def __repr__(self): ...
53
53
  def __eq__(self, other): ...
54
54
 
55
55
  def module_mount_condition(module_base: pathlib.Path): ...
56
+ def module_mount_ignore_condition(module_base: pathlib.Path): ...
56
57
 
57
58
  class _MountedPythonModule(_MountEntry):
58
59
  module_name: str
59
60
  remote_dir: typing.Union[pathlib.PurePosixPath, str]
60
- condition: typing.Optional[typing.Callable[[str], bool]]
61
+ ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]]
61
62
 
62
63
  def description(self) -> str: ...
63
64
  def _proxy_entries(self) -> list[_MountEntry]: ...
@@ -68,7 +69,7 @@ class _MountedPythonModule(_MountEntry):
68
69
  self,
69
70
  module_name: str,
70
71
  remote_dir: typing.Union[pathlib.PurePosixPath, str] = "/root",
71
- condition: typing.Optional[typing.Callable[[str], bool]] = None,
72
+ ignore: typing.Optional[typing.Callable[[pathlib.Path], bool]] = None,
72
73
  ) -> None: ...
73
74
  def __repr__(self): ...
74
75
  def __eq__(self, other): ...
@@ -90,6 +91,12 @@ class _Mount(modal.object._Object):
90
91
  def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
91
92
  def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
92
93
  def is_local(self) -> bool: ...
94
+ @staticmethod
95
+ def _add_local_dir(
96
+ local_path: pathlib.Path,
97
+ remote_path: pathlib.Path,
98
+ ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
99
+ ): ...
93
100
  def add_local_dir(
94
101
  self,
95
102
  local_path: typing.Union[str, pathlib.Path],
@@ -129,6 +136,7 @@ class _Mount(modal.object._Object):
129
136
  *module_names: str,
130
137
  remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
131
138
  condition: typing.Optional[typing.Callable[[str], bool]] = None,
139
+ ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
132
140
  ) -> _Mount: ...
133
141
  @staticmethod
134
142
  def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> _Mount: ...
@@ -165,6 +173,12 @@ class Mount(modal.object.Object):
165
173
  def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
166
174
  def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
167
175
  def is_local(self) -> bool: ...
176
+ @staticmethod
177
+ def _add_local_dir(
178
+ local_path: pathlib.Path,
179
+ remote_path: pathlib.Path,
180
+ ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool]] = [],
181
+ ): ...
168
182
  def add_local_dir(
169
183
  self,
170
184
  local_path: typing.Union[str, pathlib.Path],
@@ -214,6 +228,7 @@ class Mount(modal.object.Object):
214
228
  *module_names: str,
215
229
  remote_dir: typing.Union[str, pathlib.PurePosixPath] = "/root",
216
230
  condition: typing.Optional[typing.Callable[[str], bool]] = None,
231
+ ignore: typing.Union[typing.Sequence[str], typing.Callable[[pathlib.Path], bool], None] = None,
217
232
  ) -> Mount: ...
218
233
  @staticmethod
219
234
  def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> Mount: ...
@@ -245,7 +245,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
245
245
  if data_size > LARGE_FILE_LIMIT:
246
246
  progress_task_id = progress_cb(name=remote_path, size=data_size)
247
247
  blob_id = await blob_upload_file(
248
- fp, self._client.stub, progress_report_cb=functools.partial(progress_cb, progress_task_id)
248
+ fp,
249
+ self._client.stub,
250
+ progress_report_cb=functools.partial(progress_cb, progress_task_id),
251
+ sha256_hex=sha_hash,
249
252
  )
250
253
  req = api_pb2.SharedVolumePutFileRequest(
251
254
  shared_volume_id=self.object_id,
modal/object.py CHANGED
@@ -96,7 +96,9 @@ class _Object:
96
96
 
97
97
  def _initialize_from_other(self, other):
98
98
  # default implementation, can be overriden in subclasses
99
- pass
99
+ self._object_id = other._object_id
100
+ self._is_hydrated = other._is_hydrated
101
+ self._client = other._client
100
102
 
101
103
  def _hydrate(self, object_id: str, client: _Client, metadata: Optional[Message]):
102
104
  assert isinstance(object_id, str)
@@ -139,7 +141,7 @@ class _Object:
139
141
 
140
142
  # Object to clone must already be hydrated, otherwise from_loader is more suitable.
141
143
  self._validate_is_hydrated()
142
- obj = _Object.__new__(type(self))
144
+ obj = type(self).__new__(type(self))
143
145
  obj._initialize_from_other(self)
144
146
  return obj
145
147