modal 0.68.24__py3-none-any.whl → 0.68.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/_utils/deprecation.py +44 -0
- modal/app.py +11 -31
- modal/app.pyi +3 -4
- modal/cli/app.py +1 -1
- modal/client.pyi +2 -2
- modal/config.py +2 -1
- modal/container_process.py +2 -1
- modal/dict.py +2 -1
- modal/exception.py +0 -54
- modal/file_io.py +54 -7
- modal/file_io.pyi +18 -8
- modal/file_pattern_matcher.py +48 -15
- modal/functions.py +2 -8
- modal/functions.pyi +11 -7
- modal/image.py +21 -5
- modal/image.pyi +16 -2
- modal/network_file_system.py +2 -1
- modal/partial_function.py +2 -1
- modal/queue.py +2 -1
- modal/runner.py +2 -7
- modal/sandbox.py +23 -13
- modal/sandbox.pyi +21 -0
- modal/serving.py +1 -1
- modal/volume.py +2 -1
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/METADATA +1 -1
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/RECORD +34 -33
- modal_proto/api.proto +1 -1
- modal_proto/api_pb2.py +750 -750
- modal_proto/api_pb2.pyi +4 -4
- modal_version/_version_generated.py +1 -1
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/LICENSE +0 -0
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/WHEEL +0 -0
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/entry_points.txt +0 -0
- {modal-0.68.24.dist-info → modal-0.68.31.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
# Copyright Modal Labs 2024
|
2
|
+
import sys
|
3
|
+
import warnings
|
4
|
+
from datetime import date
|
5
|
+
|
6
|
+
from ..exception import DeprecationError, PendingDeprecationError
|
7
|
+
|
8
|
+
_INTERNAL_MODULES = ["modal", "synchronicity"]
|
9
|
+
|
10
|
+
|
11
|
+
def _is_internal_frame(frame):
|
12
|
+
module = frame.f_globals["__name__"].split(".")[0]
|
13
|
+
return module in _INTERNAL_MODULES
|
14
|
+
|
15
|
+
|
16
|
+
def deprecation_error(deprecated_on: tuple[int, int, int], msg: str):
|
17
|
+
raise DeprecationError(f"Deprecated on {date(*deprecated_on)}: {msg}")
|
18
|
+
|
19
|
+
|
20
|
+
def deprecation_warning(
|
21
|
+
deprecated_on: tuple[int, int, int], msg: str, *, pending: bool = False, show_source: bool = True
|
22
|
+
) -> None:
|
23
|
+
"""Issue a Modal deprecation warning with source optionally attributed to user code.
|
24
|
+
|
25
|
+
See the implementation of the built-in [warnings.warn](https://docs.python.org/3/library/warnings.html#available-functions).
|
26
|
+
"""
|
27
|
+
filename, lineno = "<unknown>", 0
|
28
|
+
if show_source:
|
29
|
+
# Find the last non-Modal line that triggered the warning
|
30
|
+
try:
|
31
|
+
frame = sys._getframe()
|
32
|
+
while frame is not None and _is_internal_frame(frame):
|
33
|
+
frame = frame.f_back
|
34
|
+
if frame is not None:
|
35
|
+
filename = frame.f_code.co_filename
|
36
|
+
lineno = frame.f_lineno
|
37
|
+
except ValueError:
|
38
|
+
# Use the defaults from above
|
39
|
+
pass
|
40
|
+
|
41
|
+
warning_cls = PendingDeprecationError if pending else DeprecationError
|
42
|
+
|
43
|
+
# This is a lower-level function that warnings.warn uses
|
44
|
+
warnings.warn_explicit(f"{date(*deprecated_on)}: {msg}", warning_cls, filename, lineno)
|
modal/app.py
CHANGED
@@ -22,6 +22,7 @@ from modal_proto import api_pb2
|
|
22
22
|
|
23
23
|
from ._ipython import is_notebook
|
24
24
|
from ._utils.async_utils import synchronize_api
|
25
|
+
from ._utils.deprecation import deprecation_error, deprecation_warning
|
25
26
|
from ._utils.function_utils import FunctionInfo, is_global_object, is_method_fn
|
26
27
|
from ._utils.grpc_utils import retry_transient_errors
|
27
28
|
from ._utils.mount_utils import validate_volumes
|
@@ -29,7 +30,7 @@ from .client import _Client
|
|
29
30
|
from .cloud_bucket_mount import _CloudBucketMount
|
30
31
|
from .cls import _Cls, parameter
|
31
32
|
from .config import logger
|
32
|
-
from .exception import ExecutionError, InvalidError
|
33
|
+
from .exception import ExecutionError, InvalidError
|
33
34
|
from .functions import Function, _Function
|
34
35
|
from .gpu import GPU_T
|
35
36
|
from .image import _Image
|
@@ -45,7 +46,6 @@ from .partial_function import (
|
|
45
46
|
from .proxy import _Proxy
|
46
47
|
from .retries import Retries
|
47
48
|
from .running_app import RunningApp
|
48
|
-
from .sandbox import _Sandbox
|
49
49
|
from .schedule import Schedule
|
50
50
|
from .scheduler_placement import SchedulerPlacement
|
51
51
|
from .secret import _Secret
|
@@ -964,36 +964,16 @@ class _App:
|
|
964
964
|
_experimental_scheduler_placement: Optional[
|
965
965
|
SchedulerPlacement
|
966
966
|
] = None, # Experimental controls over fine-grained scheduling (alpha).
|
967
|
-
) ->
|
968
|
-
"""
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
return await _Sandbox.create(
|
977
|
-
*entrypoint_args,
|
978
|
-
app=self,
|
979
|
-
environment_name=self._running_app.environment_name,
|
980
|
-
image=image or _default_image,
|
981
|
-
mounts=mounts,
|
982
|
-
secrets=secrets,
|
983
|
-
timeout=timeout,
|
984
|
-
workdir=workdir,
|
985
|
-
gpu=gpu,
|
986
|
-
cloud=cloud,
|
987
|
-
region=region,
|
988
|
-
cpu=cpu,
|
989
|
-
memory=memory,
|
990
|
-
network_file_systems=network_file_systems,
|
991
|
-
block_network=block_network,
|
992
|
-
volumes=volumes,
|
993
|
-
pty_info=pty_info,
|
994
|
-
_experimental_scheduler_placement=_experimental_scheduler_placement,
|
995
|
-
client=self._client,
|
967
|
+
) -> None:
|
968
|
+
"""mdmd:hidden"""
|
969
|
+
arglist = ", ".join(repr(s) for s in entrypoint_args)
|
970
|
+
message = (
|
971
|
+
"`App.spawn_sandbox` is deprecated.\n\n"
|
972
|
+
"Sandboxes can be created using the `Sandbox` object:\n\n"
|
973
|
+
f"```\nsb = Sandbox.create({arglist}, app=app)\n```\n\n"
|
974
|
+
"See https://modal.com/docs/guide/sandbox for more info on working with sandboxes."
|
996
975
|
)
|
976
|
+
deprecation_error((2024, 7, 5), message)
|
997
977
|
|
998
978
|
def include(self, /, other_app: "_App"):
|
999
979
|
"""Include another App's objects in this one.
|
modal/app.pyi
CHANGED
@@ -13,7 +13,6 @@ import modal.partial_function
|
|
13
13
|
import modal.proxy
|
14
14
|
import modal.retries
|
15
15
|
import modal.running_app
|
16
|
-
import modal.sandbox
|
17
16
|
import modal.schedule
|
18
17
|
import modal.scheduler_placement
|
19
18
|
import modal.secret
|
@@ -261,7 +260,7 @@ class _App:
|
|
261
260
|
] = {},
|
262
261
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
263
262
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
264
|
-
) ->
|
263
|
+
) -> None: ...
|
265
264
|
def include(self, /, other_app: _App): ...
|
266
265
|
def _logs(
|
267
266
|
self, client: typing.Optional[modal.client._Client] = None
|
@@ -491,7 +490,7 @@ class App:
|
|
491
490
|
] = {},
|
492
491
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
493
492
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
494
|
-
) ->
|
493
|
+
) -> None: ...
|
495
494
|
async def aio(
|
496
495
|
self,
|
497
496
|
*entrypoint_args: str,
|
@@ -515,7 +514,7 @@ class App:
|
|
515
514
|
] = {},
|
516
515
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
517
516
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
518
|
-
) ->
|
517
|
+
) -> None: ...
|
519
518
|
|
520
519
|
spawn_sandbox: __spawn_sandbox_spec
|
521
520
|
|
modal/cli/app.py
CHANGED
@@ -10,9 +10,9 @@ from rich.text import Text
|
|
10
10
|
from typer import Argument
|
11
11
|
|
12
12
|
from modal._utils.async_utils import synchronizer
|
13
|
+
from modal._utils.deprecation import deprecation_warning
|
13
14
|
from modal.client import _Client
|
14
15
|
from modal.environments import ensure_env
|
15
|
-
from modal.exception import deprecation_warning
|
16
16
|
from modal.object import _get_environment_name
|
17
17
|
from modal_proto import api_pb2
|
18
18
|
|
modal/client.pyi
CHANGED
@@ -26,7 +26,7 @@ class _Client:
|
|
26
26
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
27
27
|
|
28
28
|
def __init__(
|
29
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
29
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
|
30
30
|
): ...
|
31
31
|
def is_closed(self) -> bool: ...
|
32
32
|
@property
|
@@ -81,7 +81,7 @@ class Client:
|
|
81
81
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
82
82
|
|
83
83
|
def __init__(
|
84
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
84
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
|
85
85
|
): ...
|
86
86
|
def is_closed(self) -> bool: ...
|
87
87
|
@property
|
modal/config.py
CHANGED
@@ -86,8 +86,9 @@ from google.protobuf.empty_pb2 import Empty
|
|
86
86
|
|
87
87
|
from modal_proto import api_pb2
|
88
88
|
|
89
|
+
from ._utils.deprecation import deprecation_error
|
89
90
|
from ._utils.logger import configure_logger
|
90
|
-
from .exception import InvalidError
|
91
|
+
from .exception import InvalidError
|
91
92
|
|
92
93
|
# Locate config file and read it
|
93
94
|
|
modal/container_process.py
CHANGED
@@ -6,10 +6,11 @@ from typing import Generic, Optional, TypeVar
|
|
6
6
|
from modal_proto import api_pb2
|
7
7
|
|
8
8
|
from ._utils.async_utils import TaskContext, synchronize_api
|
9
|
+
from ._utils.deprecation import deprecation_error
|
9
10
|
from ._utils.grpc_utils import retry_transient_errors
|
10
11
|
from ._utils.shell_utils import stream_from_stdin, write_to_fd
|
11
12
|
from .client import _Client
|
12
|
-
from .exception import InteractiveTimeoutError, InvalidError
|
13
|
+
from .exception import InteractiveTimeoutError, InvalidError
|
13
14
|
from .io_streams import _StreamReader, _StreamWriter
|
14
15
|
from .stream_type import StreamType
|
15
16
|
|
modal/dict.py
CHANGED
@@ -10,11 +10,12 @@ from modal_proto import api_pb2
|
|
10
10
|
from ._resolver import Resolver
|
11
11
|
from ._serialization import deserialize, serialize
|
12
12
|
from ._utils.async_utils import TaskContext, synchronize_api
|
13
|
+
from ._utils.deprecation import deprecation_error
|
13
14
|
from ._utils.grpc_utils import retry_transient_errors
|
14
15
|
from ._utils.name_utils import check_object_name
|
15
16
|
from .client import _Client
|
16
17
|
from .config import logger
|
17
|
-
from .exception import RequestSizeError
|
18
|
+
from .exception import RequestSizeError
|
18
19
|
from .object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
|
19
20
|
|
20
21
|
|
modal/exception.py
CHANGED
@@ -1,12 +1,6 @@
|
|
1
1
|
# Copyright Modal Labs 2022
|
2
2
|
import random
|
3
3
|
import signal
|
4
|
-
import sys
|
5
|
-
import warnings
|
6
|
-
from datetime import date
|
7
|
-
from typing import Iterable
|
8
|
-
|
9
|
-
from modal_proto import api_pb2
|
10
4
|
|
11
5
|
|
12
6
|
class Error(Exception):
|
@@ -129,45 +123,6 @@ class _CliUserExecutionError(Exception):
|
|
129
123
|
self.user_source = user_source
|
130
124
|
|
131
125
|
|
132
|
-
# TODO(erikbern): we have something similready in function_utils.py
|
133
|
-
_INTERNAL_MODULES = ["modal", "synchronicity"]
|
134
|
-
|
135
|
-
|
136
|
-
def _is_internal_frame(frame):
|
137
|
-
module = frame.f_globals["__name__"].split(".")[0]
|
138
|
-
return module in _INTERNAL_MODULES
|
139
|
-
|
140
|
-
|
141
|
-
def deprecation_error(deprecated_on: tuple[int, int, int], msg: str):
|
142
|
-
raise DeprecationError(f"Deprecated on {date(*deprecated_on)}: {msg}")
|
143
|
-
|
144
|
-
|
145
|
-
def deprecation_warning(
|
146
|
-
deprecated_on: tuple[int, int, int], msg: str, *, pending: bool = False, show_source: bool = True
|
147
|
-
) -> None:
|
148
|
-
"""Utility for getting the proper stack entry.
|
149
|
-
|
150
|
-
See the implementation of the built-in [warnings.warn](https://docs.python.org/3/library/warnings.html#available-functions).
|
151
|
-
"""
|
152
|
-
filename, lineno = "<unknown>", 0
|
153
|
-
if show_source:
|
154
|
-
# Find the last non-Modal line that triggered the warning
|
155
|
-
try:
|
156
|
-
frame = sys._getframe()
|
157
|
-
while frame is not None and _is_internal_frame(frame):
|
158
|
-
frame = frame.f_back
|
159
|
-
filename = frame.f_code.co_filename
|
160
|
-
lineno = frame.f_lineno
|
161
|
-
except ValueError:
|
162
|
-
# Use the defaults from above
|
163
|
-
pass
|
164
|
-
|
165
|
-
warning_cls: type = PendingDeprecationError if pending else DeprecationError
|
166
|
-
|
167
|
-
# This is a lower-level function that warnings.warn uses
|
168
|
-
warnings.warn_explicit(f"{date(*deprecated_on)}: {msg}", warning_cls, filename, lineno)
|
169
|
-
|
170
|
-
|
171
126
|
def _simulate_preemption_interrupt(signum, frame):
|
172
127
|
signal.alarm(30) # simulate a SIGKILL after 30s
|
173
128
|
raise KeyboardInterrupt("Simulated preemption interrupt from modal-client!")
|
@@ -224,12 +179,3 @@ class ClientClosed(Error):
|
|
224
179
|
|
225
180
|
class FilesystemExecutionError(Error):
|
226
181
|
"""Raised when an unknown error is thrown during a container filesystem operation."""
|
227
|
-
|
228
|
-
|
229
|
-
def print_server_warnings(server_warnings: Iterable[api_pb2.Warning]):
|
230
|
-
# TODO(erikbern): move this to modal._utils.deprecation
|
231
|
-
for warning in server_warnings:
|
232
|
-
if warning.type == api_pb2.Warning.WARNING_TYPE_CLIENT_DEPRECATION:
|
233
|
-
warnings.warn_explicit(warning.message, DeprecationError, "<unknown>", 0)
|
234
|
-
else:
|
235
|
-
warnings.warn_explicit(warning.message, UserWarning, "<unknown>", 0)
|
modal/file_io.py
CHANGED
@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, AsyncIterator, Generic, Optional, Sequence, Ty
|
|
6
6
|
if TYPE_CHECKING:
|
7
7
|
import _typeshed
|
8
8
|
|
9
|
+
import json
|
10
|
+
|
9
11
|
from grpclib.exceptions import GRPCError, StreamTerminatedError
|
10
12
|
|
11
13
|
from modal._utils.grpc_utils import retry_transient_errors
|
@@ -267,12 +269,12 @@ class _FileIO(Generic[T]):
|
|
267
269
|
output = await self._make_read_request(None)
|
268
270
|
if self._binary:
|
269
271
|
lines_bytes = output.split(b"\n")
|
270
|
-
|
271
|
-
return cast(Sequence[T],
|
272
|
+
return_bytes = [line + b"\n" for line in lines_bytes[:-1]] + ([lines_bytes[-1]] if lines_bytes[-1] else [])
|
273
|
+
return cast(Sequence[T], return_bytes)
|
272
274
|
else:
|
273
275
|
lines = output.decode("utf-8").split("\n")
|
274
|
-
|
275
|
-
return cast(Sequence[T],
|
276
|
+
return_strs = [line + "\n" for line in lines[:-1]] + ([lines[-1]] if lines[-1] else [])
|
277
|
+
return cast(Sequence[T], return_strs)
|
276
278
|
|
277
279
|
async def write(self, data: Union[bytes, str]) -> None:
|
278
280
|
"""Write data to the current position.
|
@@ -337,6 +339,52 @@ class _FileIO(Generic[T]):
|
|
337
339
|
)
|
338
340
|
await self._wait(resp.exec_id)
|
339
341
|
|
342
|
+
@classmethod
|
343
|
+
async def ls(cls, path: str, client: _Client, task_id: str) -> list[str]:
|
344
|
+
"""List the contents of the provided directory."""
|
345
|
+
self = cls.__new__(cls)
|
346
|
+
self._client = client
|
347
|
+
self._task_id = task_id
|
348
|
+
resp = await self._make_request(
|
349
|
+
api_pb2.ContainerFilesystemExecRequest(
|
350
|
+
file_ls_request=api_pb2.ContainerFileLsRequest(path=path),
|
351
|
+
task_id=task_id,
|
352
|
+
)
|
353
|
+
)
|
354
|
+
output = await self._wait(resp.exec_id)
|
355
|
+
try:
|
356
|
+
return json.loads(output.decode("utf-8"))["paths"]
|
357
|
+
except json.JSONDecodeError:
|
358
|
+
raise FilesystemExecutionError("failed to parse list output")
|
359
|
+
|
360
|
+
@classmethod
|
361
|
+
async def mkdir(cls, path: str, client: _Client, task_id: str, parents: bool = False) -> None:
|
362
|
+
"""Create a new directory."""
|
363
|
+
self = cls.__new__(cls)
|
364
|
+
self._client = client
|
365
|
+
self._task_id = task_id
|
366
|
+
resp = await self._make_request(
|
367
|
+
api_pb2.ContainerFilesystemExecRequest(
|
368
|
+
file_mkdir_request=api_pb2.ContainerFileMkdirRequest(path=path, make_parents=parents),
|
369
|
+
task_id=self._task_id,
|
370
|
+
)
|
371
|
+
)
|
372
|
+
await self._wait(resp.exec_id)
|
373
|
+
|
374
|
+
@classmethod
|
375
|
+
async def rm(cls, path: str, client: _Client, task_id: str, recursive: bool = False) -> None:
|
376
|
+
"""Remove a file or directory in the Sandbox."""
|
377
|
+
self = cls.__new__(cls)
|
378
|
+
self._client = client
|
379
|
+
self._task_id = task_id
|
380
|
+
resp = await self._make_request(
|
381
|
+
api_pb2.ContainerFilesystemExecRequest(
|
382
|
+
file_rm_request=api_pb2.ContainerFileRmRequest(path=path, recursive=recursive),
|
383
|
+
task_id=self._task_id,
|
384
|
+
)
|
385
|
+
)
|
386
|
+
await self._wait(resp.exec_id)
|
387
|
+
|
340
388
|
async def _close(self) -> None:
|
341
389
|
# Buffer is flushed by the runner on close
|
342
390
|
resp = await self._make_request(
|
@@ -367,11 +415,10 @@ class _FileIO(Generic[T]):
|
|
367
415
|
if self._closed:
|
368
416
|
raise ValueError("I/O operation on closed file")
|
369
417
|
|
370
|
-
def
|
371
|
-
self._check_closed()
|
418
|
+
async def __aenter__(self) -> "_FileIO":
|
372
419
|
return self
|
373
420
|
|
374
|
-
async def
|
421
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
|
375
422
|
await self._close()
|
376
423
|
|
377
424
|
|
modal/file_io.pyi
CHANGED
@@ -43,13 +43,19 @@ class _FileIO(typing.Generic[T]):
|
|
43
43
|
async def flush(self) -> None: ...
|
44
44
|
def _get_whence(self, whence: int): ...
|
45
45
|
async def seek(self, offset: int, whence: int = 0) -> None: ...
|
46
|
+
@classmethod
|
47
|
+
async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]: ...
|
48
|
+
@classmethod
|
49
|
+
async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None: ...
|
50
|
+
@classmethod
|
51
|
+
async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None: ...
|
46
52
|
async def _close(self) -> None: ...
|
47
53
|
async def close(self) -> None: ...
|
48
54
|
def _check_writable(self) -> None: ...
|
49
55
|
def _check_readable(self) -> None: ...
|
50
56
|
def _check_closed(self) -> None: ...
|
51
|
-
def
|
52
|
-
async def
|
57
|
+
async def __aenter__(self) -> _FileIO: ...
|
58
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
|
53
59
|
|
54
60
|
class __delete_bytes_spec(typing_extensions.Protocol):
|
55
61
|
def __call__(self, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None: ...
|
@@ -161,6 +167,13 @@ class FileIO(typing.Generic[T]):
|
|
161
167
|
|
162
168
|
seek: __seek_spec
|
163
169
|
|
170
|
+
@classmethod
|
171
|
+
def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]: ...
|
172
|
+
@classmethod
|
173
|
+
def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None: ...
|
174
|
+
@classmethod
|
175
|
+
def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None: ...
|
176
|
+
|
164
177
|
class ___close_spec(typing_extensions.Protocol):
|
165
178
|
def __call__(self) -> None: ...
|
166
179
|
async def aio(self) -> None: ...
|
@@ -177,9 +190,6 @@ class FileIO(typing.Generic[T]):
|
|
177
190
|
def _check_readable(self) -> None: ...
|
178
191
|
def _check_closed(self) -> None: ...
|
179
192
|
def __enter__(self) -> FileIO: ...
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
async def aio(self, exc_type, exc_value, traceback) -> None: ...
|
184
|
-
|
185
|
-
__exit__: ____exit___spec
|
193
|
+
async def __aenter__(self) -> FileIO: ...
|
194
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None: ...
|
195
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
|
modal/file_pattern_matcher.py
CHANGED
@@ -10,13 +10,56 @@ then asking it whether file paths match any of its patterns.
|
|
10
10
|
"""
|
11
11
|
|
12
12
|
import os
|
13
|
+
from abc import abstractmethod
|
13
14
|
from pathlib import Path
|
14
|
-
from typing import Callable
|
15
|
+
from typing import Callable, Optional
|
15
16
|
|
16
17
|
from ._utils.pattern_utils import Pattern
|
17
18
|
|
18
19
|
|
19
|
-
class
|
20
|
+
class _AbstractPatternMatcher:
|
21
|
+
_custom_repr: Optional[str] = None
|
22
|
+
|
23
|
+
def __invert__(self) -> "_AbstractPatternMatcher":
|
24
|
+
"""Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
|
25
|
+
|
26
|
+
Usage:
|
27
|
+
```python
|
28
|
+
from pathlib import Path
|
29
|
+
from modal import FilePatternMatcher
|
30
|
+
|
31
|
+
inverted_matcher = ~FilePatternMatcher("**/*.py")
|
32
|
+
|
33
|
+
assert not inverted_matcher(Path("foo.py"))
|
34
|
+
```
|
35
|
+
"""
|
36
|
+
return _CustomPatternMatcher(lambda path: not self(path))
|
37
|
+
|
38
|
+
def with_repr(self, custom_repr) -> "_AbstractPatternMatcher":
|
39
|
+
# use to give an instance of a matcher a custom name - useful for visualizing default values in signatures
|
40
|
+
self._custom_repr = custom_repr
|
41
|
+
return self
|
42
|
+
|
43
|
+
def __repr__(self) -> str:
|
44
|
+
if self._custom_repr:
|
45
|
+
return self._custom_repr
|
46
|
+
|
47
|
+
return super().__repr__()
|
48
|
+
|
49
|
+
@abstractmethod
|
50
|
+
def __call__(self, path: Path) -> bool:
|
51
|
+
...
|
52
|
+
|
53
|
+
|
54
|
+
class _CustomPatternMatcher(_AbstractPatternMatcher):
|
55
|
+
def __init__(self, predicate: Callable[[Path], bool]):
|
56
|
+
self._predicate = predicate
|
57
|
+
|
58
|
+
def __call__(self, path: Path) -> bool:
|
59
|
+
return self._predicate(path)
|
60
|
+
|
61
|
+
|
62
|
+
class FilePatternMatcher(_AbstractPatternMatcher):
|
20
63
|
"""Allows matching file paths against a list of patterns."""
|
21
64
|
|
22
65
|
def __init__(self, *pattern: str) -> None:
|
@@ -105,17 +148,7 @@ class FilePatternMatcher:
|
|
105
148
|
"""
|
106
149
|
return self._matches(str(file_path))
|
107
150
|
|
108
|
-
def __invert__(self) -> Callable[[Path], bool]:
|
109
|
-
"""Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
|
110
|
-
|
111
|
-
Usage:
|
112
|
-
```python
|
113
|
-
from pathlib import Path
|
114
|
-
from modal import FilePatternMatcher
|
115
|
-
|
116
|
-
inverted_matcher = ~FilePatternMatcher("**/*.py")
|
117
151
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
return lambda path: not self(path)
|
152
|
+
# with_repr allows us to use this matcher as a default value in a function signature
|
153
|
+
# and get a nice repr in the docs and auto-generated type stubs:
|
154
|
+
NON_PYTHON_FILES = (~FilePatternMatcher("**/*.py")).with_repr(f"{__name__}.NON_PYTHON_FILES")
|
modal/functions.py
CHANGED
@@ -41,6 +41,7 @@ from ._utils.async_utils import (
|
|
41
41
|
synchronizer,
|
42
42
|
warn_if_generator_is_not_consumed,
|
43
43
|
)
|
44
|
+
from ._utils.deprecation import deprecation_warning
|
44
45
|
from ._utils.function_utils import (
|
45
46
|
ATTEMPT_TIMEOUT_GRACE_PERIOD,
|
46
47
|
OUTPUTS_TIMEOUT,
|
@@ -58,14 +59,7 @@ from .call_graph import InputInfo, _reconstruct_call_graph
|
|
58
59
|
from .client import _Client
|
59
60
|
from .cloud_bucket_mount import _CloudBucketMount, cloud_bucket_mounts_to_proto
|
60
61
|
from .config import config
|
61
|
-
from .exception import
|
62
|
-
ExecutionError,
|
63
|
-
FunctionTimeoutError,
|
64
|
-
InvalidError,
|
65
|
-
NotFoundError,
|
66
|
-
OutputExpiredError,
|
67
|
-
deprecation_warning,
|
68
|
-
)
|
62
|
+
from .exception import ExecutionError, FunctionTimeoutError, InvalidError, NotFoundError, OutputExpiredError
|
69
63
|
from .gpu import GPU_T, parse_gpu_config
|
70
64
|
from .image import _Image
|
71
65
|
from .mount import _get_client_mount, _Mount, get_auto_mounts
|
modal/functions.pyi
CHANGED
@@ -448,7 +448,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
448
448
|
|
449
449
|
_call_function_nowait: ___call_function_nowait_spec
|
450
450
|
|
451
|
-
|
451
|
+
class ___call_generator_spec(typing_extensions.Protocol):
|
452
|
+
def __call__(self, args, kwargs): ...
|
453
|
+
def aio(self, args, kwargs): ...
|
454
|
+
|
455
|
+
_call_generator: ___call_generator_spec
|
452
456
|
|
453
457
|
class ___call_generator_nowait_spec(typing_extensions.Protocol):
|
454
458
|
def __call__(self, args, kwargs): ...
|
@@ -456,11 +460,11 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
456
460
|
|
457
461
|
_call_generator_nowait: ___call_generator_nowait_spec
|
458
462
|
|
459
|
-
class __remote_spec(typing_extensions.Protocol[
|
463
|
+
class __remote_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
|
460
464
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
461
465
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> ReturnType_INNER: ...
|
462
466
|
|
463
|
-
remote: __remote_spec[
|
467
|
+
remote: __remote_spec[P, ReturnType]
|
464
468
|
|
465
469
|
class __remote_gen_spec(typing_extensions.Protocol):
|
466
470
|
def __call__(self, *args, **kwargs) -> typing.Generator[typing.Any, None, None]: ...
|
@@ -473,17 +477,17 @@ class Function(typing.Generic[P, ReturnType, OriginalReturnType], modal.object.O
|
|
473
477
|
def _get_obj(self) -> typing.Optional[modal.cls.Obj]: ...
|
474
478
|
def local(self, *args: P.args, **kwargs: P.kwargs) -> OriginalReturnType: ...
|
475
479
|
|
476
|
-
class ___experimental_spawn_spec(typing_extensions.Protocol[
|
480
|
+
class ___experimental_spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
|
477
481
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
478
482
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
479
483
|
|
480
|
-
_experimental_spawn: ___experimental_spawn_spec[
|
484
|
+
_experimental_spawn: ___experimental_spawn_spec[P, ReturnType]
|
481
485
|
|
482
|
-
class __spawn_spec(typing_extensions.Protocol[
|
486
|
+
class __spawn_spec(typing_extensions.Protocol[P_INNER, ReturnType_INNER]):
|
483
487
|
def __call__(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
484
488
|
async def aio(self, *args: P_INNER.args, **kwargs: P_INNER.kwargs) -> FunctionCall[ReturnType_INNER]: ...
|
485
489
|
|
486
|
-
spawn: __spawn_spec[
|
490
|
+
spawn: __spawn_spec[P, ReturnType]
|
487
491
|
|
488
492
|
def get_raw_f(self) -> typing.Callable[..., typing.Any]: ...
|
489
493
|
|
modal/image.py
CHANGED
@@ -30,14 +30,15 @@ from ._resolver import Resolver
|
|
30
30
|
from ._serialization import serialize
|
31
31
|
from ._utils.async_utils import synchronize_api
|
32
32
|
from ._utils.blob_utils import MAX_OBJECT_SIZE_BYTES
|
33
|
+
from ._utils.deprecation import deprecation_error, deprecation_warning
|
33
34
|
from ._utils.function_utils import FunctionInfo
|
34
35
|
from ._utils.grpc_utils import RETRYABLE_GRPC_STATUS_CODES, retry_transient_errors
|
35
36
|
from .client import _Client
|
36
37
|
from .cloud_bucket_mount import _CloudBucketMount
|
37
38
|
from .config import config, logger, user_config_path
|
38
39
|
from .environments import _get_environment_cached
|
39
|
-
from .exception import InvalidError, NotFoundError, RemoteError, VersionError
|
40
|
-
from .file_pattern_matcher import
|
40
|
+
from .exception import InvalidError, NotFoundError, RemoteError, VersionError
|
41
|
+
from .file_pattern_matcher import NON_PYTHON_FILES
|
41
42
|
from .gpu import GPU_T, parse_gpu_config
|
42
43
|
from .mount import _Mount, python_standalone_mount_name
|
43
44
|
from .network_file_system import _NetworkFileSystem
|
@@ -720,7 +721,9 @@ class _Image(_Object, type_prefix="im"):
|
|
720
721
|
context_mount=mount,
|
721
722
|
)
|
722
723
|
|
723
|
-
def add_local_python_source(
|
724
|
+
def add_local_python_source(
|
725
|
+
self, *modules: str, copy: bool = False, ignore: Union[Sequence[str], Callable[[Path], bool]] = NON_PYTHON_FILES
|
726
|
+
) -> "_Image":
|
724
727
|
"""Adds locally available Python packages/modules to containers
|
725
728
|
|
726
729
|
Adds all files from the specified Python package or module to containers running the Image.
|
@@ -738,9 +741,22 @@ class _Image(_Object, type_prefix="im"):
|
|
738
741
|
**Note:** This excludes all dot-prefixed subdirectories or files and all `.pyc`/`__pycache__` files.
|
739
742
|
To add full directories with finer control, use `.add_local_dir()` instead and specify `/root` as
|
740
743
|
the destination directory.
|
741
|
-
"""
|
742
744
|
|
743
|
-
|
745
|
+
By default only includes `.py`-files in the source modules. Set the `ignore` argument to a list of patterns
|
746
|
+
or a callable to override this behavior, e.g.:
|
747
|
+
|
748
|
+
```py
|
749
|
+
# includes everything except data.json
|
750
|
+
modal.Image.debian_slim().add_local_python_source("mymodule", ignore=["data.json"])
|
751
|
+
|
752
|
+
# exclude large files
|
753
|
+
modal.Image.debian_slim().add_local_python_source(
|
754
|
+
"mymodule",
|
755
|
+
ignore=lambda p: p.stat().st_size > 1e9
|
756
|
+
)
|
757
|
+
```
|
758
|
+
"""
|
759
|
+
mount = _Mount.from_local_python_packages(*modules, ignore=ignore)
|
744
760
|
return self._add_mount_layer_or_copy(mount, copy=copy)
|
745
761
|
|
746
762
|
def copy_local_dir(
|