modal 0.68.11__py3-none-any.whl → 0.68.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +2 -0
- modal/_ipython.py +3 -13
- modal/_runtime/asgi.py +4 -0
- modal/_runtime/user_code_imports.py +13 -18
- modal/_utils/blob_utils.py +27 -92
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/deprecation.py +44 -0
- modal/_utils/hash_utils.py +38 -9
- modal/_utils/http_utils.py +19 -10
- modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
- modal/_utils/shell_utils.py +11 -5
- modal/app.py +11 -31
- modal/app.pyi +3 -4
- modal/cli/app.py +1 -1
- modal/cli/run.py +25 -5
- modal/client.py +1 -1
- modal/client.pyi +2 -2
- modal/config.py +2 -1
- modal/container_process.py +2 -1
- modal/dict.py +2 -1
- modal/exception.py +0 -54
- modal/file_io.py +54 -7
- modal/file_io.pyi +18 -8
- modal/file_pattern_matcher.py +154 -0
- modal/functions.py +2 -8
- modal/functions.pyi +5 -1
- modal/image.py +106 -10
- modal/image.pyi +36 -6
- modal/mount.py +49 -9
- modal/mount.pyi +19 -4
- modal/network_file_system.py +6 -2
- modal/partial_function.py +10 -1
- modal/partial_function.pyi +8 -0
- modal/queue.py +2 -1
- modal/runner.py +2 -7
- modal/sandbox.py +23 -13
- modal/sandbox.pyi +21 -0
- modal/serving.py +1 -1
- modal/volume.py +7 -2
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/METADATA +1 -1
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/RECORD +49 -46
- modal_proto/api.proto +8 -0
- modal_proto/api_pb2.py +781 -745
- modal_proto/api_pb2.pyi +65 -3
- modal_version/_version_generated.py +1 -1
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/LICENSE +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/WHEEL +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/entry_points.txt +0 -0
- {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/top_level.txt +0 -0
modal/_utils/shell_utils.py
CHANGED
@@ -19,14 +19,20 @@ def write_to_fd(fd: int, data: bytes):
|
|
19
19
|
future = loop.create_future()
|
20
20
|
|
21
21
|
def try_write():
|
22
|
+
nonlocal data
|
22
23
|
try:
|
23
24
|
nbytes = os.write(fd, data)
|
24
|
-
|
25
|
-
|
25
|
+
data = data[nbytes:]
|
26
|
+
if not data:
|
27
|
+
loop.remove_writer(fd)
|
28
|
+
future.set_result(None)
|
26
29
|
except OSError as e:
|
27
|
-
if e.errno
|
28
|
-
|
29
|
-
|
30
|
+
if e.errno == errno.EAGAIN:
|
31
|
+
# Wait for the next write notification
|
32
|
+
return
|
33
|
+
# Fail if it's not EAGAIN
|
34
|
+
loop.remove_writer(fd)
|
35
|
+
future.set_exception(e)
|
30
36
|
|
31
37
|
loop.add_writer(fd, try_write)
|
32
38
|
return future
|
modal/app.py
CHANGED
@@ -22,6 +22,7 @@ from modal_proto import api_pb2
|
|
22
22
|
|
23
23
|
from ._ipython import is_notebook
|
24
24
|
from ._utils.async_utils import synchronize_api
|
25
|
+
from ._utils.deprecation import deprecation_error, deprecation_warning
|
25
26
|
from ._utils.function_utils import FunctionInfo, is_global_object, is_method_fn
|
26
27
|
from ._utils.grpc_utils import retry_transient_errors
|
27
28
|
from ._utils.mount_utils import validate_volumes
|
@@ -29,7 +30,7 @@ from .client import _Client
|
|
29
30
|
from .cloud_bucket_mount import _CloudBucketMount
|
30
31
|
from .cls import _Cls, parameter
|
31
32
|
from .config import logger
|
32
|
-
from .exception import ExecutionError, InvalidError
|
33
|
+
from .exception import ExecutionError, InvalidError
|
33
34
|
from .functions import Function, _Function
|
34
35
|
from .gpu import GPU_T
|
35
36
|
from .image import _Image
|
@@ -45,7 +46,6 @@ from .partial_function import (
|
|
45
46
|
from .proxy import _Proxy
|
46
47
|
from .retries import Retries
|
47
48
|
from .running_app import RunningApp
|
48
|
-
from .sandbox import _Sandbox
|
49
49
|
from .schedule import Schedule
|
50
50
|
from .scheduler_placement import SchedulerPlacement
|
51
51
|
from .secret import _Secret
|
@@ -964,36 +964,16 @@ class _App:
|
|
964
964
|
_experimental_scheduler_placement: Optional[
|
965
965
|
SchedulerPlacement
|
966
966
|
] = None, # Experimental controls over fine-grained scheduling (alpha).
|
967
|
-
) ->
|
968
|
-
"""
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
return await _Sandbox.create(
|
977
|
-
*entrypoint_args,
|
978
|
-
app=self,
|
979
|
-
environment_name=self._running_app.environment_name,
|
980
|
-
image=image or _default_image,
|
981
|
-
mounts=mounts,
|
982
|
-
secrets=secrets,
|
983
|
-
timeout=timeout,
|
984
|
-
workdir=workdir,
|
985
|
-
gpu=gpu,
|
986
|
-
cloud=cloud,
|
987
|
-
region=region,
|
988
|
-
cpu=cpu,
|
989
|
-
memory=memory,
|
990
|
-
network_file_systems=network_file_systems,
|
991
|
-
block_network=block_network,
|
992
|
-
volumes=volumes,
|
993
|
-
pty_info=pty_info,
|
994
|
-
_experimental_scheduler_placement=_experimental_scheduler_placement,
|
995
|
-
client=self._client,
|
967
|
+
) -> None:
|
968
|
+
"""mdmd:hidden"""
|
969
|
+
arglist = ", ".join(repr(s) for s in entrypoint_args)
|
970
|
+
message = (
|
971
|
+
"`App.spawn_sandbox` is deprecated.\n\n"
|
972
|
+
"Sandboxes can be created using the `Sandbox` object:\n\n"
|
973
|
+
f"```\nsb = Sandbox.create({arglist}, app=app)\n```\n\n"
|
974
|
+
"See https://modal.com/docs/guide/sandbox for more info on working with sandboxes."
|
996
975
|
)
|
976
|
+
deprecation_error((2024, 7, 5), message)
|
997
977
|
|
998
978
|
def include(self, /, other_app: "_App"):
|
999
979
|
"""Include another App's objects in this one.
|
modal/app.pyi
CHANGED
@@ -13,7 +13,6 @@ import modal.partial_function
|
|
13
13
|
import modal.proxy
|
14
14
|
import modal.retries
|
15
15
|
import modal.running_app
|
16
|
-
import modal.sandbox
|
17
16
|
import modal.schedule
|
18
17
|
import modal.scheduler_placement
|
19
18
|
import modal.secret
|
@@ -261,7 +260,7 @@ class _App:
|
|
261
260
|
] = {},
|
262
261
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
263
262
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
264
|
-
) ->
|
263
|
+
) -> None: ...
|
265
264
|
def include(self, /, other_app: _App): ...
|
266
265
|
def _logs(
|
267
266
|
self, client: typing.Optional[modal.client._Client] = None
|
@@ -491,7 +490,7 @@ class App:
|
|
491
490
|
] = {},
|
492
491
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
493
492
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
494
|
-
) ->
|
493
|
+
) -> None: ...
|
495
494
|
async def aio(
|
496
495
|
self,
|
497
496
|
*entrypoint_args: str,
|
@@ -515,7 +514,7 @@ class App:
|
|
515
514
|
] = {},
|
516
515
|
pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
|
517
516
|
_experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
|
518
|
-
) ->
|
517
|
+
) -> None: ...
|
519
518
|
|
520
519
|
spawn_sandbox: __spawn_sandbox_spec
|
521
520
|
|
modal/cli/app.py
CHANGED
@@ -10,9 +10,9 @@ from rich.text import Text
|
|
10
10
|
from typer import Argument
|
11
11
|
|
12
12
|
from modal._utils.async_utils import synchronizer
|
13
|
+
from modal._utils.deprecation import deprecation_warning
|
13
14
|
from modal.client import _Client
|
14
15
|
from modal.environments import ensure_env
|
15
|
-
from modal.exception import deprecation_warning
|
16
16
|
from modal.object import _get_environment_name
|
17
17
|
from modal_proto import api_pb2
|
18
18
|
|
modal/cli/run.py
CHANGED
@@ -133,6 +133,18 @@ def _get_clean_app_description(func_ref: str) -> str:
|
|
133
133
|
return " ".join(sys.argv)
|
134
134
|
|
135
135
|
|
136
|
+
def _write_local_result(result_path: str, res: Any):
|
137
|
+
if isinstance(res, str):
|
138
|
+
mode = "wt"
|
139
|
+
elif isinstance(res, bytes):
|
140
|
+
mode = "wb"
|
141
|
+
else:
|
142
|
+
res_type = type(res).__name__
|
143
|
+
raise InvalidError(f"Function must return str or bytes when using `--write-result`; got {res_type}.")
|
144
|
+
with open(result_path, mode) as fid:
|
145
|
+
fid.write(res)
|
146
|
+
|
147
|
+
|
136
148
|
def _get_click_command_for_function(app: App, function_tag):
|
137
149
|
function = app.registered_functions.get(function_tag)
|
138
150
|
if not function or (isinstance(function, Function) and function.info.user_cls is not None):
|
@@ -177,7 +189,7 @@ def _get_click_command_for_function(app: App, function_tag):
|
|
177
189
|
interactive=ctx.obj["interactive"],
|
178
190
|
):
|
179
191
|
if cls is None:
|
180
|
-
function.remote(**kwargs)
|
192
|
+
res = function.remote(**kwargs)
|
181
193
|
else:
|
182
194
|
# unpool class and method arguments
|
183
195
|
# TODO(erikbern): this code is a bit hacky
|
@@ -186,7 +198,10 @@ def _get_click_command_for_function(app: App, function_tag):
|
|
186
198
|
|
187
199
|
instance = cls(**cls_kwargs)
|
188
200
|
method: Function = getattr(instance, method_name)
|
189
|
-
method.remote(**fun_kwargs)
|
201
|
+
res = method.remote(**fun_kwargs)
|
202
|
+
|
203
|
+
if result_path := ctx.obj["result_path"]:
|
204
|
+
_write_local_result(result_path, res)
|
190
205
|
|
191
206
|
with_click_options = _add_click_options(f, signature)
|
192
207
|
return click.command(with_click_options)
|
@@ -214,12 +229,15 @@ def _get_click_command_for_local_entrypoint(app: App, entrypoint: LocalEntrypoin
|
|
214
229
|
):
|
215
230
|
try:
|
216
231
|
if isasync:
|
217
|
-
asyncio.run(func(*args, **kwargs))
|
232
|
+
res = asyncio.run(func(*args, **kwargs))
|
218
233
|
else:
|
219
|
-
func(*args, **kwargs)
|
234
|
+
res = func(*args, **kwargs)
|
220
235
|
except Exception as exc:
|
221
236
|
raise _CliUserExecutionError(inspect.getsourcefile(func)) from exc
|
222
237
|
|
238
|
+
if result_path := ctx.obj["result_path"]:
|
239
|
+
_write_local_result(result_path, res)
|
240
|
+
|
223
241
|
with_click_options = _add_click_options(f, _get_signature(func))
|
224
242
|
return click.command(with_click_options)
|
225
243
|
|
@@ -248,12 +266,13 @@ class RunGroup(click.Group):
|
|
248
266
|
cls=RunGroup,
|
249
267
|
subcommand_metavar="FUNC_REF",
|
250
268
|
)
|
269
|
+
@click.option("-w", "--write-result", help="Write return value (which must be str or bytes) to this local path.")
|
251
270
|
@click.option("-q", "--quiet", is_flag=True, help="Don't show Modal progress indicators.")
|
252
271
|
@click.option("-d", "--detach", is_flag=True, help="Don't stop the app if the local process dies or disconnects.")
|
253
272
|
@click.option("-i", "--interactive", is_flag=True, help="Run the app in interactive mode.")
|
254
273
|
@click.option("-e", "--env", help=ENV_OPTION_HELP, default=None)
|
255
274
|
@click.pass_context
|
256
|
-
def run(ctx, detach, quiet, interactive, env):
|
275
|
+
def run(ctx, write_result, detach, quiet, interactive, env):
|
257
276
|
"""Run a Modal function or local entrypoint.
|
258
277
|
|
259
278
|
`FUNC_REF` should be of the format `{file or module}::{function name}`.
|
@@ -284,6 +303,7 @@ def run(ctx, detach, quiet, interactive, env):
|
|
284
303
|
```
|
285
304
|
"""
|
286
305
|
ctx.ensure_object(dict)
|
306
|
+
ctx.obj["result_path"] = write_result
|
287
307
|
ctx.obj["detach"] = detach # if subcommand would be a click command...
|
288
308
|
ctx.obj["show_progress"] = False if quiet else True
|
289
309
|
ctx.obj["interactive"] = interactive
|
modal/client.py
CHANGED
@@ -236,7 +236,7 @@ class _Client:
|
|
236
236
|
Check whether can the client can connect to this server with these credentials; raise if not.
|
237
237
|
"""
|
238
238
|
async with cls(server_url, api_pb2.CLIENT_TYPE_CLIENT, credentials) as client:
|
239
|
-
client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
|
239
|
+
await client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
|
240
240
|
|
241
241
|
@classmethod
|
242
242
|
def set_env_client(cls, client: Optional["_Client"]):
|
modal/client.pyi
CHANGED
@@ -26,7 +26,7 @@ class _Client:
|
|
26
26
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
27
27
|
|
28
28
|
def __init__(
|
29
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
29
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
|
30
30
|
): ...
|
31
31
|
def is_closed(self) -> bool: ...
|
32
32
|
@property
|
@@ -81,7 +81,7 @@ class Client:
|
|
81
81
|
_stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
|
82
82
|
|
83
83
|
def __init__(
|
84
|
-
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.
|
84
|
+
self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
|
85
85
|
): ...
|
86
86
|
def is_closed(self) -> bool: ...
|
87
87
|
@property
|
modal/config.py
CHANGED
@@ -86,8 +86,9 @@ from google.protobuf.empty_pb2 import Empty
|
|
86
86
|
|
87
87
|
from modal_proto import api_pb2
|
88
88
|
|
89
|
+
from ._utils.deprecation import deprecation_error
|
89
90
|
from ._utils.logger import configure_logger
|
90
|
-
from .exception import InvalidError
|
91
|
+
from .exception import InvalidError
|
91
92
|
|
92
93
|
# Locate config file and read it
|
93
94
|
|
modal/container_process.py
CHANGED
@@ -6,10 +6,11 @@ from typing import Generic, Optional, TypeVar
|
|
6
6
|
from modal_proto import api_pb2
|
7
7
|
|
8
8
|
from ._utils.async_utils import TaskContext, synchronize_api
|
9
|
+
from ._utils.deprecation import deprecation_error
|
9
10
|
from ._utils.grpc_utils import retry_transient_errors
|
10
11
|
from ._utils.shell_utils import stream_from_stdin, write_to_fd
|
11
12
|
from .client import _Client
|
12
|
-
from .exception import InteractiveTimeoutError, InvalidError
|
13
|
+
from .exception import InteractiveTimeoutError, InvalidError
|
13
14
|
from .io_streams import _StreamReader, _StreamWriter
|
14
15
|
from .stream_type import StreamType
|
15
16
|
|
modal/dict.py
CHANGED
@@ -10,11 +10,12 @@ from modal_proto import api_pb2
|
|
10
10
|
from ._resolver import Resolver
|
11
11
|
from ._serialization import deserialize, serialize
|
12
12
|
from ._utils.async_utils import TaskContext, synchronize_api
|
13
|
+
from ._utils.deprecation import deprecation_error
|
13
14
|
from ._utils.grpc_utils import retry_transient_errors
|
14
15
|
from ._utils.name_utils import check_object_name
|
15
16
|
from .client import _Client
|
16
17
|
from .config import logger
|
17
|
-
from .exception import RequestSizeError
|
18
|
+
from .exception import RequestSizeError
|
18
19
|
from .object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
|
19
20
|
|
20
21
|
|
modal/exception.py
CHANGED
@@ -1,12 +1,6 @@
|
|
1
1
|
# Copyright Modal Labs 2022
|
2
2
|
import random
|
3
3
|
import signal
|
4
|
-
import sys
|
5
|
-
import warnings
|
6
|
-
from datetime import date
|
7
|
-
from typing import Iterable
|
8
|
-
|
9
|
-
from modal_proto import api_pb2
|
10
4
|
|
11
5
|
|
12
6
|
class Error(Exception):
|
@@ -129,45 +123,6 @@ class _CliUserExecutionError(Exception):
|
|
129
123
|
self.user_source = user_source
|
130
124
|
|
131
125
|
|
132
|
-
# TODO(erikbern): we have something similready in function_utils.py
|
133
|
-
_INTERNAL_MODULES = ["modal", "synchronicity"]
|
134
|
-
|
135
|
-
|
136
|
-
def _is_internal_frame(frame):
|
137
|
-
module = frame.f_globals["__name__"].split(".")[0]
|
138
|
-
return module in _INTERNAL_MODULES
|
139
|
-
|
140
|
-
|
141
|
-
def deprecation_error(deprecated_on: tuple[int, int, int], msg: str):
|
142
|
-
raise DeprecationError(f"Deprecated on {date(*deprecated_on)}: {msg}")
|
143
|
-
|
144
|
-
|
145
|
-
def deprecation_warning(
|
146
|
-
deprecated_on: tuple[int, int, int], msg: str, *, pending: bool = False, show_source: bool = True
|
147
|
-
) -> None:
|
148
|
-
"""Utility for getting the proper stack entry.
|
149
|
-
|
150
|
-
See the implementation of the built-in [warnings.warn](https://docs.python.org/3/library/warnings.html#available-functions).
|
151
|
-
"""
|
152
|
-
filename, lineno = "<unknown>", 0
|
153
|
-
if show_source:
|
154
|
-
# Find the last non-Modal line that triggered the warning
|
155
|
-
try:
|
156
|
-
frame = sys._getframe()
|
157
|
-
while frame is not None and _is_internal_frame(frame):
|
158
|
-
frame = frame.f_back
|
159
|
-
filename = frame.f_code.co_filename
|
160
|
-
lineno = frame.f_lineno
|
161
|
-
except ValueError:
|
162
|
-
# Use the defaults from above
|
163
|
-
pass
|
164
|
-
|
165
|
-
warning_cls: type = PendingDeprecationError if pending else DeprecationError
|
166
|
-
|
167
|
-
# This is a lower-level function that warnings.warn uses
|
168
|
-
warnings.warn_explicit(f"{date(*deprecated_on)}: {msg}", warning_cls, filename, lineno)
|
169
|
-
|
170
|
-
|
171
126
|
def _simulate_preemption_interrupt(signum, frame):
|
172
127
|
signal.alarm(30) # simulate a SIGKILL after 30s
|
173
128
|
raise KeyboardInterrupt("Simulated preemption interrupt from modal-client!")
|
@@ -224,12 +179,3 @@ class ClientClosed(Error):
|
|
224
179
|
|
225
180
|
class FilesystemExecutionError(Error):
|
226
181
|
"""Raised when an unknown error is thrown during a container filesystem operation."""
|
227
|
-
|
228
|
-
|
229
|
-
def print_server_warnings(server_warnings: Iterable[api_pb2.Warning]):
|
230
|
-
# TODO(erikbern): move this to modal._utils.deprecation
|
231
|
-
for warning in server_warnings:
|
232
|
-
if warning.type == api_pb2.Warning.WARNING_TYPE_CLIENT_DEPRECATION:
|
233
|
-
warnings.warn_explicit(warning.message, DeprecationError, "<unknown>", 0)
|
234
|
-
else:
|
235
|
-
warnings.warn_explicit(warning.message, UserWarning, "<unknown>", 0)
|
modal/file_io.py
CHANGED
@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, AsyncIterator, Generic, Optional, Sequence, Ty
|
|
6
6
|
if TYPE_CHECKING:
|
7
7
|
import _typeshed
|
8
8
|
|
9
|
+
import json
|
10
|
+
|
9
11
|
from grpclib.exceptions import GRPCError, StreamTerminatedError
|
10
12
|
|
11
13
|
from modal._utils.grpc_utils import retry_transient_errors
|
@@ -267,12 +269,12 @@ class _FileIO(Generic[T]):
|
|
267
269
|
output = await self._make_read_request(None)
|
268
270
|
if self._binary:
|
269
271
|
lines_bytes = output.split(b"\n")
|
270
|
-
|
271
|
-
return cast(Sequence[T],
|
272
|
+
return_bytes = [line + b"\n" for line in lines_bytes[:-1]] + ([lines_bytes[-1]] if lines_bytes[-1] else [])
|
273
|
+
return cast(Sequence[T], return_bytes)
|
272
274
|
else:
|
273
275
|
lines = output.decode("utf-8").split("\n")
|
274
|
-
|
275
|
-
return cast(Sequence[T],
|
276
|
+
return_strs = [line + "\n" for line in lines[:-1]] + ([lines[-1]] if lines[-1] else [])
|
277
|
+
return cast(Sequence[T], return_strs)
|
276
278
|
|
277
279
|
async def write(self, data: Union[bytes, str]) -> None:
|
278
280
|
"""Write data to the current position.
|
@@ -337,6 +339,52 @@ class _FileIO(Generic[T]):
|
|
337
339
|
)
|
338
340
|
await self._wait(resp.exec_id)
|
339
341
|
|
342
|
+
@classmethod
|
343
|
+
async def ls(cls, path: str, client: _Client, task_id: str) -> list[str]:
|
344
|
+
"""List the contents of the provided directory."""
|
345
|
+
self = cls.__new__(cls)
|
346
|
+
self._client = client
|
347
|
+
self._task_id = task_id
|
348
|
+
resp = await self._make_request(
|
349
|
+
api_pb2.ContainerFilesystemExecRequest(
|
350
|
+
file_ls_request=api_pb2.ContainerFileLsRequest(path=path),
|
351
|
+
task_id=task_id,
|
352
|
+
)
|
353
|
+
)
|
354
|
+
output = await self._wait(resp.exec_id)
|
355
|
+
try:
|
356
|
+
return json.loads(output.decode("utf-8"))["paths"]
|
357
|
+
except json.JSONDecodeError:
|
358
|
+
raise FilesystemExecutionError("failed to parse list output")
|
359
|
+
|
360
|
+
@classmethod
|
361
|
+
async def mkdir(cls, path: str, client: _Client, task_id: str, parents: bool = False) -> None:
|
362
|
+
"""Create a new directory."""
|
363
|
+
self = cls.__new__(cls)
|
364
|
+
self._client = client
|
365
|
+
self._task_id = task_id
|
366
|
+
resp = await self._make_request(
|
367
|
+
api_pb2.ContainerFilesystemExecRequest(
|
368
|
+
file_mkdir_request=api_pb2.ContainerFileMkdirRequest(path=path, make_parents=parents),
|
369
|
+
task_id=self._task_id,
|
370
|
+
)
|
371
|
+
)
|
372
|
+
await self._wait(resp.exec_id)
|
373
|
+
|
374
|
+
@classmethod
|
375
|
+
async def rm(cls, path: str, client: _Client, task_id: str, recursive: bool = False) -> None:
|
376
|
+
"""Remove a file or directory in the Sandbox."""
|
377
|
+
self = cls.__new__(cls)
|
378
|
+
self._client = client
|
379
|
+
self._task_id = task_id
|
380
|
+
resp = await self._make_request(
|
381
|
+
api_pb2.ContainerFilesystemExecRequest(
|
382
|
+
file_rm_request=api_pb2.ContainerFileRmRequest(path=path, recursive=recursive),
|
383
|
+
task_id=self._task_id,
|
384
|
+
)
|
385
|
+
)
|
386
|
+
await self._wait(resp.exec_id)
|
387
|
+
|
340
388
|
async def _close(self) -> None:
|
341
389
|
# Buffer is flushed by the runner on close
|
342
390
|
resp = await self._make_request(
|
@@ -367,11 +415,10 @@ class _FileIO(Generic[T]):
|
|
367
415
|
if self._closed:
|
368
416
|
raise ValueError("I/O operation on closed file")
|
369
417
|
|
370
|
-
def
|
371
|
-
self._check_closed()
|
418
|
+
async def __aenter__(self) -> "_FileIO":
|
372
419
|
return self
|
373
420
|
|
374
|
-
async def
|
421
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
|
375
422
|
await self._close()
|
376
423
|
|
377
424
|
|
modal/file_io.pyi
CHANGED
@@ -43,13 +43,19 @@ class _FileIO(typing.Generic[T]):
|
|
43
43
|
async def flush(self) -> None: ...
|
44
44
|
def _get_whence(self, whence: int): ...
|
45
45
|
async def seek(self, offset: int, whence: int = 0) -> None: ...
|
46
|
+
@classmethod
|
47
|
+
async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]: ...
|
48
|
+
@classmethod
|
49
|
+
async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None: ...
|
50
|
+
@classmethod
|
51
|
+
async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None: ...
|
46
52
|
async def _close(self) -> None: ...
|
47
53
|
async def close(self) -> None: ...
|
48
54
|
def _check_writable(self) -> None: ...
|
49
55
|
def _check_readable(self) -> None: ...
|
50
56
|
def _check_closed(self) -> None: ...
|
51
|
-
def
|
52
|
-
async def
|
57
|
+
async def __aenter__(self) -> _FileIO: ...
|
58
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
|
53
59
|
|
54
60
|
class __delete_bytes_spec(typing_extensions.Protocol):
|
55
61
|
def __call__(self, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None: ...
|
@@ -161,6 +167,13 @@ class FileIO(typing.Generic[T]):
|
|
161
167
|
|
162
168
|
seek: __seek_spec
|
163
169
|
|
170
|
+
@classmethod
|
171
|
+
def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]: ...
|
172
|
+
@classmethod
|
173
|
+
def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None: ...
|
174
|
+
@classmethod
|
175
|
+
def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None: ...
|
176
|
+
|
164
177
|
class ___close_spec(typing_extensions.Protocol):
|
165
178
|
def __call__(self) -> None: ...
|
166
179
|
async def aio(self) -> None: ...
|
@@ -177,9 +190,6 @@ class FileIO(typing.Generic[T]):
|
|
177
190
|
def _check_readable(self) -> None: ...
|
178
191
|
def _check_closed(self) -> None: ...
|
179
192
|
def __enter__(self) -> FileIO: ...
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
async def aio(self, exc_type, exc_value, traceback) -> None: ...
|
184
|
-
|
185
|
-
__exit__: ____exit___spec
|
193
|
+
async def __aenter__(self) -> FileIO: ...
|
194
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None: ...
|
195
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# Copyright Modal Labs 2024
|
2
|
+
"""Pattern matching library ported from https://github.com/moby/patternmatcher.
|
3
|
+
|
4
|
+
This is the same pattern-matching logic used by Docker, except it is written in
|
5
|
+
Python rather than Go. Also, the original Go library has a couple deprecated
|
6
|
+
functions that we don't implement in this port.
|
7
|
+
|
8
|
+
The main way to use this library is by constructing a `FilePatternMatcher` object,
|
9
|
+
then asking it whether file paths match any of its patterns.
|
10
|
+
"""
|
11
|
+
|
12
|
+
import os
|
13
|
+
from abc import abstractmethod
|
14
|
+
from pathlib import Path
|
15
|
+
from typing import Callable, Optional
|
16
|
+
|
17
|
+
from ._utils.pattern_utils import Pattern
|
18
|
+
|
19
|
+
|
20
|
+
class _AbstractPatternMatcher:
|
21
|
+
_custom_repr: Optional[str] = None
|
22
|
+
|
23
|
+
def __invert__(self) -> "_AbstractPatternMatcher":
|
24
|
+
"""Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
|
25
|
+
|
26
|
+
Usage:
|
27
|
+
```python
|
28
|
+
from pathlib import Path
|
29
|
+
from modal import FilePatternMatcher
|
30
|
+
|
31
|
+
inverted_matcher = ~FilePatternMatcher("**/*.py")
|
32
|
+
|
33
|
+
assert not inverted_matcher(Path("foo.py"))
|
34
|
+
```
|
35
|
+
"""
|
36
|
+
return _CustomPatternMatcher(lambda path: not self(path))
|
37
|
+
|
38
|
+
def with_repr(self, custom_repr) -> "_AbstractPatternMatcher":
|
39
|
+
# use to give an instance of a matcher a custom name - useful for visualizing default values in signatures
|
40
|
+
self._custom_repr = custom_repr
|
41
|
+
return self
|
42
|
+
|
43
|
+
def __repr__(self) -> str:
|
44
|
+
if self._custom_repr:
|
45
|
+
return self._custom_repr
|
46
|
+
|
47
|
+
return super().__repr__()
|
48
|
+
|
49
|
+
@abstractmethod
|
50
|
+
def __call__(self, path: Path) -> bool:
|
51
|
+
...
|
52
|
+
|
53
|
+
|
54
|
+
class _CustomPatternMatcher(_AbstractPatternMatcher):
|
55
|
+
def __init__(self, predicate: Callable[[Path], bool]):
|
56
|
+
self._predicate = predicate
|
57
|
+
|
58
|
+
def __call__(self, path: Path) -> bool:
|
59
|
+
return self._predicate(path)
|
60
|
+
|
61
|
+
|
62
|
+
class FilePatternMatcher(_AbstractPatternMatcher):
|
63
|
+
"""Allows matching file paths against a list of patterns."""
|
64
|
+
|
65
|
+
def __init__(self, *pattern: str) -> None:
|
66
|
+
"""Initialize a new FilePatternMatcher instance.
|
67
|
+
|
68
|
+
Args:
|
69
|
+
pattern (str): One or more pattern strings.
|
70
|
+
|
71
|
+
Raises:
|
72
|
+
ValueError: If an illegal exclusion pattern is provided.
|
73
|
+
"""
|
74
|
+
self.patterns: list[Pattern] = []
|
75
|
+
self.exclusions = False
|
76
|
+
for p in list(pattern):
|
77
|
+
p = p.strip()
|
78
|
+
if not p:
|
79
|
+
continue
|
80
|
+
p = os.path.normpath(p)
|
81
|
+
new_pattern = Pattern()
|
82
|
+
if p[0] == "!":
|
83
|
+
if len(p) == 1:
|
84
|
+
raise ValueError('Illegal exclusion pattern: "!"')
|
85
|
+
new_pattern.exclusion = True
|
86
|
+
p = p[1:]
|
87
|
+
self.exclusions = True
|
88
|
+
# In Python, we can proceed without explicit syntax checking
|
89
|
+
new_pattern.cleaned_pattern = p
|
90
|
+
new_pattern.dirs = p.split(os.path.sep)
|
91
|
+
self.patterns.append(new_pattern)
|
92
|
+
|
93
|
+
def _matches(self, file_path: str) -> bool:
|
94
|
+
"""Check if the file path or any of its parent directories match the patterns.
|
95
|
+
|
96
|
+
This is equivalent to `MatchesOrParentMatches()` in the original Go
|
97
|
+
library. The reason is that `Matches()` in the original library is
|
98
|
+
deprecated due to buggy behavior.
|
99
|
+
"""
|
100
|
+
matched = False
|
101
|
+
file_path = os.path.normpath(file_path)
|
102
|
+
if file_path == ".":
|
103
|
+
# Don't let them exclude everything; kind of silly.
|
104
|
+
return False
|
105
|
+
parent_path = os.path.dirname(file_path)
|
106
|
+
if parent_path == "":
|
107
|
+
parent_path = "."
|
108
|
+
parent_path_dirs = parent_path.split(os.path.sep)
|
109
|
+
|
110
|
+
for pattern in self.patterns:
|
111
|
+
# Skip evaluation based on current match status and pattern exclusion
|
112
|
+
if pattern.exclusion != matched:
|
113
|
+
continue
|
114
|
+
|
115
|
+
match = pattern.match(file_path)
|
116
|
+
|
117
|
+
if not match and parent_path != ".":
|
118
|
+
# Check if the pattern matches any of the parent directories
|
119
|
+
for i in range(len(parent_path_dirs)):
|
120
|
+
dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
|
121
|
+
if pattern.match(dir_path):
|
122
|
+
match = True
|
123
|
+
break
|
124
|
+
|
125
|
+
if match:
|
126
|
+
matched = not pattern.exclusion
|
127
|
+
|
128
|
+
return matched
|
129
|
+
|
130
|
+
def __call__(self, file_path: Path) -> bool:
|
131
|
+
"""Check if the path matches any of the patterns.
|
132
|
+
|
133
|
+
Args:
|
134
|
+
file_path (Path): The path to check.
|
135
|
+
|
136
|
+
Returns:
|
137
|
+
True if the path matches any of the patterns.
|
138
|
+
|
139
|
+
Usage:
|
140
|
+
```python
|
141
|
+
from pathlib import Path
|
142
|
+
from modal import FilePatternMatcher
|
143
|
+
|
144
|
+
matcher = FilePatternMatcher("*.py")
|
145
|
+
|
146
|
+
assert matcher(Path("foo.py"))
|
147
|
+
```
|
148
|
+
"""
|
149
|
+
return self._matches(str(file_path))
|
150
|
+
|
151
|
+
|
152
|
+
# with_repr allows us to use this matcher as a default value in a function signature
|
153
|
+
# and get a nice repr in the docs and auto-generated type stubs:
|
154
|
+
NON_PYTHON_FILES = (~FilePatternMatcher("**/*.py")).with_repr(f"{__name__}.NON_PYTHON_FILES")
|