modal 0.67.1__py3-none-any.whl → 0.67.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/_clustered_functions.py +2 -2
- modal/_clustered_functions.pyi +2 -2
- modal/_container_entrypoint.py +8 -5
- modal/_output.py +29 -28
- modal/_pty.py +2 -2
- modal/_resolver.py +6 -5
- modal/_resources.py +3 -3
- modal/_runtime/asgi.py +46 -6
- modal/_runtime/container_io_manager.py +22 -26
- modal/_runtime/execution_context.py +2 -2
- modal/_runtime/telemetry.py +1 -2
- modal/_runtime/user_code_imports.py +12 -14
- modal/_serialization.py +3 -7
- modal/_traceback.py +5 -5
- modal/_tunnel.py +5 -4
- modal/_tunnel.pyi +2 -2
- modal/_utils/async_utils.py +53 -17
- modal/_utils/blob_utils.py +22 -7
- modal/_utils/function_utils.py +20 -10
- modal/_utils/grpc_testing.py +7 -6
- modal/_utils/grpc_utils.py +2 -3
- modal/_utils/hash_utils.py +2 -2
- modal/_utils/mount_utils.py +5 -4
- modal/_utils/package_utils.py +2 -3
- modal/_utils/pattern_matcher.py +6 -6
- modal/_utils/rand_pb_testing.py +3 -3
- modal/_utils/shell_utils.py +2 -1
- modal/_vendor/a2wsgi_wsgi.py +62 -72
- modal/_vendor/cloudpickle.py +1 -1
- modal/_watcher.py +8 -7
- modal/app.py +68 -62
- modal/app.pyi +104 -99
- modal/call_graph.py +6 -6
- modal/cli/_download.py +3 -2
- modal/cli/_traceback.py +4 -4
- modal/cli/app.py +4 -4
- modal/cli/container.py +4 -4
- modal/cli/dict.py +1 -1
- modal/cli/environment.py +2 -3
- modal/cli/import_refs.py +1 -1
- modal/cli/launch.py +2 -2
- modal/cli/network_file_system.py +1 -1
- modal/cli/profile.py +1 -1
- modal/cli/programs/run_jupyter.py +2 -2
- modal/cli/programs/vscode.py +3 -3
- modal/cli/queues.py +1 -1
- modal/cli/run.py +6 -6
- modal/cli/secret.py +3 -3
- modal/cli/utils.py +2 -1
- modal/cli/volume.py +3 -3
- modal/client.py +6 -11
- modal/client.pyi +18 -27
- modal/cloud_bucket_mount.py +3 -3
- modal/cloud_bucket_mount.pyi +2 -2
- modal/cls.py +100 -47
- modal/cls.pyi +40 -40
- modal/config.py +3 -2
- modal/container_process.py +6 -2
- modal/dict.py +6 -3
- modal/dict.pyi +10 -9
- modal/environments.py +3 -3
- modal/environments.pyi +3 -3
- modal/exception.py +2 -3
- modal/functions.py +112 -104
- modal/functions.pyi +77 -58
- modal/image.py +59 -57
- modal/image.pyi +104 -103
- modal/io_streams.py +20 -12
- modal/io_streams.pyi +24 -14
- modal/mount.py +24 -24
- modal/mount.pyi +28 -29
- modal/network_file_system.py +14 -11
- modal/network_file_system.pyi +12 -11
- modal/object.py +9 -8
- modal/object.pyi +47 -34
- modal/output.py +2 -1
- modal/parallel_map.py +4 -4
- modal/partial_function.py +10 -14
- modal/partial_function.pyi +17 -18
- modal/queue.py +11 -8
- modal/queue.pyi +23 -22
- modal/retries.py +38 -0
- modal/runner.py +8 -7
- modal/runner.pyi +8 -14
- modal/running_app.py +3 -3
- modal/sandbox.py +20 -13
- modal/sandbox.pyi +73 -72
- modal/scheduler_placement.py +2 -1
- modal/secret.py +7 -7
- modal/secret.pyi +12 -12
- modal/serving.py +4 -3
- modal/serving.pyi +5 -4
- modal/token_flow.py +3 -2
- modal/token_flow.pyi +3 -3
- modal/volume.py +16 -23
- modal/volume.pyi +17 -16
- {modal-0.67.1.dist-info → modal-0.67.33.dist-info}/METADATA +2 -2
- modal-0.67.33.dist-info/RECORD +168 -0
- modal_docs/mdmd/signatures.py +1 -2
- modal_global_objects/mounts/python_standalone.py +1 -1
- modal_proto/api.proto +15 -0
- modal_proto/api_grpc.py +32 -0
- modal_proto/api_pb2.py +674 -654
- modal_proto/api_pb2.pyi +45 -1
- modal_proto/api_pb2_grpc.py +66 -0
- modal_proto/api_pb2_grpc.pyi +20 -0
- modal_proto/modal_api_grpc.py +2 -0
- modal_version/_version_generated.py +1 -1
- modal-0.67.1.dist-info/RECORD +0 -168
- {modal-0.67.1.dist-info → modal-0.67.33.dist-info}/LICENSE +0 -0
- {modal-0.67.1.dist-info → modal-0.67.33.dist-info}/WHEEL +0 -0
- {modal-0.67.1.dist-info → modal-0.67.33.dist-info}/entry_points.txt +0 -0
- {modal-0.67.1.dist-info → modal-0.67.33.dist-info}/top_level.txt +0 -0
modal/io_streams.pyi
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import collections.abc
|
1
2
|
import modal.client
|
2
3
|
import modal.stream_type
|
3
4
|
import typing
|
@@ -5,15 +6,15 @@ import typing_extensions
|
|
5
6
|
|
6
7
|
def _sandbox_logs_iterator(
|
7
8
|
sandbox_id: str, file_descriptor: int, last_entry_id: str, client: modal.client._Client
|
8
|
-
) ->
|
9
|
+
) -> collections.abc.AsyncGenerator[tuple[typing.Optional[bytes], str], None]: ...
|
9
10
|
def _container_process_logs_iterator(
|
10
11
|
process_id: str, file_descriptor: int, client: modal.client._Client
|
11
|
-
) ->
|
12
|
+
) -> collections.abc.AsyncGenerator[typing.Optional[bytes], None]: ...
|
12
13
|
|
13
14
|
T = typing.TypeVar("T")
|
14
15
|
|
15
16
|
class _StreamReader(typing.Generic[T]):
|
16
|
-
_stream: typing.Optional[
|
17
|
+
_stream: typing.Optional[collections.abc.AsyncGenerator[typing.Optional[bytes], None]]
|
17
18
|
|
18
19
|
def __init__(
|
19
20
|
self,
|
@@ -29,11 +30,14 @@ class _StreamReader(typing.Generic[T]):
|
|
29
30
|
def file_descriptor(self) -> int: ...
|
30
31
|
async def read(self) -> T: ...
|
31
32
|
async def _consume_container_process_stream(self): ...
|
32
|
-
def _stream_container_process(self) ->
|
33
|
-
def _get_logs(
|
34
|
-
|
35
|
-
|
33
|
+
def _stream_container_process(self) -> collections.abc.AsyncGenerator[tuple[typing.Optional[bytes], str], None]: ...
|
34
|
+
def _get_logs(
|
35
|
+
self, skip_empty_messages: bool = True
|
36
|
+
) -> collections.abc.AsyncGenerator[typing.Optional[bytes], None]: ...
|
37
|
+
def _get_logs_by_line(self) -> collections.abc.AsyncGenerator[typing.Optional[bytes], None]: ...
|
38
|
+
def __aiter__(self) -> collections.abc.AsyncIterator[T]: ...
|
36
39
|
async def __anext__(self) -> T: ...
|
40
|
+
async def aclose(self): ...
|
37
41
|
|
38
42
|
class _StreamWriter:
|
39
43
|
def __init__(
|
@@ -47,7 +51,7 @@ class _StreamWriter:
|
|
47
51
|
T_INNER = typing.TypeVar("T_INNER", covariant=True)
|
48
52
|
|
49
53
|
class StreamReader(typing.Generic[T]):
|
50
|
-
_stream: typing.Optional[
|
54
|
+
_stream: typing.Optional[collections.abc.AsyncGenerator[typing.Optional[bytes], None]]
|
51
55
|
|
52
56
|
def __init__(
|
53
57
|
self,
|
@@ -75,27 +79,33 @@ class StreamReader(typing.Generic[T]):
|
|
75
79
|
_consume_container_process_stream: ___consume_container_process_stream_spec
|
76
80
|
|
77
81
|
class ___stream_container_process_spec(typing_extensions.Protocol):
|
78
|
-
def __call__(self) -> typing.Generator[
|
79
|
-
def aio(self) ->
|
82
|
+
def __call__(self) -> typing.Generator[tuple[typing.Optional[bytes], str], None, None]: ...
|
83
|
+
def aio(self) -> collections.abc.AsyncGenerator[tuple[typing.Optional[bytes], str], None]: ...
|
80
84
|
|
81
85
|
_stream_container_process: ___stream_container_process_spec
|
82
86
|
|
83
87
|
class ___get_logs_spec(typing_extensions.Protocol):
|
84
|
-
def __call__(
|
85
|
-
|
88
|
+
def __call__(
|
89
|
+
self, skip_empty_messages: bool = True
|
90
|
+
) -> typing.Generator[typing.Optional[bytes], None, None]: ...
|
91
|
+
def aio(
|
92
|
+
self, skip_empty_messages: bool = True
|
93
|
+
) -> collections.abc.AsyncGenerator[typing.Optional[bytes], None]: ...
|
86
94
|
|
87
95
|
_get_logs: ___get_logs_spec
|
88
96
|
|
89
97
|
class ___get_logs_by_line_spec(typing_extensions.Protocol):
|
90
98
|
def __call__(self) -> typing.Generator[typing.Optional[bytes], None, None]: ...
|
91
|
-
def aio(self) ->
|
99
|
+
def aio(self) -> collections.abc.AsyncGenerator[typing.Optional[bytes], None]: ...
|
92
100
|
|
93
101
|
_get_logs_by_line: ___get_logs_by_line_spec
|
94
102
|
|
95
103
|
def __iter__(self) -> typing.Iterator[T]: ...
|
96
|
-
def __aiter__(self) ->
|
104
|
+
def __aiter__(self) -> collections.abc.AsyncIterator[T]: ...
|
97
105
|
def __next__(self) -> T: ...
|
98
106
|
async def __anext__(self) -> T: ...
|
107
|
+
def close(self): ...
|
108
|
+
async def aclose(self): ...
|
99
109
|
|
100
110
|
class StreamWriter:
|
101
111
|
def __init__(
|
modal/mount.py
CHANGED
@@ -9,8 +9,9 @@ import sys
|
|
9
9
|
import sysconfig
|
10
10
|
import time
|
11
11
|
import typing
|
12
|
+
from collections.abc import AsyncGenerator
|
12
13
|
from pathlib import Path, PurePosixPath
|
13
|
-
from typing import
|
14
|
+
from typing import Callable, Optional, Union
|
14
15
|
|
15
16
|
from google.protobuf.message import Message
|
16
17
|
|
@@ -36,7 +37,7 @@ MOUNT_PUT_FILE_CLIENT_TIMEOUT = 10 * 60 # 10 min max for transferring files
|
|
36
37
|
#
|
37
38
|
# These can be updated safely, but changes will trigger a rebuild for all images
|
38
39
|
# that rely on `add_python()` in their constructor.
|
39
|
-
PYTHON_STANDALONE_VERSIONS:
|
40
|
+
PYTHON_STANDALONE_VERSIONS: dict[str, tuple[str, str]] = {
|
40
41
|
"3.9": ("20230826", "3.9.18"),
|
41
42
|
"3.10": ("20230826", "3.10.13"),
|
42
43
|
"3.11": ("20230826", "3.11.5"),
|
@@ -73,21 +74,21 @@ class _MountEntry(metaclass=abc.ABCMeta):
|
|
73
74
|
...
|
74
75
|
|
75
76
|
@abc.abstractmethod
|
76
|
-
def get_files_to_upload(self) -> typing.Iterator[
|
77
|
+
def get_files_to_upload(self) -> typing.Iterator[tuple[Path, str]]:
|
77
78
|
...
|
78
79
|
|
79
80
|
@abc.abstractmethod
|
80
|
-
def watch_entry(self) ->
|
81
|
+
def watch_entry(self) -> tuple[Path, Path]:
|
81
82
|
...
|
82
83
|
|
83
84
|
@abc.abstractmethod
|
84
|
-
def top_level_paths(self) ->
|
85
|
+
def top_level_paths(self) -> list[tuple[Path, PurePosixPath]]:
|
85
86
|
...
|
86
87
|
|
87
88
|
|
88
|
-
def _select_files(entries:
|
89
|
+
def _select_files(entries: list[_MountEntry]) -> list[tuple[Path, PurePosixPath]]:
|
89
90
|
# TODO: make this async
|
90
|
-
all_files:
|
91
|
+
all_files: set[tuple[Path, PurePosixPath]] = set()
|
91
92
|
for entry in entries:
|
92
93
|
all_files |= set(entry.get_files_to_upload())
|
93
94
|
return list(all_files)
|
@@ -113,7 +114,7 @@ class _MountFile(_MountEntry):
|
|
113
114
|
safe_path = self.local_file.expanduser().absolute()
|
114
115
|
return safe_path.parent, safe_path
|
115
116
|
|
116
|
-
def top_level_paths(self) ->
|
117
|
+
def top_level_paths(self) -> list[tuple[Path, PurePosixPath]]:
|
117
118
|
return [(self.local_file, self.remote_path)]
|
118
119
|
|
119
120
|
|
@@ -150,7 +151,7 @@ class _MountDir(_MountEntry):
|
|
150
151
|
def watch_entry(self):
|
151
152
|
return self.local_dir.resolve().expanduser(), None
|
152
153
|
|
153
|
-
def top_level_paths(self) ->
|
154
|
+
def top_level_paths(self) -> list[tuple[Path, PurePosixPath]]:
|
154
155
|
return [(self.local_dir, self.remote_path)]
|
155
156
|
|
156
157
|
|
@@ -194,7 +195,7 @@ class _MountedPythonModule(_MountEntry):
|
|
194
195
|
def description(self) -> str:
|
195
196
|
return f"PythonPackage:{self.module_name}"
|
196
197
|
|
197
|
-
def _proxy_entries(self) ->
|
198
|
+
def _proxy_entries(self) -> list[_MountEntry]:
|
198
199
|
mount_infos = get_module_mount_info(self.module_name)
|
199
200
|
entries = []
|
200
201
|
for mount_info in mount_infos:
|
@@ -220,16 +221,16 @@ class _MountedPythonModule(_MountEntry):
|
|
220
221
|
)
|
221
222
|
return entries
|
222
223
|
|
223
|
-
def get_files_to_upload(self) -> typing.Iterator[
|
224
|
+
def get_files_to_upload(self) -> typing.Iterator[tuple[Path, str]]:
|
224
225
|
for entry in self._proxy_entries():
|
225
226
|
yield from entry.get_files_to_upload()
|
226
227
|
|
227
|
-
def watch_entry(self) ->
|
228
|
+
def watch_entry(self) -> tuple[Path, Path]:
|
228
229
|
for entry in self._proxy_entries():
|
229
230
|
# TODO: fix watch for mounts of multi-path packages
|
230
231
|
return entry.watch_entry()
|
231
232
|
|
232
|
-
def top_level_paths(self) ->
|
233
|
+
def top_level_paths(self) -> list[tuple[Path, PurePosixPath]]:
|
233
234
|
paths = []
|
234
235
|
for sub in self._proxy_entries():
|
235
236
|
paths.extend(sub.top_level_paths())
|
@@ -262,14 +263,14 @@ class _Mount(_Object, type_prefix="mo"):
|
|
262
263
|
the file's contents to skip uploading files that have been uploaded before.
|
263
264
|
"""
|
264
265
|
|
265
|
-
_entries: Optional[
|
266
|
+
_entries: Optional[list[_MountEntry]] = None
|
266
267
|
_deployment_name: Optional[str] = None
|
267
268
|
_namespace: Optional[int] = None
|
268
269
|
_environment_name: Optional[str] = None
|
269
270
|
_content_checksum_sha256_hex: Optional[str] = None
|
270
271
|
|
271
272
|
@staticmethod
|
272
|
-
def _new(entries:
|
273
|
+
def _new(entries: list[_MountEntry] = []) -> "_Mount":
|
273
274
|
rep = f"Mount({entries})"
|
274
275
|
|
275
276
|
async def mount_content_deduplication_key():
|
@@ -298,10 +299,10 @@ class _Mount(_Object, type_prefix="mo"):
|
|
298
299
|
assert isinstance(handle_metadata, api_pb2.MountHandleMetadata)
|
299
300
|
self._content_checksum_sha256_hex = handle_metadata.content_checksum_sha256_hex
|
300
301
|
|
301
|
-
def _top_level_paths(self) ->
|
302
|
+
def _top_level_paths(self) -> list[tuple[Path, PurePosixPath]]:
|
302
303
|
# Returns [(local_absolute_path, remote_path), ...] for all top level entries in the Mount
|
303
304
|
# Used to determine if a package mount is installed in a sys directory or not
|
304
|
-
res:
|
305
|
+
res: list[tuple[Path, PurePosixPath]] = []
|
305
306
|
for entry in self.entries:
|
306
307
|
res.extend(entry.top_level_paths())
|
307
308
|
return res
|
@@ -413,12 +414,12 @@ class _Mount(_Object, type_prefix="mo"):
|
|
413
414
|
return _Mount._new().add_local_file(local_path, remote_path=remote_path)
|
414
415
|
|
415
416
|
@staticmethod
|
416
|
-
def _description(entries:
|
417
|
+
def _description(entries: list[_MountEntry]) -> str:
|
417
418
|
local_contents = [e.description() for e in entries]
|
418
419
|
return ", ".join(local_contents)
|
419
420
|
|
420
421
|
@staticmethod
|
421
|
-
async def _get_files(entries:
|
422
|
+
async def _get_files(entries: list[_MountEntry]) -> AsyncGenerator[FileUploadSpec, None]:
|
422
423
|
loop = asyncio.get_event_loop()
|
423
424
|
with concurrent.futures.ThreadPoolExecutor() as exe:
|
424
425
|
all_files = await loop.run_in_executor(exe, _select_files, entries)
|
@@ -502,7 +503,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
502
503
|
|
503
504
|
# Upload files, or check if they already exist.
|
504
505
|
n_concurrent_uploads = 512
|
505
|
-
files:
|
506
|
+
files: list[api_pb2.MountFile] = []
|
506
507
|
async with aclosing(
|
507
508
|
async_map(_Mount._get_files(self._entries), _put_file, concurrency=n_concurrent_uploads)
|
508
509
|
) as stream:
|
@@ -602,7 +603,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
602
603
|
|
603
604
|
@classmethod
|
604
605
|
async def lookup(
|
605
|
-
cls:
|
606
|
+
cls: type["_Mount"],
|
606
607
|
label: str,
|
607
608
|
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
|
608
609
|
client: Optional[_Client] = None,
|
@@ -715,7 +716,7 @@ def _is_modal_path(remote_path: PurePosixPath):
|
|
715
716
|
return False
|
716
717
|
|
717
718
|
|
718
|
-
def get_auto_mounts() ->
|
719
|
+
def get_auto_mounts() -> list[_Mount]:
|
719
720
|
"""mdmd:hidden
|
720
721
|
|
721
722
|
Auto-mount local modules that have been imported in global scope.
|
@@ -748,8 +749,7 @@ def get_auto_mounts() -> typing.List[_Mount]:
|
|
748
749
|
continue
|
749
750
|
|
750
751
|
for local_path, remote_path in mount_paths:
|
751
|
-
|
752
|
-
if any(str(local_path).startswith(str(p)) for p in SYS_PREFIXES) or _is_modal_path(remote_path):
|
752
|
+
if any(local_path.is_relative_to(p) for p in SYS_PREFIXES) or _is_modal_path(remote_path):
|
753
753
|
# skip any module that has paths in SYS_PREFIXES, or would overwrite the modal Package in the container
|
754
754
|
break
|
755
755
|
else:
|
modal/mount.pyi
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import collections.abc
|
1
2
|
import google.protobuf.message
|
2
3
|
import modal._resolver
|
3
4
|
import modal._utils.blob_utils
|
@@ -13,13 +14,11 @@ def python_standalone_mount_name(version: str) -> str: ...
|
|
13
14
|
|
14
15
|
class _MountEntry:
|
15
16
|
def description(self) -> str: ...
|
16
|
-
def get_files_to_upload(self) -> typing.Iterator[
|
17
|
-
def watch_entry(self) ->
|
18
|
-
def top_level_paths(self) ->
|
17
|
+
def get_files_to_upload(self) -> typing.Iterator[tuple[pathlib.Path, str]]: ...
|
18
|
+
def watch_entry(self) -> tuple[pathlib.Path, pathlib.Path]: ...
|
19
|
+
def top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
19
20
|
|
20
|
-
def _select_files(
|
21
|
-
entries: typing.List[_MountEntry],
|
22
|
-
) -> typing.List[typing.Tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
21
|
+
def _select_files(entries: list[_MountEntry]) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
23
22
|
|
24
23
|
class _MountFile(_MountEntry):
|
25
24
|
local_file: pathlib.Path
|
@@ -28,7 +27,7 @@ class _MountFile(_MountEntry):
|
|
28
27
|
def description(self) -> str: ...
|
29
28
|
def get_files_to_upload(self): ...
|
30
29
|
def watch_entry(self): ...
|
31
|
-
def top_level_paths(self) ->
|
30
|
+
def top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
32
31
|
def __init__(self, local_file: pathlib.Path, remote_path: pathlib.PurePosixPath) -> None: ...
|
33
32
|
def __repr__(self): ...
|
34
33
|
def __eq__(self, other): ...
|
@@ -42,7 +41,7 @@ class _MountDir(_MountEntry):
|
|
42
41
|
def description(self): ...
|
43
42
|
def get_files_to_upload(self): ...
|
44
43
|
def watch_entry(self): ...
|
45
|
-
def top_level_paths(self) ->
|
44
|
+
def top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
46
45
|
def __init__(
|
47
46
|
self,
|
48
47
|
local_dir: pathlib.Path,
|
@@ -61,10 +60,10 @@ class _MountedPythonModule(_MountEntry):
|
|
61
60
|
condition: typing.Optional[typing.Callable[[str], bool]]
|
62
61
|
|
63
62
|
def description(self) -> str: ...
|
64
|
-
def _proxy_entries(self) ->
|
65
|
-
def get_files_to_upload(self) -> typing.Iterator[
|
66
|
-
def watch_entry(self) ->
|
67
|
-
def top_level_paths(self) ->
|
63
|
+
def _proxy_entries(self) -> list[_MountEntry]: ...
|
64
|
+
def get_files_to_upload(self) -> typing.Iterator[tuple[pathlib.Path, str]]: ...
|
65
|
+
def watch_entry(self) -> tuple[pathlib.Path, pathlib.Path]: ...
|
66
|
+
def top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
68
67
|
def __init__(
|
69
68
|
self,
|
70
69
|
module_name: str,
|
@@ -77,19 +76,19 @@ class _MountedPythonModule(_MountEntry):
|
|
77
76
|
class NonLocalMountError(Exception): ...
|
78
77
|
|
79
78
|
class _Mount(modal.object._Object):
|
80
|
-
_entries: typing.Optional[
|
79
|
+
_entries: typing.Optional[list[_MountEntry]]
|
81
80
|
_deployment_name: typing.Optional[str]
|
82
81
|
_namespace: typing.Optional[int]
|
83
82
|
_environment_name: typing.Optional[str]
|
84
83
|
_content_checksum_sha256_hex: typing.Optional[str]
|
85
84
|
|
86
85
|
@staticmethod
|
87
|
-
def _new(entries:
|
86
|
+
def _new(entries: list[_MountEntry] = []) -> _Mount: ...
|
88
87
|
def _extend(self, entry: _MountEntry) -> _Mount: ...
|
89
88
|
@property
|
90
89
|
def entries(self): ...
|
91
90
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
92
|
-
def _top_level_paths(self) ->
|
91
|
+
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
93
92
|
def is_local(self) -> bool: ...
|
94
93
|
def add_local_dir(
|
95
94
|
self,
|
@@ -117,11 +116,11 @@ class _Mount(modal.object._Object):
|
|
117
116
|
local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None
|
118
117
|
) -> _Mount: ...
|
119
118
|
@staticmethod
|
120
|
-
def _description(entries:
|
119
|
+
def _description(entries: list[_MountEntry]) -> str: ...
|
121
120
|
@staticmethod
|
122
121
|
def _get_files(
|
123
|
-
entries:
|
124
|
-
) ->
|
122
|
+
entries: list[_MountEntry],
|
123
|
+
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
125
124
|
async def _load_mount(
|
126
125
|
self: _Mount, resolver: modal._resolver.Resolver, existing_object_id: typing.Optional[str]
|
127
126
|
): ...
|
@@ -135,7 +134,7 @@ class _Mount(modal.object._Object):
|
|
135
134
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> _Mount: ...
|
136
135
|
@classmethod
|
137
136
|
async def lookup(
|
138
|
-
cls:
|
137
|
+
cls: type[_Mount],
|
139
138
|
label: str,
|
140
139
|
namespace=1,
|
141
140
|
client: typing.Optional[modal.client._Client] = None,
|
@@ -151,7 +150,7 @@ class _Mount(modal.object._Object):
|
|
151
150
|
def _get_metadata(self) -> modal_proto.api_pb2.MountHandleMetadata: ...
|
152
151
|
|
153
152
|
class Mount(modal.object.Object):
|
154
|
-
_entries: typing.Optional[
|
153
|
+
_entries: typing.Optional[list[_MountEntry]]
|
155
154
|
_deployment_name: typing.Optional[str]
|
156
155
|
_namespace: typing.Optional[int]
|
157
156
|
_environment_name: typing.Optional[str]
|
@@ -159,12 +158,12 @@ class Mount(modal.object.Object):
|
|
159
158
|
|
160
159
|
def __init__(self, *args, **kwargs): ...
|
161
160
|
@staticmethod
|
162
|
-
def _new(entries:
|
161
|
+
def _new(entries: list[_MountEntry] = []) -> Mount: ...
|
163
162
|
def _extend(self, entry: _MountEntry) -> Mount: ...
|
164
163
|
@property
|
165
164
|
def entries(self): ...
|
166
165
|
def _hydrate_metadata(self, handle_metadata: typing.Optional[google.protobuf.message.Message]): ...
|
167
|
-
def _top_level_paths(self) ->
|
166
|
+
def _top_level_paths(self) -> list[tuple[pathlib.Path, pathlib.PurePosixPath]]: ...
|
168
167
|
def is_local(self) -> bool: ...
|
169
168
|
def add_local_dir(
|
170
169
|
self,
|
@@ -192,15 +191,15 @@ class Mount(modal.object.Object):
|
|
192
191
|
local_path: typing.Union[str, pathlib.Path], remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None
|
193
192
|
) -> Mount: ...
|
194
193
|
@staticmethod
|
195
|
-
def _description(entries:
|
194
|
+
def _description(entries: list[_MountEntry]) -> str: ...
|
196
195
|
|
197
196
|
class ___get_files_spec(typing_extensions.Protocol):
|
198
197
|
def __call__(
|
199
|
-
self, entries:
|
198
|
+
self, entries: list[_MountEntry]
|
200
199
|
) -> typing.Generator[modal._utils.blob_utils.FileUploadSpec, None, None]: ...
|
201
200
|
def aio(
|
202
|
-
self, entries:
|
203
|
-
) ->
|
201
|
+
self, entries: list[_MountEntry]
|
202
|
+
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
204
203
|
|
205
204
|
_get_files: ___get_files_spec
|
206
205
|
|
@@ -220,7 +219,7 @@ class Mount(modal.object.Object):
|
|
220
219
|
def from_name(label: str, namespace=1, environment_name: typing.Optional[str] = None) -> Mount: ...
|
221
220
|
@classmethod
|
222
221
|
def lookup(
|
223
|
-
cls:
|
222
|
+
cls: type[Mount],
|
224
223
|
label: str,
|
225
224
|
namespace=1,
|
226
225
|
client: typing.Optional[modal.client.Client] = None,
|
@@ -251,8 +250,8 @@ def _create_client_mount(): ...
|
|
251
250
|
def create_client_mount(): ...
|
252
251
|
def _get_client_mount(): ...
|
253
252
|
def _is_modal_path(remote_path: pathlib.PurePosixPath): ...
|
254
|
-
def get_auto_mounts() ->
|
253
|
+
def get_auto_mounts() -> list[_Mount]: ...
|
255
254
|
|
256
255
|
ROOT_DIR: pathlib.PurePosixPath
|
257
256
|
|
258
|
-
PYTHON_STANDALONE_VERSIONS:
|
257
|
+
PYTHON_STANDALONE_VERSIONS: dict[str, tuple[str, str]]
|
modal/network_file_system.py
CHANGED
@@ -2,8 +2,9 @@
|
|
2
2
|
import functools
|
3
3
|
import os
|
4
4
|
import time
|
5
|
+
from collections.abc import AsyncIterator
|
5
6
|
from pathlib import Path, PurePosixPath
|
6
|
-
from typing import Any,
|
7
|
+
from typing import Any, BinaryIO, Callable, Optional, Union
|
7
8
|
|
8
9
|
from grpclib import GRPCError, Status
|
9
10
|
from synchronicity.async_wrap import asynccontextmanager
|
@@ -34,9 +35,9 @@ NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT = (
|
|
34
35
|
|
35
36
|
|
36
37
|
def network_file_system_mount_protos(
|
37
|
-
validated_network_file_systems:
|
38
|
+
validated_network_file_systems: list[tuple[str, "_NetworkFileSystem"]],
|
38
39
|
allow_cross_region_volumes: bool,
|
39
|
-
) ->
|
40
|
+
) -> list[api_pb2.SharedVolumeMount]:
|
40
41
|
network_file_system_mounts = []
|
41
42
|
# Relies on dicts being ordered (true as of Python 3.6).
|
42
43
|
for path, volume in validated_network_file_systems:
|
@@ -143,7 +144,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
143
144
|
@classmethod
|
144
145
|
@asynccontextmanager
|
145
146
|
async def ephemeral(
|
146
|
-
cls:
|
147
|
+
cls: type["_NetworkFileSystem"],
|
147
148
|
client: Optional[_Client] = None,
|
148
149
|
environment_name: Optional[str] = None,
|
149
150
|
_heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
@@ -152,11 +153,13 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
152
153
|
|
153
154
|
Usage:
|
154
155
|
```python
|
155
|
-
with NetworkFileSystem.ephemeral() as nfs:
|
156
|
-
assert nfs.listdir() == []
|
156
|
+
with modal.NetworkFileSystem.ephemeral() as nfs:
|
157
|
+
assert nfs.listdir("/") == []
|
158
|
+
```
|
157
159
|
|
158
|
-
|
159
|
-
|
160
|
+
```python notest
|
161
|
+
async with modal.NetworkFileSystem.ephemeral() as nfs:
|
162
|
+
assert await nfs.listdir("/") == []
|
160
163
|
```
|
161
164
|
"""
|
162
165
|
if client is None:
|
@@ -184,7 +187,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
184
187
|
In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
|
185
188
|
that will hydrate the local object with metadata from Modal servers.
|
186
189
|
|
187
|
-
```python
|
190
|
+
```python notest
|
188
191
|
nfs = modal.NetworkFileSystem.lookup("my-nfs")
|
189
192
|
print(nfs.listdir("/"))
|
190
193
|
```
|
@@ -327,7 +330,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
327
330
|
relpath_str = subpath.relative_to(_local_path).as_posix()
|
328
331
|
yield subpath, PurePosixPath(remote_path, relpath_str)
|
329
332
|
|
330
|
-
async def _add_local_file(paths:
|
333
|
+
async def _add_local_file(paths: tuple[Path, PurePosixPath]) -> int:
|
331
334
|
return await self.add_local_file(paths[0], paths[1], progress_cb)
|
332
335
|
|
333
336
|
async with aclosing(async_map(sync_or_async_iter(gen_transfers()), _add_local_file, concurrency=20)) as stream:
|
@@ -335,7 +338,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
335
338
|
pass
|
336
339
|
|
337
340
|
@live_method
|
338
|
-
async def listdir(self, path: str) ->
|
341
|
+
async def listdir(self, path: str) -> list[FileEntry]:
|
339
342
|
"""List all files in a directory in the network file system.
|
340
343
|
|
341
344
|
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
modal/network_file_system.pyi
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import collections.abc
|
1
2
|
import modal.client
|
2
3
|
import modal.object
|
3
4
|
import modal.volume
|
@@ -8,8 +9,8 @@ import typing
|
|
8
9
|
import typing_extensions
|
9
10
|
|
10
11
|
def network_file_system_mount_protos(
|
11
|
-
validated_network_file_systems:
|
12
|
-
) ->
|
12
|
+
validated_network_file_systems: list[tuple[str, _NetworkFileSystem]], allow_cross_region_volumes: bool
|
13
|
+
) -> list[modal_proto.api_pb2.SharedVolumeMount]: ...
|
13
14
|
|
14
15
|
class _NetworkFileSystem(modal.object._Object):
|
15
16
|
@staticmethod
|
@@ -20,7 +21,7 @@ class _NetworkFileSystem(modal.object._Object):
|
|
20
21
|
) -> _NetworkFileSystem: ...
|
21
22
|
@classmethod
|
22
23
|
def ephemeral(
|
23
|
-
cls:
|
24
|
+
cls: type[_NetworkFileSystem],
|
24
25
|
client: typing.Optional[modal.client._Client] = None,
|
25
26
|
environment_name: typing.Optional[str] = None,
|
26
27
|
_heartbeat_sleep: float = 300,
|
@@ -46,8 +47,8 @@ class _NetworkFileSystem(modal.object._Object):
|
|
46
47
|
fp: typing.BinaryIO,
|
47
48
|
progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
|
48
49
|
) -> int: ...
|
49
|
-
def read_file(self, path: str) ->
|
50
|
-
def iterdir(self, path: str) ->
|
50
|
+
def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
|
51
|
+
def iterdir(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
|
51
52
|
async def add_local_file(
|
52
53
|
self,
|
53
54
|
local_path: typing.Union[pathlib.Path, str],
|
@@ -60,7 +61,7 @@ class _NetworkFileSystem(modal.object._Object):
|
|
60
61
|
remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
|
61
62
|
progress_cb: typing.Optional[typing.Callable[..., typing.Any]] = None,
|
62
63
|
): ...
|
63
|
-
async def listdir(self, path: str) ->
|
64
|
+
async def listdir(self, path: str) -> list[modal.volume.FileEntry]: ...
|
64
65
|
async def remove_file(self, path: str, recursive=False): ...
|
65
66
|
|
66
67
|
class NetworkFileSystem(modal.object.Object):
|
@@ -73,7 +74,7 @@ class NetworkFileSystem(modal.object.Object):
|
|
73
74
|
) -> NetworkFileSystem: ...
|
74
75
|
@classmethod
|
75
76
|
def ephemeral(
|
76
|
-
cls:
|
77
|
+
cls: type[NetworkFileSystem],
|
77
78
|
client: typing.Optional[modal.client.Client] = None,
|
78
79
|
environment_name: typing.Optional[str] = None,
|
79
80
|
_heartbeat_sleep: float = 300,
|
@@ -135,13 +136,13 @@ class NetworkFileSystem(modal.object.Object):
|
|
135
136
|
|
136
137
|
class __read_file_spec(typing_extensions.Protocol):
|
137
138
|
def __call__(self, path: str) -> typing.Iterator[bytes]: ...
|
138
|
-
def aio(self, path: str) ->
|
139
|
+
def aio(self, path: str) -> collections.abc.AsyncIterator[bytes]: ...
|
139
140
|
|
140
141
|
read_file: __read_file_spec
|
141
142
|
|
142
143
|
class __iterdir_spec(typing_extensions.Protocol):
|
143
144
|
def __call__(self, path: str) -> typing.Iterator[modal.volume.FileEntry]: ...
|
144
|
-
def aio(self, path: str) ->
|
145
|
+
def aio(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]: ...
|
145
146
|
|
146
147
|
iterdir: __iterdir_spec
|
147
148
|
|
@@ -178,8 +179,8 @@ class NetworkFileSystem(modal.object.Object):
|
|
178
179
|
add_local_dir: __add_local_dir_spec
|
179
180
|
|
180
181
|
class __listdir_spec(typing_extensions.Protocol):
|
181
|
-
def __call__(self, path: str) ->
|
182
|
-
async def aio(self, path: str) ->
|
182
|
+
def __call__(self, path: str) -> list[modal.volume.FileEntry]: ...
|
183
|
+
async def aio(self, path: str) -> list[modal.volume.FileEntry]: ...
|
183
184
|
|
184
185
|
listdir: __listdir_spec
|
185
186
|
|
modal/object.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# Copyright Modal Labs 2022
|
2
2
|
import uuid
|
3
|
+
from collections.abc import Awaitable, Hashable, Sequence
|
3
4
|
from functools import wraps
|
4
|
-
from typing import
|
5
|
+
from typing import Callable, ClassVar, Optional, TypeVar
|
5
6
|
|
6
7
|
from google.protobuf.message import Message
|
7
8
|
|
@@ -31,7 +32,7 @@ def _get_environment_name(environment_name: Optional[str] = None, resolver: Opti
|
|
31
32
|
|
32
33
|
class _Object:
|
33
34
|
_type_prefix: ClassVar[Optional[str]] = None
|
34
|
-
_prefix_to_type: ClassVar[
|
35
|
+
_prefix_to_type: ClassVar[dict[str, type]] = {}
|
35
36
|
|
36
37
|
# For constructors
|
37
38
|
_load: Optional[Callable[[O, Resolver, Optional[str]], Awaitable[None]]]
|
@@ -39,7 +40,7 @@ class _Object:
|
|
39
40
|
_rep: str
|
40
41
|
_is_another_app: bool
|
41
42
|
_hydrate_lazily: bool
|
42
|
-
_deps: Optional[Callable[...,
|
43
|
+
_deps: Optional[Callable[..., list["_Object"]]]
|
43
44
|
_deduplication_key: Optional[Callable[[], Awaitable[Hashable]]] = None
|
44
45
|
|
45
46
|
# For hydrated objects
|
@@ -65,7 +66,7 @@ class _Object:
|
|
65
66
|
is_another_app: bool = False,
|
66
67
|
preload: Optional[Callable[[O, Resolver, Optional[str]], Awaitable[None]]] = None,
|
67
68
|
hydrate_lazily: bool = False,
|
68
|
-
deps: Optional[Callable[...,
|
69
|
+
deps: Optional[Callable[..., list["_Object"]]] = None,
|
69
70
|
deduplication_key: Optional[Callable[[], Awaitable[Hashable]]] = None,
|
70
71
|
):
|
71
72
|
self._local_uuid = str(uuid.uuid4())
|
@@ -159,7 +160,7 @@ class _Object:
|
|
159
160
|
return obj
|
160
161
|
|
161
162
|
@classmethod
|
162
|
-
def _get_type_from_id(cls:
|
163
|
+
def _get_type_from_id(cls: type[O], object_id: str) -> type[O]:
|
163
164
|
parts = object_id.split("-")
|
164
165
|
if len(parts) != 2:
|
165
166
|
raise InvalidError(f"Object id {object_id} has no dash in it")
|
@@ -169,12 +170,12 @@ class _Object:
|
|
169
170
|
return cls._prefix_to_type[prefix]
|
170
171
|
|
171
172
|
@classmethod
|
172
|
-
def _is_id_type(cls:
|
173
|
+
def _is_id_type(cls: type[O], object_id) -> bool:
|
173
174
|
return cls._get_type_from_id(object_id) == cls
|
174
175
|
|
175
176
|
@classmethod
|
176
177
|
def _new_hydrated(
|
177
|
-
cls:
|
178
|
+
cls: type[O], object_id: str, client: _Client, handle_metadata: Optional[Message], is_another_app: bool = False
|
178
179
|
) -> O:
|
179
180
|
if cls._type_prefix is not None:
|
180
181
|
# This is called directly on a subclass, e.g. Secret.from_id
|
@@ -215,7 +216,7 @@ class _Object:
|
|
215
216
|
return self._is_hydrated
|
216
217
|
|
217
218
|
@property
|
218
|
-
def deps(self) -> Callable[...,
|
219
|
+
def deps(self) -> Callable[..., list["_Object"]]:
|
219
220
|
"""mdmd:hidden"""
|
220
221
|
return self._deps if self._deps is not None else lambda: []
|
221
222
|
|