modal 0.62.115__py3-none-any.whl → 0.72.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +13 -9
- modal/__main__.py +41 -3
- modal/_clustered_functions.py +80 -0
- modal/_clustered_functions.pyi +22 -0
- modal/_container_entrypoint.py +402 -398
- modal/_ipython.py +3 -13
- modal/_location.py +17 -10
- modal/_output.py +243 -99
- modal/_pty.py +2 -2
- modal/_resolver.py +55 -60
- modal/_resources.py +26 -7
- modal/_runtime/__init__.py +1 -0
- modal/_runtime/asgi.py +519 -0
- modal/_runtime/container_io_manager.py +1025 -0
- modal/{execution_context.py → _runtime/execution_context.py} +11 -2
- modal/_runtime/telemetry.py +169 -0
- modal/_runtime/user_code_imports.py +356 -0
- modal/_serialization.py +123 -6
- modal/_traceback.py +47 -187
- modal/_tunnel.py +50 -14
- modal/_tunnel.pyi +19 -36
- modal/_utils/app_utils.py +3 -17
- modal/_utils/async_utils.py +386 -104
- modal/_utils/blob_utils.py +157 -186
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/deprecation.py +89 -0
- modal/_utils/docker_utils.py +98 -0
- modal/_utils/function_utils.py +299 -98
- modal/_utils/grpc_testing.py +47 -34
- modal/_utils/grpc_utils.py +54 -21
- modal/_utils/hash_utils.py +51 -10
- modal/_utils/http_utils.py +39 -9
- modal/_utils/logger.py +2 -1
- modal/_utils/mount_utils.py +34 -16
- modal/_utils/name_utils.py +58 -0
- modal/_utils/package_utils.py +14 -1
- modal/_utils/pattern_utils.py +205 -0
- modal/_utils/rand_pb_testing.py +3 -3
- modal/_utils/shell_utils.py +15 -49
- modal/_vendor/a2wsgi_wsgi.py +62 -72
- modal/_vendor/cloudpickle.py +1 -1
- modal/_watcher.py +12 -10
- modal/app.py +561 -323
- modal/app.pyi +474 -262
- modal/call_graph.py +7 -6
- modal/cli/_download.py +22 -6
- modal/cli/_traceback.py +200 -0
- modal/cli/app.py +203 -42
- modal/cli/config.py +12 -5
- modal/cli/container.py +61 -13
- modal/cli/dict.py +128 -0
- modal/cli/entry_point.py +26 -13
- modal/cli/environment.py +40 -9
- modal/cli/import_refs.py +21 -48
- modal/cli/launch.py +28 -14
- modal/cli/network_file_system.py +57 -21
- modal/cli/profile.py +1 -1
- modal/cli/programs/run_jupyter.py +34 -9
- modal/cli/programs/vscode.py +58 -8
- modal/cli/queues.py +131 -0
- modal/cli/run.py +199 -96
- modal/cli/secret.py +5 -4
- modal/cli/token.py +7 -2
- modal/cli/utils.py +74 -8
- modal/cli/volume.py +97 -56
- modal/client.py +248 -144
- modal/client.pyi +156 -124
- modal/cloud_bucket_mount.py +43 -30
- modal/cloud_bucket_mount.pyi +32 -25
- modal/cls.py +528 -141
- modal/cls.pyi +189 -145
- modal/config.py +32 -15
- modal/container_process.py +177 -0
- modal/container_process.pyi +82 -0
- modal/dict.py +50 -54
- modal/dict.pyi +120 -164
- modal/environments.py +106 -5
- modal/environments.pyi +77 -25
- modal/exception.py +30 -43
- modal/experimental.py +62 -2
- modal/file_io.py +537 -0
- modal/file_io.pyi +235 -0
- modal/file_pattern_matcher.py +196 -0
- modal/functions.py +846 -428
- modal/functions.pyi +446 -387
- modal/gpu.py +57 -44
- modal/image.py +943 -417
- modal/image.pyi +584 -245
- modal/io_streams.py +434 -0
- modal/io_streams.pyi +122 -0
- modal/mount.py +223 -90
- modal/mount.pyi +241 -243
- modal/network_file_system.py +85 -86
- modal/network_file_system.pyi +151 -110
- modal/object.py +66 -36
- modal/object.pyi +166 -143
- modal/output.py +63 -0
- modal/parallel_map.py +73 -47
- modal/parallel_map.pyi +51 -63
- modal/partial_function.py +272 -107
- modal/partial_function.pyi +219 -120
- modal/proxy.py +15 -12
- modal/proxy.pyi +3 -8
- modal/queue.py +96 -72
- modal/queue.pyi +210 -135
- modal/requirements/2024.04.txt +2 -1
- modal/requirements/2024.10.txt +16 -0
- modal/requirements/README.md +21 -0
- modal/requirements/base-images.json +22 -0
- modal/retries.py +45 -4
- modal/runner.py +325 -203
- modal/runner.pyi +124 -110
- modal/running_app.py +27 -4
- modal/sandbox.py +509 -231
- modal/sandbox.pyi +396 -169
- modal/schedule.py +2 -2
- modal/scheduler_placement.py +20 -3
- modal/secret.py +41 -25
- modal/secret.pyi +62 -42
- modal/serving.py +39 -49
- modal/serving.pyi +37 -43
- modal/stream_type.py +15 -0
- modal/token_flow.py +5 -3
- modal/token_flow.pyi +37 -32
- modal/volume.py +123 -137
- modal/volume.pyi +228 -221
- {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/METADATA +5 -5
- modal-0.72.13.dist-info/RECORD +174 -0
- {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/top_level.txt +0 -1
- modal_docs/gen_reference_docs.py +3 -1
- modal_docs/mdmd/mdmd.py +0 -1
- modal_docs/mdmd/signatures.py +1 -2
- modal_global_objects/images/base_images.py +28 -0
- modal_global_objects/mounts/python_standalone.py +2 -2
- modal_proto/__init__.py +1 -1
- modal_proto/api.proto +1231 -531
- modal_proto/api_grpc.py +750 -430
- modal_proto/api_pb2.py +2102 -1176
- modal_proto/api_pb2.pyi +8859 -0
- modal_proto/api_pb2_grpc.py +1329 -675
- modal_proto/api_pb2_grpc.pyi +1416 -0
- modal_proto/modal_api_grpc.py +149 -0
- modal_proto/modal_options_grpc.py +3 -0
- modal_proto/options_pb2.pyi +20 -0
- modal_proto/options_pb2_grpc.pyi +7 -0
- modal_proto/py.typed +0 -0
- modal_version/__init__.py +1 -1
- modal_version/_version_generated.py +2 -2
- modal/_asgi.py +0 -370
- modal/_container_exec.py +0 -128
- modal/_container_io_manager.py +0 -646
- modal/_container_io_manager.pyi +0 -412
- modal/_sandbox_shell.py +0 -49
- modal/app_utils.py +0 -20
- modal/app_utils.pyi +0 -17
- modal/execution_context.pyi +0 -37
- modal/shared_volume.py +0 -23
- modal/shared_volume.pyi +0 -24
- modal-0.62.115.dist-info/RECORD +0 -207
- modal_global_objects/images/conda.py +0 -15
- modal_global_objects/images/debian_slim.py +0 -15
- modal_global_objects/images/micromamba.py +0 -15
- test/__init__.py +0 -1
- test/aio_test.py +0 -12
- test/async_utils_test.py +0 -279
- test/blob_test.py +0 -67
- test/cli_imports_test.py +0 -149
- test/cli_test.py +0 -674
- test/client_test.py +0 -203
- test/cloud_bucket_mount_test.py +0 -22
- test/cls_test.py +0 -636
- test/config_test.py +0 -149
- test/conftest.py +0 -1485
- test/container_app_test.py +0 -50
- test/container_test.py +0 -1405
- test/cpu_test.py +0 -23
- test/decorator_test.py +0 -85
- test/deprecation_test.py +0 -34
- test/dict_test.py +0 -51
- test/e2e_test.py +0 -68
- test/error_test.py +0 -7
- test/function_serialization_test.py +0 -32
- test/function_test.py +0 -791
- test/function_utils_test.py +0 -101
- test/gpu_test.py +0 -159
- test/grpc_utils_test.py +0 -82
- test/helpers.py +0 -47
- test/image_test.py +0 -814
- test/live_reload_test.py +0 -80
- test/lookup_test.py +0 -70
- test/mdmd_test.py +0 -329
- test/mount_test.py +0 -162
- test/mounted_files_test.py +0 -327
- test/network_file_system_test.py +0 -188
- test/notebook_test.py +0 -66
- test/object_test.py +0 -41
- test/package_utils_test.py +0 -25
- test/queue_test.py +0 -115
- test/resolver_test.py +0 -59
- test/retries_test.py +0 -67
- test/runner_test.py +0 -85
- test/sandbox_test.py +0 -191
- test/schedule_test.py +0 -15
- test/scheduler_placement_test.py +0 -57
- test/secret_test.py +0 -89
- test/serialization_test.py +0 -50
- test/stub_composition_test.py +0 -10
- test/stub_test.py +0 -361
- test/test_asgi_wrapper.py +0 -234
- test/token_flow_test.py +0 -18
- test/traceback_test.py +0 -135
- test/tunnel_test.py +0 -29
- test/utils_test.py +0 -88
- test/version_test.py +0 -14
- test/volume_test.py +0 -397
- test/watcher_test.py +0 -58
- test/webhook_test.py +0 -145
- {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/LICENSE +0 -0
- {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/WHEEL +0 -0
- {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/entry_points.txt +0 -0
test/traceback_test.py
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2024
|
2
|
-
import pytest
|
3
|
-
from pathlib import Path
|
4
|
-
from traceback import extract_tb
|
5
|
-
from typing import Dict, List, Tuple
|
6
|
-
|
7
|
-
from modal._traceback import append_modal_tb, extract_traceback, reduce_traceback_to_user_code
|
8
|
-
from modal._vendor import tblib
|
9
|
-
|
10
|
-
from .supports.raise_error import raise_error
|
11
|
-
|
12
|
-
SUPPORT_MODULE = "supports.raise_error"
|
13
|
-
|
14
|
-
|
15
|
-
def call_raise_error():
|
16
|
-
raise_error()
|
17
|
-
|
18
|
-
|
19
|
-
def test_extract_traceback():
|
20
|
-
task_id = "ta-123"
|
21
|
-
try:
|
22
|
-
call_raise_error()
|
23
|
-
except Exception as exc:
|
24
|
-
tb_dict, line_cache = extract_traceback(exc, task_id)
|
25
|
-
|
26
|
-
test_path = Path(__file__)
|
27
|
-
support_path = test_path.parent / (SUPPORT_MODULE.replace(".", "/") + ".py")
|
28
|
-
|
29
|
-
frame = tb_dict["tb_frame"]
|
30
|
-
assert tb_dict["tb_lineno"] == frame["f_lineno"] - 2
|
31
|
-
assert frame["f_code"]["co_filename"] == f"<{task_id}>:{test_path}"
|
32
|
-
assert frame["f_code"]["co_name"] == "test_extract_traceback"
|
33
|
-
assert frame["f_globals"]["__file__"] == str(test_path)
|
34
|
-
assert frame["f_globals"]["__name__"] == f"test.{test_path.name[:-3]}"
|
35
|
-
assert frame["f_locals"] == {}
|
36
|
-
|
37
|
-
frame = tb_dict["tb_next"]["tb_frame"]
|
38
|
-
assert frame["f_code"]["co_filename"] == f"<{task_id}>:{test_path}"
|
39
|
-
assert frame["f_code"]["co_name"] == "call_raise_error"
|
40
|
-
assert frame["f_globals"]["__file__"] == str(test_path)
|
41
|
-
assert frame["f_globals"]["__name__"] == f"test.{test_path.name[:-3]}"
|
42
|
-
assert frame["f_locals"] == {}
|
43
|
-
|
44
|
-
frame = tb_dict["tb_next"]["tb_next"]["tb_frame"]
|
45
|
-
assert frame["f_code"]["co_filename"] == f"<{task_id}>:{support_path}"
|
46
|
-
assert frame["f_code"]["co_name"] == "raise_error"
|
47
|
-
assert frame["f_globals"]["__file__"] == str(support_path)
|
48
|
-
assert frame["f_globals"]["__name__"] == f"test.{SUPPORT_MODULE}"
|
49
|
-
assert frame["f_locals"] == {}
|
50
|
-
|
51
|
-
assert tb_dict["tb_next"]["tb_next"]["tb_next"] is None
|
52
|
-
|
53
|
-
line_cache_list = list(line_cache.items())
|
54
|
-
assert line_cache_list[0][0][0] == str(test_path)
|
55
|
-
assert line_cache_list[0][1] == "call_raise_error()"
|
56
|
-
assert line_cache_list[1][0][0] == str(test_path)
|
57
|
-
assert line_cache_list[1][1] == "raise_error()"
|
58
|
-
assert line_cache_list[2][0][0] == str(support_path)
|
59
|
-
assert line_cache_list[2][1] == 'raise RuntimeError("Boo!")'
|
60
|
-
|
61
|
-
|
62
|
-
def test_append_modal_tb():
|
63
|
-
task_id = "ta-123"
|
64
|
-
try:
|
65
|
-
call_raise_error()
|
66
|
-
except Exception as exc:
|
67
|
-
tb_dict, line_cache = extract_traceback(exc, task_id)
|
68
|
-
|
69
|
-
try:
|
70
|
-
raise RuntimeError("Remote error")
|
71
|
-
except Exception as exc:
|
72
|
-
remote_exc = exc
|
73
|
-
append_modal_tb(exc, tb_dict, line_cache)
|
74
|
-
|
75
|
-
assert remote_exc.__line_cache__ == line_cache # type: ignore
|
76
|
-
frames = [f.name for f in extract_tb(remote_exc.__traceback__)]
|
77
|
-
assert frames == ["test_append_modal_tb", "call_raise_error", "raise_error"]
|
78
|
-
|
79
|
-
|
80
|
-
def make_tb_stack(frames: List[Tuple[str, str]]) -> List[Dict]:
|
81
|
-
"""Given a minimal specification of (code filename, code name), return dict formatted for tblib."""
|
82
|
-
tb_frames = []
|
83
|
-
for lineno, (filename, name) in enumerate(frames):
|
84
|
-
tb_frames.append(
|
85
|
-
{
|
86
|
-
"tb_lineno": lineno,
|
87
|
-
"tb_frame": {
|
88
|
-
"f_lineno": lineno,
|
89
|
-
"f_globals": {},
|
90
|
-
"f_locals": {},
|
91
|
-
"f_code": {"co_filename": filename, "co_name": name},
|
92
|
-
},
|
93
|
-
}
|
94
|
-
)
|
95
|
-
return tb_frames
|
96
|
-
|
97
|
-
|
98
|
-
def tb_dict_from_stack_dicts(stack: List[Dict]) -> Dict:
|
99
|
-
tb_root = tb = stack.pop(0)
|
100
|
-
while stack:
|
101
|
-
tb["tb_next"] = stack.pop(0)
|
102
|
-
tb = tb["tb_next"]
|
103
|
-
tb["tb_next"] = None
|
104
|
-
return tb_root
|
105
|
-
|
106
|
-
|
107
|
-
@pytest.mark.parametrize("user_mode", ["script", "module"])
|
108
|
-
def test_reduce_traceback_to_user_code(user_mode):
|
109
|
-
if user_mode == "script":
|
110
|
-
user_source, user_filename, user_name = ("/root/user/ai.py", "/root/user/ai.py", "train")
|
111
|
-
elif user_mode == "module":
|
112
|
-
user_source, user_filename, user_name = ("ai.training", "/root/user/ai/training.py", "<module>")
|
113
|
-
|
114
|
-
stack = [
|
115
|
-
("/modal/__main__.py", "main"),
|
116
|
-
("/modal/entrypoint.py", "run"),
|
117
|
-
("/site-packages/synchronicity/wizard.py", "magic"),
|
118
|
-
(user_filename, user_name),
|
119
|
-
("/modal/function.py", "execute"),
|
120
|
-
("/site-packages/synchronicity/devil.py", "pitchfork"),
|
121
|
-
]
|
122
|
-
|
123
|
-
tb_dict = tb_dict_from_stack_dicts(make_tb_stack(stack))
|
124
|
-
tb = tblib.Traceback.from_dict(tb_dict)
|
125
|
-
tb_out = reduce_traceback_to_user_code(tb, user_source)
|
126
|
-
|
127
|
-
f = tb_out.tb_frame
|
128
|
-
assert f.f_code.co_filename == user_filename
|
129
|
-
assert f.f_code.co_name == user_name
|
130
|
-
|
131
|
-
f = tb_out.tb_next.tb_frame
|
132
|
-
assert f.f_code.co_filename == "/modal/function.py"
|
133
|
-
assert f.f_code.co_name == "execute"
|
134
|
-
|
135
|
-
assert tb_out.tb_next.tb_next is None
|
test/tunnel_test.py
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2023
|
2
|
-
|
3
|
-
import pytest
|
4
|
-
|
5
|
-
from modal import forward
|
6
|
-
from modal.exception import InvalidError
|
7
|
-
|
8
|
-
from .supports.skip import skip_windows_unix_socket
|
9
|
-
|
10
|
-
|
11
|
-
def test_tunnel_outside_container(client):
|
12
|
-
with pytest.raises(InvalidError):
|
13
|
-
with forward(8000, client=client):
|
14
|
-
pass
|
15
|
-
|
16
|
-
|
17
|
-
@skip_windows_unix_socket
|
18
|
-
def test_invalid_port_numbers(container_client):
|
19
|
-
for port in (-1, 0, 65536):
|
20
|
-
with pytest.raises(InvalidError):
|
21
|
-
with forward(port, client=container_client):
|
22
|
-
pass
|
23
|
-
|
24
|
-
|
25
|
-
@skip_windows_unix_socket
|
26
|
-
def test_create_tunnel(container_client):
|
27
|
-
with forward(8000, client=container_client) as tunnel:
|
28
|
-
assert tunnel.host == "8000.modal.test"
|
29
|
-
assert tunnel.url == "https://8000.modal.test"
|
test/utils_test.py
DELETED
@@ -1,88 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import asyncio
|
3
|
-
import hashlib
|
4
|
-
import io
|
5
|
-
import pytest
|
6
|
-
|
7
|
-
from modal._utils.app_utils import is_valid_app_name, is_valid_subdomain_label
|
8
|
-
from modal._utils.blob_utils import BytesIOSegmentPayload
|
9
|
-
|
10
|
-
|
11
|
-
def test_subdomain_label():
|
12
|
-
assert is_valid_subdomain_label("banana")
|
13
|
-
assert is_valid_subdomain_label("foo-123-456")
|
14
|
-
assert not is_valid_subdomain_label("BaNaNa")
|
15
|
-
assert not is_valid_subdomain_label(" ")
|
16
|
-
assert not is_valid_subdomain_label("ban/ana")
|
17
|
-
|
18
|
-
|
19
|
-
def test_app_name():
|
20
|
-
assert is_valid_app_name("baNaNa")
|
21
|
-
assert is_valid_app_name("foo-123_456")
|
22
|
-
assert is_valid_app_name("a" * 64)
|
23
|
-
assert not is_valid_app_name("hello world")
|
24
|
-
assert not is_valid_app_name("a" * 65)
|
25
|
-
|
26
|
-
|
27
|
-
@pytest.mark.asyncio
|
28
|
-
async def test_file_segment_payloads():
|
29
|
-
data = io.BytesIO(b"abc123")
|
30
|
-
data2 = io.BytesIO(data.getbuffer())
|
31
|
-
|
32
|
-
class DummyOutput: # AbstractStreamWriter
|
33
|
-
def __init__(self):
|
34
|
-
self.value = b""
|
35
|
-
|
36
|
-
async def write(self, chunk: bytes):
|
37
|
-
self.value += chunk
|
38
|
-
|
39
|
-
out1 = DummyOutput()
|
40
|
-
out2 = DummyOutput()
|
41
|
-
p1 = BytesIOSegmentPayload(data, 0, 3)
|
42
|
-
p2 = BytesIOSegmentPayload(data2, 3, 3)
|
43
|
-
|
44
|
-
# "out of order" writes
|
45
|
-
await p2.write(out2) # type: ignore
|
46
|
-
await p1.write(out1) # type: ignore
|
47
|
-
assert out1.value == b"abc"
|
48
|
-
assert out2.value == b"123"
|
49
|
-
assert p1.md5_checksum().digest() == hashlib.md5(b"abc").digest()
|
50
|
-
assert p2.md5_checksum().digest() == hashlib.md5(b"123").digest()
|
51
|
-
|
52
|
-
data = io.BytesIO(b"abc123")
|
53
|
-
|
54
|
-
# test reset_on_error
|
55
|
-
all_data = BytesIOSegmentPayload(data, 0, 6)
|
56
|
-
|
57
|
-
class DummyExc(Exception):
|
58
|
-
pass
|
59
|
-
|
60
|
-
try:
|
61
|
-
with all_data.reset_on_error():
|
62
|
-
await all_data.write(DummyOutput()) # type: ignore
|
63
|
-
except DummyExc:
|
64
|
-
pass
|
65
|
-
|
66
|
-
out = DummyOutput()
|
67
|
-
await all_data.write(out) # type: ignore
|
68
|
-
assert out.value == b"abc123"
|
69
|
-
|
70
|
-
|
71
|
-
@pytest.mark.asyncio
|
72
|
-
async def test_file_segment_payloads_concurrency():
|
73
|
-
data = io.BytesIO((b"123" * 1024 * 350)[: 1024 * 1024]) # 1 MiB
|
74
|
-
data2 = io.BytesIO(data.getbuffer())
|
75
|
-
|
76
|
-
class DummyOutput: # AbstractStreamWriter
|
77
|
-
def __init__(self):
|
78
|
-
self.value = b""
|
79
|
-
|
80
|
-
async def write(self, chunk: bytes):
|
81
|
-
self.value += chunk
|
82
|
-
|
83
|
-
out1 = DummyOutput()
|
84
|
-
out2 = DummyOutput()
|
85
|
-
p1 = BytesIOSegmentPayload(data, 0, len(data.getvalue()) // 2, chunk_size=100 * 1024) # 100 KiB chunks
|
86
|
-
p2 = BytesIOSegmentPayload(data2, len(data.getvalue()) // 2, len(data.getvalue()) // 2, chunk_size=100 * 1024)
|
87
|
-
await asyncio.gather(p2.write(out2), p1.write(out1)) # type: ignore
|
88
|
-
assert out1.value + out2.value == data.getvalue()
|
test/version_test.py
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import pkg_resources
|
3
|
-
|
4
|
-
import modal
|
5
|
-
|
6
|
-
|
7
|
-
def test_version():
|
8
|
-
mod_version = modal.__version__
|
9
|
-
pkg_version = pkg_resources.require("modal")[0].version
|
10
|
-
|
11
|
-
assert pkg_resources.parse_version(mod_version) > pkg_resources.parse_version("0.0.0")
|
12
|
-
assert pkg_resources.parse_version(pkg_version) > pkg_resources.parse_version("0.0.0")
|
13
|
-
|
14
|
-
assert mod_version == pkg_version
|
test/volume_test.py
DELETED
@@ -1,397 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2023
|
2
|
-
import asyncio
|
3
|
-
import io
|
4
|
-
import os
|
5
|
-
import platform
|
6
|
-
import pytest
|
7
|
-
import re
|
8
|
-
import sys
|
9
|
-
import time
|
10
|
-
from pathlib import Path
|
11
|
-
from unittest import mock
|
12
|
-
|
13
|
-
import modal
|
14
|
-
from modal.exception import DeprecationError, InvalidError, NotFoundError, VolumeUploadTimeoutError
|
15
|
-
from modal.runner import deploy_app
|
16
|
-
from modal.volume import _open_files_error_annotation
|
17
|
-
from modal_proto import api_pb2
|
18
|
-
|
19
|
-
|
20
|
-
def dummy():
|
21
|
-
pass
|
22
|
-
|
23
|
-
|
24
|
-
def test_volume_mount(client, servicer):
|
25
|
-
app = modal.App()
|
26
|
-
vol = modal.Volume.from_name("xyz", create_if_missing=True)
|
27
|
-
|
28
|
-
_ = app.function(volumes={"/root/foo": vol})(dummy)
|
29
|
-
|
30
|
-
with app.run(client=client):
|
31
|
-
pass
|
32
|
-
|
33
|
-
|
34
|
-
def test_volume_bad_paths():
|
35
|
-
app = modal.App()
|
36
|
-
vol = modal.Volume.from_name("xyz")
|
37
|
-
|
38
|
-
with pytest.raises(InvalidError):
|
39
|
-
app.function(volumes={"/root/../../foo": vol})(dummy)
|
40
|
-
|
41
|
-
with pytest.raises(InvalidError):
|
42
|
-
app.function(volumes={"/": vol})(dummy)
|
43
|
-
|
44
|
-
with pytest.raises(InvalidError):
|
45
|
-
app.function(volumes={"/tmp/": vol})(dummy)
|
46
|
-
|
47
|
-
|
48
|
-
def test_volume_duplicate_mount():
|
49
|
-
app = modal.App()
|
50
|
-
vol = modal.Volume.from_name("xyz")
|
51
|
-
|
52
|
-
with pytest.raises(InvalidError):
|
53
|
-
app.function(volumes={"/foo": vol, "/bar": vol})(dummy)
|
54
|
-
|
55
|
-
|
56
|
-
@pytest.mark.parametrize("skip_reload", [False, True])
|
57
|
-
def test_volume_commit(client, servicer, skip_reload):
|
58
|
-
with servicer.intercept() as ctx:
|
59
|
-
ctx.add_response("VolumeCommit", api_pb2.VolumeCommitResponse(skip_reload=skip_reload))
|
60
|
-
ctx.add_response("VolumeCommit", api_pb2.VolumeCommitResponse(skip_reload=skip_reload))
|
61
|
-
|
62
|
-
with modal.Volume.ephemeral(client=client) as vol:
|
63
|
-
# Note that in practice this will not work unless run in a task.
|
64
|
-
vol.commit()
|
65
|
-
|
66
|
-
# Make sure we can commit through the provider too
|
67
|
-
vol.commit()
|
68
|
-
|
69
|
-
assert ctx.pop_request("VolumeCommit").volume_id == vol.object_id
|
70
|
-
assert ctx.pop_request("VolumeCommit").volume_id == vol.object_id
|
71
|
-
|
72
|
-
# commit should implicitly reload on successful commit if skip_reload=False
|
73
|
-
assert servicer.volume_reloads[vol.object_id] == 0 if skip_reload else 2
|
74
|
-
|
75
|
-
|
76
|
-
@pytest.mark.asyncio
|
77
|
-
async def test_volume_get(servicer, client, tmp_path):
|
78
|
-
await modal.Volume.create_deployed.aio("my-vol", client=client)
|
79
|
-
vol = await modal.Volume.lookup.aio("my-vol", client=client) # type: ignore
|
80
|
-
|
81
|
-
file_contents = b"hello world"
|
82
|
-
file_path = b"foo.txt"
|
83
|
-
local_file_path = tmp_path / file_path.decode("utf-8")
|
84
|
-
local_file_path.write_bytes(file_contents)
|
85
|
-
|
86
|
-
async with vol.batch_upload() as batch:
|
87
|
-
batch.put_file(local_file_path, file_path.decode("utf-8"))
|
88
|
-
|
89
|
-
data = b""
|
90
|
-
for chunk in vol.read_file(file_path):
|
91
|
-
data += chunk
|
92
|
-
assert data == file_contents
|
93
|
-
|
94
|
-
output = io.BytesIO()
|
95
|
-
vol.read_file_into_fileobj(file_path, output)
|
96
|
-
assert output.getvalue() == file_contents
|
97
|
-
|
98
|
-
with pytest.raises(FileNotFoundError):
|
99
|
-
for _ in vol.read_file("/abc/def/i-dont-exist-at-all"):
|
100
|
-
...
|
101
|
-
|
102
|
-
|
103
|
-
def test_volume_reload(client, servicer):
|
104
|
-
with modal.Volume.ephemeral(client=client) as vol:
|
105
|
-
# Note that in practice this will not work unless run in a task.
|
106
|
-
vol.reload()
|
107
|
-
|
108
|
-
assert servicer.volume_reloads[vol.object_id] == 1
|
109
|
-
|
110
|
-
|
111
|
-
def test_redeploy(servicer, client):
|
112
|
-
app = modal.App()
|
113
|
-
|
114
|
-
with pytest.warns(DeprecationError):
|
115
|
-
v1 = modal.Volume.new()
|
116
|
-
v2 = modal.Volume.new()
|
117
|
-
v3 = modal.Volume.new()
|
118
|
-
app.v1, app.v2, app.v3 = v1, v2, v3
|
119
|
-
|
120
|
-
# Deploy app once
|
121
|
-
deploy_app(app, "my-app", client=client)
|
122
|
-
app1_ids = [v1.object_id, v2.object_id, v3.object_id]
|
123
|
-
|
124
|
-
# Deploy app again
|
125
|
-
deploy_app(app, "my-app", client=client)
|
126
|
-
app2_ids = [v1.object_id, v2.object_id, v3.object_id]
|
127
|
-
|
128
|
-
# Make sure ids are stable
|
129
|
-
assert app1_ids == app2_ids
|
130
|
-
|
131
|
-
# Make sure ids are unique
|
132
|
-
assert len(set(app1_ids)) == 3
|
133
|
-
assert len(set(app2_ids)) == 3
|
134
|
-
|
135
|
-
# Deploy to a different app
|
136
|
-
deploy_app(app, "my-other-app", client=client)
|
137
|
-
app3_ids = [v1.object_id, v2.object_id, v3.object_id]
|
138
|
-
|
139
|
-
# Should be unique and different
|
140
|
-
assert len(set(app3_ids)) == 3
|
141
|
-
assert set(app1_ids) & set(app3_ids) == set()
|
142
|
-
|
143
|
-
|
144
|
-
@pytest.mark.asyncio
|
145
|
-
async def test_volume_batch_upload(servicer, client, tmp_path):
|
146
|
-
local_file_path = tmp_path / "some_file"
|
147
|
-
local_file_path.write_text("hello world")
|
148
|
-
|
149
|
-
local_dir = tmp_path / "some_dir"
|
150
|
-
local_dir.mkdir()
|
151
|
-
(local_dir / "smol").write_text("###")
|
152
|
-
|
153
|
-
subdir = local_dir / "subdir"
|
154
|
-
subdir.mkdir()
|
155
|
-
(subdir / "other").write_text("####")
|
156
|
-
|
157
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
158
|
-
with open(local_file_path, "rb") as fp:
|
159
|
-
with vol.batch_upload() as batch:
|
160
|
-
batch.put_file(local_file_path, "/some_file")
|
161
|
-
batch.put_directory(local_dir, "/some_dir")
|
162
|
-
batch.put_file(io.BytesIO(b"data from a file-like object"), "/filelike", mode=0o600)
|
163
|
-
batch.put_directory(local_dir, "/non-recursive", recursive=False)
|
164
|
-
batch.put_file(fp, "/filelike2")
|
165
|
-
object_id = vol.object_id
|
166
|
-
|
167
|
-
assert servicer.volume_files[object_id].keys() == {
|
168
|
-
"/some_file",
|
169
|
-
"/some_dir/smol",
|
170
|
-
"/some_dir/subdir/other",
|
171
|
-
"/filelike",
|
172
|
-
"/non-recursive/smol",
|
173
|
-
"/filelike2",
|
174
|
-
}
|
175
|
-
assert servicer.volume_files[object_id]["/some_file"].data == b"hello world"
|
176
|
-
assert servicer.volume_files[object_id]["/some_dir/smol"].data == b"###"
|
177
|
-
assert servicer.volume_files[object_id]["/some_dir/subdir/other"].data == b"####"
|
178
|
-
assert servicer.volume_files[object_id]["/filelike"].data == b"data from a file-like object"
|
179
|
-
assert servicer.volume_files[object_id]["/filelike"].mode == 0o600
|
180
|
-
assert servicer.volume_files[object_id]["/non-recursive/smol"].data == b"###"
|
181
|
-
assert servicer.volume_files[object_id]["/filelike2"].data == b"hello world"
|
182
|
-
assert servicer.volume_files[object_id]["/filelike2"].mode == 0o644
|
183
|
-
|
184
|
-
|
185
|
-
@pytest.mark.asyncio
|
186
|
-
async def test_volume_batch_upload_force(servicer, client, tmp_path):
|
187
|
-
local_file_path = tmp_path / "some_file"
|
188
|
-
local_file_path.write_text("hello world")
|
189
|
-
|
190
|
-
local_file_path2 = tmp_path / "some_file2"
|
191
|
-
local_file_path2.write_text("overwritten")
|
192
|
-
|
193
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
194
|
-
with servicer.intercept() as ctx:
|
195
|
-
# Seed the volume
|
196
|
-
with vol.batch_upload() as batch:
|
197
|
-
batch.put_file(local_file_path, "/some_file")
|
198
|
-
assert ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
|
199
|
-
|
200
|
-
# Attempting to overwrite the file with force=False should result in an error
|
201
|
-
with pytest.raises(FileExistsError):
|
202
|
-
with vol.batch_upload(force=False) as batch:
|
203
|
-
batch.put_file(local_file_path, "/some_file")
|
204
|
-
assert ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
|
205
|
-
assert servicer.volume_files[vol.object_id]["/some_file"].data == b"hello world"
|
206
|
-
|
207
|
-
# Overwriting should work with force=True
|
208
|
-
with vol.batch_upload(force=True) as batch:
|
209
|
-
batch.put_file(local_file_path2, "/some_file")
|
210
|
-
assert not ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
|
211
|
-
assert servicer.volume_files[vol.object_id]["/some_file"].data == b"overwritten"
|
212
|
-
|
213
|
-
|
214
|
-
@pytest.mark.asyncio
|
215
|
-
async def test_volume_upload_removed_file(servicer, client, tmp_path):
|
216
|
-
local_file_path = tmp_path / "some_file"
|
217
|
-
local_file_path.write_text("hello world")
|
218
|
-
|
219
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
220
|
-
with pytest.raises(FileNotFoundError):
|
221
|
-
with vol.batch_upload() as batch:
|
222
|
-
batch.put_file(local_file_path, "/dest")
|
223
|
-
local_file_path.unlink()
|
224
|
-
|
225
|
-
|
226
|
-
@pytest.mark.asyncio
|
227
|
-
async def test_volume_upload_large_file(client, tmp_path, servicer, blob_server, *args):
|
228
|
-
with mock.patch("modal._utils.blob_utils.LARGE_FILE_LIMIT", 10):
|
229
|
-
local_file_path = tmp_path / "bigfile"
|
230
|
-
local_file_path.write_text("hello world, this is a lot of text")
|
231
|
-
|
232
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
233
|
-
async with vol.batch_upload() as batch:
|
234
|
-
batch.put_file(local_file_path, "/a")
|
235
|
-
object_id = vol.object_id
|
236
|
-
|
237
|
-
assert servicer.volume_files[object_id].keys() == {"/a"}
|
238
|
-
assert servicer.volume_files[object_id]["/a"].data == b""
|
239
|
-
assert servicer.volume_files[object_id]["/a"].data_blob_id == "bl-1"
|
240
|
-
|
241
|
-
_, blobs = blob_server
|
242
|
-
assert blobs["bl-1"] == b"hello world, this is a lot of text"
|
243
|
-
|
244
|
-
|
245
|
-
@pytest.mark.asyncio
|
246
|
-
async def test_volume_upload_file_timeout(client, tmp_path, servicer, blob_server, *args):
|
247
|
-
call_count = 0
|
248
|
-
|
249
|
-
async def mount_put_file(self, stream):
|
250
|
-
await stream.recv_message()
|
251
|
-
nonlocal call_count
|
252
|
-
call_count += 1
|
253
|
-
await stream.send_message(api_pb2.MountPutFileResponse(exists=False))
|
254
|
-
|
255
|
-
with servicer.intercept() as ctx:
|
256
|
-
ctx.set_responder("MountPutFile", mount_put_file)
|
257
|
-
with mock.patch("modal._utils.blob_utils.LARGE_FILE_LIMIT", 10):
|
258
|
-
with mock.patch("modal.volume.VOLUME_PUT_FILE_CLIENT_TIMEOUT", 0.5):
|
259
|
-
local_file_path = tmp_path / "bigfile"
|
260
|
-
local_file_path.write_text("hello world, this is a lot of text")
|
261
|
-
|
262
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
263
|
-
with pytest.raises(VolumeUploadTimeoutError):
|
264
|
-
async with vol.batch_upload() as batch:
|
265
|
-
batch.put_file(local_file_path, "/dest")
|
266
|
-
|
267
|
-
assert call_count > 2
|
268
|
-
|
269
|
-
|
270
|
-
@pytest.mark.asyncio
|
271
|
-
async def test_volume_copy_1(client, tmp_path, servicer):
|
272
|
-
## test 1: copy src path to dst path ##
|
273
|
-
src_path = "original.txt"
|
274
|
-
dst_path = "copied.txt"
|
275
|
-
local_file_path = tmp_path / src_path
|
276
|
-
local_file_path.write_text("test copy")
|
277
|
-
|
278
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
279
|
-
# add local file to volume
|
280
|
-
async with vol.batch_upload() as batch:
|
281
|
-
batch.put_file(local_file_path, src_path)
|
282
|
-
object_id = vol.object_id
|
283
|
-
|
284
|
-
# copy file from src_path to dst_path
|
285
|
-
vol.copy_files([src_path], dst_path)
|
286
|
-
|
287
|
-
assert servicer.volume_files[object_id].keys() == {src_path, dst_path}
|
288
|
-
|
289
|
-
assert servicer.volume_files[object_id][src_path].data == b"test copy"
|
290
|
-
assert servicer.volume_files[object_id][dst_path].data == b"test copy"
|
291
|
-
|
292
|
-
|
293
|
-
@pytest.mark.asyncio
|
294
|
-
async def test_volume_copy_2(client, tmp_path, servicer):
|
295
|
-
## test 2: copy multiple files into a directory ##
|
296
|
-
file_paths = ["file1.txt", "file2.txt"]
|
297
|
-
|
298
|
-
async with modal.Volume.ephemeral(client=client) as vol:
|
299
|
-
for file_path in file_paths:
|
300
|
-
local_file_path = tmp_path / file_path
|
301
|
-
local_file_path.write_text("test copy")
|
302
|
-
async with vol.batch_upload() as batch:
|
303
|
-
batch.put_file(local_file_path, file_path)
|
304
|
-
object_id = vol.object_id
|
305
|
-
|
306
|
-
vol.copy_files(file_paths, "test_dir")
|
307
|
-
|
308
|
-
returned_volume_files = [Path(file) for file in servicer.volume_files[object_id].keys()]
|
309
|
-
expected_volume_files = [
|
310
|
-
Path(file) for file in ["file1.txt", "file2.txt", "test_dir/file1.txt", "test_dir/file2.txt"]
|
311
|
-
]
|
312
|
-
|
313
|
-
assert returned_volume_files == expected_volume_files
|
314
|
-
|
315
|
-
returned_file_data = {
|
316
|
-
Path(entry): servicer.volume_files[object_id][entry] for entry in servicer.volume_files[object_id]
|
317
|
-
}
|
318
|
-
assert returned_file_data[Path("test_dir/file1.txt")].data == b"test copy"
|
319
|
-
assert returned_file_data[Path("test_dir/file2.txt")].data == b"test copy"
|
320
|
-
|
321
|
-
|
322
|
-
@pytest.mark.parametrize("delete_as_instance_method", [True, False])
|
323
|
-
def test_persisted(servicer, client, delete_as_instance_method):
|
324
|
-
# Lookup should fail since it doesn't exist
|
325
|
-
with pytest.raises(NotFoundError):
|
326
|
-
modal.Volume.lookup("xyz", client=client)
|
327
|
-
|
328
|
-
# Create it
|
329
|
-
modal.Volume.lookup("xyz", create_if_missing=True, client=client)
|
330
|
-
|
331
|
-
# Lookup should succeed now
|
332
|
-
v = modal.Volume.lookup("xyz", client=client)
|
333
|
-
|
334
|
-
# Delete it
|
335
|
-
if delete_as_instance_method:
|
336
|
-
with pytest.warns(DeprecationError):
|
337
|
-
v.delete()
|
338
|
-
else:
|
339
|
-
modal.Volume.delete("xyz", client=client)
|
340
|
-
|
341
|
-
# Lookup should fail again
|
342
|
-
with pytest.raises(NotFoundError):
|
343
|
-
modal.Volume.lookup("xyz", client=client)
|
344
|
-
|
345
|
-
|
346
|
-
def test_ephemeral(servicer, client):
|
347
|
-
assert servicer.n_vol_heartbeats == 0
|
348
|
-
with modal.Volume.ephemeral(client=client, _heartbeat_sleep=1) as vol:
|
349
|
-
assert vol.listdir("/") == []
|
350
|
-
# TODO(erikbern): perform some operations
|
351
|
-
time.sleep(1.5) # Make time for 2 heartbeats
|
352
|
-
assert servicer.n_vol_heartbeats == 2
|
353
|
-
|
354
|
-
|
355
|
-
def test_lazy_hydration_from_named(set_env_client):
|
356
|
-
vol = modal.Volume.from_name("my-vol", create_if_missing=True)
|
357
|
-
assert vol.listdir("/") == []
|
358
|
-
|
359
|
-
|
360
|
-
@pytest.mark.skipif(platform.system() != "Linux", reason="needs /proc")
|
361
|
-
@pytest.mark.asyncio
|
362
|
-
async def test_open_files_error_annotation(tmp_path):
|
363
|
-
assert _open_files_error_annotation(tmp_path) is None
|
364
|
-
|
365
|
-
# Current process keeps file open
|
366
|
-
with (tmp_path / "foo.txt").open("w") as _f:
|
367
|
-
assert _open_files_error_annotation(tmp_path) == "path foo.txt is open"
|
368
|
-
|
369
|
-
# cwd of current process is inside volume
|
370
|
-
cwd = os.getcwd()
|
371
|
-
os.chdir(tmp_path)
|
372
|
-
assert _open_files_error_annotation(tmp_path) == "cwd is inside volume"
|
373
|
-
os.chdir(cwd)
|
374
|
-
|
375
|
-
# Subprocess keeps open file
|
376
|
-
open_path = tmp_path / "bar.txt"
|
377
|
-
open_path.write_text("")
|
378
|
-
proc = await asyncio.create_subprocess_exec("tail", "-f", open_path.as_posix())
|
379
|
-
await asyncio.sleep(0.01) # Give process some time to start
|
380
|
-
assert _open_files_error_annotation(tmp_path) == f"path bar.txt is open from 'tail -f {open_path.as_posix()}'"
|
381
|
-
proc.kill()
|
382
|
-
await proc.wait()
|
383
|
-
assert _open_files_error_annotation(tmp_path) is None
|
384
|
-
|
385
|
-
# Subprocess cwd inside volume
|
386
|
-
proc = await asyncio.create_subprocess_exec(
|
387
|
-
sys.executable, "-c", f"import time; import os; os.chdir('{tmp_path}'); time.sleep(60)"
|
388
|
-
)
|
389
|
-
# Wait for process to chdir
|
390
|
-
for _ in range(100):
|
391
|
-
if os.readlink(f"/proc/{proc.pid}/cwd") == tmp_path.as_posix():
|
392
|
-
break
|
393
|
-
await asyncio.sleep(0.05)
|
394
|
-
assert re.match(f"^cwd of '{sys.executable} -c .*' is inside volume$", _open_files_error_annotation(tmp_path))
|
395
|
-
proc.kill()
|
396
|
-
await proc.wait()
|
397
|
-
assert _open_files_error_annotation(tmp_path) is None
|