modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. modal/__init__.py +17 -13
  2. modal/__main__.py +41 -3
  3. modal/_clustered_functions.py +80 -0
  4. modal/_clustered_functions.pyi +22 -0
  5. modal/_container_entrypoint.py +420 -937
  6. modal/_ipython.py +3 -13
  7. modal/_location.py +17 -10
  8. modal/_output.py +243 -99
  9. modal/_pty.py +2 -2
  10. modal/_resolver.py +55 -59
  11. modal/_resources.py +51 -0
  12. modal/_runtime/__init__.py +1 -0
  13. modal/_runtime/asgi.py +519 -0
  14. modal/_runtime/container_io_manager.py +1036 -0
  15. modal/_runtime/execution_context.py +89 -0
  16. modal/_runtime/telemetry.py +169 -0
  17. modal/_runtime/user_code_imports.py +356 -0
  18. modal/_serialization.py +134 -9
  19. modal/_traceback.py +47 -187
  20. modal/_tunnel.py +52 -16
  21. modal/_tunnel.pyi +19 -36
  22. modal/_utils/app_utils.py +3 -17
  23. modal/_utils/async_utils.py +479 -100
  24. modal/_utils/blob_utils.py +157 -186
  25. modal/_utils/bytes_io_segment_payload.py +97 -0
  26. modal/_utils/deprecation.py +89 -0
  27. modal/_utils/docker_utils.py +98 -0
  28. modal/_utils/function_utils.py +460 -171
  29. modal/_utils/grpc_testing.py +47 -31
  30. modal/_utils/grpc_utils.py +62 -109
  31. modal/_utils/hash_utils.py +61 -19
  32. modal/_utils/http_utils.py +39 -9
  33. modal/_utils/logger.py +2 -1
  34. modal/_utils/mount_utils.py +34 -16
  35. modal/_utils/name_utils.py +58 -0
  36. modal/_utils/package_utils.py +14 -1
  37. modal/_utils/pattern_utils.py +205 -0
  38. modal/_utils/rand_pb_testing.py +5 -7
  39. modal/_utils/shell_utils.py +15 -49
  40. modal/_vendor/a2wsgi_wsgi.py +62 -72
  41. modal/_vendor/cloudpickle.py +1 -1
  42. modal/_watcher.py +14 -12
  43. modal/app.py +1003 -314
  44. modal/app.pyi +540 -264
  45. modal/call_graph.py +7 -6
  46. modal/cli/_download.py +63 -53
  47. modal/cli/_traceback.py +200 -0
  48. modal/cli/app.py +205 -45
  49. modal/cli/config.py +12 -5
  50. modal/cli/container.py +62 -14
  51. modal/cli/dict.py +128 -0
  52. modal/cli/entry_point.py +26 -13
  53. modal/cli/environment.py +40 -9
  54. modal/cli/import_refs.py +64 -58
  55. modal/cli/launch.py +32 -18
  56. modal/cli/network_file_system.py +64 -83
  57. modal/cli/profile.py +1 -1
  58. modal/cli/programs/run_jupyter.py +35 -10
  59. modal/cli/programs/vscode.py +60 -10
  60. modal/cli/queues.py +131 -0
  61. modal/cli/run.py +234 -131
  62. modal/cli/secret.py +8 -7
  63. modal/cli/token.py +7 -2
  64. modal/cli/utils.py +79 -10
  65. modal/cli/volume.py +110 -109
  66. modal/client.py +250 -144
  67. modal/client.pyi +157 -118
  68. modal/cloud_bucket_mount.py +108 -34
  69. modal/cloud_bucket_mount.pyi +32 -38
  70. modal/cls.py +535 -148
  71. modal/cls.pyi +190 -146
  72. modal/config.py +41 -19
  73. modal/container_process.py +177 -0
  74. modal/container_process.pyi +82 -0
  75. modal/dict.py +111 -65
  76. modal/dict.pyi +136 -131
  77. modal/environments.py +106 -5
  78. modal/environments.pyi +77 -25
  79. modal/exception.py +34 -43
  80. modal/experimental.py +61 -2
  81. modal/extensions/ipython.py +5 -5
  82. modal/file_io.py +537 -0
  83. modal/file_io.pyi +235 -0
  84. modal/file_pattern_matcher.py +197 -0
  85. modal/functions.py +906 -911
  86. modal/functions.pyi +466 -430
  87. modal/gpu.py +57 -44
  88. modal/image.py +1089 -479
  89. modal/image.pyi +584 -228
  90. modal/io_streams.py +434 -0
  91. modal/io_streams.pyi +122 -0
  92. modal/mount.py +314 -101
  93. modal/mount.pyi +241 -235
  94. modal/network_file_system.py +92 -92
  95. modal/network_file_system.pyi +152 -110
  96. modal/object.py +67 -36
  97. modal/object.pyi +166 -143
  98. modal/output.py +63 -0
  99. modal/parallel_map.py +434 -0
  100. modal/parallel_map.pyi +75 -0
  101. modal/partial_function.py +282 -117
  102. modal/partial_function.pyi +222 -129
  103. modal/proxy.py +15 -12
  104. modal/proxy.pyi +3 -8
  105. modal/queue.py +182 -65
  106. modal/queue.pyi +218 -118
  107. modal/requirements/2024.04.txt +29 -0
  108. modal/requirements/2024.10.txt +16 -0
  109. modal/requirements/README.md +21 -0
  110. modal/requirements/base-images.json +22 -0
  111. modal/retries.py +48 -7
  112. modal/runner.py +459 -156
  113. modal/runner.pyi +135 -71
  114. modal/running_app.py +38 -0
  115. modal/sandbox.py +514 -236
  116. modal/sandbox.pyi +397 -169
  117. modal/schedule.py +4 -4
  118. modal/scheduler_placement.py +20 -3
  119. modal/secret.py +56 -31
  120. modal/secret.pyi +62 -42
  121. modal/serving.py +51 -56
  122. modal/serving.pyi +44 -36
  123. modal/stream_type.py +15 -0
  124. modal/token_flow.py +5 -3
  125. modal/token_flow.pyi +37 -32
  126. modal/volume.py +285 -157
  127. modal/volume.pyi +249 -184
  128. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/METADATA +7 -7
  129. modal-0.72.11.dist-info/RECORD +174 -0
  130. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/top_level.txt +0 -1
  131. modal_docs/gen_reference_docs.py +3 -1
  132. modal_docs/mdmd/mdmd.py +0 -1
  133. modal_docs/mdmd/signatures.py +5 -2
  134. modal_global_objects/images/base_images.py +28 -0
  135. modal_global_objects/mounts/python_standalone.py +2 -2
  136. modal_proto/__init__.py +1 -1
  137. modal_proto/api.proto +1288 -533
  138. modal_proto/api_grpc.py +856 -456
  139. modal_proto/api_pb2.py +2165 -1157
  140. modal_proto/api_pb2.pyi +8859 -0
  141. modal_proto/api_pb2_grpc.py +1674 -855
  142. modal_proto/api_pb2_grpc.pyi +1416 -0
  143. modal_proto/modal_api_grpc.py +149 -0
  144. modal_proto/modal_options_grpc.py +3 -0
  145. modal_proto/options_pb2.pyi +20 -0
  146. modal_proto/options_pb2_grpc.pyi +7 -0
  147. modal_proto/py.typed +0 -0
  148. modal_version/__init__.py +1 -1
  149. modal_version/_version_generated.py +2 -2
  150. modal/_asgi.py +0 -370
  151. modal/_container_entrypoint.pyi +0 -378
  152. modal/_container_exec.py +0 -128
  153. modal/_sandbox_shell.py +0 -49
  154. modal/shared_volume.py +0 -23
  155. modal/shared_volume.pyi +0 -24
  156. modal/stub.py +0 -783
  157. modal/stub.pyi +0 -332
  158. modal-0.62.16.dist-info/RECORD +0 -198
  159. modal_global_objects/images/conda.py +0 -15
  160. modal_global_objects/images/debian_slim.py +0 -15
  161. modal_global_objects/images/micromamba.py +0 -15
  162. test/__init__.py +0 -1
  163. test/aio_test.py +0 -12
  164. test/async_utils_test.py +0 -262
  165. test/blob_test.py +0 -67
  166. test/cli_imports_test.py +0 -149
  167. test/cli_test.py +0 -659
  168. test/client_test.py +0 -194
  169. test/cls_test.py +0 -630
  170. test/config_test.py +0 -137
  171. test/conftest.py +0 -1420
  172. test/container_app_test.py +0 -32
  173. test/container_test.py +0 -1389
  174. test/cpu_test.py +0 -23
  175. test/decorator_test.py +0 -85
  176. test/deprecation_test.py +0 -34
  177. test/dict_test.py +0 -33
  178. test/e2e_test.py +0 -68
  179. test/error_test.py +0 -7
  180. test/function_serialization_test.py +0 -32
  181. test/function_test.py +0 -653
  182. test/function_utils_test.py +0 -101
  183. test/gpu_test.py +0 -159
  184. test/grpc_utils_test.py +0 -141
  185. test/helpers.py +0 -42
  186. test/image_test.py +0 -669
  187. test/live_reload_test.py +0 -80
  188. test/lookup_test.py +0 -70
  189. test/mdmd_test.py +0 -329
  190. test/mount_test.py +0 -162
  191. test/mounted_files_test.py +0 -329
  192. test/network_file_system_test.py +0 -181
  193. test/notebook_test.py +0 -66
  194. test/object_test.py +0 -41
  195. test/package_utils_test.py +0 -25
  196. test/queue_test.py +0 -97
  197. test/resolver_test.py +0 -58
  198. test/retries_test.py +0 -67
  199. test/runner_test.py +0 -85
  200. test/sandbox_test.py +0 -191
  201. test/schedule_test.py +0 -15
  202. test/scheduler_placement_test.py +0 -29
  203. test/secret_test.py +0 -78
  204. test/serialization_test.py +0 -42
  205. test/stub_composition_test.py +0 -10
  206. test/stub_test.py +0 -360
  207. test/test_asgi_wrapper.py +0 -234
  208. test/token_flow_test.py +0 -18
  209. test/traceback_test.py +0 -135
  210. test/tunnel_test.py +0 -29
  211. test/utils_test.py +0 -88
  212. test/version_test.py +0 -14
  213. test/volume_test.py +0 -341
  214. test/watcher_test.py +0 -30
  215. test/webhook_test.py +0 -146
  216. /modal/{requirements.312.txt → requirements/2023.12.312.txt} +0 -0
  217. /modal/{requirements.txt → requirements/2023.12.txt} +0 -0
  218. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/LICENSE +0 -0
  219. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/WHEEL +0 -0
  220. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/entry_points.txt +0 -0
test/traceback_test.py DELETED
@@ -1,135 +0,0 @@
1
- # Copyright Modal Labs 2024
2
- import pytest
3
- from pathlib import Path
4
- from traceback import extract_tb
5
- from typing import Dict, List, Tuple
6
-
7
- from modal._traceback import append_modal_tb, extract_traceback, reduce_traceback_to_user_code
8
- from modal._vendor import tblib
9
-
10
- from .supports.raise_error import raise_error
11
-
12
- SUPPORT_MODULE = "supports.raise_error"
13
-
14
-
15
- def call_raise_error():
16
- raise_error()
17
-
18
-
19
- def test_extract_traceback():
20
- task_id = "ta-123"
21
- try:
22
- call_raise_error()
23
- except Exception as exc:
24
- tb_dict, line_cache = extract_traceback(exc, task_id)
25
-
26
- test_path = Path(__file__)
27
- support_path = test_path.parent / (SUPPORT_MODULE.replace(".", "/") + ".py")
28
-
29
- frame = tb_dict["tb_frame"]
30
- assert tb_dict["tb_lineno"] == frame["f_lineno"] - 2
31
- assert frame["f_code"]["co_filename"] == f"<{task_id}>:{test_path}"
32
- assert frame["f_code"]["co_name"] == "test_extract_traceback"
33
- assert frame["f_globals"]["__file__"] == str(test_path)
34
- assert frame["f_globals"]["__name__"] == f"test.{test_path.name[:-3]}"
35
- assert frame["f_locals"] == {}
36
-
37
- frame = tb_dict["tb_next"]["tb_frame"]
38
- assert frame["f_code"]["co_filename"] == f"<{task_id}>:{test_path}"
39
- assert frame["f_code"]["co_name"] == "call_raise_error"
40
- assert frame["f_globals"]["__file__"] == str(test_path)
41
- assert frame["f_globals"]["__name__"] == f"test.{test_path.name[:-3]}"
42
- assert frame["f_locals"] == {}
43
-
44
- frame = tb_dict["tb_next"]["tb_next"]["tb_frame"]
45
- assert frame["f_code"]["co_filename"] == f"<{task_id}>:{support_path}"
46
- assert frame["f_code"]["co_name"] == "raise_error"
47
- assert frame["f_globals"]["__file__"] == str(support_path)
48
- assert frame["f_globals"]["__name__"] == f"test.{SUPPORT_MODULE}"
49
- assert frame["f_locals"] == {}
50
-
51
- assert tb_dict["tb_next"]["tb_next"]["tb_next"] is None
52
-
53
- line_cache_list = list(line_cache.items())
54
- assert line_cache_list[0][0][0] == str(test_path)
55
- assert line_cache_list[0][1] == "call_raise_error()"
56
- assert line_cache_list[1][0][0] == str(test_path)
57
- assert line_cache_list[1][1] == "raise_error()"
58
- assert line_cache_list[2][0][0] == str(support_path)
59
- assert line_cache_list[2][1] == 'raise RuntimeError("Boo!")'
60
-
61
-
62
- def test_append_modal_tb():
63
- task_id = "ta-123"
64
- try:
65
- call_raise_error()
66
- except Exception as exc:
67
- tb_dict, line_cache = extract_traceback(exc, task_id)
68
-
69
- try:
70
- raise RuntimeError("Remote error")
71
- except Exception as exc:
72
- remote_exc = exc
73
- append_modal_tb(exc, tb_dict, line_cache)
74
-
75
- assert remote_exc.__line_cache__ == line_cache # type: ignore
76
- frames = [f.name for f in extract_tb(remote_exc.__traceback__)]
77
- assert frames == ["test_append_modal_tb", "call_raise_error", "raise_error"]
78
-
79
-
80
- def make_tb_stack(frames: List[Tuple[str, str]]) -> List[Dict]:
81
- """Given a minimal specification of (code filename, code name), return dict formatted for tblib."""
82
- tb_frames = []
83
- for lineno, (filename, name) in enumerate(frames):
84
- tb_frames.append(
85
- {
86
- "tb_lineno": lineno,
87
- "tb_frame": {
88
- "f_lineno": lineno,
89
- "f_globals": {},
90
- "f_locals": {},
91
- "f_code": {"co_filename": filename, "co_name": name},
92
- },
93
- }
94
- )
95
- return tb_frames
96
-
97
-
98
- def tb_dict_from_stack_dicts(stack: List[Dict]) -> Dict:
99
- tb_root = tb = stack.pop(0)
100
- while stack:
101
- tb["tb_next"] = stack.pop(0)
102
- tb = tb["tb_next"]
103
- tb["tb_next"] = None
104
- return tb_root
105
-
106
-
107
- @pytest.mark.parametrize("user_mode", ["script", "module"])
108
- def test_reduce_traceback_to_user_code(user_mode):
109
- if user_mode == "script":
110
- user_source, user_filename, user_name = ("/root/user/ai.py", "/root/user/ai.py", "train")
111
- elif user_mode == "module":
112
- user_source, user_filename, user_name = ("ai.training", "/root/user/ai/training.py", "<module>")
113
-
114
- stack = [
115
- ("/modal/__main__.py", "main"),
116
- ("/modal/entrypoint.py", "run"),
117
- ("/site-packages/synchronicity/wizard.py", "magic"),
118
- (user_filename, user_name),
119
- ("/modal/function.py", "execute"),
120
- ("/site-packages/synchronicity/devil.py", "pitchfork"),
121
- ]
122
-
123
- tb_dict = tb_dict_from_stack_dicts(make_tb_stack(stack))
124
- tb = tblib.Traceback.from_dict(tb_dict)
125
- tb_out = reduce_traceback_to_user_code(tb, user_source)
126
-
127
- f = tb_out.tb_frame
128
- assert f.f_code.co_filename == user_filename
129
- assert f.f_code.co_name == user_name
130
-
131
- f = tb_out.tb_next.tb_frame
132
- assert f.f_code.co_filename == "/modal/function.py"
133
- assert f.f_code.co_name == "execute"
134
-
135
- assert tb_out.tb_next.tb_next is None
test/tunnel_test.py DELETED
@@ -1,29 +0,0 @@
1
- # Copyright Modal Labs 2023
2
-
3
- import pytest
4
-
5
- from modal import forward
6
- from modal.exception import InvalidError
7
-
8
- from .supports.skip import skip_windows_unix_socket
9
-
10
-
11
- def test_tunnel_outside_container(client):
12
- with pytest.raises(InvalidError):
13
- with forward(8000, client=client):
14
- pass
15
-
16
-
17
- @skip_windows_unix_socket
18
- def test_invalid_port_numbers(container_client):
19
- for port in (-1, 0, 65536):
20
- with pytest.raises(InvalidError):
21
- with forward(port, client=container_client):
22
- pass
23
-
24
-
25
- @skip_windows_unix_socket
26
- def test_create_tunnel(container_client):
27
- with forward(8000, client=container_client) as tunnel:
28
- assert tunnel.host == "8000.modal.test"
29
- assert tunnel.url == "https://8000.modal.test"
test/utils_test.py DELETED
@@ -1,88 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import asyncio
3
- import hashlib
4
- import io
5
- import pytest
6
-
7
- from modal._utils.app_utils import is_valid_app_name, is_valid_subdomain_label
8
- from modal._utils.blob_utils import BytesIOSegmentPayload
9
-
10
-
11
- def test_subdomain_label():
12
- assert is_valid_subdomain_label("banana")
13
- assert is_valid_subdomain_label("foo-123-456")
14
- assert not is_valid_subdomain_label("BaNaNa")
15
- assert not is_valid_subdomain_label(" ")
16
- assert not is_valid_subdomain_label("ban/ana")
17
-
18
-
19
- def test_app_name():
20
- assert is_valid_app_name("baNaNa")
21
- assert is_valid_app_name("foo-123_456")
22
- assert is_valid_app_name("a" * 64)
23
- assert not is_valid_app_name("hello world")
24
- assert not is_valid_app_name("a" * 65)
25
-
26
-
27
- @pytest.mark.asyncio
28
- async def test_file_segment_payloads():
29
- data = io.BytesIO(b"abc123")
30
- data2 = io.BytesIO(data.getbuffer())
31
-
32
- class DummyOutput: # AbstractStreamWriter
33
- def __init__(self):
34
- self.value = b""
35
-
36
- async def write(self, chunk: bytes):
37
- self.value += chunk
38
-
39
- out1 = DummyOutput()
40
- out2 = DummyOutput()
41
- p1 = BytesIOSegmentPayload(data, 0, 3)
42
- p2 = BytesIOSegmentPayload(data2, 3, 3)
43
-
44
- # "out of order" writes
45
- await p2.write(out2) # type: ignore
46
- await p1.write(out1) # type: ignore
47
- assert out1.value == b"abc"
48
- assert out2.value == b"123"
49
- assert p1.md5_checksum().digest() == hashlib.md5(b"abc").digest()
50
- assert p2.md5_checksum().digest() == hashlib.md5(b"123").digest()
51
-
52
- data = io.BytesIO(b"abc123")
53
-
54
- # test reset_on_error
55
- all_data = BytesIOSegmentPayload(data, 0, 6)
56
-
57
- class DummyExc(Exception):
58
- pass
59
-
60
- try:
61
- with all_data.reset_on_error():
62
- await all_data.write(DummyOutput()) # type: ignore
63
- except DummyExc:
64
- pass
65
-
66
- out = DummyOutput()
67
- await all_data.write(out) # type: ignore
68
- assert out.value == b"abc123"
69
-
70
-
71
- @pytest.mark.asyncio
72
- async def test_file_segment_payloads_concurrency():
73
- data = io.BytesIO((b"123" * 1024 * 350)[: 1024 * 1024]) # 1 MiB
74
- data2 = io.BytesIO(data.getbuffer())
75
-
76
- class DummyOutput: # AbstractStreamWriter
77
- def __init__(self):
78
- self.value = b""
79
-
80
- async def write(self, chunk: bytes):
81
- self.value += chunk
82
-
83
- out1 = DummyOutput()
84
- out2 = DummyOutput()
85
- p1 = BytesIOSegmentPayload(data, 0, len(data.getvalue()) // 2, chunk_size=100 * 1024) # 100 KiB chunks
86
- p2 = BytesIOSegmentPayload(data2, len(data.getvalue()) // 2, len(data.getvalue()) // 2, chunk_size=100 * 1024)
87
- await asyncio.gather(p2.write(out2), p1.write(out1)) # type: ignore
88
- assert out1.value + out2.value == data.getvalue()
test/version_test.py DELETED
@@ -1,14 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pkg_resources
3
-
4
- import modal
5
-
6
-
7
- def test_version():
8
- mod_version = modal.__version__
9
- pkg_version = pkg_resources.require("modal")[0].version
10
-
11
- assert pkg_resources.parse_version(mod_version) > pkg_resources.parse_version("0.0.0")
12
- assert pkg_resources.parse_version(pkg_version) > pkg_resources.parse_version("0.0.0")
13
-
14
- assert mod_version == pkg_version
test/volume_test.py DELETED
@@ -1,341 +0,0 @@
1
- # Copyright Modal Labs 2023
2
- import io
3
- import pytest
4
- import time
5
- from pathlib import Path
6
- from unittest import mock
7
-
8
- import modal
9
- from modal.exception import DeprecationError, InvalidError, NotFoundError, VolumeUploadTimeoutError
10
- from modal.runner import deploy_stub
11
- from modal_proto import api_pb2
12
-
13
-
14
- def dummy():
15
- pass
16
-
17
-
18
- def test_volume_mount(client, servicer):
19
- stub = modal.Stub()
20
- vol = modal.Volume.from_name("xyz", create_if_missing=True)
21
-
22
- _ = stub.function(volumes={"/root/foo": vol})(dummy)
23
-
24
- with stub.run(client=client):
25
- pass
26
-
27
-
28
- def test_volume_bad_paths():
29
- stub = modal.Stub()
30
- vol = modal.Volume.from_name("xyz")
31
-
32
- with pytest.raises(InvalidError):
33
- stub.function(volumes={"/root/../../foo": vol})(dummy)
34
-
35
- with pytest.raises(InvalidError):
36
- stub.function(volumes={"/": vol})(dummy)
37
-
38
- with pytest.raises(InvalidError):
39
- stub.function(volumes={"/tmp/": vol})(dummy)
40
-
41
-
42
- def test_volume_duplicate_mount():
43
- stub = modal.Stub()
44
- vol = modal.Volume.from_name("xyz")
45
-
46
- with pytest.raises(InvalidError):
47
- stub.function(volumes={"/foo": vol, "/bar": vol})(dummy)
48
-
49
-
50
- @pytest.mark.parametrize("skip_reload", [False, True])
51
- def test_volume_commit(client, servicer, skip_reload):
52
- with servicer.intercept() as ctx:
53
- ctx.add_response("VolumeCommit", api_pb2.VolumeCommitResponse(skip_reload=skip_reload))
54
- ctx.add_response("VolumeCommit", api_pb2.VolumeCommitResponse(skip_reload=skip_reload))
55
-
56
- with modal.Volume.ephemeral(client=client) as vol:
57
- # Note that in practice this will not work unless run in a task.
58
- vol.commit()
59
-
60
- # Make sure we can commit through the provider too
61
- vol.commit()
62
-
63
- assert ctx.pop_request("VolumeCommit").volume_id == vol.object_id
64
- assert ctx.pop_request("VolumeCommit").volume_id == vol.object_id
65
-
66
- # commit should implicitly reload on successful commit if skip_reload=False
67
- assert servicer.volume_reloads[vol.object_id] == 0 if skip_reload else 2
68
-
69
-
70
- @pytest.mark.asyncio
71
- async def test_volume_get(servicer, client, tmp_path):
72
- await modal.Volume.create_deployed.aio("my-vol", client=client)
73
- vol = await modal.Volume.lookup.aio("my-vol", client=client) # type: ignore
74
-
75
- file_contents = b"hello world"
76
- file_path = b"foo.txt"
77
- local_file_path = tmp_path / file_path.decode("utf-8")
78
- local_file_path.write_bytes(file_contents)
79
-
80
- async with vol.batch_upload() as batch:
81
- batch.put_file(local_file_path, file_path.decode("utf-8"))
82
-
83
- data = b""
84
- for chunk in vol.read_file(file_path):
85
- data += chunk
86
- assert data == file_contents
87
-
88
- output = io.BytesIO()
89
- vol.read_file_into_fileobj(file_path, output)
90
- assert output.getvalue() == file_contents
91
-
92
- with pytest.raises(FileNotFoundError):
93
- for _ in vol.read_file("/abc/def/i-dont-exist-at-all"):
94
- ...
95
-
96
-
97
- def test_volume_reload(client, servicer):
98
- with modal.Volume.ephemeral(client=client) as vol:
99
- # Note that in practice this will not work unless run in a task.
100
- vol.reload()
101
-
102
- assert servicer.volume_reloads[vol.object_id] == 1
103
-
104
-
105
- def test_redeploy(servicer, client):
106
- stub = modal.Stub()
107
-
108
- with pytest.warns(DeprecationError):
109
- v1 = modal.Volume.new()
110
- v2 = modal.Volume.new()
111
- v3 = modal.Volume.new()
112
- stub.v1, stub.v2, stub.v3 = v1, v2, v3
113
-
114
- # Deploy app once
115
- deploy_stub(stub, "my-app", client=client)
116
- app1_ids = [v1.object_id, v2.object_id, v3.object_id]
117
-
118
- # Deploy app again
119
- deploy_stub(stub, "my-app", client=client)
120
- app2_ids = [v1.object_id, v2.object_id, v3.object_id]
121
-
122
- # Make sure ids are stable
123
- assert app1_ids == app2_ids
124
-
125
- # Make sure ids are unique
126
- assert len(set(app1_ids)) == 3
127
- assert len(set(app2_ids)) == 3
128
-
129
- # Deploy to a different app
130
- deploy_stub(stub, "my-other-app", client=client)
131
- app3_ids = [v1.object_id, v2.object_id, v3.object_id]
132
-
133
- # Should be unique and different
134
- assert len(set(app3_ids)) == 3
135
- assert set(app1_ids) & set(app3_ids) == set()
136
-
137
-
138
- @pytest.mark.asyncio
139
- async def test_volume_batch_upload(servicer, client, tmp_path):
140
- local_file_path = tmp_path / "some_file"
141
- local_file_path.write_text("hello world")
142
-
143
- local_dir = tmp_path / "some_dir"
144
- local_dir.mkdir()
145
- (local_dir / "smol").write_text("###")
146
-
147
- subdir = local_dir / "subdir"
148
- subdir.mkdir()
149
- (subdir / "other").write_text("####")
150
-
151
- async with modal.Volume.ephemeral(client=client) as vol:
152
- with open(local_file_path, "rb") as fp:
153
- with vol.batch_upload() as batch:
154
- batch.put_file(local_file_path, "/some_file")
155
- batch.put_directory(local_dir, "/some_dir")
156
- batch.put_file(io.BytesIO(b"data from a file-like object"), "/filelike", mode=0o600)
157
- batch.put_directory(local_dir, "/non-recursive", recursive=False)
158
- batch.put_file(fp, "/filelike2")
159
- object_id = vol.object_id
160
-
161
- assert servicer.volume_files[object_id].keys() == {
162
- "/some_file",
163
- "/some_dir/smol",
164
- "/some_dir/subdir/other",
165
- "/filelike",
166
- "/non-recursive/smol",
167
- "/filelike2",
168
- }
169
- assert servicer.volume_files[object_id]["/some_file"].data == b"hello world"
170
- assert servicer.volume_files[object_id]["/some_dir/smol"].data == b"###"
171
- assert servicer.volume_files[object_id]["/some_dir/subdir/other"].data == b"####"
172
- assert servicer.volume_files[object_id]["/filelike"].data == b"data from a file-like object"
173
- assert servicer.volume_files[object_id]["/filelike"].mode == 0o600
174
- assert servicer.volume_files[object_id]["/non-recursive/smol"].data == b"###"
175
- assert servicer.volume_files[object_id]["/filelike2"].data == b"hello world"
176
- assert servicer.volume_files[object_id]["/filelike2"].mode == 0o644
177
-
178
-
179
- @pytest.mark.asyncio
180
- async def test_volume_batch_upload_force(servicer, client, tmp_path):
181
- local_file_path = tmp_path / "some_file"
182
- local_file_path.write_text("hello world")
183
-
184
- local_file_path2 = tmp_path / "some_file2"
185
- local_file_path2.write_text("overwritten")
186
-
187
- async with modal.Volume.ephemeral(client=client) as vol:
188
- with servicer.intercept() as ctx:
189
- # Seed the volume
190
- with vol.batch_upload() as batch:
191
- batch.put_file(local_file_path, "/some_file")
192
- assert ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
193
-
194
- # Attempting to overwrite the file with force=False should result in an error
195
- with pytest.raises(FileExistsError):
196
- with vol.batch_upload(force=False) as batch:
197
- batch.put_file(local_file_path, "/some_file")
198
- assert ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
199
- assert servicer.volume_files[vol.object_id]["/some_file"].data == b"hello world"
200
-
201
- # Overwriting should work with force=True
202
- with vol.batch_upload(force=True) as batch:
203
- batch.put_file(local_file_path2, "/some_file")
204
- assert not ctx.pop_request("VolumePutFiles").disallow_overwrite_existing_files
205
- assert servicer.volume_files[vol.object_id]["/some_file"].data == b"overwritten"
206
-
207
-
208
- @pytest.mark.asyncio
209
- async def test_volume_upload_removed_file(servicer, client, tmp_path):
210
- local_file_path = tmp_path / "some_file"
211
- local_file_path.write_text("hello world")
212
-
213
- async with modal.Volume.ephemeral(client=client) as vol:
214
- with pytest.raises(FileNotFoundError):
215
- with vol.batch_upload() as batch:
216
- batch.put_file(local_file_path, "/dest")
217
- local_file_path.unlink()
218
-
219
-
220
- @pytest.mark.asyncio
221
- async def test_volume_upload_large_file(client, tmp_path, servicer, blob_server, *args):
222
- with mock.patch("modal._utils.blob_utils.LARGE_FILE_LIMIT", 10):
223
- local_file_path = tmp_path / "bigfile"
224
- local_file_path.write_text("hello world, this is a lot of text")
225
-
226
- async with modal.Volume.ephemeral(client=client) as vol:
227
- async with vol.batch_upload() as batch:
228
- batch.put_file(local_file_path, "/a")
229
- object_id = vol.object_id
230
-
231
- assert servicer.volume_files[object_id].keys() == {"/a"}
232
- assert servicer.volume_files[object_id]["/a"].data == b""
233
- assert servicer.volume_files[object_id]["/a"].data_blob_id == "bl-1"
234
-
235
- _, blobs = blob_server
236
- assert blobs["bl-1"] == b"hello world, this is a lot of text"
237
-
238
-
239
- @pytest.mark.asyncio
240
- async def test_volume_upload_file_timeout(client, tmp_path, servicer, blob_server, *args):
241
- call_count = 0
242
-
243
- async def mount_put_file(self, stream):
244
- await stream.recv_message()
245
- nonlocal call_count
246
- call_count += 1
247
- await stream.send_message(api_pb2.MountPutFileResponse(exists=False))
248
-
249
- with servicer.intercept() as ctx:
250
- ctx.set_responder("MountPutFile", mount_put_file)
251
- with mock.patch("modal._utils.blob_utils.LARGE_FILE_LIMIT", 10):
252
- with mock.patch("modal.volume.VOLUME_PUT_FILE_CLIENT_TIMEOUT", 0.5):
253
- local_file_path = tmp_path / "bigfile"
254
- local_file_path.write_text("hello world, this is a lot of text")
255
-
256
- async with modal.Volume.ephemeral(client=client) as vol:
257
- with pytest.raises(VolumeUploadTimeoutError):
258
- async with vol.batch_upload() as batch:
259
- batch.put_file(local_file_path, "/dest")
260
-
261
- assert call_count > 2
262
-
263
-
264
- @pytest.mark.asyncio
265
- async def test_volume_copy_1(client, tmp_path, servicer):
266
- ## test 1: copy src path to dst path ##
267
- src_path = "original.txt"
268
- dst_path = "copied.txt"
269
- local_file_path = tmp_path / src_path
270
- local_file_path.write_text("test copy")
271
-
272
- async with modal.Volume.ephemeral(client=client) as vol:
273
- # add local file to volume
274
- async with vol.batch_upload() as batch:
275
- batch.put_file(local_file_path, src_path)
276
- object_id = vol.object_id
277
-
278
- # copy file from src_path to dst_path
279
- vol.copy_files([src_path], dst_path)
280
-
281
- assert servicer.volume_files[object_id].keys() == {src_path, dst_path}
282
-
283
- assert servicer.volume_files[object_id][src_path].data == b"test copy"
284
- assert servicer.volume_files[object_id][dst_path].data == b"test copy"
285
-
286
-
287
- @pytest.mark.asyncio
288
- async def test_volume_copy_2(client, tmp_path, servicer):
289
- ## test 2: copy multiple files into a directory ##
290
- file_paths = ["file1.txt", "file2.txt"]
291
-
292
- async with modal.Volume.ephemeral(client=client) as vol:
293
- for file_path in file_paths:
294
- local_file_path = tmp_path / file_path
295
- local_file_path.write_text("test copy")
296
- async with vol.batch_upload() as batch:
297
- batch.put_file(local_file_path, file_path)
298
- object_id = vol.object_id
299
-
300
- vol.copy_files(file_paths, "test_dir")
301
-
302
- returned_volume_files = [Path(file) for file in servicer.volume_files[object_id].keys()]
303
- expected_volume_files = [
304
- Path(file) for file in ["file1.txt", "file2.txt", "test_dir/file1.txt", "test_dir/file2.txt"]
305
- ]
306
-
307
- assert returned_volume_files == expected_volume_files
308
-
309
- returned_file_data = {
310
- Path(entry): servicer.volume_files[object_id][entry] for entry in servicer.volume_files[object_id]
311
- }
312
- assert returned_file_data[Path("test_dir/file1.txt")].data == b"test copy"
313
- assert returned_file_data[Path("test_dir/file2.txt")].data == b"test copy"
314
-
315
-
316
- def test_persisted(servicer, client):
317
- # Lookup should fail since it doesn't exist
318
- with pytest.raises(NotFoundError):
319
- modal.Volume.lookup("xyz", client=client)
320
-
321
- # Create it
322
- modal.Volume.lookup("xyz", create_if_missing=True, client=client)
323
-
324
- # Lookup should succeed now
325
- v = modal.Volume.lookup("xyz", client=client)
326
-
327
- # Delete it
328
- v.delete()
329
-
330
- # Lookup should fail again
331
- with pytest.raises(NotFoundError):
332
- modal.Volume.lookup("xyz", client=client)
333
-
334
-
335
- def test_ephemeral(servicer, client):
336
- assert servicer.n_vol_heartbeats == 0
337
- with modal.Volume.ephemeral(client=client, _heartbeat_sleep=1) as vol:
338
- assert vol.listdir("**") == []
339
- # TODO(erikbern): perform some operations
340
- time.sleep(1.5) # Make time for 2 heartbeats
341
- assert servicer.n_vol_heartbeats == 2
test/watcher_test.py DELETED
@@ -1,30 +0,0 @@
1
- # Copyright Modal Labs 2023
2
- import pytest
3
- import random
4
- import string
5
- from pathlib import Path
6
-
7
- from watchfiles import Change
8
-
9
- from modal._watcher import _watch_args_from_mounts
10
- from modal.mount import _Mount
11
-
12
-
13
- @pytest.mark.asyncio
14
- async def test__watch_args_from_mounts(monkeypatch, test_dir):
15
- paths, watch_filter = _watch_args_from_mounts(
16
- mounts=[
17
- _Mount.from_local_file("/x/foo.py", remote_path="/foo.py"),
18
- _Mount.from_local_dir("/one/two/bucklemyshoe", remote_path="/"),
19
- _Mount.from_local_dir("/x/z", remote_path="/z"),
20
- ]
21
- )
22
-
23
- assert paths == {Path("/x").absolute(), Path("/one/two/bucklemyshoe").absolute(), Path("/x/z").absolute()}
24
- assert watch_filter(Change.modified, "/x/foo.py")
25
- assert not watch_filter(Change.modified, "/x/notwatched.py")
26
- assert not watch_filter(Change.modified, "/x/y/foo.py")
27
- assert watch_filter(Change.modified, "/x/z/bar.py")
28
- random_filename = "".join(random.choices(string.ascii_uppercase + string.digits, k=10))
29
- assert watch_filter(Change.modified, f"/one/two/bucklemyshoe/{random_filename}")
30
- assert not watch_filter(Change.modified, "/one/two/bucklemyshoe/.DS_Store")