modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. modal/__init__.py +17 -13
  2. modal/__main__.py +41 -3
  3. modal/_clustered_functions.py +80 -0
  4. modal/_clustered_functions.pyi +22 -0
  5. modal/_container_entrypoint.py +420 -937
  6. modal/_ipython.py +3 -13
  7. modal/_location.py +17 -10
  8. modal/_output.py +243 -99
  9. modal/_pty.py +2 -2
  10. modal/_resolver.py +55 -59
  11. modal/_resources.py +51 -0
  12. modal/_runtime/__init__.py +1 -0
  13. modal/_runtime/asgi.py +519 -0
  14. modal/_runtime/container_io_manager.py +1036 -0
  15. modal/_runtime/execution_context.py +89 -0
  16. modal/_runtime/telemetry.py +169 -0
  17. modal/_runtime/user_code_imports.py +356 -0
  18. modal/_serialization.py +134 -9
  19. modal/_traceback.py +47 -187
  20. modal/_tunnel.py +52 -16
  21. modal/_tunnel.pyi +19 -36
  22. modal/_utils/app_utils.py +3 -17
  23. modal/_utils/async_utils.py +479 -100
  24. modal/_utils/blob_utils.py +157 -186
  25. modal/_utils/bytes_io_segment_payload.py +97 -0
  26. modal/_utils/deprecation.py +89 -0
  27. modal/_utils/docker_utils.py +98 -0
  28. modal/_utils/function_utils.py +460 -171
  29. modal/_utils/grpc_testing.py +47 -31
  30. modal/_utils/grpc_utils.py +62 -109
  31. modal/_utils/hash_utils.py +61 -19
  32. modal/_utils/http_utils.py +39 -9
  33. modal/_utils/logger.py +2 -1
  34. modal/_utils/mount_utils.py +34 -16
  35. modal/_utils/name_utils.py +58 -0
  36. modal/_utils/package_utils.py +14 -1
  37. modal/_utils/pattern_utils.py +205 -0
  38. modal/_utils/rand_pb_testing.py +5 -7
  39. modal/_utils/shell_utils.py +15 -49
  40. modal/_vendor/a2wsgi_wsgi.py +62 -72
  41. modal/_vendor/cloudpickle.py +1 -1
  42. modal/_watcher.py +14 -12
  43. modal/app.py +1003 -314
  44. modal/app.pyi +540 -264
  45. modal/call_graph.py +7 -6
  46. modal/cli/_download.py +63 -53
  47. modal/cli/_traceback.py +200 -0
  48. modal/cli/app.py +205 -45
  49. modal/cli/config.py +12 -5
  50. modal/cli/container.py +62 -14
  51. modal/cli/dict.py +128 -0
  52. modal/cli/entry_point.py +26 -13
  53. modal/cli/environment.py +40 -9
  54. modal/cli/import_refs.py +64 -58
  55. modal/cli/launch.py +32 -18
  56. modal/cli/network_file_system.py +64 -83
  57. modal/cli/profile.py +1 -1
  58. modal/cli/programs/run_jupyter.py +35 -10
  59. modal/cli/programs/vscode.py +60 -10
  60. modal/cli/queues.py +131 -0
  61. modal/cli/run.py +234 -131
  62. modal/cli/secret.py +8 -7
  63. modal/cli/token.py +7 -2
  64. modal/cli/utils.py +79 -10
  65. modal/cli/volume.py +110 -109
  66. modal/client.py +250 -144
  67. modal/client.pyi +157 -118
  68. modal/cloud_bucket_mount.py +108 -34
  69. modal/cloud_bucket_mount.pyi +32 -38
  70. modal/cls.py +535 -148
  71. modal/cls.pyi +190 -146
  72. modal/config.py +41 -19
  73. modal/container_process.py +177 -0
  74. modal/container_process.pyi +82 -0
  75. modal/dict.py +111 -65
  76. modal/dict.pyi +136 -131
  77. modal/environments.py +106 -5
  78. modal/environments.pyi +77 -25
  79. modal/exception.py +34 -43
  80. modal/experimental.py +61 -2
  81. modal/extensions/ipython.py +5 -5
  82. modal/file_io.py +537 -0
  83. modal/file_io.pyi +235 -0
  84. modal/file_pattern_matcher.py +197 -0
  85. modal/functions.py +906 -911
  86. modal/functions.pyi +466 -430
  87. modal/gpu.py +57 -44
  88. modal/image.py +1089 -479
  89. modal/image.pyi +584 -228
  90. modal/io_streams.py +434 -0
  91. modal/io_streams.pyi +122 -0
  92. modal/mount.py +314 -101
  93. modal/mount.pyi +241 -235
  94. modal/network_file_system.py +92 -92
  95. modal/network_file_system.pyi +152 -110
  96. modal/object.py +67 -36
  97. modal/object.pyi +166 -143
  98. modal/output.py +63 -0
  99. modal/parallel_map.py +434 -0
  100. modal/parallel_map.pyi +75 -0
  101. modal/partial_function.py +282 -117
  102. modal/partial_function.pyi +222 -129
  103. modal/proxy.py +15 -12
  104. modal/proxy.pyi +3 -8
  105. modal/queue.py +182 -65
  106. modal/queue.pyi +218 -118
  107. modal/requirements/2024.04.txt +29 -0
  108. modal/requirements/2024.10.txt +16 -0
  109. modal/requirements/README.md +21 -0
  110. modal/requirements/base-images.json +22 -0
  111. modal/retries.py +48 -7
  112. modal/runner.py +459 -156
  113. modal/runner.pyi +135 -71
  114. modal/running_app.py +38 -0
  115. modal/sandbox.py +514 -236
  116. modal/sandbox.pyi +397 -169
  117. modal/schedule.py +4 -4
  118. modal/scheduler_placement.py +20 -3
  119. modal/secret.py +56 -31
  120. modal/secret.pyi +62 -42
  121. modal/serving.py +51 -56
  122. modal/serving.pyi +44 -36
  123. modal/stream_type.py +15 -0
  124. modal/token_flow.py +5 -3
  125. modal/token_flow.pyi +37 -32
  126. modal/volume.py +285 -157
  127. modal/volume.pyi +249 -184
  128. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/METADATA +7 -7
  129. modal-0.72.11.dist-info/RECORD +174 -0
  130. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/top_level.txt +0 -1
  131. modal_docs/gen_reference_docs.py +3 -1
  132. modal_docs/mdmd/mdmd.py +0 -1
  133. modal_docs/mdmd/signatures.py +5 -2
  134. modal_global_objects/images/base_images.py +28 -0
  135. modal_global_objects/mounts/python_standalone.py +2 -2
  136. modal_proto/__init__.py +1 -1
  137. modal_proto/api.proto +1288 -533
  138. modal_proto/api_grpc.py +856 -456
  139. modal_proto/api_pb2.py +2165 -1157
  140. modal_proto/api_pb2.pyi +8859 -0
  141. modal_proto/api_pb2_grpc.py +1674 -855
  142. modal_proto/api_pb2_grpc.pyi +1416 -0
  143. modal_proto/modal_api_grpc.py +149 -0
  144. modal_proto/modal_options_grpc.py +3 -0
  145. modal_proto/options_pb2.pyi +20 -0
  146. modal_proto/options_pb2_grpc.pyi +7 -0
  147. modal_proto/py.typed +0 -0
  148. modal_version/__init__.py +1 -1
  149. modal_version/_version_generated.py +2 -2
  150. modal/_asgi.py +0 -370
  151. modal/_container_entrypoint.pyi +0 -378
  152. modal/_container_exec.py +0 -128
  153. modal/_sandbox_shell.py +0 -49
  154. modal/shared_volume.py +0 -23
  155. modal/shared_volume.pyi +0 -24
  156. modal/stub.py +0 -783
  157. modal/stub.pyi +0 -332
  158. modal-0.62.16.dist-info/RECORD +0 -198
  159. modal_global_objects/images/conda.py +0 -15
  160. modal_global_objects/images/debian_slim.py +0 -15
  161. modal_global_objects/images/micromamba.py +0 -15
  162. test/__init__.py +0 -1
  163. test/aio_test.py +0 -12
  164. test/async_utils_test.py +0 -262
  165. test/blob_test.py +0 -67
  166. test/cli_imports_test.py +0 -149
  167. test/cli_test.py +0 -659
  168. test/client_test.py +0 -194
  169. test/cls_test.py +0 -630
  170. test/config_test.py +0 -137
  171. test/conftest.py +0 -1420
  172. test/container_app_test.py +0 -32
  173. test/container_test.py +0 -1389
  174. test/cpu_test.py +0 -23
  175. test/decorator_test.py +0 -85
  176. test/deprecation_test.py +0 -34
  177. test/dict_test.py +0 -33
  178. test/e2e_test.py +0 -68
  179. test/error_test.py +0 -7
  180. test/function_serialization_test.py +0 -32
  181. test/function_test.py +0 -653
  182. test/function_utils_test.py +0 -101
  183. test/gpu_test.py +0 -159
  184. test/grpc_utils_test.py +0 -141
  185. test/helpers.py +0 -42
  186. test/image_test.py +0 -669
  187. test/live_reload_test.py +0 -80
  188. test/lookup_test.py +0 -70
  189. test/mdmd_test.py +0 -329
  190. test/mount_test.py +0 -162
  191. test/mounted_files_test.py +0 -329
  192. test/network_file_system_test.py +0 -181
  193. test/notebook_test.py +0 -66
  194. test/object_test.py +0 -41
  195. test/package_utils_test.py +0 -25
  196. test/queue_test.py +0 -97
  197. test/resolver_test.py +0 -58
  198. test/retries_test.py +0 -67
  199. test/runner_test.py +0 -85
  200. test/sandbox_test.py +0 -191
  201. test/schedule_test.py +0 -15
  202. test/scheduler_placement_test.py +0 -29
  203. test/secret_test.py +0 -78
  204. test/serialization_test.py +0 -42
  205. test/stub_composition_test.py +0 -10
  206. test/stub_test.py +0 -360
  207. test/test_asgi_wrapper.py +0 -234
  208. test/token_flow_test.py +0 -18
  209. test/traceback_test.py +0 -135
  210. test/tunnel_test.py +0 -29
  211. test/utils_test.py +0 -88
  212. test/version_test.py +0 -14
  213. test/volume_test.py +0 -341
  214. test/watcher_test.py +0 -30
  215. test/webhook_test.py +0 -146
  216. /modal/{requirements.312.txt → requirements/2023.12.312.txt} +0 -0
  217. /modal/{requirements.txt → requirements/2023.12.txt} +0 -0
  218. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/LICENSE +0 -0
  219. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/WHEEL +0 -0
  220. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/entry_points.txt +0 -0
test/live_reload_test.py DELETED
@@ -1,80 +0,0 @@
1
- # Copyright Modal Labs 2023
2
- import asyncio
3
- import pytest
4
- import threading
5
- import time
6
- from unittest import mock
7
-
8
- from modal import Function
9
- from modal.serving import serve_stub
10
-
11
- from .supports.app_run_tests.webhook import stub
12
- from .supports.skip import skip_windows
13
-
14
-
15
- @pytest.fixture
16
- def stub_ref(test_dir):
17
- return str(test_dir / "supports" / "app_run_tests" / "webhook.py")
18
-
19
-
20
- @pytest.mark.asyncio
21
- async def test_live_reload(stub_ref, server_url_env, servicer):
22
- async with serve_stub.aio(stub, stub_ref):
23
- await asyncio.sleep(3.0)
24
- assert servicer.app_set_objects_count == 1
25
- assert servicer.app_client_disconnect_count == 1
26
- assert servicer.app_get_logs_initial_count == 1
27
-
28
-
29
- @skip_windows("live-reload not supported on windows")
30
- def test_file_changes_trigger_reloads(stub_ref, server_url_env, servicer):
31
- watcher_done = threading.Event()
32
-
33
- async def fake_watch():
34
- for i in range(3):
35
- yield {"/some/file"}
36
- watcher_done.set()
37
-
38
- with serve_stub(stub, stub_ref, _watcher=fake_watch()):
39
- watcher_done.wait() # wait until watcher loop is done
40
-
41
- # TODO ideally we would assert the specific expected number here, but this test
42
- # is consistently flaking in CI and I cannot reproduce locally to debug.
43
- # I'm relaxing the assertion for now to stop the test from blocking deployments.
44
- # assert servicer.app_set_objects_count == 4 # 1 + number of file changes
45
- assert servicer.app_set_objects_count > 1
46
- assert servicer.app_client_disconnect_count == 1
47
- assert servicer.app_get_logs_initial_count == 1
48
- foo = stub.indexed_objects["foo"]
49
- assert isinstance(foo, Function)
50
- assert foo.web_url.startswith("http://")
51
-
52
-
53
- @pytest.mark.asyncio
54
- async def test_no_change(stub_ref, server_url_env, servicer):
55
- async def fake_watch():
56
- # Iterator that returns immediately, yielding nothing
57
- if False:
58
- yield
59
-
60
- async with serve_stub.aio(stub, stub_ref, _watcher=fake_watch()):
61
- pass
62
-
63
- assert servicer.app_set_objects_count == 1 # Should create the initial app once
64
- assert servicer.app_client_disconnect_count == 1
65
- assert servicer.app_get_logs_initial_count == 1
66
-
67
-
68
- @pytest.mark.asyncio
69
- async def test_heartbeats(stub_ref, server_url_env, servicer):
70
- with mock.patch("modal.runner.HEARTBEAT_INTERVAL", 1):
71
- t0 = time.time()
72
- async with serve_stub.aio(stub, stub_ref):
73
- await asyncio.sleep(3.1)
74
- total_secs = int(time.time() - t0)
75
-
76
- apps = list(servicer.app_heartbeats.keys())
77
- assert len(apps) == 1
78
- # Typically [0s, 1s, 2s, 3s], but asyncio.sleep may lag.
79
- actual_heartbeats = servicer.app_heartbeats[apps[0]]
80
- assert abs(actual_heartbeats - (total_secs + 1)) <= 1
test/lookup_test.py DELETED
@@ -1,70 +0,0 @@
1
- # Copyright Modal Labs 2023
2
- import pytest
3
-
4
- from modal import Function, Stub, Volume, web_endpoint
5
- from modal.exception import ExecutionError, NotFoundError
6
- from modal.runner import deploy_stub
7
-
8
-
9
- def test_persistent_object(servicer, client):
10
- volume_id = Volume.create_deployed("my-volume", client=client)
11
-
12
- v: Volume = Volume.lookup("my-volume", client=client)
13
- assert isinstance(v, Volume)
14
- assert v.object_id == volume_id
15
-
16
- with pytest.raises(NotFoundError):
17
- Volume.lookup("bazbazbaz", client=client)
18
-
19
-
20
- def square(x):
21
- # This function isn't deployed anyway
22
- pass
23
-
24
-
25
- def test_lookup_function(servicer, client):
26
- stub = Stub()
27
-
28
- stub.function()(square)
29
- deploy_stub(stub, "my-function", client=client)
30
-
31
- f = Function.lookup("my-function", "square", client=client)
32
- assert f.object_id == "fu-1"
33
-
34
- # Call it using two arguments
35
- f = Function.lookup("my-function", "square", client=client)
36
- assert f.object_id == "fu-1"
37
- with pytest.raises(NotFoundError):
38
- f = Function.lookup("my-function", "cube", client=client)
39
-
40
- # Make sure we can call this function
41
- assert f.remote(2, 4) == 20
42
- assert [r for r in f.map([5, 2], [4, 3])] == [41, 13]
43
-
44
- # Make sure the new-style local calls raise an error
45
- with pytest.raises(ExecutionError):
46
- assert f.local(2, 4) == 20
47
-
48
-
49
- def test_webhook_lookup(servicer, client):
50
- stub = Stub()
51
- stub.function()(web_endpoint(method="POST")(square))
52
- deploy_stub(stub, "my-webhook", client=client)
53
-
54
- f = Function.lookup("my-webhook", "square", client=client)
55
- assert f.web_url
56
-
57
-
58
- def test_deploy_exists(servicer, client):
59
- with pytest.raises(NotFoundError):
60
- Volume.lookup("my-volume", client=client)
61
- Volume.create_deployed("my-volume", client=client)
62
- v1: Volume = Volume.lookup("my-volume", client=client)
63
- v2: Volume = Volume.lookup("my-volume", client=client)
64
- assert v1.object_id == v2.object_id
65
-
66
-
67
- def test_create_if_missing(servicer, client):
68
- v1: Volume = Volume.lookup("my-volume", create_if_missing=True, client=client)
69
- v2: Volume = Volume.lookup("my-volume", client=client)
70
- assert v1.object_id == v2.object_id
test/mdmd_test.py DELETED
@@ -1,329 +0,0 @@
1
- # Copyright Modal Labs 2023
2
- import importlib
3
- import os
4
- import pytest
5
- import sys
6
- from enum import IntEnum
7
-
8
- from modal_docs.mdmd import mdmd
9
-
10
- # Skipping a few tests on 3.7 - doesn't matter since we don't generate docs on 3.7
11
- skip_37 = pytest.mark.skipif(sys.version_info < (3, 8), reason="Requires Python 3.8")
12
-
13
-
14
- def test_simple_function():
15
- def foo():
16
- pass
17
-
18
- assert (
19
- mdmd.function_str("bar", foo)
20
- == """```python
21
- def bar():
22
- ```\n\n"""
23
- )
24
-
25
-
26
- def test_simple_async_function():
27
- async def foo():
28
- pass
29
-
30
- assert (
31
- mdmd.function_str("bar", foo)
32
- == """```python
33
- async def bar():
34
- ```\n\n"""
35
- )
36
-
37
-
38
- def test_async_gen_function():
39
- async def foo():
40
- yield
41
-
42
- assert (
43
- mdmd.function_str("bar", foo)
44
- == """```python
45
- async def bar():
46
- ```\n\n"""
47
- )
48
-
49
-
50
- def test_complex_function_signature():
51
- def foo(a: str, *args, **kwargs):
52
- pass
53
-
54
- assert (
55
- mdmd.function_str("foo", foo)
56
- == """```python
57
- def foo(a: str, *args, **kwargs):
58
- ```\n\n"""
59
- )
60
-
61
-
62
- @skip_37
63
- def test_function_has_docstring():
64
- def foo():
65
- """short description
66
-
67
- longer description"""
68
-
69
- assert (
70
- mdmd.function_str("foo", foo)
71
- == """```python
72
- def foo():
73
- ```
74
-
75
- short description
76
-
77
- longer description
78
- """
79
- )
80
-
81
-
82
- def test_simple_class_with_docstring():
83
- class Foo:
84
- """The all important Foo"""
85
-
86
- def bar(self, baz: str):
87
- """Bars the foo with the baz"""
88
-
89
- assert (
90
- mdmd.class_str("Foo", Foo)
91
- == """```python
92
- class Foo(object)
93
- ```
94
-
95
- The all important Foo
96
-
97
- ### bar
98
-
99
- ```python
100
- def bar(self, baz: str):
101
- ```
102
-
103
- Bars the foo with the baz
104
- """
105
- )
106
-
107
-
108
- def test_enum():
109
- class Eee(IntEnum):
110
- FOO = 1
111
- BAR = 2
112
- XYZ = 3
113
-
114
- expected = """```python
115
- class bar(enum.IntEnum)
116
- ```
117
-
118
- An enumeration.
119
-
120
- The possible values are:
121
-
122
- * `FOO`
123
- * `BAR`
124
- * `XYZ`
125
- """
126
-
127
- assert mdmd.class_str("bar", Eee) == expected
128
-
129
-
130
- def test_class_with_classmethod():
131
- class Foo:
132
- @classmethod
133
- def create_foo(cls, some_arg):
134
- pass
135
-
136
- assert (
137
- mdmd.class_str("Foo", Foo)
138
- == """```python
139
- class Foo(object)
140
- ```
141
-
142
- ### create_foo
143
-
144
- ```python
145
- @classmethod
146
- def create_foo(cls, some_arg):
147
- ```
148
-
149
- """
150
- )
151
-
152
-
153
- def test_class_with_baseclass_includes_base_methods():
154
- class Foo:
155
- def foo(self):
156
- pass
157
-
158
- class Bar(Foo):
159
- def bar(self):
160
- pass
161
-
162
- out = mdmd.class_str("Bar", Bar)
163
- assert "def foo(self):" in out
164
-
165
-
166
- @skip_37
167
- def test_module(monkeypatch):
168
- test_data_dir = os.path.join(os.path.dirname(__file__), "mdmd_data")
169
- monkeypatch.chdir(test_data_dir)
170
- monkeypatch.syspath_prepend(test_data_dir)
171
- test_module = importlib.import_module("foo")
172
- expected_output = open("./foo-expected.md").read()
173
- assert mdmd.module_str("foo", test_module) == expected_output
174
-
175
-
176
- def test_docstring_format_reindents_code():
177
- assert (
178
- mdmd.format_docstring(
179
- """```python
180
- foo
181
- bar
182
- ```"""
183
- )
184
- == """```python
185
- foo
186
- bar
187
- ```
188
- """
189
- )
190
-
191
-
192
- def test_synchronicity_async_and_blocking_interfaces():
193
- from synchronicity import Synchronizer
194
-
195
- class Foo:
196
- """docky mcdocface"""
197
-
198
- async def foo(self):
199
- pass
200
-
201
- def bar(self):
202
- pass
203
-
204
- s = Synchronizer()
205
- AsyncFoo = s.create_async(Foo, "AsyncFoo")
206
- BlockingFoo = s.create_blocking(Foo, "BlockingFoo")
207
-
208
- assert (
209
- mdmd.class_str("AsyncFoo", AsyncFoo)
210
- == """```python
211
- class AsyncFoo(object)
212
- ```
213
-
214
- docky mcdocface
215
-
216
- ### foo
217
-
218
- ```python
219
- async def foo(self):
220
- ```
221
-
222
- ### bar
223
-
224
- ```python
225
- def bar(self):
226
- ```
227
-
228
- """
229
- )
230
-
231
- assert (
232
- mdmd.class_str("BlockingFoo", BlockingFoo)
233
- == """```python
234
- class BlockingFoo(object)
235
- ```
236
-
237
- docky mcdocface
238
-
239
- ### foo
240
-
241
- ```python
242
- def foo(self):
243
- ```
244
-
245
- ### bar
246
-
247
- ```python
248
- def bar(self):
249
- ```
250
-
251
- """
252
- )
253
-
254
-
255
- def test_synchronicity_constructors():
256
- from synchronicity import Synchronizer
257
-
258
- class Foo:
259
- """docky mcdocface"""
260
-
261
- def __init__(self):
262
- """constructy mcconstructorface"""
263
-
264
- s = Synchronizer()
265
- AsyncFoo = s.create_async(Foo, "AsyncFoo")
266
-
267
- assert (
268
- mdmd.class_str("AsyncFoo", AsyncFoo)
269
- == """```python
270
- class AsyncFoo(object)
271
- ```
272
-
273
- docky mcdocface
274
-
275
- ```python
276
- def __init__(self):
277
- ```
278
-
279
- constructy mcconstructorface
280
- """
281
- )
282
-
283
-
284
- def test_get_all_signature_comments():
285
- def foo(
286
- # prefix comment
287
- one, # one comment
288
- two, # two comment
289
- # postfix comment
290
- ) -> str: # return value comment
291
- pass
292
-
293
- assert (
294
- mdmd.function_str("foo", foo)
295
- == """```python
296
- def foo(
297
- # prefix comment
298
- one, # one comment
299
- two, # two comment
300
- # postfix comment
301
- ) -> str: # return value comment
302
- ```
303
-
304
- """
305
- )
306
-
307
-
308
- def test_get_decorators():
309
- BLA = 1
310
-
311
- def my_deco(arg):
312
- def wrapper(f):
313
- return f
314
-
315
- return wrapper
316
-
317
- @my_deco(BLA)
318
- def foo():
319
- pass
320
-
321
- assert (
322
- mdmd.function_str("foo", foo)
323
- == """```python
324
- @my_deco(BLA)
325
- def foo():
326
- ```
327
-
328
- """
329
- )
test/mount_test.py DELETED
@@ -1,162 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import hashlib
3
- import os
4
- import platform
5
- import pytest
6
- import sys
7
- from pathlib import Path
8
-
9
- from modal import Stub
10
- from modal._utils.blob_utils import LARGE_FILE_LIMIT
11
- from modal.exception import ModuleNotMountable
12
- from modal.mount import Mount
13
-
14
-
15
- @pytest.mark.asyncio
16
- async def test_get_files(servicer, client, tmpdir):
17
- small_content = b"# not much here"
18
- large_content = b"a" * (LARGE_FILE_LIMIT + 1)
19
-
20
- tmpdir.join("small.py").write(small_content)
21
- tmpdir.join("large.py").write(large_content)
22
- tmpdir.join("fluff").write("hello")
23
-
24
- files = {}
25
- m = Mount.from_local_dir(Path(tmpdir), remote_path="/", condition=lambda fn: fn.endswith(".py"), recursive=True)
26
- async for upload_spec in Mount._get_files.aio(m.entries):
27
- files[upload_spec.mount_filename] = upload_spec
28
-
29
- os.umask(umask := os.umask(0o022)) # Get the current umask
30
- expected_mode = 0o644 if platform.system() == "Windows" else 0o666 - umask
31
-
32
- assert "/small.py" in files
33
- assert "/large.py" in files
34
- assert "/fluff" not in files
35
- assert files["/small.py"].use_blob is False
36
- assert files["/small.py"].content == small_content
37
- assert files["/small.py"].sha256_hex == hashlib.sha256(small_content).hexdigest()
38
- assert files["/small.py"].mode == expected_mode
39
-
40
- assert files["/large.py"].use_blob is True
41
- assert files["/large.py"].content is None
42
- assert files["/large.py"].sha256_hex == hashlib.sha256(large_content).hexdigest()
43
- assert files["/large.py"].mode == expected_mode
44
-
45
- await m._deploy.aio("my-mount", client=client)
46
- blob_id = max(servicer.blobs.keys()) # last uploaded one
47
- assert len(servicer.blobs[blob_id]) == len(large_content)
48
- assert servicer.blobs[blob_id] == large_content
49
-
50
- assert servicer.files_sha2data[files["/large.py"].sha256_hex] == {"data": b"", "data_blob_id": blob_id}
51
- assert servicer.files_sha2data[files["/small.py"].sha256_hex] == {
52
- "data": small_content,
53
- "data_blob_id": "",
54
- }
55
-
56
-
57
- def test_create_mount(servicer, client):
58
- local_dir, cur_filename = os.path.split(__file__)
59
-
60
- def condition(fn):
61
- return fn.endswith(".py")
62
-
63
- m = Mount.from_local_dir(local_dir, remote_path="/foo", condition=condition)
64
-
65
- m._deploy("my-mount", client=client)
66
-
67
- assert m.object_id == "mo-1"
68
- assert f"/foo/{cur_filename}" in servicer.files_name2sha
69
- sha256_hex = servicer.files_name2sha[f"/foo/{cur_filename}"]
70
- assert sha256_hex in servicer.files_sha2data
71
- assert servicer.files_sha2data[sha256_hex]["data"] == open(__file__, "rb").read()
72
- assert repr(Path(local_dir)) in repr(m)
73
-
74
-
75
- def test_create_mount_file_errors(servicer, tmpdir, client):
76
- m = Mount.from_local_dir(Path(tmpdir) / "xyz", remote_path="/xyz")
77
- with pytest.raises(FileNotFoundError):
78
- m._deploy("my-mount", client=client)
79
-
80
- with open(tmpdir / "abc", "w"):
81
- pass
82
- m = Mount.from_local_dir(Path(tmpdir) / "abc", remote_path="/abc")
83
- with pytest.raises(NotADirectoryError):
84
- m._deploy("my-mount", client=client)
85
-
86
-
87
- def dummy():
88
- pass
89
-
90
-
91
- def test_from_local_python_packages(servicer, client, test_dir):
92
- stub = Stub()
93
-
94
- sys.path.append((test_dir / "supports").as_posix())
95
-
96
- stub.function(mounts=[Mount.from_local_python_packages("pkg_a", "pkg_b", "standalone_file")])(dummy)
97
-
98
- with stub.run(client=client):
99
- files = set(servicer.files_name2sha.keys())
100
- expected_files = {
101
- "/root/pkg_a/a.py",
102
- "/root/pkg_a/b/c.py",
103
- "/root/pkg_b/f.py",
104
- "/root/pkg_b/g/h.py",
105
- "/root/standalone_file.py",
106
- }
107
- assert expected_files.issubset(files)
108
-
109
- assert "/root/pkg_c/i.py" not in files
110
- assert "/root/pkg_c/j/k.py" not in files
111
-
112
-
113
- def test_stub_mounts(servicer, client, test_dir):
114
- sys.path.append((test_dir / "supports").as_posix())
115
-
116
- stub = Stub(mounts=[Mount.from_local_python_packages("pkg_b")])
117
-
118
- stub.function(mounts=[Mount.from_local_python_packages("pkg_a")])(dummy)
119
-
120
- with stub.run(client=client):
121
- files = set(servicer.files_name2sha.keys())
122
- expected_files = {
123
- "/root/pkg_a/a.py",
124
- "/root/pkg_a/b/c.py",
125
- "/root/pkg_b/f.py",
126
- "/root/pkg_b/g/h.py",
127
- }
128
- assert expected_files.issubset(files)
129
-
130
- assert "/root/pkg_c/i.py" not in files
131
- assert "/root/pkg_c/j/k.py" not in files
132
-
133
-
134
- def test_from_local_python_packages_missing_module(servicer, client, test_dir, server_url_env):
135
- stub = Stub()
136
- stub.function(mounts=[Mount.from_local_python_packages("nonexistent_package")])(dummy)
137
-
138
- with pytest.raises(ModuleNotMountable):
139
- with stub.run(client=client):
140
- pass
141
-
142
-
143
- def test_chained_entries(test_dir):
144
- a_txt = str(test_dir / "a.txt")
145
- b_txt = str(test_dir / "b.txt")
146
- with open(a_txt, "w") as f:
147
- f.write("A")
148
- with open(b_txt, "w") as f:
149
- f.write("B")
150
- mount = Mount.from_local_file(a_txt).add_local_file(b_txt)
151
- entries = mount.entries
152
- assert len(entries) == 2
153
- files = [file for file in Mount._get_files(entries)]
154
- assert len(files) == 2
155
- files.sort(key=lambda file: file.source_description)
156
- assert files[0].source_description.name == "a.txt"
157
- assert files[0].mount_filename.endswith("/a.txt")
158
- assert files[0].content == b"A"
159
- m = hashlib.sha256()
160
- m.update(b"A")
161
- assert files[0].sha256_hex == m.hexdigest()
162
- assert files[0].use_blob is False