modal 0.62.16__py3-none-any.whl → 0.72.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. modal/__init__.py +17 -13
  2. modal/__main__.py +41 -3
  3. modal/_clustered_functions.py +80 -0
  4. modal/_clustered_functions.pyi +22 -0
  5. modal/_container_entrypoint.py +420 -937
  6. modal/_ipython.py +3 -13
  7. modal/_location.py +17 -10
  8. modal/_output.py +243 -99
  9. modal/_pty.py +2 -2
  10. modal/_resolver.py +55 -59
  11. modal/_resources.py +51 -0
  12. modal/_runtime/__init__.py +1 -0
  13. modal/_runtime/asgi.py +519 -0
  14. modal/_runtime/container_io_manager.py +1036 -0
  15. modal/_runtime/execution_context.py +89 -0
  16. modal/_runtime/telemetry.py +169 -0
  17. modal/_runtime/user_code_imports.py +356 -0
  18. modal/_serialization.py +134 -9
  19. modal/_traceback.py +47 -187
  20. modal/_tunnel.py +52 -16
  21. modal/_tunnel.pyi +19 -36
  22. modal/_utils/app_utils.py +3 -17
  23. modal/_utils/async_utils.py +479 -100
  24. modal/_utils/blob_utils.py +157 -186
  25. modal/_utils/bytes_io_segment_payload.py +97 -0
  26. modal/_utils/deprecation.py +89 -0
  27. modal/_utils/docker_utils.py +98 -0
  28. modal/_utils/function_utils.py +460 -171
  29. modal/_utils/grpc_testing.py +47 -31
  30. modal/_utils/grpc_utils.py +62 -109
  31. modal/_utils/hash_utils.py +61 -19
  32. modal/_utils/http_utils.py +39 -9
  33. modal/_utils/logger.py +2 -1
  34. modal/_utils/mount_utils.py +34 -16
  35. modal/_utils/name_utils.py +58 -0
  36. modal/_utils/package_utils.py +14 -1
  37. modal/_utils/pattern_utils.py +205 -0
  38. modal/_utils/rand_pb_testing.py +5 -7
  39. modal/_utils/shell_utils.py +15 -49
  40. modal/_vendor/a2wsgi_wsgi.py +62 -72
  41. modal/_vendor/cloudpickle.py +1 -1
  42. modal/_watcher.py +14 -12
  43. modal/app.py +1003 -314
  44. modal/app.pyi +540 -264
  45. modal/call_graph.py +7 -6
  46. modal/cli/_download.py +63 -53
  47. modal/cli/_traceback.py +200 -0
  48. modal/cli/app.py +205 -45
  49. modal/cli/config.py +12 -5
  50. modal/cli/container.py +62 -14
  51. modal/cli/dict.py +128 -0
  52. modal/cli/entry_point.py +26 -13
  53. modal/cli/environment.py +40 -9
  54. modal/cli/import_refs.py +64 -58
  55. modal/cli/launch.py +32 -18
  56. modal/cli/network_file_system.py +64 -83
  57. modal/cli/profile.py +1 -1
  58. modal/cli/programs/run_jupyter.py +35 -10
  59. modal/cli/programs/vscode.py +60 -10
  60. modal/cli/queues.py +131 -0
  61. modal/cli/run.py +234 -131
  62. modal/cli/secret.py +8 -7
  63. modal/cli/token.py +7 -2
  64. modal/cli/utils.py +79 -10
  65. modal/cli/volume.py +110 -109
  66. modal/client.py +250 -144
  67. modal/client.pyi +157 -118
  68. modal/cloud_bucket_mount.py +108 -34
  69. modal/cloud_bucket_mount.pyi +32 -38
  70. modal/cls.py +535 -148
  71. modal/cls.pyi +190 -146
  72. modal/config.py +41 -19
  73. modal/container_process.py +177 -0
  74. modal/container_process.pyi +82 -0
  75. modal/dict.py +111 -65
  76. modal/dict.pyi +136 -131
  77. modal/environments.py +106 -5
  78. modal/environments.pyi +77 -25
  79. modal/exception.py +34 -43
  80. modal/experimental.py +61 -2
  81. modal/extensions/ipython.py +5 -5
  82. modal/file_io.py +537 -0
  83. modal/file_io.pyi +235 -0
  84. modal/file_pattern_matcher.py +197 -0
  85. modal/functions.py +906 -911
  86. modal/functions.pyi +466 -430
  87. modal/gpu.py +57 -44
  88. modal/image.py +1089 -479
  89. modal/image.pyi +584 -228
  90. modal/io_streams.py +434 -0
  91. modal/io_streams.pyi +122 -0
  92. modal/mount.py +314 -101
  93. modal/mount.pyi +241 -235
  94. modal/network_file_system.py +92 -92
  95. modal/network_file_system.pyi +152 -110
  96. modal/object.py +67 -36
  97. modal/object.pyi +166 -143
  98. modal/output.py +63 -0
  99. modal/parallel_map.py +434 -0
  100. modal/parallel_map.pyi +75 -0
  101. modal/partial_function.py +282 -117
  102. modal/partial_function.pyi +222 -129
  103. modal/proxy.py +15 -12
  104. modal/proxy.pyi +3 -8
  105. modal/queue.py +182 -65
  106. modal/queue.pyi +218 -118
  107. modal/requirements/2024.04.txt +29 -0
  108. modal/requirements/2024.10.txt +16 -0
  109. modal/requirements/README.md +21 -0
  110. modal/requirements/base-images.json +22 -0
  111. modal/retries.py +48 -7
  112. modal/runner.py +459 -156
  113. modal/runner.pyi +135 -71
  114. modal/running_app.py +38 -0
  115. modal/sandbox.py +514 -236
  116. modal/sandbox.pyi +397 -169
  117. modal/schedule.py +4 -4
  118. modal/scheduler_placement.py +20 -3
  119. modal/secret.py +56 -31
  120. modal/secret.pyi +62 -42
  121. modal/serving.py +51 -56
  122. modal/serving.pyi +44 -36
  123. modal/stream_type.py +15 -0
  124. modal/token_flow.py +5 -3
  125. modal/token_flow.pyi +37 -32
  126. modal/volume.py +285 -157
  127. modal/volume.pyi +249 -184
  128. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/METADATA +7 -7
  129. modal-0.72.11.dist-info/RECORD +174 -0
  130. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/top_level.txt +0 -1
  131. modal_docs/gen_reference_docs.py +3 -1
  132. modal_docs/mdmd/mdmd.py +0 -1
  133. modal_docs/mdmd/signatures.py +5 -2
  134. modal_global_objects/images/base_images.py +28 -0
  135. modal_global_objects/mounts/python_standalone.py +2 -2
  136. modal_proto/__init__.py +1 -1
  137. modal_proto/api.proto +1288 -533
  138. modal_proto/api_grpc.py +856 -456
  139. modal_proto/api_pb2.py +2165 -1157
  140. modal_proto/api_pb2.pyi +8859 -0
  141. modal_proto/api_pb2_grpc.py +1674 -855
  142. modal_proto/api_pb2_grpc.pyi +1416 -0
  143. modal_proto/modal_api_grpc.py +149 -0
  144. modal_proto/modal_options_grpc.py +3 -0
  145. modal_proto/options_pb2.pyi +20 -0
  146. modal_proto/options_pb2_grpc.pyi +7 -0
  147. modal_proto/py.typed +0 -0
  148. modal_version/__init__.py +1 -1
  149. modal_version/_version_generated.py +2 -2
  150. modal/_asgi.py +0 -370
  151. modal/_container_entrypoint.pyi +0 -378
  152. modal/_container_exec.py +0 -128
  153. modal/_sandbox_shell.py +0 -49
  154. modal/shared_volume.py +0 -23
  155. modal/shared_volume.pyi +0 -24
  156. modal/stub.py +0 -783
  157. modal/stub.pyi +0 -332
  158. modal-0.62.16.dist-info/RECORD +0 -198
  159. modal_global_objects/images/conda.py +0 -15
  160. modal_global_objects/images/debian_slim.py +0 -15
  161. modal_global_objects/images/micromamba.py +0 -15
  162. test/__init__.py +0 -1
  163. test/aio_test.py +0 -12
  164. test/async_utils_test.py +0 -262
  165. test/blob_test.py +0 -67
  166. test/cli_imports_test.py +0 -149
  167. test/cli_test.py +0 -659
  168. test/client_test.py +0 -194
  169. test/cls_test.py +0 -630
  170. test/config_test.py +0 -137
  171. test/conftest.py +0 -1420
  172. test/container_app_test.py +0 -32
  173. test/container_test.py +0 -1389
  174. test/cpu_test.py +0 -23
  175. test/decorator_test.py +0 -85
  176. test/deprecation_test.py +0 -34
  177. test/dict_test.py +0 -33
  178. test/e2e_test.py +0 -68
  179. test/error_test.py +0 -7
  180. test/function_serialization_test.py +0 -32
  181. test/function_test.py +0 -653
  182. test/function_utils_test.py +0 -101
  183. test/gpu_test.py +0 -159
  184. test/grpc_utils_test.py +0 -141
  185. test/helpers.py +0 -42
  186. test/image_test.py +0 -669
  187. test/live_reload_test.py +0 -80
  188. test/lookup_test.py +0 -70
  189. test/mdmd_test.py +0 -329
  190. test/mount_test.py +0 -162
  191. test/mounted_files_test.py +0 -329
  192. test/network_file_system_test.py +0 -181
  193. test/notebook_test.py +0 -66
  194. test/object_test.py +0 -41
  195. test/package_utils_test.py +0 -25
  196. test/queue_test.py +0 -97
  197. test/resolver_test.py +0 -58
  198. test/retries_test.py +0 -67
  199. test/runner_test.py +0 -85
  200. test/sandbox_test.py +0 -191
  201. test/schedule_test.py +0 -15
  202. test/scheduler_placement_test.py +0 -29
  203. test/secret_test.py +0 -78
  204. test/serialization_test.py +0 -42
  205. test/stub_composition_test.py +0 -10
  206. test/stub_test.py +0 -360
  207. test/test_asgi_wrapper.py +0 -234
  208. test/token_flow_test.py +0 -18
  209. test/traceback_test.py +0 -135
  210. test/tunnel_test.py +0 -29
  211. test/utils_test.py +0 -88
  212. test/version_test.py +0 -14
  213. test/volume_test.py +0 -341
  214. test/watcher_test.py +0 -30
  215. test/webhook_test.py +0 -146
  216. /modal/{requirements.312.txt → requirements/2023.12.312.txt} +0 -0
  217. /modal/{requirements.txt → requirements/2023.12.txt} +0 -0
  218. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/LICENSE +0 -0
  219. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/WHEEL +0 -0
  220. {modal-0.62.16.dist-info → modal-0.72.11.dist-info}/entry_points.txt +0 -0
@@ -1,329 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import os
3
- import pytest
4
- import subprocess
5
- import sys
6
- from pathlib import Path
7
-
8
- import pytest_asyncio
9
-
10
- import modal
11
- from modal import Mount
12
- from modal._utils.function_utils import FunctionInfo
13
-
14
- from . import helpers
15
- from .supports.skip import skip_windows
16
-
17
-
18
- @pytest.fixture
19
- def venv_path(tmp_path, repo_root):
20
- venv_path = tmp_path
21
- subprocess.run([sys.executable, "-m", "venv", venv_path, "--copies", "--system-site-packages"], check=True)
22
- # Install Modal and a tiny package in the venv.
23
- subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "-e", repo_root], check=True)
24
- subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "--force-reinstall", "six"], check=True)
25
- yield venv_path
26
-
27
-
28
- @pytest.fixture
29
- def path_with_symlinked_files(tmp_path):
30
- src = tmp_path / "foo.txt"
31
- src.write_text("Hello")
32
- trg = tmp_path / "bar.txt"
33
- trg.symlink_to(src)
34
- return tmp_path, {src, trg}
35
-
36
-
37
- script_path = "pkg_a/script.py"
38
-
39
-
40
- def f():
41
- pass
42
-
43
-
44
- @pytest_asyncio.fixture
45
- async def env_mount_files():
46
- # If something is installed using pip -e, it will be bundled up as a part of the environment.
47
- # Those are env-specific so we ignore those as a part of the test
48
- fn_info = FunctionInfo(f)
49
-
50
- filenames = []
51
- for mount in fn_info.get_auto_mounts():
52
- async for file_info in mount._get_files(mount.entries):
53
- filenames.append(file_info.mount_filename)
54
-
55
- return filenames
56
-
57
-
58
- def test_mounted_files_script(servicer, supports_dir, env_mount_files, server_url_env):
59
- helpers.deploy_stub_externally(servicer, script_path, cwd=supports_dir)
60
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
61
-
62
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
63
- assert files == {
64
- "/root/a.py",
65
- "/root/b/c.py",
66
- "/root/b/e.py",
67
- "/root/pkg_b/__init__.py",
68
- "/root/pkg_b/f.py",
69
- "/root/pkg_b/g/h.py",
70
- "/root/script.py",
71
- }
72
-
73
-
74
- serialized_fn_path = "pkg_a/serialized_fn.py"
75
-
76
-
77
- def test_mounted_files_serialized(servicer, supports_dir, env_mount_files, server_url_env):
78
- helpers.deploy_stub_externally(servicer, serialized_fn_path, cwd=supports_dir)
79
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
80
-
81
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
82
- assert (
83
- files
84
- == {
85
- "/root/serialized_fn.py", # should serialized_fn be included? It's not needed to run the function, but it's loaded into sys.modules at definition time...
86
- "/root/b/c.py", # this is mounted under root since it's imported as `import b` and not `import pkg_a.b` from serialized_fn.py
87
- "/root/b/e.py", # same as above
88
- "/root/a.py", # same as above
89
- "/root/pkg_b/__init__.py",
90
- "/root/pkg_b/f.py",
91
- "/root/pkg_b/g/h.py",
92
- }
93
- )
94
-
95
-
96
- def test_mounted_files_package(supports_dir, env_mount_files, servicer, server_url_env):
97
- p = subprocess.run(["modal", "run", "pkg_a.package"], cwd=supports_dir)
98
- assert p.returncode == 0
99
-
100
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
101
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
102
- assert files == {
103
- "/root/pkg_a/__init__.py",
104
- "/root/pkg_a/a.py",
105
- "/root/pkg_a/b/c.py",
106
- "/root/pkg_a/d.py",
107
- "/root/pkg_a/b/e.py",
108
- "/root/pkg_a/script.py",
109
- "/root/pkg_a/serialized_fn.py",
110
- "/root/pkg_a/package.py",
111
- "/root/pkg_b/__init__.py",
112
- "/root/pkg_b/f.py",
113
- "/root/pkg_b/g/h.py",
114
- }
115
-
116
-
117
- def test_mounted_files_package_no_automount(supports_dir, env_mount_files, servicer, server_url_env):
118
- # when triggered like a module, the target module should be put at the correct package path
119
- p = subprocess.run(
120
- ["modal", "run", "pkg_a.package"],
121
- cwd=supports_dir,
122
- capture_output=True,
123
- env={**os.environ, "MODAL_AUTOMOUNT": "0"},
124
- )
125
- assert p.returncode == 0
126
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
127
- assert files == {
128
- "/root/pkg_a/__init__.py",
129
- "/root/pkg_a/package.py",
130
- }
131
-
132
-
133
- @skip_windows("venvs behave differently on Windows.")
134
- def test_mounted_files_sys_prefix(servicer, supports_dir, venv_path, env_mount_files, server_url_env):
135
- # Run with venv activated, so it's on sys.prefix, and modal is dev-installed in the VM
136
- subprocess.run(
137
- [venv_path / "bin" / "modal", "run", script_path],
138
- cwd=supports_dir,
139
- )
140
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
141
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
142
- assert files == {
143
- "/root/a.py",
144
- "/root/b/c.py",
145
- "/root/b/e.py",
146
- "/root/script.py",
147
- "/root/pkg_b/__init__.py",
148
- "/root/pkg_b/f.py",
149
- "/root/pkg_b/g/h.py",
150
- }
151
-
152
-
153
- @pytest.fixture
154
- def symlinked_python_installation_venv_path(tmp_path, repo_root):
155
- # sets up a symlink to the python *installation* (not just the python binary)
156
- # and initialize the virtualenv using a path via that symlink
157
- # This makes the file paths of any stdlib modules use the symlinked path
158
- # instead of the original, which is similar to what some tools do (e.g. mise)
159
- # and has the potential to break automounting behavior, so we keep this
160
- # test as a regression test for that
161
- venv_path = tmp_path / "venv"
162
- actual_executable = Path(sys.executable).resolve()
163
- assert actual_executable.parent.name == "bin"
164
- python_install_dir = actual_executable.parent.parent
165
- # create a symlink to the python install *root*
166
- symlink_python_install = tmp_path / "python-install"
167
- symlink_python_install.symlink_to(python_install_dir)
168
-
169
- # use a python executable specified via the above symlink
170
- symlink_python_executable = symlink_python_install / "bin" / "python"
171
- # create a new venv
172
- subprocess.check_call([symlink_python_executable, "-m", "venv", venv_path, "--copies"])
173
- # check that a builtin module, like ast, is indeed identified to be in the non-resolved install path
174
- # since this is the source of bugs that we want to assert we don't run into!
175
- ast_path = subprocess.check_output(
176
- [venv_path / "bin" / "python", "-c", "import ast; print(ast.__file__);"], encoding="utf8"
177
- )
178
- assert ast_path != Path(ast_path).resolve()
179
-
180
- # install modal from current dir
181
- subprocess.check_call([venv_path / "bin" / "pip", "install", repo_root])
182
- yield venv_path
183
-
184
-
185
- @skip_windows("venvs behave differently on Windows.")
186
- def test_mounted_files_symlinked_python_install(
187
- symlinked_python_installation_venv_path, supports_dir, server_url_env, servicer
188
- ):
189
- subprocess.check_call(
190
- [symlinked_python_installation_venv_path / "bin" / "modal", "run", supports_dir / "imports_ast.py"]
191
- )
192
- assert "/root/ast.py" not in servicer.files_name2sha
193
-
194
-
195
- def test_mounted_files_config(servicer, supports_dir, env_mount_files, server_url_env):
196
- p = subprocess.run(
197
- ["modal", "run", "pkg_a/script.py"], cwd=supports_dir, env={**os.environ, "MODAL_AUTOMOUNT": "0"}
198
- )
199
- assert p.returncode == 0
200
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
201
- assert files == {
202
- "/root/script.py",
203
- }
204
-
205
-
206
- def test_e2e_modal_run_py_file_mounts(servicer, supports_dir):
207
- helpers.deploy_stub_externally(servicer, "hello.py", cwd=supports_dir)
208
- # Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
209
- # assert len(servicer.files_name2sha) == 1
210
- # assert servicer.n_mounts == 1 # there should be a single mount
211
- # assert servicer.n_mount_files == 1
212
- assert "/root/hello.py" in servicer.files_name2sha
213
-
214
-
215
- def test_e2e_modal_run_py_module_mounts(servicer, supports_dir):
216
- helpers.deploy_stub_externally(servicer, "hello", cwd=supports_dir)
217
- # Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
218
- # assert len(servicer.files_name2sha) == 1
219
- # assert servicer.n_mounts == 1 # there should be a single mount
220
- # assert servicer.n_mount_files == 1
221
- assert "/root/hello.py" in servicer.files_name2sha
222
-
223
-
224
- def foo():
225
- pass
226
-
227
-
228
- def test_mounts_are_not_traversed_on_declaration(test_dir, monkeypatch, client, server_url_env):
229
- return_values = []
230
- original = modal.mount._MountDir.get_files_to_upload
231
-
232
- def mock_get_files_to_upload(self):
233
- r = list(original(self))
234
- return_values.append(r)
235
- return r
236
-
237
- monkeypatch.setattr("modal.mount._MountDir.get_files_to_upload", mock_get_files_to_upload)
238
- stub = modal.Stub()
239
- mount_with_many_files = Mount.from_local_dir(test_dir, remote_path="/test")
240
- stub.function(mounts=[mount_with_many_files])(foo)
241
- assert len(return_values) == 0 # ensure we don't look at the files yet
242
-
243
- with stub.run(client=client):
244
- pass
245
-
246
- assert return_values # at this point we should have gotten all the mount files
247
- # flatten inspected files
248
- files = set()
249
- for r in return_values:
250
- for fn, _ in r:
251
- files.add(fn)
252
- # sanity check - this test file should be included since we mounted the test dir
253
- assert __file__ in files # this test file should have been included
254
-
255
-
256
- def test_mount_dedupe(servicer, test_dir, server_url_env):
257
- supports_dir = test_dir / "supports"
258
- normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
259
- normally_not_included_file.touch(exist_ok=True)
260
- print(
261
- helpers.deploy_stub_externally(
262
- # no explicit mounts, rely on auto-mounting
263
- servicer,
264
- "mount_dedupe.py",
265
- cwd=test_dir / "supports",
266
- env={"USE_EXPLICIT": "0"},
267
- )
268
- )
269
- assert servicer.n_mounts == 2
270
- assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
271
- pkg_a_mount = servicer.mount_contents["mo-2"]
272
- for fn in pkg_a_mount.keys():
273
- assert fn.startswith("/root/pkg_a")
274
- assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
275
-
276
-
277
- def test_mount_dedupe_explicit(servicer, test_dir, server_url_env):
278
- supports_dir = test_dir / "supports"
279
- normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
280
- normally_not_included_file.touch(exist_ok=True)
281
- print(
282
- helpers.deploy_stub_externally(
283
- # two explicit mounts of the same package
284
- servicer,
285
- "mount_dedupe.py",
286
- cwd=supports_dir,
287
- env={"USE_EXPLICIT": "1"},
288
- )
289
- )
290
- assert servicer.n_mounts == 3
291
- assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
292
- pkg_a_mount = servicer.mount_contents["mo-2"]
293
- for fn in pkg_a_mount.keys():
294
- assert fn.startswith("/root/pkg_a")
295
- assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
296
-
297
- custom_pkg_a_mount = servicer.mount_contents["mo-3"]
298
- assert len(custom_pkg_a_mount) == len(pkg_a_mount) + 1
299
- assert "/root/pkg_a/normally_not_included.pyc" in custom_pkg_a_mount.keys()
300
-
301
-
302
- @skip_windows("pip-installed pdm seems somewhat broken on windows")
303
- def test_pdm_cache_automount_exclude(tmp_path, monkeypatch, supports_dir, servicer, server_url_env):
304
- # check that `pdm`'s cached packages are not included in automounts
305
- project_dir = Path(__file__).parent.parent
306
- monkeypatch.chdir(tmp_path)
307
- subprocess.run(["pdm", "init", "-n"], check=True)
308
- subprocess.run(
309
- ["pdm", "add", "--dev", project_dir], check=True
310
- ) # install workdir modal into venv, not using cache...
311
- subprocess.run(["pdm", "config", "--local", "install.cache", "on"], check=True)
312
- subprocess.run(["pdm", "add", "six"], check=True) # single file module
313
- subprocess.run(
314
- ["pdm", "run", "modal", "deploy", supports_dir / "imports_six.py"], check=True
315
- ) # deploy a basically empty function
316
-
317
- files = set(servicer.files_name2sha.keys())
318
- assert files == {
319
- "/root/imports_six.py",
320
- }
321
-
322
-
323
- def test_mount_directory_with_symlinked_file(path_with_symlinked_files, servicer, server_url_env):
324
- path, files = path_with_symlinked_files
325
- mount = Mount.from_local_dir(path)
326
- mount._deploy("mo-1")
327
- pkg_a_mount = servicer.mount_contents["mo-1"]
328
- for src_f in files:
329
- assert any(mnt_f.endswith(src_f.name) for mnt_f in pkg_a_mount)
@@ -1,181 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
- import time
4
- from unittest import mock
5
-
6
- import modal
7
- from modal.exception import DeprecationError, InvalidError, NotFoundError
8
- from modal.runner import deploy_stub
9
-
10
-
11
- def dummy():
12
- pass
13
-
14
-
15
- def test_network_file_system_files(client, test_dir, servicer):
16
- stub = modal.Stub()
17
- nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
18
-
19
- dummy_modal = stub.function(network_file_systems={"/root/foo": nfs})(dummy)
20
-
21
- with stub.run(client=client):
22
- dummy_modal.remote()
23
-
24
-
25
- def test_network_file_system_bad_paths():
26
- stub = modal.Stub()
27
- nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
28
-
29
- def _f():
30
- pass
31
-
32
- with pytest.raises(InvalidError):
33
- stub.function(network_file_systems={"/root/../../foo": nfs})(dummy)
34
-
35
- with pytest.raises(InvalidError):
36
- stub.function(network_file_systems={"/": nfs})(dummy)
37
-
38
- with pytest.raises(InvalidError):
39
- stub.function(network_file_systems={"/tmp/": nfs})(dummy)
40
-
41
-
42
- def test_network_file_system_handle_single_file(client, tmp_path, servicer):
43
- local_file_path = tmp_path / "some_file"
44
- local_file_path.write_text("hello world")
45
-
46
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
47
- nfs.add_local_file(local_file_path)
48
- nfs.add_local_file(local_file_path.as_posix(), remote_path="/foo/other_destination")
49
- object_id = nfs.object_id
50
-
51
- assert servicer.nfs_files[object_id].keys() == {
52
- "/some_file",
53
- "/foo/other_destination",
54
- }
55
- assert servicer.nfs_files[object_id]["/some_file"].data == b"hello world"
56
- assert servicer.nfs_files[object_id]["/foo/other_destination"].data == b"hello world"
57
-
58
-
59
- @pytest.mark.asyncio
60
- async def test_network_file_system_handle_dir(client, tmp_path, servicer):
61
- local_dir = tmp_path / "some_dir"
62
- local_dir.mkdir()
63
- (local_dir / "smol").write_text("###")
64
-
65
- subdir = local_dir / "subdir"
66
- subdir.mkdir()
67
- (subdir / "other").write_text("####")
68
-
69
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
70
- nfs.add_local_dir(local_dir)
71
- object_id = nfs.object_id
72
-
73
- assert servicer.nfs_files[object_id].keys() == {
74
- "/some_dir/smol",
75
- "/some_dir/subdir/other",
76
- }
77
- assert servicer.nfs_files[object_id]["/some_dir/smol"].data == b"###"
78
- assert servicer.nfs_files[object_id]["/some_dir/subdir/other"].data == b"####"
79
-
80
-
81
- @pytest.mark.asyncio
82
- async def test_network_file_system_handle_big_file(client, tmp_path, servicer, blob_server, *args):
83
- with mock.patch("modal.network_file_system.LARGE_FILE_LIMIT", 10):
84
- local_file_path = tmp_path / "bigfile"
85
- local_file_path.write_text("hello world, this is a lot of text")
86
-
87
- async with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
88
- await nfs.add_local_file.aio(local_file_path)
89
- object_id = nfs.object_id
90
-
91
- assert servicer.nfs_files[object_id].keys() == {"/bigfile"}
92
- assert servicer.nfs_files[object_id]["/bigfile"].data == b""
93
- assert servicer.nfs_files[object_id]["/bigfile"].data_blob_id == "bl-1"
94
-
95
- _, blobs = blob_server
96
- assert blobs["bl-1"] == b"hello world, this is a lot of text"
97
-
98
-
99
- def test_old_syntax(client, servicer):
100
- stub = modal.Stub()
101
- with pytest.raises(DeprecationError):
102
- stub.vol1 = modal.SharedVolume() # type: ignore # This is just a post-deprecation husk
103
- with pytest.raises(DeprecationError):
104
- stub.vol2 = modal.SharedVolume.new()
105
-
106
-
107
- def test_redeploy(servicer, client):
108
- stub = modal.Stub()
109
- with pytest.warns(DeprecationError):
110
- n1 = modal.NetworkFileSystem.new()
111
- n2 = modal.NetworkFileSystem.new()
112
- n3 = modal.NetworkFileSystem.new()
113
- stub.n1, stub.n2, stub.n3 = n1, n2, n3
114
-
115
- # Deploy app once
116
- deploy_stub(stub, "my-app", client=client)
117
- app1_ids = [n1.object_id, n2.object_id, n3.object_id]
118
-
119
- # Deploy app again
120
- deploy_stub(stub, "my-app", client=client)
121
- app2_ids = [n1.object_id, n2.object_id, n3.object_id]
122
-
123
- # Make sure ids are stable
124
- assert app1_ids == app2_ids
125
-
126
- # Make sure ids are unique
127
- assert len(set(app1_ids)) == 3
128
- assert len(set(app2_ids)) == 3
129
-
130
- # Deploy to a different app
131
- deploy_stub(stub, "my-other-app", client=client)
132
- app3_ids = [n1.object_id, n2.object_id, n3.object_id]
133
-
134
- # Should be unique and different
135
- assert len(set(app3_ids)) == 3
136
- assert set(app1_ids) & set(app3_ids) == set()
137
-
138
-
139
- def test_read_file(client, tmp_path, servicer):
140
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
141
- with pytest.raises(FileNotFoundError):
142
- for _ in nfs.read_file("idontexist.txt"):
143
- ...
144
-
145
-
146
- def test_write_file(client, tmp_path, servicer):
147
- local_file_path = tmp_path / "some_file"
148
- local_file_path.write_text("hello world")
149
-
150
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
151
- nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
152
-
153
- # Make sure we can write through the provider too
154
- nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
155
-
156
-
157
- def test_persisted(servicer, client):
158
- # Lookup should fail since it doesn't exist
159
- with pytest.raises(NotFoundError):
160
- modal.NetworkFileSystem.lookup("xyz", client=client)
161
-
162
- # Create it
163
- modal.NetworkFileSystem.lookup("xyz", create_if_missing=True, client=client)
164
-
165
- # Lookup should succeed now
166
- modal.NetworkFileSystem.lookup("xyz", client=client)
167
-
168
-
169
- def test_nfs_ephemeral(servicer, client, tmp_path):
170
- local_file_path = tmp_path / "some_file"
171
- local_file_path.write_text("hello world")
172
-
173
- assert servicer.n_nfs_heartbeats == 0
174
- with modal.NetworkFileSystem.ephemeral(client=client, _heartbeat_sleep=1) as nfs:
175
- assert nfs.listdir("/") == []
176
- nfs.write_file("xyz.txt", open(local_file_path, "rb"))
177
- (entry,) = nfs.listdir("/")
178
- assert entry.path == "xyz.txt"
179
-
180
- time.sleep(1.5) # Make time for 2 heartbeats
181
- assert servicer.n_nfs_heartbeats == 2
test/notebook_test.py DELETED
@@ -1,66 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
- import warnings
4
- from pathlib import Path
5
-
6
- import jupytext
7
-
8
- try:
9
- from nbclient.exceptions import CellExecutionError
10
- except ModuleNotFoundError:
11
- # TODO(erikbern): sometimes my local jupyter packages end up in a bad state,
12
- # but we don't want that to cause pytest to fail on startup.
13
- warnings.warn("failed importing nbclient")
14
-
15
-
16
- @pytest.fixture
17
- def notebook_runner(servicer):
18
- import nbformat
19
- from nbclient import NotebookClient
20
-
21
- def runner(notebook_path: Path):
22
- output_notebook_path = notebook_path.with_suffix(".output.ipynb")
23
-
24
- nb = jupytext.read(
25
- notebook_path,
26
- )
27
-
28
- parameter_cell = nb["cells"][0]
29
- assert "parameters" in parameter_cell["metadata"]["tags"] # like in papermill
30
- parameter_cell["source"] = f'server_addr = "{servicer.remote_addr}"'
31
-
32
- client = NotebookClient(nb)
33
-
34
- try:
35
- client.execute()
36
- except CellExecutionError:
37
- nbformat.write(nb, output_notebook_path)
38
- pytest.fail(
39
- f"""There was an error when executing the notebook.
40
-
41
- Inspect the output notebook: {output_notebook_path}
42
- """
43
- )
44
- tagged_cells = {}
45
- for cell in nb["cells"]:
46
- for tag in cell["metadata"].get("tags", []):
47
- tagged_cells[tag] = cell
48
-
49
- return tagged_cells
50
-
51
- return runner
52
-
53
-
54
- # for some reason this import is failing due to a circular import of IPython.terminal.embed
55
- # but only when running in CI (sometimes?), causing these tests to fail:
56
- # from IPython.terminal import interactiveshell
57
-
58
-
59
- @pytest.mark.skip("temporarily disabled until IPython import issues in CI are resolved")
60
- def test_notebook_outputs_status(notebook_runner, test_dir):
61
- input_notebook_path = test_dir / "supports" / "notebooks" / "simple.notebook.py"
62
- tagged_cells = notebook_runner(input_notebook_path)
63
- combined_output = "\n".join(c["data"]["text/plain"] for c in tagged_cells["main"]["outputs"])
64
- assert "Initialized" in combined_output
65
- assert "Created objects." in combined_output
66
- assert "App completed." in combined_output
test/object_test.py DELETED
@@ -1,41 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
-
4
- from modal import Queue, Secret, Stub
5
- from modal.exception import DeprecationError, InvalidError
6
-
7
-
8
- @pytest.mark.asyncio
9
- async def test_async_factory(client):
10
- stub = Stub()
11
- with pytest.warns(DeprecationError):
12
- stub.my_factory = Queue.new()
13
- async with stub.run(client=client):
14
- assert isinstance(stub.my_factory, Queue)
15
- assert stub.my_factory.object_id == "qu-1"
16
-
17
-
18
- def test_new_hydrated(client):
19
- from modal.dict import _Dict
20
- from modal.object import _Object
21
- from modal.queue import _Queue
22
-
23
- assert isinstance(_Dict._new_hydrated("di-123", client, None), _Dict)
24
- assert isinstance(_Queue._new_hydrated("qu-123", client, None), _Queue)
25
-
26
- with pytest.raises(InvalidError):
27
- _Queue._new_hydrated("di-123", client, None) # Wrong prefix for type
28
-
29
- assert isinstance(_Object._new_hydrated("qu-123", client, None), _Queue)
30
- assert isinstance(_Object._new_hydrated("di-123", client, None), _Dict)
31
-
32
- with pytest.raises(InvalidError):
33
- _Object._new_hydrated("xy-123", client, None)
34
-
35
-
36
- def test_constructor():
37
- with pytest.raises(InvalidError) as excinfo:
38
- Secret({"foo": 123})
39
-
40
- assert "Secret" in str(excinfo.value)
41
- assert "constructor" in str(excinfo.value)
@@ -1,25 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import platform
3
- import pytest
4
-
5
- from modal._utils.package_utils import get_module_mount_info
6
- from modal.exception import ModuleNotMountable
7
-
8
-
9
- def test_get_module_mount_info():
10
- res = get_module_mount_info("modal")
11
- assert len(res) == 1
12
- assert res[0][0] == True
13
-
14
- res = get_module_mount_info("asyncio")
15
- assert len(res) == 1
16
- assert res[0][0] == True
17
-
18
- res = get_module_mount_info("six")
19
- assert len(res) == 1
20
- assert res[0][0] == False
21
-
22
- if platform.system() != "Windows":
23
- # TODO This assertion fails on windows; I assume that compiled file formats are different there?
24
- with pytest.raises(ModuleNotMountable, match="aiohttp can't be mounted because it contains binary file"):
25
- get_module_mount_info("aiohttp")