modal 0.62.115__py3-none-any.whl → 0.72.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. modal/__init__.py +13 -9
  2. modal/__main__.py +41 -3
  3. modal/_clustered_functions.py +80 -0
  4. modal/_clustered_functions.pyi +22 -0
  5. modal/_container_entrypoint.py +402 -398
  6. modal/_ipython.py +3 -13
  7. modal/_location.py +17 -10
  8. modal/_output.py +243 -99
  9. modal/_pty.py +2 -2
  10. modal/_resolver.py +55 -60
  11. modal/_resources.py +26 -7
  12. modal/_runtime/__init__.py +1 -0
  13. modal/_runtime/asgi.py +519 -0
  14. modal/_runtime/container_io_manager.py +1025 -0
  15. modal/{execution_context.py → _runtime/execution_context.py} +11 -2
  16. modal/_runtime/telemetry.py +169 -0
  17. modal/_runtime/user_code_imports.py +356 -0
  18. modal/_serialization.py +123 -6
  19. modal/_traceback.py +47 -187
  20. modal/_tunnel.py +50 -14
  21. modal/_tunnel.pyi +19 -36
  22. modal/_utils/app_utils.py +3 -17
  23. modal/_utils/async_utils.py +386 -104
  24. modal/_utils/blob_utils.py +157 -186
  25. modal/_utils/bytes_io_segment_payload.py +97 -0
  26. modal/_utils/deprecation.py +89 -0
  27. modal/_utils/docker_utils.py +98 -0
  28. modal/_utils/function_utils.py +299 -98
  29. modal/_utils/grpc_testing.py +47 -34
  30. modal/_utils/grpc_utils.py +54 -21
  31. modal/_utils/hash_utils.py +51 -10
  32. modal/_utils/http_utils.py +39 -9
  33. modal/_utils/logger.py +2 -1
  34. modal/_utils/mount_utils.py +34 -16
  35. modal/_utils/name_utils.py +58 -0
  36. modal/_utils/package_utils.py +14 -1
  37. modal/_utils/pattern_utils.py +205 -0
  38. modal/_utils/rand_pb_testing.py +3 -3
  39. modal/_utils/shell_utils.py +15 -49
  40. modal/_vendor/a2wsgi_wsgi.py +62 -72
  41. modal/_vendor/cloudpickle.py +1 -1
  42. modal/_watcher.py +12 -10
  43. modal/app.py +561 -323
  44. modal/app.pyi +474 -262
  45. modal/call_graph.py +7 -6
  46. modal/cli/_download.py +22 -6
  47. modal/cli/_traceback.py +200 -0
  48. modal/cli/app.py +203 -42
  49. modal/cli/config.py +12 -5
  50. modal/cli/container.py +61 -13
  51. modal/cli/dict.py +128 -0
  52. modal/cli/entry_point.py +26 -13
  53. modal/cli/environment.py +40 -9
  54. modal/cli/import_refs.py +21 -48
  55. modal/cli/launch.py +28 -14
  56. modal/cli/network_file_system.py +57 -21
  57. modal/cli/profile.py +1 -1
  58. modal/cli/programs/run_jupyter.py +34 -9
  59. modal/cli/programs/vscode.py +58 -8
  60. modal/cli/queues.py +131 -0
  61. modal/cli/run.py +199 -96
  62. modal/cli/secret.py +5 -4
  63. modal/cli/token.py +7 -2
  64. modal/cli/utils.py +74 -8
  65. modal/cli/volume.py +97 -56
  66. modal/client.py +248 -144
  67. modal/client.pyi +156 -124
  68. modal/cloud_bucket_mount.py +43 -30
  69. modal/cloud_bucket_mount.pyi +32 -25
  70. modal/cls.py +528 -141
  71. modal/cls.pyi +189 -145
  72. modal/config.py +32 -15
  73. modal/container_process.py +177 -0
  74. modal/container_process.pyi +82 -0
  75. modal/dict.py +50 -54
  76. modal/dict.pyi +120 -164
  77. modal/environments.py +106 -5
  78. modal/environments.pyi +77 -25
  79. modal/exception.py +30 -43
  80. modal/experimental.py +62 -2
  81. modal/file_io.py +537 -0
  82. modal/file_io.pyi +235 -0
  83. modal/file_pattern_matcher.py +196 -0
  84. modal/functions.py +846 -428
  85. modal/functions.pyi +446 -387
  86. modal/gpu.py +57 -44
  87. modal/image.py +943 -417
  88. modal/image.pyi +584 -245
  89. modal/io_streams.py +434 -0
  90. modal/io_streams.pyi +122 -0
  91. modal/mount.py +223 -90
  92. modal/mount.pyi +241 -243
  93. modal/network_file_system.py +85 -86
  94. modal/network_file_system.pyi +151 -110
  95. modal/object.py +66 -36
  96. modal/object.pyi +166 -143
  97. modal/output.py +63 -0
  98. modal/parallel_map.py +73 -47
  99. modal/parallel_map.pyi +51 -63
  100. modal/partial_function.py +272 -107
  101. modal/partial_function.pyi +219 -120
  102. modal/proxy.py +15 -12
  103. modal/proxy.pyi +3 -8
  104. modal/queue.py +96 -72
  105. modal/queue.pyi +210 -135
  106. modal/requirements/2024.04.txt +2 -1
  107. modal/requirements/2024.10.txt +16 -0
  108. modal/requirements/README.md +21 -0
  109. modal/requirements/base-images.json +22 -0
  110. modal/retries.py +45 -4
  111. modal/runner.py +325 -203
  112. modal/runner.pyi +124 -110
  113. modal/running_app.py +27 -4
  114. modal/sandbox.py +509 -231
  115. modal/sandbox.pyi +396 -169
  116. modal/schedule.py +2 -2
  117. modal/scheduler_placement.py +20 -3
  118. modal/secret.py +41 -25
  119. modal/secret.pyi +62 -42
  120. modal/serving.py +39 -49
  121. modal/serving.pyi +37 -43
  122. modal/stream_type.py +15 -0
  123. modal/token_flow.py +5 -3
  124. modal/token_flow.pyi +37 -32
  125. modal/volume.py +123 -137
  126. modal/volume.pyi +228 -221
  127. {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/METADATA +5 -5
  128. modal-0.72.13.dist-info/RECORD +174 -0
  129. {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/top_level.txt +0 -1
  130. modal_docs/gen_reference_docs.py +3 -1
  131. modal_docs/mdmd/mdmd.py +0 -1
  132. modal_docs/mdmd/signatures.py +1 -2
  133. modal_global_objects/images/base_images.py +28 -0
  134. modal_global_objects/mounts/python_standalone.py +2 -2
  135. modal_proto/__init__.py +1 -1
  136. modal_proto/api.proto +1231 -531
  137. modal_proto/api_grpc.py +750 -430
  138. modal_proto/api_pb2.py +2102 -1176
  139. modal_proto/api_pb2.pyi +8859 -0
  140. modal_proto/api_pb2_grpc.py +1329 -675
  141. modal_proto/api_pb2_grpc.pyi +1416 -0
  142. modal_proto/modal_api_grpc.py +149 -0
  143. modal_proto/modal_options_grpc.py +3 -0
  144. modal_proto/options_pb2.pyi +20 -0
  145. modal_proto/options_pb2_grpc.pyi +7 -0
  146. modal_proto/py.typed +0 -0
  147. modal_version/__init__.py +1 -1
  148. modal_version/_version_generated.py +2 -2
  149. modal/_asgi.py +0 -370
  150. modal/_container_exec.py +0 -128
  151. modal/_container_io_manager.py +0 -646
  152. modal/_container_io_manager.pyi +0 -412
  153. modal/_sandbox_shell.py +0 -49
  154. modal/app_utils.py +0 -20
  155. modal/app_utils.pyi +0 -17
  156. modal/execution_context.pyi +0 -37
  157. modal/shared_volume.py +0 -23
  158. modal/shared_volume.pyi +0 -24
  159. modal-0.62.115.dist-info/RECORD +0 -207
  160. modal_global_objects/images/conda.py +0 -15
  161. modal_global_objects/images/debian_slim.py +0 -15
  162. modal_global_objects/images/micromamba.py +0 -15
  163. test/__init__.py +0 -1
  164. test/aio_test.py +0 -12
  165. test/async_utils_test.py +0 -279
  166. test/blob_test.py +0 -67
  167. test/cli_imports_test.py +0 -149
  168. test/cli_test.py +0 -674
  169. test/client_test.py +0 -203
  170. test/cloud_bucket_mount_test.py +0 -22
  171. test/cls_test.py +0 -636
  172. test/config_test.py +0 -149
  173. test/conftest.py +0 -1485
  174. test/container_app_test.py +0 -50
  175. test/container_test.py +0 -1405
  176. test/cpu_test.py +0 -23
  177. test/decorator_test.py +0 -85
  178. test/deprecation_test.py +0 -34
  179. test/dict_test.py +0 -51
  180. test/e2e_test.py +0 -68
  181. test/error_test.py +0 -7
  182. test/function_serialization_test.py +0 -32
  183. test/function_test.py +0 -791
  184. test/function_utils_test.py +0 -101
  185. test/gpu_test.py +0 -159
  186. test/grpc_utils_test.py +0 -82
  187. test/helpers.py +0 -47
  188. test/image_test.py +0 -814
  189. test/live_reload_test.py +0 -80
  190. test/lookup_test.py +0 -70
  191. test/mdmd_test.py +0 -329
  192. test/mount_test.py +0 -162
  193. test/mounted_files_test.py +0 -327
  194. test/network_file_system_test.py +0 -188
  195. test/notebook_test.py +0 -66
  196. test/object_test.py +0 -41
  197. test/package_utils_test.py +0 -25
  198. test/queue_test.py +0 -115
  199. test/resolver_test.py +0 -59
  200. test/retries_test.py +0 -67
  201. test/runner_test.py +0 -85
  202. test/sandbox_test.py +0 -191
  203. test/schedule_test.py +0 -15
  204. test/scheduler_placement_test.py +0 -57
  205. test/secret_test.py +0 -89
  206. test/serialization_test.py +0 -50
  207. test/stub_composition_test.py +0 -10
  208. test/stub_test.py +0 -361
  209. test/test_asgi_wrapper.py +0 -234
  210. test/token_flow_test.py +0 -18
  211. test/traceback_test.py +0 -135
  212. test/tunnel_test.py +0 -29
  213. test/utils_test.py +0 -88
  214. test/version_test.py +0 -14
  215. test/volume_test.py +0 -397
  216. test/watcher_test.py +0 -58
  217. test/webhook_test.py +0 -145
  218. {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/LICENSE +0 -0
  219. {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/WHEEL +0 -0
  220. {modal-0.62.115.dist-info → modal-0.72.13.dist-info}/entry_points.txt +0 -0
@@ -1,327 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import os
3
- import pytest
4
- import subprocess
5
- import sys
6
- from pathlib import Path
7
-
8
- import pytest_asyncio
9
-
10
- import modal
11
- from modal import Mount
12
- from modal.mount import get_auto_mounts
13
-
14
- from . import helpers
15
- from .supports.skip import skip_windows
16
-
17
-
18
- @pytest.fixture
19
- def venv_path(tmp_path, repo_root):
20
- venv_path = tmp_path
21
- subprocess.run([sys.executable, "-m", "venv", venv_path, "--copies", "--system-site-packages"], check=True)
22
- # Install Modal and a tiny package in the venv.
23
- subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "-e", repo_root], check=True)
24
- subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "--force-reinstall", "six"], check=True)
25
- yield venv_path
26
-
27
-
28
- @pytest.fixture
29
- def path_with_symlinked_files(tmp_path):
30
- src = tmp_path / "foo.txt"
31
- src.write_text("Hello")
32
- trg = tmp_path / "bar.txt"
33
- trg.symlink_to(src)
34
- return tmp_path, {src, trg}
35
-
36
-
37
- script_path = "pkg_a/script.py"
38
-
39
-
40
- def f():
41
- pass
42
-
43
-
44
- @pytest_asyncio.fixture
45
- async def env_mount_files():
46
- # If something is installed using pip -e, it will be bundled up as a part of the environment.
47
- # Those are env-specific so we ignore those as a part of the test
48
- filenames = []
49
- for mount in get_auto_mounts():
50
- async for file_info in mount._get_files(mount.entries):
51
- filenames.append(file_info.mount_filename)
52
-
53
- return filenames
54
-
55
-
56
- def test_mounted_files_script(servicer, supports_dir, env_mount_files, server_url_env):
57
- helpers.deploy_app_externally(servicer, script_path, cwd=supports_dir)
58
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
59
-
60
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
61
- assert files == {
62
- "/root/a.py",
63
- "/root/b/c.py",
64
- "/root/b/e.py",
65
- "/root/pkg_b/__init__.py",
66
- "/root/pkg_b/f.py",
67
- "/root/pkg_b/g/h.py",
68
- "/root/script.py",
69
- }
70
-
71
-
72
- serialized_fn_path = "pkg_a/serialized_fn.py"
73
-
74
-
75
- def test_mounted_files_serialized(servicer, supports_dir, env_mount_files, server_url_env):
76
- helpers.deploy_app_externally(servicer, serialized_fn_path, cwd=supports_dir)
77
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
78
-
79
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
80
- assert (
81
- files
82
- == {
83
- "/root/serialized_fn.py", # should serialized_fn be included? It's not needed to run the function, but it's loaded into sys.modules at definition time...
84
- "/root/b/c.py", # this is mounted under root since it's imported as `import b` and not `import pkg_a.b` from serialized_fn.py
85
- "/root/b/e.py", # same as above
86
- "/root/a.py", # same as above
87
- "/root/pkg_b/__init__.py",
88
- "/root/pkg_b/f.py",
89
- "/root/pkg_b/g/h.py",
90
- }
91
- )
92
-
93
-
94
- def test_mounted_files_package(supports_dir, env_mount_files, servicer, server_url_env):
95
- p = subprocess.run(["modal", "run", "pkg_a.package"], cwd=supports_dir)
96
- assert p.returncode == 0
97
-
98
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
99
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
100
- assert files == {
101
- "/root/pkg_a/__init__.py",
102
- "/root/pkg_a/a.py",
103
- "/root/pkg_a/b/c.py",
104
- "/root/pkg_a/d.py",
105
- "/root/pkg_a/b/e.py",
106
- "/root/pkg_a/script.py",
107
- "/root/pkg_a/serialized_fn.py",
108
- "/root/pkg_a/package.py",
109
- "/root/pkg_b/__init__.py",
110
- "/root/pkg_b/f.py",
111
- "/root/pkg_b/g/h.py",
112
- }
113
-
114
-
115
- def test_mounted_files_package_no_automount(supports_dir, env_mount_files, servicer, server_url_env):
116
- # when triggered like a module, the target module should be put at the correct package path
117
- p = subprocess.run(
118
- ["modal", "run", "pkg_a.package"],
119
- cwd=supports_dir,
120
- capture_output=True,
121
- env={**os.environ, "MODAL_AUTOMOUNT": "0"},
122
- )
123
- assert p.returncode == 0
124
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
125
- assert files == {
126
- "/root/pkg_a/__init__.py",
127
- "/root/pkg_a/package.py",
128
- }
129
-
130
-
131
- @skip_windows("venvs behave differently on Windows.")
132
- def test_mounted_files_sys_prefix(servicer, supports_dir, venv_path, env_mount_files, server_url_env):
133
- # Run with venv activated, so it's on sys.prefix, and modal is dev-installed in the VM
134
- subprocess.run(
135
- [venv_path / "bin" / "modal", "run", script_path],
136
- cwd=supports_dir,
137
- )
138
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
139
- # Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
140
- assert files == {
141
- "/root/a.py",
142
- "/root/b/c.py",
143
- "/root/b/e.py",
144
- "/root/script.py",
145
- "/root/pkg_b/__init__.py",
146
- "/root/pkg_b/f.py",
147
- "/root/pkg_b/g/h.py",
148
- }
149
-
150
-
151
- @pytest.fixture
152
- def symlinked_python_installation_venv_path(tmp_path, repo_root):
153
- # sets up a symlink to the python *installation* (not just the python binary)
154
- # and initialize the virtualenv using a path via that symlink
155
- # This makes the file paths of any stdlib modules use the symlinked path
156
- # instead of the original, which is similar to what some tools do (e.g. mise)
157
- # and has the potential to break automounting behavior, so we keep this
158
- # test as a regression test for that
159
- venv_path = tmp_path / "venv"
160
- actual_executable = Path(sys.executable).resolve()
161
- assert actual_executable.parent.name == "bin"
162
- python_install_dir = actual_executable.parent.parent
163
- # create a symlink to the python install *root*
164
- symlink_python_install = tmp_path / "python-install"
165
- symlink_python_install.symlink_to(python_install_dir)
166
-
167
- # use a python executable specified via the above symlink
168
- symlink_python_executable = symlink_python_install / "bin" / "python"
169
- # create a new venv
170
- subprocess.check_call([symlink_python_executable, "-m", "venv", venv_path, "--copies"])
171
- # check that a builtin module, like ast, is indeed identified to be in the non-resolved install path
172
- # since this is the source of bugs that we want to assert we don't run into!
173
- ast_path = subprocess.check_output(
174
- [venv_path / "bin" / "python", "-c", "import ast; print(ast.__file__);"], encoding="utf8"
175
- )
176
- assert ast_path != Path(ast_path).resolve()
177
-
178
- # install modal from current dir
179
- subprocess.check_call([venv_path / "bin" / "pip", "install", repo_root])
180
- yield venv_path
181
-
182
-
183
- @skip_windows("venvs behave differently on Windows.")
184
- def test_mounted_files_symlinked_python_install(
185
- symlinked_python_installation_venv_path, supports_dir, server_url_env, servicer
186
- ):
187
- subprocess.check_call(
188
- [symlinked_python_installation_venv_path / "bin" / "modal", "run", supports_dir / "imports_ast.py"]
189
- )
190
- assert "/root/ast.py" not in servicer.files_name2sha
191
-
192
-
193
- def test_mounted_files_config(servicer, supports_dir, env_mount_files, server_url_env):
194
- p = subprocess.run(
195
- ["modal", "run", "pkg_a/script.py"], cwd=supports_dir, env={**os.environ, "MODAL_AUTOMOUNT": "0"}
196
- )
197
- assert p.returncode == 0
198
- files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
199
- assert files == {
200
- "/root/script.py",
201
- }
202
-
203
-
204
- def test_e2e_modal_run_py_file_mounts(servicer, supports_dir):
205
- helpers.deploy_app_externally(servicer, "hello.py", cwd=supports_dir)
206
- # Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
207
- # assert len(servicer.files_name2sha) == 1
208
- # assert servicer.n_mounts == 1 # there should be a single mount
209
- # assert servicer.n_mount_files == 1
210
- assert "/root/hello.py" in servicer.files_name2sha
211
-
212
-
213
- def test_e2e_modal_run_py_module_mounts(servicer, supports_dir):
214
- helpers.deploy_app_externally(servicer, "hello", cwd=supports_dir)
215
- # Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
216
- # assert len(servicer.files_name2sha) == 1
217
- # assert servicer.n_mounts == 1 # there should be a single mount
218
- # assert servicer.n_mount_files == 1
219
- assert "/root/hello.py" in servicer.files_name2sha
220
-
221
-
222
- def foo():
223
- pass
224
-
225
-
226
- def test_mounts_are_not_traversed_on_declaration(test_dir, monkeypatch, client, server_url_env):
227
- return_values = []
228
- original = modal.mount._MountDir.get_files_to_upload
229
-
230
- def mock_get_files_to_upload(self):
231
- r = list(original(self))
232
- return_values.append(r)
233
- return r
234
-
235
- monkeypatch.setattr("modal.mount._MountDir.get_files_to_upload", mock_get_files_to_upload)
236
- app = modal.App()
237
- mount_with_many_files = Mount.from_local_dir(test_dir, remote_path="/test")
238
- app.function(mounts=[mount_with_many_files])(foo)
239
- assert len(return_values) == 0 # ensure we don't look at the files yet
240
-
241
- with app.run(client=client):
242
- pass
243
-
244
- assert return_values # at this point we should have gotten all the mount files
245
- # flatten inspected files
246
- files = set()
247
- for r in return_values:
248
- for fn, _ in r:
249
- files.add(fn)
250
- # sanity check - this test file should be included since we mounted the test dir
251
- assert __file__ in files # this test file should have been included
252
-
253
-
254
- def test_mount_dedupe(servicer, test_dir, server_url_env):
255
- supports_dir = test_dir / "supports"
256
- normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
257
- normally_not_included_file.touch(exist_ok=True)
258
- print(
259
- helpers.deploy_app_externally(
260
- # no explicit mounts, rely on auto-mounting
261
- servicer,
262
- "mount_dedupe.py",
263
- cwd=test_dir / "supports",
264
- env={"USE_EXPLICIT": "0"},
265
- )
266
- )
267
- assert servicer.n_mounts == 2
268
- assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
269
- pkg_a_mount = servicer.mount_contents["mo-2"]
270
- for fn in pkg_a_mount.keys():
271
- assert fn.startswith("/root/pkg_a")
272
- assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
273
-
274
-
275
- def test_mount_dedupe_explicit(servicer, test_dir, server_url_env):
276
- supports_dir = test_dir / "supports"
277
- normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
278
- normally_not_included_file.touch(exist_ok=True)
279
- print(
280
- helpers.deploy_app_externally(
281
- # two explicit mounts of the same package
282
- servicer,
283
- "mount_dedupe.py",
284
- cwd=supports_dir,
285
- env={"USE_EXPLICIT": "1"},
286
- )
287
- )
288
- assert servicer.n_mounts == 3
289
- assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
290
- pkg_a_mount = servicer.mount_contents["mo-2"]
291
- for fn in pkg_a_mount.keys():
292
- assert fn.startswith("/root/pkg_a")
293
- assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
294
-
295
- custom_pkg_a_mount = servicer.mount_contents["mo-3"]
296
- assert len(custom_pkg_a_mount) == len(pkg_a_mount) + 1
297
- assert "/root/pkg_a/normally_not_included.pyc" in custom_pkg_a_mount.keys()
298
-
299
-
300
- @skip_windows("pip-installed pdm seems somewhat broken on windows")
301
- def test_pdm_cache_automount_exclude(tmp_path, monkeypatch, supports_dir, servicer, server_url_env):
302
- # check that `pdm`'s cached packages are not included in automounts
303
- project_dir = Path(__file__).parent.parent
304
- monkeypatch.chdir(tmp_path)
305
- subprocess.run(["pdm", "init", "-n"], check=True)
306
- subprocess.run(
307
- ["pdm", "add", "--dev", project_dir], check=True
308
- ) # install workdir modal into venv, not using cache...
309
- subprocess.run(["pdm", "config", "--local", "install.cache", "on"], check=True)
310
- subprocess.run(["pdm", "add", "six"], check=True) # single file module
311
- subprocess.run(
312
- ["pdm", "run", "modal", "deploy", supports_dir / "imports_six.py"], check=True
313
- ) # deploy a basically empty function
314
-
315
- files = set(servicer.files_name2sha.keys())
316
- assert files == {
317
- "/root/imports_six.py",
318
- }
319
-
320
-
321
- def test_mount_directory_with_symlinked_file(path_with_symlinked_files, servicer, server_url_env):
322
- path, files = path_with_symlinked_files
323
- mount = Mount.from_local_dir(path)
324
- mount._deploy("mo-1")
325
- pkg_a_mount = servicer.mount_contents["mo-1"]
326
- for src_f in files:
327
- assert any(mnt_f.endswith(src_f.name) for mnt_f in pkg_a_mount)
@@ -1,188 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
- import time
4
- from io import BytesIO
5
- from unittest import mock
6
-
7
- import modal
8
- from modal.exception import DeprecationError, InvalidError, NotFoundError
9
- from modal.runner import deploy_app
10
-
11
-
12
- def dummy():
13
- pass
14
-
15
-
16
- def test_network_file_system_files(client, test_dir, servicer):
17
- app = modal.App()
18
- nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
19
-
20
- dummy_modal = app.function(network_file_systems={"/root/foo": nfs})(dummy)
21
-
22
- with app.run(client=client):
23
- dummy_modal.remote()
24
-
25
-
26
- def test_network_file_system_bad_paths():
27
- app = modal.App()
28
- nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
29
-
30
- def _f():
31
- pass
32
-
33
- with pytest.raises(InvalidError):
34
- app.function(network_file_systems={"/root/../../foo": nfs})(dummy)
35
-
36
- with pytest.raises(InvalidError):
37
- app.function(network_file_systems={"/": nfs})(dummy)
38
-
39
- with pytest.raises(InvalidError):
40
- app.function(network_file_systems={"/tmp/": nfs})(dummy)
41
-
42
-
43
- def test_network_file_system_handle_single_file(client, tmp_path, servicer):
44
- local_file_path = tmp_path / "some_file"
45
- local_file_path.write_text("hello world")
46
-
47
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
48
- nfs.add_local_file(local_file_path)
49
- nfs.add_local_file(local_file_path.as_posix(), remote_path="/foo/other_destination")
50
- object_id = nfs.object_id
51
-
52
- assert servicer.nfs_files[object_id].keys() == {
53
- "/some_file",
54
- "/foo/other_destination",
55
- }
56
- assert servicer.nfs_files[object_id]["/some_file"].data == b"hello world"
57
- assert servicer.nfs_files[object_id]["/foo/other_destination"].data == b"hello world"
58
-
59
-
60
- @pytest.mark.asyncio
61
- async def test_network_file_system_handle_dir(client, tmp_path, servicer):
62
- local_dir = tmp_path / "some_dir"
63
- local_dir.mkdir()
64
- (local_dir / "smol").write_text("###")
65
-
66
- subdir = local_dir / "subdir"
67
- subdir.mkdir()
68
- (subdir / "other").write_text("####")
69
-
70
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
71
- nfs.add_local_dir(local_dir)
72
- object_id = nfs.object_id
73
-
74
- assert servicer.nfs_files[object_id].keys() == {
75
- "/some_dir/smol",
76
- "/some_dir/subdir/other",
77
- }
78
- assert servicer.nfs_files[object_id]["/some_dir/smol"].data == b"###"
79
- assert servicer.nfs_files[object_id]["/some_dir/subdir/other"].data == b"####"
80
-
81
-
82
- @pytest.mark.asyncio
83
- async def test_network_file_system_handle_big_file(client, tmp_path, servicer, blob_server, *args):
84
- with mock.patch("modal.network_file_system.LARGE_FILE_LIMIT", 10):
85
- local_file_path = tmp_path / "bigfile"
86
- local_file_path.write_text("hello world, this is a lot of text")
87
-
88
- async with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
89
- await nfs.add_local_file.aio(local_file_path)
90
- object_id = nfs.object_id
91
-
92
- assert servicer.nfs_files[object_id].keys() == {"/bigfile"}
93
- assert servicer.nfs_files[object_id]["/bigfile"].data == b""
94
- assert servicer.nfs_files[object_id]["/bigfile"].data_blob_id == "bl-1"
95
-
96
- _, blobs = blob_server
97
- assert blobs["bl-1"] == b"hello world, this is a lot of text"
98
-
99
-
100
- def test_old_syntax(client, servicer):
101
- app = modal.App()
102
- with pytest.raises(DeprecationError):
103
- app.vol1 = modal.SharedVolume() # type: ignore # This is just a post-deprecation husk
104
- with pytest.raises(DeprecationError):
105
- app.vol2 = modal.SharedVolume.new()
106
-
107
-
108
- def test_redeploy(servicer, client):
109
- app = modal.App()
110
- with pytest.warns(DeprecationError):
111
- n1 = modal.NetworkFileSystem.new()
112
- n2 = modal.NetworkFileSystem.new()
113
- n3 = modal.NetworkFileSystem.new()
114
- app.n1, app.n2, app.n3 = n1, n2, n3
115
-
116
- # Deploy app once
117
- deploy_app(app, "my-app", client=client)
118
- app1_ids = [n1.object_id, n2.object_id, n3.object_id]
119
-
120
- # Deploy app again
121
- deploy_app(app, "my-app", client=client)
122
- app2_ids = [n1.object_id, n2.object_id, n3.object_id]
123
-
124
- # Make sure ids are stable
125
- assert app1_ids == app2_ids
126
-
127
- # Make sure ids are unique
128
- assert len(set(app1_ids)) == 3
129
- assert len(set(app2_ids)) == 3
130
-
131
- # Deploy to a different app
132
- deploy_app(app, "my-other-app", client=client)
133
- app3_ids = [n1.object_id, n2.object_id, n3.object_id]
134
-
135
- # Should be unique and different
136
- assert len(set(app3_ids)) == 3
137
- assert set(app1_ids) & set(app3_ids) == set()
138
-
139
-
140
- def test_read_file(client, tmp_path, servicer):
141
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
142
- with pytest.raises(FileNotFoundError):
143
- for _ in nfs.read_file("idontexist.txt"):
144
- ...
145
-
146
-
147
- def test_write_file(client, tmp_path, servicer):
148
- local_file_path = tmp_path / "some_file"
149
- local_file_path.write_text("hello world")
150
-
151
- with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
152
- nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
153
-
154
- # Make sure we can write through the provider too
155
- nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
156
-
157
-
158
- def test_persisted(servicer, client):
159
- # Lookup should fail since it doesn't exist
160
- with pytest.raises(NotFoundError):
161
- modal.NetworkFileSystem.lookup("xyz", client=client)
162
-
163
- # Create it
164
- modal.NetworkFileSystem.lookup("xyz", create_if_missing=True, client=client)
165
-
166
- # Lookup should succeed now
167
- modal.NetworkFileSystem.lookup("xyz", client=client)
168
-
169
-
170
- def test_nfs_ephemeral(servicer, client, tmp_path):
171
- local_file_path = tmp_path / "some_file"
172
- local_file_path.write_text("hello world")
173
-
174
- assert servicer.n_nfs_heartbeats == 0
175
- with modal.NetworkFileSystem.ephemeral(client=client, _heartbeat_sleep=1) as nfs:
176
- assert nfs.listdir("/") == []
177
- nfs.write_file("xyz.txt", open(local_file_path, "rb"))
178
- (entry,) = nfs.listdir("/")
179
- assert entry.path == "xyz.txt"
180
-
181
- time.sleep(1.5) # Make time for 2 heartbeats
182
- assert servicer.n_nfs_heartbeats == 2
183
-
184
-
185
- def test_nfs_lazy_hydration_from_name(set_env_client):
186
- nfs = modal.NetworkFileSystem.from_name("nfs", create_if_missing=True)
187
- bio = BytesIO(b"content")
188
- nfs.write_file("blah", bio)
test/notebook_test.py DELETED
@@ -1,66 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
- import warnings
4
- from pathlib import Path
5
-
6
- import jupytext
7
-
8
- try:
9
- from nbclient.exceptions import CellExecutionError
10
- except ModuleNotFoundError:
11
- # TODO(erikbern): sometimes my local jupyter packages end up in a bad state,
12
- # but we don't want that to cause pytest to fail on startup.
13
- warnings.warn("failed importing nbclient")
14
-
15
-
16
- @pytest.fixture
17
- def notebook_runner(servicer):
18
- import nbformat
19
- from nbclient import NotebookClient
20
-
21
- def runner(notebook_path: Path):
22
- output_notebook_path = notebook_path.with_suffix(".output.ipynb")
23
-
24
- nb = jupytext.read(
25
- notebook_path,
26
- )
27
-
28
- parameter_cell = nb["cells"][0]
29
- assert "parameters" in parameter_cell["metadata"]["tags"] # like in papermill
30
- parameter_cell["source"] = f'server_addr = "{servicer.remote_addr}"'
31
-
32
- client = NotebookClient(nb)
33
-
34
- try:
35
- client.execute()
36
- except CellExecutionError:
37
- nbformat.write(nb, output_notebook_path)
38
- pytest.fail(
39
- f"""There was an error when executing the notebook.
40
-
41
- Inspect the output notebook: {output_notebook_path}
42
- """
43
- )
44
- tagged_cells = {}
45
- for cell in nb["cells"]:
46
- for tag in cell["metadata"].get("tags", []):
47
- tagged_cells[tag] = cell
48
-
49
- return tagged_cells
50
-
51
- return runner
52
-
53
-
54
- # for some reason this import is failing due to a circular import of IPython.terminal.embed
55
- # but only when running in CI (sometimes?), causing these tests to fail:
56
- # from IPython.terminal import interactiveshell
57
-
58
-
59
- @pytest.mark.skip("temporarily disabled until IPython import issues in CI are resolved")
60
- def test_notebook_outputs_status(notebook_runner, test_dir):
61
- input_notebook_path = test_dir / "supports" / "notebooks" / "simple.notebook.py"
62
- tagged_cells = notebook_runner(input_notebook_path)
63
- combined_output = "\n".join(c["data"]["text/plain"] for c in tagged_cells["main"]["outputs"])
64
- assert "Initialized" in combined_output
65
- assert "Created objects." in combined_output
66
- assert "App completed." in combined_output
test/object_test.py DELETED
@@ -1,41 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import pytest
3
-
4
- from modal import App, Queue, Secret
5
- from modal.exception import DeprecationError, InvalidError
6
-
7
-
8
- @pytest.mark.asyncio
9
- async def test_async_factory(client):
10
- app = App()
11
- with pytest.warns(DeprecationError):
12
- app.my_factory = Queue.new()
13
- async with app.run(client=client):
14
- assert isinstance(app.my_factory, Queue)
15
- assert app.my_factory.object_id == "qu-1"
16
-
17
-
18
- def test_new_hydrated(client):
19
- from modal.dict import _Dict
20
- from modal.object import _Object
21
- from modal.queue import _Queue
22
-
23
- assert isinstance(_Dict._new_hydrated("di-123", client, None), _Dict)
24
- assert isinstance(_Queue._new_hydrated("qu-123", client, None), _Queue)
25
-
26
- with pytest.raises(InvalidError):
27
- _Queue._new_hydrated("di-123", client, None) # Wrong prefix for type
28
-
29
- assert isinstance(_Object._new_hydrated("qu-123", client, None), _Queue)
30
- assert isinstance(_Object._new_hydrated("di-123", client, None), _Dict)
31
-
32
- with pytest.raises(InvalidError):
33
- _Object._new_hydrated("xy-123", client, None)
34
-
35
-
36
- def test_constructor():
37
- with pytest.raises(InvalidError) as excinfo:
38
- Secret({"foo": 123})
39
-
40
- assert "Secret" in str(excinfo.value)
41
- assert "constructor" in str(excinfo.value)
@@ -1,25 +0,0 @@
1
- # Copyright Modal Labs 2022
2
- import platform
3
- import pytest
4
-
5
- from modal._utils.package_utils import get_module_mount_info
6
- from modal.exception import ModuleNotMountable
7
-
8
-
9
- def test_get_module_mount_info():
10
- res = get_module_mount_info("modal")
11
- assert len(res) == 1
12
- assert res[0][0] == True
13
-
14
- res = get_module_mount_info("asyncio")
15
- assert len(res) == 1
16
- assert res[0][0] == True
17
-
18
- res = get_module_mount_info("six")
19
- assert len(res) == 1
20
- assert res[0][0] == False
21
-
22
- if platform.system() != "Windows":
23
- # TODO This assertion fails on windows; I assume that compiled file formats are different there?
24
- with pytest.raises(ModuleNotMountable, match="aiohttp can't be mounted because it contains binary file"):
25
- get_module_mount_info("aiohttp")