modal 0.62.115__py3-none-any.whl → 0.72.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modal/__init__.py +13 -9
- modal/__main__.py +41 -3
- modal/_clustered_functions.py +80 -0
- modal/_clustered_functions.pyi +22 -0
- modal/_container_entrypoint.py +407 -398
- modal/_ipython.py +3 -13
- modal/_location.py +17 -10
- modal/_output.py +243 -99
- modal/_pty.py +2 -2
- modal/_resolver.py +55 -60
- modal/_resources.py +26 -7
- modal/_runtime/__init__.py +1 -0
- modal/_runtime/asgi.py +519 -0
- modal/_runtime/container_io_manager.py +1036 -0
- modal/{execution_context.py → _runtime/execution_context.py} +11 -2
- modal/_runtime/telemetry.py +169 -0
- modal/_runtime/user_code_imports.py +356 -0
- modal/_serialization.py +123 -6
- modal/_traceback.py +47 -187
- modal/_tunnel.py +50 -14
- modal/_tunnel.pyi +19 -36
- modal/_utils/app_utils.py +3 -17
- modal/_utils/async_utils.py +386 -104
- modal/_utils/blob_utils.py +157 -186
- modal/_utils/bytes_io_segment_payload.py +97 -0
- modal/_utils/deprecation.py +89 -0
- modal/_utils/docker_utils.py +98 -0
- modal/_utils/function_utils.py +299 -98
- modal/_utils/grpc_testing.py +47 -34
- modal/_utils/grpc_utils.py +54 -21
- modal/_utils/hash_utils.py +51 -10
- modal/_utils/http_utils.py +39 -9
- modal/_utils/logger.py +2 -1
- modal/_utils/mount_utils.py +34 -16
- modal/_utils/name_utils.py +58 -0
- modal/_utils/package_utils.py +14 -1
- modal/_utils/pattern_utils.py +205 -0
- modal/_utils/rand_pb_testing.py +3 -3
- modal/_utils/shell_utils.py +15 -49
- modal/_vendor/a2wsgi_wsgi.py +62 -72
- modal/_vendor/cloudpickle.py +1 -1
- modal/_watcher.py +12 -10
- modal/app.py +561 -323
- modal/app.pyi +474 -262
- modal/call_graph.py +7 -6
- modal/cli/_download.py +22 -6
- modal/cli/_traceback.py +200 -0
- modal/cli/app.py +203 -42
- modal/cli/config.py +12 -5
- modal/cli/container.py +61 -13
- modal/cli/dict.py +128 -0
- modal/cli/entry_point.py +26 -13
- modal/cli/environment.py +40 -9
- modal/cli/import_refs.py +21 -48
- modal/cli/launch.py +28 -14
- modal/cli/network_file_system.py +57 -21
- modal/cli/profile.py +1 -1
- modal/cli/programs/run_jupyter.py +34 -9
- modal/cli/programs/vscode.py +58 -8
- modal/cli/queues.py +131 -0
- modal/cli/run.py +199 -96
- modal/cli/secret.py +5 -4
- modal/cli/token.py +7 -2
- modal/cli/utils.py +74 -8
- modal/cli/volume.py +97 -56
- modal/client.py +248 -144
- modal/client.pyi +156 -124
- modal/cloud_bucket_mount.py +43 -30
- modal/cloud_bucket_mount.pyi +32 -25
- modal/cls.py +528 -141
- modal/cls.pyi +189 -145
- modal/config.py +32 -15
- modal/container_process.py +177 -0
- modal/container_process.pyi +82 -0
- modal/dict.py +50 -54
- modal/dict.pyi +120 -164
- modal/environments.py +106 -5
- modal/environments.pyi +77 -25
- modal/exception.py +30 -43
- modal/experimental.py +62 -2
- modal/file_io.py +537 -0
- modal/file_io.pyi +235 -0
- modal/file_pattern_matcher.py +197 -0
- modal/functions.py +846 -428
- modal/functions.pyi +446 -387
- modal/gpu.py +57 -44
- modal/image.py +946 -417
- modal/image.pyi +584 -245
- modal/io_streams.py +434 -0
- modal/io_streams.pyi +122 -0
- modal/mount.py +223 -90
- modal/mount.pyi +241 -243
- modal/network_file_system.py +85 -86
- modal/network_file_system.pyi +151 -110
- modal/object.py +66 -36
- modal/object.pyi +166 -143
- modal/output.py +63 -0
- modal/parallel_map.py +73 -47
- modal/parallel_map.pyi +51 -63
- modal/partial_function.py +272 -107
- modal/partial_function.pyi +219 -120
- modal/proxy.py +15 -12
- modal/proxy.pyi +3 -8
- modal/queue.py +96 -72
- modal/queue.pyi +210 -135
- modal/requirements/2024.04.txt +2 -1
- modal/requirements/2024.10.txt +16 -0
- modal/requirements/README.md +21 -0
- modal/requirements/base-images.json +22 -0
- modal/retries.py +45 -4
- modal/runner.py +325 -203
- modal/runner.pyi +124 -110
- modal/running_app.py +27 -4
- modal/sandbox.py +509 -231
- modal/sandbox.pyi +396 -169
- modal/schedule.py +2 -2
- modal/scheduler_placement.py +20 -3
- modal/secret.py +41 -25
- modal/secret.pyi +62 -42
- modal/serving.py +39 -49
- modal/serving.pyi +37 -43
- modal/stream_type.py +15 -0
- modal/token_flow.py +5 -3
- modal/token_flow.pyi +37 -32
- modal/volume.py +123 -137
- modal/volume.pyi +228 -221
- {modal-0.62.115.dist-info → modal-0.72.11.dist-info}/METADATA +5 -5
- modal-0.72.11.dist-info/RECORD +174 -0
- {modal-0.62.115.dist-info → modal-0.72.11.dist-info}/top_level.txt +0 -1
- modal_docs/gen_reference_docs.py +3 -1
- modal_docs/mdmd/mdmd.py +0 -1
- modal_docs/mdmd/signatures.py +1 -2
- modal_global_objects/images/base_images.py +28 -0
- modal_global_objects/mounts/python_standalone.py +2 -2
- modal_proto/__init__.py +1 -1
- modal_proto/api.proto +1231 -531
- modal_proto/api_grpc.py +750 -430
- modal_proto/api_pb2.py +2102 -1176
- modal_proto/api_pb2.pyi +8859 -0
- modal_proto/api_pb2_grpc.py +1329 -675
- modal_proto/api_pb2_grpc.pyi +1416 -0
- modal_proto/modal_api_grpc.py +149 -0
- modal_proto/modal_options_grpc.py +3 -0
- modal_proto/options_pb2.pyi +20 -0
- modal_proto/options_pb2_grpc.pyi +7 -0
- modal_proto/py.typed +0 -0
- modal_version/__init__.py +1 -1
- modal_version/_version_generated.py +2 -2
- modal/_asgi.py +0 -370
- modal/_container_exec.py +0 -128
- modal/_container_io_manager.py +0 -646
- modal/_container_io_manager.pyi +0 -412
- modal/_sandbox_shell.py +0 -49
- modal/app_utils.py +0 -20
- modal/app_utils.pyi +0 -17
- modal/execution_context.pyi +0 -37
- modal/shared_volume.py +0 -23
- modal/shared_volume.pyi +0 -24
- modal-0.62.115.dist-info/RECORD +0 -207
- modal_global_objects/images/conda.py +0 -15
- modal_global_objects/images/debian_slim.py +0 -15
- modal_global_objects/images/micromamba.py +0 -15
- test/__init__.py +0 -1
- test/aio_test.py +0 -12
- test/async_utils_test.py +0 -279
- test/blob_test.py +0 -67
- test/cli_imports_test.py +0 -149
- test/cli_test.py +0 -674
- test/client_test.py +0 -203
- test/cloud_bucket_mount_test.py +0 -22
- test/cls_test.py +0 -636
- test/config_test.py +0 -149
- test/conftest.py +0 -1485
- test/container_app_test.py +0 -50
- test/container_test.py +0 -1405
- test/cpu_test.py +0 -23
- test/decorator_test.py +0 -85
- test/deprecation_test.py +0 -34
- test/dict_test.py +0 -51
- test/e2e_test.py +0 -68
- test/error_test.py +0 -7
- test/function_serialization_test.py +0 -32
- test/function_test.py +0 -791
- test/function_utils_test.py +0 -101
- test/gpu_test.py +0 -159
- test/grpc_utils_test.py +0 -82
- test/helpers.py +0 -47
- test/image_test.py +0 -814
- test/live_reload_test.py +0 -80
- test/lookup_test.py +0 -70
- test/mdmd_test.py +0 -329
- test/mount_test.py +0 -162
- test/mounted_files_test.py +0 -327
- test/network_file_system_test.py +0 -188
- test/notebook_test.py +0 -66
- test/object_test.py +0 -41
- test/package_utils_test.py +0 -25
- test/queue_test.py +0 -115
- test/resolver_test.py +0 -59
- test/retries_test.py +0 -67
- test/runner_test.py +0 -85
- test/sandbox_test.py +0 -191
- test/schedule_test.py +0 -15
- test/scheduler_placement_test.py +0 -57
- test/secret_test.py +0 -89
- test/serialization_test.py +0 -50
- test/stub_composition_test.py +0 -10
- test/stub_test.py +0 -361
- test/test_asgi_wrapper.py +0 -234
- test/token_flow_test.py +0 -18
- test/traceback_test.py +0 -135
- test/tunnel_test.py +0 -29
- test/utils_test.py +0 -88
- test/version_test.py +0 -14
- test/volume_test.py +0 -397
- test/watcher_test.py +0 -58
- test/webhook_test.py +0 -145
- {modal-0.62.115.dist-info → modal-0.72.11.dist-info}/LICENSE +0 -0
- {modal-0.62.115.dist-info → modal-0.72.11.dist-info}/WHEEL +0 -0
- {modal-0.62.115.dist-info → modal-0.72.11.dist-info}/entry_points.txt +0 -0
test/mounted_files_test.py
DELETED
@@ -1,327 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import os
|
3
|
-
import pytest
|
4
|
-
import subprocess
|
5
|
-
import sys
|
6
|
-
from pathlib import Path
|
7
|
-
|
8
|
-
import pytest_asyncio
|
9
|
-
|
10
|
-
import modal
|
11
|
-
from modal import Mount
|
12
|
-
from modal.mount import get_auto_mounts
|
13
|
-
|
14
|
-
from . import helpers
|
15
|
-
from .supports.skip import skip_windows
|
16
|
-
|
17
|
-
|
18
|
-
@pytest.fixture
|
19
|
-
def venv_path(tmp_path, repo_root):
|
20
|
-
venv_path = tmp_path
|
21
|
-
subprocess.run([sys.executable, "-m", "venv", venv_path, "--copies", "--system-site-packages"], check=True)
|
22
|
-
# Install Modal and a tiny package in the venv.
|
23
|
-
subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "-e", repo_root], check=True)
|
24
|
-
subprocess.run([venv_path / "bin" / "python", "-m", "pip", "install", "--force-reinstall", "six"], check=True)
|
25
|
-
yield venv_path
|
26
|
-
|
27
|
-
|
28
|
-
@pytest.fixture
|
29
|
-
def path_with_symlinked_files(tmp_path):
|
30
|
-
src = tmp_path / "foo.txt"
|
31
|
-
src.write_text("Hello")
|
32
|
-
trg = tmp_path / "bar.txt"
|
33
|
-
trg.symlink_to(src)
|
34
|
-
return tmp_path, {src, trg}
|
35
|
-
|
36
|
-
|
37
|
-
script_path = "pkg_a/script.py"
|
38
|
-
|
39
|
-
|
40
|
-
def f():
|
41
|
-
pass
|
42
|
-
|
43
|
-
|
44
|
-
@pytest_asyncio.fixture
|
45
|
-
async def env_mount_files():
|
46
|
-
# If something is installed using pip -e, it will be bundled up as a part of the environment.
|
47
|
-
# Those are env-specific so we ignore those as a part of the test
|
48
|
-
filenames = []
|
49
|
-
for mount in get_auto_mounts():
|
50
|
-
async for file_info in mount._get_files(mount.entries):
|
51
|
-
filenames.append(file_info.mount_filename)
|
52
|
-
|
53
|
-
return filenames
|
54
|
-
|
55
|
-
|
56
|
-
def test_mounted_files_script(servicer, supports_dir, env_mount_files, server_url_env):
|
57
|
-
helpers.deploy_app_externally(servicer, script_path, cwd=supports_dir)
|
58
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
59
|
-
|
60
|
-
# Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
|
61
|
-
assert files == {
|
62
|
-
"/root/a.py",
|
63
|
-
"/root/b/c.py",
|
64
|
-
"/root/b/e.py",
|
65
|
-
"/root/pkg_b/__init__.py",
|
66
|
-
"/root/pkg_b/f.py",
|
67
|
-
"/root/pkg_b/g/h.py",
|
68
|
-
"/root/script.py",
|
69
|
-
}
|
70
|
-
|
71
|
-
|
72
|
-
serialized_fn_path = "pkg_a/serialized_fn.py"
|
73
|
-
|
74
|
-
|
75
|
-
def test_mounted_files_serialized(servicer, supports_dir, env_mount_files, server_url_env):
|
76
|
-
helpers.deploy_app_externally(servicer, serialized_fn_path, cwd=supports_dir)
|
77
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
78
|
-
|
79
|
-
# Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
|
80
|
-
assert (
|
81
|
-
files
|
82
|
-
== {
|
83
|
-
"/root/serialized_fn.py", # should serialized_fn be included? It's not needed to run the function, but it's loaded into sys.modules at definition time...
|
84
|
-
"/root/b/c.py", # this is mounted under root since it's imported as `import b` and not `import pkg_a.b` from serialized_fn.py
|
85
|
-
"/root/b/e.py", # same as above
|
86
|
-
"/root/a.py", # same as above
|
87
|
-
"/root/pkg_b/__init__.py",
|
88
|
-
"/root/pkg_b/f.py",
|
89
|
-
"/root/pkg_b/g/h.py",
|
90
|
-
}
|
91
|
-
)
|
92
|
-
|
93
|
-
|
94
|
-
def test_mounted_files_package(supports_dir, env_mount_files, servicer, server_url_env):
|
95
|
-
p = subprocess.run(["modal", "run", "pkg_a.package"], cwd=supports_dir)
|
96
|
-
assert p.returncode == 0
|
97
|
-
|
98
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
99
|
-
# Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
|
100
|
-
assert files == {
|
101
|
-
"/root/pkg_a/__init__.py",
|
102
|
-
"/root/pkg_a/a.py",
|
103
|
-
"/root/pkg_a/b/c.py",
|
104
|
-
"/root/pkg_a/d.py",
|
105
|
-
"/root/pkg_a/b/e.py",
|
106
|
-
"/root/pkg_a/script.py",
|
107
|
-
"/root/pkg_a/serialized_fn.py",
|
108
|
-
"/root/pkg_a/package.py",
|
109
|
-
"/root/pkg_b/__init__.py",
|
110
|
-
"/root/pkg_b/f.py",
|
111
|
-
"/root/pkg_b/g/h.py",
|
112
|
-
}
|
113
|
-
|
114
|
-
|
115
|
-
def test_mounted_files_package_no_automount(supports_dir, env_mount_files, servicer, server_url_env):
|
116
|
-
# when triggered like a module, the target module should be put at the correct package path
|
117
|
-
p = subprocess.run(
|
118
|
-
["modal", "run", "pkg_a.package"],
|
119
|
-
cwd=supports_dir,
|
120
|
-
capture_output=True,
|
121
|
-
env={**os.environ, "MODAL_AUTOMOUNT": "0"},
|
122
|
-
)
|
123
|
-
assert p.returncode == 0
|
124
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
125
|
-
assert files == {
|
126
|
-
"/root/pkg_a/__init__.py",
|
127
|
-
"/root/pkg_a/package.py",
|
128
|
-
}
|
129
|
-
|
130
|
-
|
131
|
-
@skip_windows("venvs behave differently on Windows.")
|
132
|
-
def test_mounted_files_sys_prefix(servicer, supports_dir, venv_path, env_mount_files, server_url_env):
|
133
|
-
# Run with venv activated, so it's on sys.prefix, and modal is dev-installed in the VM
|
134
|
-
subprocess.run(
|
135
|
-
[venv_path / "bin" / "modal", "run", script_path],
|
136
|
-
cwd=supports_dir,
|
137
|
-
)
|
138
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
139
|
-
# Assert we include everything from `pkg_a` and `pkg_b` but not `pkg_c`:
|
140
|
-
assert files == {
|
141
|
-
"/root/a.py",
|
142
|
-
"/root/b/c.py",
|
143
|
-
"/root/b/e.py",
|
144
|
-
"/root/script.py",
|
145
|
-
"/root/pkg_b/__init__.py",
|
146
|
-
"/root/pkg_b/f.py",
|
147
|
-
"/root/pkg_b/g/h.py",
|
148
|
-
}
|
149
|
-
|
150
|
-
|
151
|
-
@pytest.fixture
|
152
|
-
def symlinked_python_installation_venv_path(tmp_path, repo_root):
|
153
|
-
# sets up a symlink to the python *installation* (not just the python binary)
|
154
|
-
# and initialize the virtualenv using a path via that symlink
|
155
|
-
# This makes the file paths of any stdlib modules use the symlinked path
|
156
|
-
# instead of the original, which is similar to what some tools do (e.g. mise)
|
157
|
-
# and has the potential to break automounting behavior, so we keep this
|
158
|
-
# test as a regression test for that
|
159
|
-
venv_path = tmp_path / "venv"
|
160
|
-
actual_executable = Path(sys.executable).resolve()
|
161
|
-
assert actual_executable.parent.name == "bin"
|
162
|
-
python_install_dir = actual_executable.parent.parent
|
163
|
-
# create a symlink to the python install *root*
|
164
|
-
symlink_python_install = tmp_path / "python-install"
|
165
|
-
symlink_python_install.symlink_to(python_install_dir)
|
166
|
-
|
167
|
-
# use a python executable specified via the above symlink
|
168
|
-
symlink_python_executable = symlink_python_install / "bin" / "python"
|
169
|
-
# create a new venv
|
170
|
-
subprocess.check_call([symlink_python_executable, "-m", "venv", venv_path, "--copies"])
|
171
|
-
# check that a builtin module, like ast, is indeed identified to be in the non-resolved install path
|
172
|
-
# since this is the source of bugs that we want to assert we don't run into!
|
173
|
-
ast_path = subprocess.check_output(
|
174
|
-
[venv_path / "bin" / "python", "-c", "import ast; print(ast.__file__);"], encoding="utf8"
|
175
|
-
)
|
176
|
-
assert ast_path != Path(ast_path).resolve()
|
177
|
-
|
178
|
-
# install modal from current dir
|
179
|
-
subprocess.check_call([venv_path / "bin" / "pip", "install", repo_root])
|
180
|
-
yield venv_path
|
181
|
-
|
182
|
-
|
183
|
-
@skip_windows("venvs behave differently on Windows.")
|
184
|
-
def test_mounted_files_symlinked_python_install(
|
185
|
-
symlinked_python_installation_venv_path, supports_dir, server_url_env, servicer
|
186
|
-
):
|
187
|
-
subprocess.check_call(
|
188
|
-
[symlinked_python_installation_venv_path / "bin" / "modal", "run", supports_dir / "imports_ast.py"]
|
189
|
-
)
|
190
|
-
assert "/root/ast.py" not in servicer.files_name2sha
|
191
|
-
|
192
|
-
|
193
|
-
def test_mounted_files_config(servicer, supports_dir, env_mount_files, server_url_env):
|
194
|
-
p = subprocess.run(
|
195
|
-
["modal", "run", "pkg_a/script.py"], cwd=supports_dir, env={**os.environ, "MODAL_AUTOMOUNT": "0"}
|
196
|
-
)
|
197
|
-
assert p.returncode == 0
|
198
|
-
files = set(servicer.files_name2sha.keys()) - set(env_mount_files)
|
199
|
-
assert files == {
|
200
|
-
"/root/script.py",
|
201
|
-
}
|
202
|
-
|
203
|
-
|
204
|
-
def test_e2e_modal_run_py_file_mounts(servicer, supports_dir):
|
205
|
-
helpers.deploy_app_externally(servicer, "hello.py", cwd=supports_dir)
|
206
|
-
# Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
|
207
|
-
# assert len(servicer.files_name2sha) == 1
|
208
|
-
# assert servicer.n_mounts == 1 # there should be a single mount
|
209
|
-
# assert servicer.n_mount_files == 1
|
210
|
-
assert "/root/hello.py" in servicer.files_name2sha
|
211
|
-
|
212
|
-
|
213
|
-
def test_e2e_modal_run_py_module_mounts(servicer, supports_dir):
|
214
|
-
helpers.deploy_app_externally(servicer, "hello", cwd=supports_dir)
|
215
|
-
# Reactivate the following mount assertions when we remove auto-mounting of dev-installed packages
|
216
|
-
# assert len(servicer.files_name2sha) == 1
|
217
|
-
# assert servicer.n_mounts == 1 # there should be a single mount
|
218
|
-
# assert servicer.n_mount_files == 1
|
219
|
-
assert "/root/hello.py" in servicer.files_name2sha
|
220
|
-
|
221
|
-
|
222
|
-
def foo():
|
223
|
-
pass
|
224
|
-
|
225
|
-
|
226
|
-
def test_mounts_are_not_traversed_on_declaration(test_dir, monkeypatch, client, server_url_env):
|
227
|
-
return_values = []
|
228
|
-
original = modal.mount._MountDir.get_files_to_upload
|
229
|
-
|
230
|
-
def mock_get_files_to_upload(self):
|
231
|
-
r = list(original(self))
|
232
|
-
return_values.append(r)
|
233
|
-
return r
|
234
|
-
|
235
|
-
monkeypatch.setattr("modal.mount._MountDir.get_files_to_upload", mock_get_files_to_upload)
|
236
|
-
app = modal.App()
|
237
|
-
mount_with_many_files = Mount.from_local_dir(test_dir, remote_path="/test")
|
238
|
-
app.function(mounts=[mount_with_many_files])(foo)
|
239
|
-
assert len(return_values) == 0 # ensure we don't look at the files yet
|
240
|
-
|
241
|
-
with app.run(client=client):
|
242
|
-
pass
|
243
|
-
|
244
|
-
assert return_values # at this point we should have gotten all the mount files
|
245
|
-
# flatten inspected files
|
246
|
-
files = set()
|
247
|
-
for r in return_values:
|
248
|
-
for fn, _ in r:
|
249
|
-
files.add(fn)
|
250
|
-
# sanity check - this test file should be included since we mounted the test dir
|
251
|
-
assert __file__ in files # this test file should have been included
|
252
|
-
|
253
|
-
|
254
|
-
def test_mount_dedupe(servicer, test_dir, server_url_env):
|
255
|
-
supports_dir = test_dir / "supports"
|
256
|
-
normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
|
257
|
-
normally_not_included_file.touch(exist_ok=True)
|
258
|
-
print(
|
259
|
-
helpers.deploy_app_externally(
|
260
|
-
# no explicit mounts, rely on auto-mounting
|
261
|
-
servicer,
|
262
|
-
"mount_dedupe.py",
|
263
|
-
cwd=test_dir / "supports",
|
264
|
-
env={"USE_EXPLICIT": "0"},
|
265
|
-
)
|
266
|
-
)
|
267
|
-
assert servicer.n_mounts == 2
|
268
|
-
assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
|
269
|
-
pkg_a_mount = servicer.mount_contents["mo-2"]
|
270
|
-
for fn in pkg_a_mount.keys():
|
271
|
-
assert fn.startswith("/root/pkg_a")
|
272
|
-
assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
|
273
|
-
|
274
|
-
|
275
|
-
def test_mount_dedupe_explicit(servicer, test_dir, server_url_env):
|
276
|
-
supports_dir = test_dir / "supports"
|
277
|
-
normally_not_included_file = supports_dir / "pkg_a" / "normally_not_included.pyc"
|
278
|
-
normally_not_included_file.touch(exist_ok=True)
|
279
|
-
print(
|
280
|
-
helpers.deploy_app_externally(
|
281
|
-
# two explicit mounts of the same package
|
282
|
-
servicer,
|
283
|
-
"mount_dedupe.py",
|
284
|
-
cwd=supports_dir,
|
285
|
-
env={"USE_EXPLICIT": "1"},
|
286
|
-
)
|
287
|
-
)
|
288
|
-
assert servicer.n_mounts == 3
|
289
|
-
assert servicer.mount_contents["mo-1"].keys() == {"/root/mount_dedupe.py"}
|
290
|
-
pkg_a_mount = servicer.mount_contents["mo-2"]
|
291
|
-
for fn in pkg_a_mount.keys():
|
292
|
-
assert fn.startswith("/root/pkg_a")
|
293
|
-
assert "/root/pkg_a/normally_not_included.pyc" not in pkg_a_mount.keys()
|
294
|
-
|
295
|
-
custom_pkg_a_mount = servicer.mount_contents["mo-3"]
|
296
|
-
assert len(custom_pkg_a_mount) == len(pkg_a_mount) + 1
|
297
|
-
assert "/root/pkg_a/normally_not_included.pyc" in custom_pkg_a_mount.keys()
|
298
|
-
|
299
|
-
|
300
|
-
@skip_windows("pip-installed pdm seems somewhat broken on windows")
|
301
|
-
def test_pdm_cache_automount_exclude(tmp_path, monkeypatch, supports_dir, servicer, server_url_env):
|
302
|
-
# check that `pdm`'s cached packages are not included in automounts
|
303
|
-
project_dir = Path(__file__).parent.parent
|
304
|
-
monkeypatch.chdir(tmp_path)
|
305
|
-
subprocess.run(["pdm", "init", "-n"], check=True)
|
306
|
-
subprocess.run(
|
307
|
-
["pdm", "add", "--dev", project_dir], check=True
|
308
|
-
) # install workdir modal into venv, not using cache...
|
309
|
-
subprocess.run(["pdm", "config", "--local", "install.cache", "on"], check=True)
|
310
|
-
subprocess.run(["pdm", "add", "six"], check=True) # single file module
|
311
|
-
subprocess.run(
|
312
|
-
["pdm", "run", "modal", "deploy", supports_dir / "imports_six.py"], check=True
|
313
|
-
) # deploy a basically empty function
|
314
|
-
|
315
|
-
files = set(servicer.files_name2sha.keys())
|
316
|
-
assert files == {
|
317
|
-
"/root/imports_six.py",
|
318
|
-
}
|
319
|
-
|
320
|
-
|
321
|
-
def test_mount_directory_with_symlinked_file(path_with_symlinked_files, servicer, server_url_env):
|
322
|
-
path, files = path_with_symlinked_files
|
323
|
-
mount = Mount.from_local_dir(path)
|
324
|
-
mount._deploy("mo-1")
|
325
|
-
pkg_a_mount = servicer.mount_contents["mo-1"]
|
326
|
-
for src_f in files:
|
327
|
-
assert any(mnt_f.endswith(src_f.name) for mnt_f in pkg_a_mount)
|
test/network_file_system_test.py
DELETED
@@ -1,188 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import pytest
|
3
|
-
import time
|
4
|
-
from io import BytesIO
|
5
|
-
from unittest import mock
|
6
|
-
|
7
|
-
import modal
|
8
|
-
from modal.exception import DeprecationError, InvalidError, NotFoundError
|
9
|
-
from modal.runner import deploy_app
|
10
|
-
|
11
|
-
|
12
|
-
def dummy():
|
13
|
-
pass
|
14
|
-
|
15
|
-
|
16
|
-
def test_network_file_system_files(client, test_dir, servicer):
|
17
|
-
app = modal.App()
|
18
|
-
nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
|
19
|
-
|
20
|
-
dummy_modal = app.function(network_file_systems={"/root/foo": nfs})(dummy)
|
21
|
-
|
22
|
-
with app.run(client=client):
|
23
|
-
dummy_modal.remote()
|
24
|
-
|
25
|
-
|
26
|
-
def test_network_file_system_bad_paths():
|
27
|
-
app = modal.App()
|
28
|
-
nfs = modal.NetworkFileSystem.from_name("xyz", create_if_missing=True)
|
29
|
-
|
30
|
-
def _f():
|
31
|
-
pass
|
32
|
-
|
33
|
-
with pytest.raises(InvalidError):
|
34
|
-
app.function(network_file_systems={"/root/../../foo": nfs})(dummy)
|
35
|
-
|
36
|
-
with pytest.raises(InvalidError):
|
37
|
-
app.function(network_file_systems={"/": nfs})(dummy)
|
38
|
-
|
39
|
-
with pytest.raises(InvalidError):
|
40
|
-
app.function(network_file_systems={"/tmp/": nfs})(dummy)
|
41
|
-
|
42
|
-
|
43
|
-
def test_network_file_system_handle_single_file(client, tmp_path, servicer):
|
44
|
-
local_file_path = tmp_path / "some_file"
|
45
|
-
local_file_path.write_text("hello world")
|
46
|
-
|
47
|
-
with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
|
48
|
-
nfs.add_local_file(local_file_path)
|
49
|
-
nfs.add_local_file(local_file_path.as_posix(), remote_path="/foo/other_destination")
|
50
|
-
object_id = nfs.object_id
|
51
|
-
|
52
|
-
assert servicer.nfs_files[object_id].keys() == {
|
53
|
-
"/some_file",
|
54
|
-
"/foo/other_destination",
|
55
|
-
}
|
56
|
-
assert servicer.nfs_files[object_id]["/some_file"].data == b"hello world"
|
57
|
-
assert servicer.nfs_files[object_id]["/foo/other_destination"].data == b"hello world"
|
58
|
-
|
59
|
-
|
60
|
-
@pytest.mark.asyncio
|
61
|
-
async def test_network_file_system_handle_dir(client, tmp_path, servicer):
|
62
|
-
local_dir = tmp_path / "some_dir"
|
63
|
-
local_dir.mkdir()
|
64
|
-
(local_dir / "smol").write_text("###")
|
65
|
-
|
66
|
-
subdir = local_dir / "subdir"
|
67
|
-
subdir.mkdir()
|
68
|
-
(subdir / "other").write_text("####")
|
69
|
-
|
70
|
-
with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
|
71
|
-
nfs.add_local_dir(local_dir)
|
72
|
-
object_id = nfs.object_id
|
73
|
-
|
74
|
-
assert servicer.nfs_files[object_id].keys() == {
|
75
|
-
"/some_dir/smol",
|
76
|
-
"/some_dir/subdir/other",
|
77
|
-
}
|
78
|
-
assert servicer.nfs_files[object_id]["/some_dir/smol"].data == b"###"
|
79
|
-
assert servicer.nfs_files[object_id]["/some_dir/subdir/other"].data == b"####"
|
80
|
-
|
81
|
-
|
82
|
-
@pytest.mark.asyncio
|
83
|
-
async def test_network_file_system_handle_big_file(client, tmp_path, servicer, blob_server, *args):
|
84
|
-
with mock.patch("modal.network_file_system.LARGE_FILE_LIMIT", 10):
|
85
|
-
local_file_path = tmp_path / "bigfile"
|
86
|
-
local_file_path.write_text("hello world, this is a lot of text")
|
87
|
-
|
88
|
-
async with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
|
89
|
-
await nfs.add_local_file.aio(local_file_path)
|
90
|
-
object_id = nfs.object_id
|
91
|
-
|
92
|
-
assert servicer.nfs_files[object_id].keys() == {"/bigfile"}
|
93
|
-
assert servicer.nfs_files[object_id]["/bigfile"].data == b""
|
94
|
-
assert servicer.nfs_files[object_id]["/bigfile"].data_blob_id == "bl-1"
|
95
|
-
|
96
|
-
_, blobs = blob_server
|
97
|
-
assert blobs["bl-1"] == b"hello world, this is a lot of text"
|
98
|
-
|
99
|
-
|
100
|
-
def test_old_syntax(client, servicer):
|
101
|
-
app = modal.App()
|
102
|
-
with pytest.raises(DeprecationError):
|
103
|
-
app.vol1 = modal.SharedVolume() # type: ignore # This is just a post-deprecation husk
|
104
|
-
with pytest.raises(DeprecationError):
|
105
|
-
app.vol2 = modal.SharedVolume.new()
|
106
|
-
|
107
|
-
|
108
|
-
def test_redeploy(servicer, client):
|
109
|
-
app = modal.App()
|
110
|
-
with pytest.warns(DeprecationError):
|
111
|
-
n1 = modal.NetworkFileSystem.new()
|
112
|
-
n2 = modal.NetworkFileSystem.new()
|
113
|
-
n3 = modal.NetworkFileSystem.new()
|
114
|
-
app.n1, app.n2, app.n3 = n1, n2, n3
|
115
|
-
|
116
|
-
# Deploy app once
|
117
|
-
deploy_app(app, "my-app", client=client)
|
118
|
-
app1_ids = [n1.object_id, n2.object_id, n3.object_id]
|
119
|
-
|
120
|
-
# Deploy app again
|
121
|
-
deploy_app(app, "my-app", client=client)
|
122
|
-
app2_ids = [n1.object_id, n2.object_id, n3.object_id]
|
123
|
-
|
124
|
-
# Make sure ids are stable
|
125
|
-
assert app1_ids == app2_ids
|
126
|
-
|
127
|
-
# Make sure ids are unique
|
128
|
-
assert len(set(app1_ids)) == 3
|
129
|
-
assert len(set(app2_ids)) == 3
|
130
|
-
|
131
|
-
# Deploy to a different app
|
132
|
-
deploy_app(app, "my-other-app", client=client)
|
133
|
-
app3_ids = [n1.object_id, n2.object_id, n3.object_id]
|
134
|
-
|
135
|
-
# Should be unique and different
|
136
|
-
assert len(set(app3_ids)) == 3
|
137
|
-
assert set(app1_ids) & set(app3_ids) == set()
|
138
|
-
|
139
|
-
|
140
|
-
def test_read_file(client, tmp_path, servicer):
|
141
|
-
with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
|
142
|
-
with pytest.raises(FileNotFoundError):
|
143
|
-
for _ in nfs.read_file("idontexist.txt"):
|
144
|
-
...
|
145
|
-
|
146
|
-
|
147
|
-
def test_write_file(client, tmp_path, servicer):
|
148
|
-
local_file_path = tmp_path / "some_file"
|
149
|
-
local_file_path.write_text("hello world")
|
150
|
-
|
151
|
-
with modal.NetworkFileSystem.ephemeral(client=client) as nfs:
|
152
|
-
nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
|
153
|
-
|
154
|
-
# Make sure we can write through the provider too
|
155
|
-
nfs.write_file("remote_path.txt", open(local_file_path, "rb"))
|
156
|
-
|
157
|
-
|
158
|
-
def test_persisted(servicer, client):
|
159
|
-
# Lookup should fail since it doesn't exist
|
160
|
-
with pytest.raises(NotFoundError):
|
161
|
-
modal.NetworkFileSystem.lookup("xyz", client=client)
|
162
|
-
|
163
|
-
# Create it
|
164
|
-
modal.NetworkFileSystem.lookup("xyz", create_if_missing=True, client=client)
|
165
|
-
|
166
|
-
# Lookup should succeed now
|
167
|
-
modal.NetworkFileSystem.lookup("xyz", client=client)
|
168
|
-
|
169
|
-
|
170
|
-
def test_nfs_ephemeral(servicer, client, tmp_path):
|
171
|
-
local_file_path = tmp_path / "some_file"
|
172
|
-
local_file_path.write_text("hello world")
|
173
|
-
|
174
|
-
assert servicer.n_nfs_heartbeats == 0
|
175
|
-
with modal.NetworkFileSystem.ephemeral(client=client, _heartbeat_sleep=1) as nfs:
|
176
|
-
assert nfs.listdir("/") == []
|
177
|
-
nfs.write_file("xyz.txt", open(local_file_path, "rb"))
|
178
|
-
(entry,) = nfs.listdir("/")
|
179
|
-
assert entry.path == "xyz.txt"
|
180
|
-
|
181
|
-
time.sleep(1.5) # Make time for 2 heartbeats
|
182
|
-
assert servicer.n_nfs_heartbeats == 2
|
183
|
-
|
184
|
-
|
185
|
-
def test_nfs_lazy_hydration_from_name(set_env_client):
|
186
|
-
nfs = modal.NetworkFileSystem.from_name("nfs", create_if_missing=True)
|
187
|
-
bio = BytesIO(b"content")
|
188
|
-
nfs.write_file("blah", bio)
|
test/notebook_test.py
DELETED
@@ -1,66 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import pytest
|
3
|
-
import warnings
|
4
|
-
from pathlib import Path
|
5
|
-
|
6
|
-
import jupytext
|
7
|
-
|
8
|
-
try:
|
9
|
-
from nbclient.exceptions import CellExecutionError
|
10
|
-
except ModuleNotFoundError:
|
11
|
-
# TODO(erikbern): sometimes my local jupyter packages end up in a bad state,
|
12
|
-
# but we don't want that to cause pytest to fail on startup.
|
13
|
-
warnings.warn("failed importing nbclient")
|
14
|
-
|
15
|
-
|
16
|
-
@pytest.fixture
|
17
|
-
def notebook_runner(servicer):
|
18
|
-
import nbformat
|
19
|
-
from nbclient import NotebookClient
|
20
|
-
|
21
|
-
def runner(notebook_path: Path):
|
22
|
-
output_notebook_path = notebook_path.with_suffix(".output.ipynb")
|
23
|
-
|
24
|
-
nb = jupytext.read(
|
25
|
-
notebook_path,
|
26
|
-
)
|
27
|
-
|
28
|
-
parameter_cell = nb["cells"][0]
|
29
|
-
assert "parameters" in parameter_cell["metadata"]["tags"] # like in papermill
|
30
|
-
parameter_cell["source"] = f'server_addr = "{servicer.remote_addr}"'
|
31
|
-
|
32
|
-
client = NotebookClient(nb)
|
33
|
-
|
34
|
-
try:
|
35
|
-
client.execute()
|
36
|
-
except CellExecutionError:
|
37
|
-
nbformat.write(nb, output_notebook_path)
|
38
|
-
pytest.fail(
|
39
|
-
f"""There was an error when executing the notebook.
|
40
|
-
|
41
|
-
Inspect the output notebook: {output_notebook_path}
|
42
|
-
"""
|
43
|
-
)
|
44
|
-
tagged_cells = {}
|
45
|
-
for cell in nb["cells"]:
|
46
|
-
for tag in cell["metadata"].get("tags", []):
|
47
|
-
tagged_cells[tag] = cell
|
48
|
-
|
49
|
-
return tagged_cells
|
50
|
-
|
51
|
-
return runner
|
52
|
-
|
53
|
-
|
54
|
-
# for some reason this import is failing due to a circular import of IPython.terminal.embed
|
55
|
-
# but only when running in CI (sometimes?), causing these tests to fail:
|
56
|
-
# from IPython.terminal import interactiveshell
|
57
|
-
|
58
|
-
|
59
|
-
@pytest.mark.skip("temporarily disabled until IPython import issues in CI are resolved")
|
60
|
-
def test_notebook_outputs_status(notebook_runner, test_dir):
|
61
|
-
input_notebook_path = test_dir / "supports" / "notebooks" / "simple.notebook.py"
|
62
|
-
tagged_cells = notebook_runner(input_notebook_path)
|
63
|
-
combined_output = "\n".join(c["data"]["text/plain"] for c in tagged_cells["main"]["outputs"])
|
64
|
-
assert "Initialized" in combined_output
|
65
|
-
assert "Created objects." in combined_output
|
66
|
-
assert "App completed." in combined_output
|
test/object_test.py
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import pytest
|
3
|
-
|
4
|
-
from modal import App, Queue, Secret
|
5
|
-
from modal.exception import DeprecationError, InvalidError
|
6
|
-
|
7
|
-
|
8
|
-
@pytest.mark.asyncio
|
9
|
-
async def test_async_factory(client):
|
10
|
-
app = App()
|
11
|
-
with pytest.warns(DeprecationError):
|
12
|
-
app.my_factory = Queue.new()
|
13
|
-
async with app.run(client=client):
|
14
|
-
assert isinstance(app.my_factory, Queue)
|
15
|
-
assert app.my_factory.object_id == "qu-1"
|
16
|
-
|
17
|
-
|
18
|
-
def test_new_hydrated(client):
|
19
|
-
from modal.dict import _Dict
|
20
|
-
from modal.object import _Object
|
21
|
-
from modal.queue import _Queue
|
22
|
-
|
23
|
-
assert isinstance(_Dict._new_hydrated("di-123", client, None), _Dict)
|
24
|
-
assert isinstance(_Queue._new_hydrated("qu-123", client, None), _Queue)
|
25
|
-
|
26
|
-
with pytest.raises(InvalidError):
|
27
|
-
_Queue._new_hydrated("di-123", client, None) # Wrong prefix for type
|
28
|
-
|
29
|
-
assert isinstance(_Object._new_hydrated("qu-123", client, None), _Queue)
|
30
|
-
assert isinstance(_Object._new_hydrated("di-123", client, None), _Dict)
|
31
|
-
|
32
|
-
with pytest.raises(InvalidError):
|
33
|
-
_Object._new_hydrated("xy-123", client, None)
|
34
|
-
|
35
|
-
|
36
|
-
def test_constructor():
|
37
|
-
with pytest.raises(InvalidError) as excinfo:
|
38
|
-
Secret({"foo": 123})
|
39
|
-
|
40
|
-
assert "Secret" in str(excinfo.value)
|
41
|
-
assert "constructor" in str(excinfo.value)
|
test/package_utils_test.py
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
# Copyright Modal Labs 2022
|
2
|
-
import platform
|
3
|
-
import pytest
|
4
|
-
|
5
|
-
from modal._utils.package_utils import get_module_mount_info
|
6
|
-
from modal.exception import ModuleNotMountable
|
7
|
-
|
8
|
-
|
9
|
-
def test_get_module_mount_info():
|
10
|
-
res = get_module_mount_info("modal")
|
11
|
-
assert len(res) == 1
|
12
|
-
assert res[0][0] == True
|
13
|
-
|
14
|
-
res = get_module_mount_info("asyncio")
|
15
|
-
assert len(res) == 1
|
16
|
-
assert res[0][0] == True
|
17
|
-
|
18
|
-
res = get_module_mount_info("six")
|
19
|
-
assert len(res) == 1
|
20
|
-
assert res[0][0] == False
|
21
|
-
|
22
|
-
if platform.system() != "Windows":
|
23
|
-
# TODO This assertion fails on windows; I assume that compiled file formats are different there?
|
24
|
-
with pytest.raises(ModuleNotMountable, match="aiohttp can't be mounted because it contains binary file"):
|
25
|
-
get_module_mount_info("aiohttp")
|