modal 1.0.3.dev10__py3-none-any.whl → 1.2.3.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of modal might be problematic. Click here for more details.
- modal/__init__.py +0 -2
- modal/__main__.py +3 -4
- modal/_billing.py +80 -0
- modal/_clustered_functions.py +7 -3
- modal/_clustered_functions.pyi +15 -3
- modal/_container_entrypoint.py +51 -69
- modal/_functions.py +508 -240
- modal/_grpc_client.py +171 -0
- modal/_load_context.py +105 -0
- modal/_object.py +81 -21
- modal/_output.py +58 -45
- modal/_partial_function.py +48 -73
- modal/_pty.py +7 -3
- modal/_resolver.py +26 -46
- modal/_runtime/asgi.py +4 -3
- modal/_runtime/container_io_manager.py +358 -220
- modal/_runtime/container_io_manager.pyi +296 -101
- modal/_runtime/execution_context.py +18 -2
- modal/_runtime/execution_context.pyi +64 -7
- modal/_runtime/gpu_memory_snapshot.py +262 -57
- modal/_runtime/user_code_imports.py +28 -58
- modal/_serialization.py +90 -6
- modal/_traceback.py +42 -1
- modal/_tunnel.pyi +380 -12
- modal/_utils/async_utils.py +84 -29
- modal/_utils/auth_token_manager.py +111 -0
- modal/_utils/blob_utils.py +181 -58
- modal/_utils/deprecation.py +19 -0
- modal/_utils/function_utils.py +91 -47
- modal/_utils/grpc_utils.py +89 -66
- modal/_utils/mount_utils.py +26 -1
- modal/_utils/name_utils.py +17 -3
- modal/_utils/task_command_router_client.py +536 -0
- modal/_utils/time_utils.py +34 -6
- modal/app.py +256 -88
- modal/app.pyi +909 -92
- modal/billing.py +5 -0
- modal/builder/2025.06.txt +18 -0
- modal/builder/PREVIEW.txt +18 -0
- modal/builder/base-images.json +58 -0
- modal/cli/_download.py +19 -3
- modal/cli/_traceback.py +3 -2
- modal/cli/app.py +4 -4
- modal/cli/cluster.py +15 -7
- modal/cli/config.py +5 -3
- modal/cli/container.py +7 -6
- modal/cli/dict.py +22 -16
- modal/cli/entry_point.py +12 -5
- modal/cli/environment.py +5 -4
- modal/cli/import_refs.py +3 -3
- modal/cli/launch.py +102 -5
- modal/cli/network_file_system.py +11 -12
- modal/cli/profile.py +3 -2
- modal/cli/programs/launch_instance_ssh.py +94 -0
- modal/cli/programs/run_jupyter.py +1 -1
- modal/cli/programs/run_marimo.py +95 -0
- modal/cli/programs/vscode.py +1 -1
- modal/cli/queues.py +57 -26
- modal/cli/run.py +91 -23
- modal/cli/secret.py +48 -22
- modal/cli/token.py +7 -8
- modal/cli/utils.py +4 -7
- modal/cli/volume.py +31 -25
- modal/client.py +15 -85
- modal/client.pyi +183 -62
- modal/cloud_bucket_mount.py +5 -3
- modal/cloud_bucket_mount.pyi +197 -5
- modal/cls.py +200 -126
- modal/cls.pyi +446 -68
- modal/config.py +29 -11
- modal/container_process.py +319 -19
- modal/container_process.pyi +190 -20
- modal/dict.py +290 -71
- modal/dict.pyi +835 -83
- modal/environments.py +15 -27
- modal/environments.pyi +46 -24
- modal/exception.py +14 -2
- modal/experimental/__init__.py +194 -40
- modal/experimental/flash.py +618 -0
- modal/experimental/flash.pyi +380 -0
- modal/experimental/ipython.py +11 -7
- modal/file_io.py +29 -36
- modal/file_io.pyi +251 -53
- modal/file_pattern_matcher.py +56 -16
- modal/functions.pyi +673 -92
- modal/gpu.py +1 -1
- modal/image.py +528 -176
- modal/image.pyi +1572 -145
- modal/io_streams.py +458 -128
- modal/io_streams.pyi +433 -52
- modal/mount.py +216 -151
- modal/mount.pyi +225 -78
- modal/network_file_system.py +45 -62
- modal/network_file_system.pyi +277 -56
- modal/object.pyi +93 -17
- modal/parallel_map.py +942 -129
- modal/parallel_map.pyi +294 -15
- modal/partial_function.py +0 -2
- modal/partial_function.pyi +234 -19
- modal/proxy.py +17 -8
- modal/proxy.pyi +36 -3
- modal/queue.py +270 -65
- modal/queue.pyi +817 -57
- modal/runner.py +115 -101
- modal/runner.pyi +205 -49
- modal/sandbox.py +512 -136
- modal/sandbox.pyi +845 -111
- modal/schedule.py +1 -1
- modal/secret.py +300 -70
- modal/secret.pyi +589 -34
- modal/serving.py +7 -11
- modal/serving.pyi +7 -8
- modal/snapshot.py +11 -8
- modal/snapshot.pyi +25 -4
- modal/token_flow.py +4 -4
- modal/token_flow.pyi +28 -8
- modal/volume.py +416 -158
- modal/volume.pyi +1117 -121
- {modal-1.0.3.dev10.dist-info → modal-1.2.3.dev7.dist-info}/METADATA +10 -9
- modal-1.2.3.dev7.dist-info/RECORD +195 -0
- modal_docs/mdmd/mdmd.py +17 -4
- modal_proto/api.proto +534 -79
- modal_proto/api_grpc.py +337 -1
- modal_proto/api_pb2.py +1522 -968
- modal_proto/api_pb2.pyi +1619 -134
- modal_proto/api_pb2_grpc.py +699 -4
- modal_proto/api_pb2_grpc.pyi +226 -14
- modal_proto/modal_api_grpc.py +175 -154
- modal_proto/sandbox_router.proto +145 -0
- modal_proto/sandbox_router_grpc.py +105 -0
- modal_proto/sandbox_router_pb2.py +149 -0
- modal_proto/sandbox_router_pb2.pyi +333 -0
- modal_proto/sandbox_router_pb2_grpc.py +203 -0
- modal_proto/sandbox_router_pb2_grpc.pyi +75 -0
- modal_proto/task_command_router.proto +144 -0
- modal_proto/task_command_router_grpc.py +105 -0
- modal_proto/task_command_router_pb2.py +149 -0
- modal_proto/task_command_router_pb2.pyi +333 -0
- modal_proto/task_command_router_pb2_grpc.py +203 -0
- modal_proto/task_command_router_pb2_grpc.pyi +75 -0
- modal_version/__init__.py +1 -1
- modal/requirements/PREVIEW.txt +0 -16
- modal/requirements/base-images.json +0 -26
- modal-1.0.3.dev10.dist-info/RECORD +0 -179
- modal_proto/modal_options_grpc.py +0 -3
- modal_proto/options.proto +0 -19
- modal_proto/options_grpc.py +0 -3
- modal_proto/options_pb2.py +0 -35
- modal_proto/options_pb2.pyi +0 -20
- modal_proto/options_pb2_grpc.py +0 -4
- modal_proto/options_pb2_grpc.pyi +0 -7
- /modal/{requirements → builder}/2023.12.312.txt +0 -0
- /modal/{requirements → builder}/2023.12.txt +0 -0
- /modal/{requirements → builder}/2024.04.txt +0 -0
- /modal/{requirements → builder}/2024.10.txt +0 -0
- /modal/{requirements → builder}/README.md +0 -0
- {modal-1.0.3.dev10.dist-info → modal-1.2.3.dev7.dist-info}/WHEEL +0 -0
- {modal-1.0.3.dev10.dist-info → modal-1.2.3.dev7.dist-info}/entry_points.txt +0 -0
- {modal-1.0.3.dev10.dist-info → modal-1.2.3.dev7.dist-info}/licenses/LICENSE +0 -0
- {modal-1.0.3.dev10.dist-info → modal-1.2.3.dev7.dist-info}/top_level.txt +0 -0
modal/network_file_system.py
CHANGED
|
@@ -6,12 +6,12 @@ from collections.abc import AsyncIterator
|
|
|
6
6
|
from pathlib import Path, PurePosixPath
|
|
7
7
|
from typing import Any, BinaryIO, Callable, Optional, Union
|
|
8
8
|
|
|
9
|
-
from grpclib import GRPCError, Status
|
|
10
9
|
from synchronicity.async_wrap import asynccontextmanager
|
|
11
10
|
|
|
12
11
|
import modal
|
|
13
12
|
from modal_proto import api_pb2
|
|
14
13
|
|
|
14
|
+
from ._load_context import LoadContext
|
|
15
15
|
from ._object import (
|
|
16
16
|
EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
|
17
17
|
_get_environment_name,
|
|
@@ -22,8 +22,7 @@ from ._object import (
|
|
|
22
22
|
from ._resolver import Resolver
|
|
23
23
|
from ._utils.async_utils import TaskContext, aclosing, async_map, sync_or_async_iter, synchronize_api
|
|
24
24
|
from ._utils.blob_utils import LARGE_FILE_LIMIT, blob_iter, blob_upload_file
|
|
25
|
-
from ._utils.deprecation import
|
|
26
|
-
from ._utils.grpc_utils import retry_transient_errors
|
|
25
|
+
from ._utils.deprecation import warn_if_passing_namespace
|
|
27
26
|
from ._utils.hash_utils import get_sha256_hex
|
|
28
27
|
from ._utils.name_utils import check_object_name
|
|
29
28
|
from .client import _Client
|
|
@@ -56,6 +55,8 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
56
55
|
By attaching this file system as a mount to one or more functions, they can
|
|
57
56
|
share and persist data with each other.
|
|
58
57
|
|
|
58
|
+
**Note: `NetworkFileSystem` has been deprecated and will be removed.**
|
|
59
|
+
|
|
59
60
|
**Usage**
|
|
60
61
|
|
|
61
62
|
```python
|
|
@@ -92,15 +93,16 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
92
93
|
def from_name(
|
|
93
94
|
name: str,
|
|
94
95
|
*,
|
|
95
|
-
namespace=
|
|
96
|
+
namespace=None, # mdmd:line-hidden
|
|
96
97
|
environment_name: Optional[str] = None,
|
|
97
98
|
create_if_missing: bool = False,
|
|
99
|
+
client: Optional[_Client] = None,
|
|
98
100
|
) -> "_NetworkFileSystem":
|
|
99
101
|
"""Reference a NetworkFileSystem by its name, creating if necessary.
|
|
100
102
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
This is a lazy method that defers hydrating the local object with
|
|
104
|
+
metadata from Modal servers until the first time it is actually
|
|
105
|
+
used.
|
|
104
106
|
|
|
105
107
|
```python notest
|
|
106
108
|
nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
|
|
@@ -111,25 +113,32 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
111
113
|
```
|
|
112
114
|
"""
|
|
113
115
|
check_object_name(name, "NetworkFileSystem")
|
|
116
|
+
warn_if_passing_namespace(namespace, "modal.NetworkFileSystem.from_name")
|
|
114
117
|
|
|
115
|
-
async def _load(
|
|
118
|
+
async def _load(
|
|
119
|
+
self: _NetworkFileSystem, resolver: Resolver, load_context: LoadContext, existing_object_id: Optional[str]
|
|
120
|
+
):
|
|
116
121
|
req = api_pb2.SharedVolumeGetOrCreateRequest(
|
|
117
122
|
deployment_name=name,
|
|
118
|
-
|
|
119
|
-
environment_name=_get_environment_name(environment_name, resolver),
|
|
123
|
+
environment_name=load_context.environment_name,
|
|
120
124
|
object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
|
|
121
125
|
)
|
|
122
126
|
try:
|
|
123
|
-
response = await
|
|
124
|
-
self._hydrate(response.shared_volume_id,
|
|
125
|
-
except
|
|
126
|
-
if exc.
|
|
127
|
+
response = await load_context.client.stub.SharedVolumeGetOrCreate(req)
|
|
128
|
+
self._hydrate(response.shared_volume_id, load_context.client, None)
|
|
129
|
+
except modal.exception.NotFoundError as exc:
|
|
130
|
+
if exc.args[0] == "App has wrong entity vo":
|
|
127
131
|
raise InvalidError(
|
|
128
132
|
f"Attempted to mount: `{name}` as a NetworkFileSystem " + "which already exists as a Volume"
|
|
129
133
|
)
|
|
130
134
|
raise
|
|
131
135
|
|
|
132
|
-
return _NetworkFileSystem._from_loader(
|
|
136
|
+
return _NetworkFileSystem._from_loader(
|
|
137
|
+
_load,
|
|
138
|
+
"NetworkFileSystem()",
|
|
139
|
+
hydrate_lazily=True,
|
|
140
|
+
load_context_overrides=LoadContext(environment_name=environment_name, client=client),
|
|
141
|
+
)
|
|
133
142
|
|
|
134
143
|
@classmethod
|
|
135
144
|
@asynccontextmanager
|
|
@@ -137,7 +146,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
137
146
|
cls: type["_NetworkFileSystem"],
|
|
138
147
|
client: Optional[_Client] = None,
|
|
139
148
|
environment_name: Optional[str] = None,
|
|
140
|
-
_heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
|
149
|
+
_heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, # mdmd:line-hidden
|
|
141
150
|
) -> AsyncIterator["_NetworkFileSystem"]:
|
|
142
151
|
"""Creates a new ephemeral network filesystem within a context manager:
|
|
143
152
|
|
|
@@ -162,69 +171,39 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
162
171
|
async with TaskContext() as tc:
|
|
163
172
|
request = api_pb2.SharedVolumeHeartbeatRequest(shared_volume_id=response.shared_volume_id)
|
|
164
173
|
tc.infinite_loop(lambda: client.stub.SharedVolumeHeartbeat(request), sleep=_heartbeat_sleep)
|
|
165
|
-
yield cls._new_hydrated(
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
environment_name: Optional[str] = None,
|
|
173
|
-
create_if_missing: bool = False,
|
|
174
|
-
) -> "_NetworkFileSystem":
|
|
175
|
-
"""mdmd:hidden
|
|
176
|
-
Lookup a named NetworkFileSystem.
|
|
177
|
-
|
|
178
|
-
DEPRECATED: This method is deprecated in favor of `modal.NetworkFileSystem.from_name`.
|
|
179
|
-
|
|
180
|
-
In contrast to `modal.NetworkFileSystem.from_name`, this is an eager method
|
|
181
|
-
that will hydrate the local object with metadata from Modal servers.
|
|
182
|
-
|
|
183
|
-
```python notest
|
|
184
|
-
nfs = modal.NetworkFileSystem.lookup("my-nfs")
|
|
185
|
-
print(nfs.listdir("/"))
|
|
186
|
-
```
|
|
187
|
-
"""
|
|
188
|
-
deprecation_warning(
|
|
189
|
-
(2025, 1, 27),
|
|
190
|
-
"`modal.NetworkFileSystem.lookup` is deprecated and will be removed in a future release."
|
|
191
|
-
" It can be replaced with `modal.NetworkFileSystem.from_name`."
|
|
192
|
-
"\n\nSee https://modal.com/docs/guide/modal-1-0-migration for more information.",
|
|
193
|
-
)
|
|
194
|
-
obj = _NetworkFileSystem.from_name(
|
|
195
|
-
name, namespace=namespace, environment_name=environment_name, create_if_missing=create_if_missing
|
|
196
|
-
)
|
|
197
|
-
if client is None:
|
|
198
|
-
client = await _Client.from_env()
|
|
199
|
-
resolver = Resolver(client=client)
|
|
200
|
-
await resolver.load(obj)
|
|
201
|
-
return obj
|
|
174
|
+
yield cls._new_hydrated(
|
|
175
|
+
response.shared_volume_id,
|
|
176
|
+
client,
|
|
177
|
+
None,
|
|
178
|
+
is_another_app=True,
|
|
179
|
+
rep="modal.NetworkFileSystem.ephemeral()",
|
|
180
|
+
)
|
|
202
181
|
|
|
203
182
|
@staticmethod
|
|
204
183
|
async def create_deployed(
|
|
205
184
|
deployment_name: str,
|
|
206
|
-
namespace=
|
|
185
|
+
namespace=None, # mdmd:line-hidden
|
|
207
186
|
client: Optional[_Client] = None,
|
|
208
187
|
environment_name: Optional[str] = None,
|
|
209
188
|
) -> str:
|
|
210
189
|
"""mdmd:hidden"""
|
|
211
190
|
check_object_name(deployment_name, "NetworkFileSystem")
|
|
191
|
+
warn_if_passing_namespace(namespace, "modal.NetworkFileSystem.create_deployed")
|
|
212
192
|
if client is None:
|
|
213
193
|
client = await _Client.from_env()
|
|
214
194
|
request = api_pb2.SharedVolumeGetOrCreateRequest(
|
|
215
195
|
deployment_name=deployment_name,
|
|
216
|
-
namespace=namespace,
|
|
217
196
|
environment_name=_get_environment_name(environment_name),
|
|
218
197
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
|
|
219
198
|
)
|
|
220
|
-
resp = await
|
|
199
|
+
resp = await client.stub.SharedVolumeGetOrCreate(request)
|
|
221
200
|
return resp.shared_volume_id
|
|
222
201
|
|
|
223
202
|
@staticmethod
|
|
224
203
|
async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
|
|
225
204
|
obj = await _NetworkFileSystem.from_name(name, environment_name=environment_name).hydrate(client)
|
|
226
205
|
req = api_pb2.SharedVolumeDeleteRequest(shared_volume_id=obj.object_id)
|
|
227
|
-
await
|
|
206
|
+
await obj._client.stub.SharedVolumeDelete(req)
|
|
228
207
|
|
|
229
208
|
@live_method
|
|
230
209
|
async def write_file(self, remote_path: str, fp: BinaryIO, progress_cb: Optional[Callable[..., Any]] = None) -> int:
|
|
@@ -264,7 +243,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
264
243
|
|
|
265
244
|
t0 = time.monotonic()
|
|
266
245
|
while time.monotonic() - t0 < NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT:
|
|
267
|
-
response = await
|
|
246
|
+
response = await self._client.stub.SharedVolumePutFile(req)
|
|
268
247
|
if response.exists:
|
|
269
248
|
break
|
|
270
249
|
else:
|
|
@@ -277,9 +256,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
277
256
|
"""Read a file from the network file system"""
|
|
278
257
|
req = api_pb2.SharedVolumeGetFileRequest(shared_volume_id=self.object_id, path=path)
|
|
279
258
|
try:
|
|
280
|
-
response = await
|
|
281
|
-
except
|
|
282
|
-
raise FileNotFoundError(exc.
|
|
259
|
+
response = await self._client.stub.SharedVolumeGetFile(req)
|
|
260
|
+
except modal.exception.NotFoundError as exc:
|
|
261
|
+
raise FileNotFoundError(exc.args[0])
|
|
262
|
+
|
|
283
263
|
if response.WhichOneof("data_oneof") == "data":
|
|
284
264
|
yield response.data
|
|
285
265
|
else:
|
|
@@ -360,7 +340,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
360
340
|
async def remove_file(self, path: str, recursive=False):
|
|
361
341
|
"""Remove a file in a network file system."""
|
|
362
342
|
req = api_pb2.SharedVolumeRemoveFileRequest(shared_volume_id=self.object_id, path=path, recursive=recursive)
|
|
363
|
-
|
|
343
|
+
try:
|
|
344
|
+
await self._client.stub.SharedVolumeRemoveFile(req)
|
|
345
|
+
except modal.exception.NotFoundError as exc:
|
|
346
|
+
raise FileNotFoundError(exc.args[0])
|
|
364
347
|
|
|
365
348
|
|
|
366
349
|
NetworkFileSystem = synchronize_api(_NetworkFileSystem)
|
modal/network_file_system.pyi
CHANGED
|
@@ -14,32 +14,101 @@ def network_file_system_mount_protos(
|
|
|
14
14
|
) -> list[modal_proto.api_pb2.SharedVolumeMount]: ...
|
|
15
15
|
|
|
16
16
|
class _NetworkFileSystem(modal._object._Object):
|
|
17
|
+
"""A shared, writable file system accessible by one or more Modal functions.
|
|
18
|
+
|
|
19
|
+
By attaching this file system as a mount to one or more functions, they can
|
|
20
|
+
share and persist data with each other.
|
|
21
|
+
|
|
22
|
+
**Note: `NetworkFileSystem` has been deprecated and will be removed.**
|
|
23
|
+
|
|
24
|
+
**Usage**
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import modal
|
|
28
|
+
|
|
29
|
+
nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
|
|
30
|
+
app = modal.App()
|
|
31
|
+
|
|
32
|
+
@app.function(network_file_systems={"/root/foo": nfs})
|
|
33
|
+
def f():
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
@app.function(network_file_systems={"/root/goo": nfs})
|
|
37
|
+
def g():
|
|
38
|
+
pass
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Also see the CLI methods for accessing network file systems:
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
modal nfs --help
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
A `NetworkFileSystem` can also be useful for some local scripting scenarios, e.g.:
|
|
48
|
+
|
|
49
|
+
```python notest
|
|
50
|
+
nfs = modal.NetworkFileSystem.from_name("my-network-file-system")
|
|
51
|
+
for chunk in nfs.read_file("my_db_dump.csv"):
|
|
52
|
+
...
|
|
53
|
+
```
|
|
54
|
+
"""
|
|
17
55
|
@staticmethod
|
|
18
56
|
def from_name(
|
|
19
|
-
name: str,
|
|
20
|
-
|
|
57
|
+
name: str,
|
|
58
|
+
*,
|
|
59
|
+
namespace=None,
|
|
60
|
+
environment_name: typing.Optional[str] = None,
|
|
61
|
+
create_if_missing: bool = False,
|
|
62
|
+
client: typing.Optional[modal.client._Client] = None,
|
|
63
|
+
) -> _NetworkFileSystem:
|
|
64
|
+
"""Reference a NetworkFileSystem by its name, creating if necessary.
|
|
65
|
+
|
|
66
|
+
This is a lazy method that defers hydrating the local object with
|
|
67
|
+
metadata from Modal servers until the first time it is actually
|
|
68
|
+
used.
|
|
69
|
+
|
|
70
|
+
```python notest
|
|
71
|
+
nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
|
|
72
|
+
|
|
73
|
+
@app.function(network_file_systems={"/data": nfs})
|
|
74
|
+
def f():
|
|
75
|
+
pass
|
|
76
|
+
```
|
|
77
|
+
"""
|
|
78
|
+
...
|
|
79
|
+
|
|
21
80
|
@classmethod
|
|
22
81
|
def ephemeral(
|
|
23
82
|
cls: type[_NetworkFileSystem],
|
|
24
83
|
client: typing.Optional[modal.client._Client] = None,
|
|
25
84
|
environment_name: typing.Optional[str] = None,
|
|
26
85
|
_heartbeat_sleep: float = 300,
|
|
27
|
-
) -> typing.AsyncContextManager[_NetworkFileSystem]:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
86
|
+
) -> typing.AsyncContextManager[_NetworkFileSystem]:
|
|
87
|
+
"""Creates a new ephemeral network filesystem within a context manager:
|
|
88
|
+
|
|
89
|
+
Usage:
|
|
90
|
+
```python
|
|
91
|
+
with modal.NetworkFileSystem.ephemeral() as nfs:
|
|
92
|
+
assert nfs.listdir("/") == []
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```python notest
|
|
96
|
+
async with modal.NetworkFileSystem.ephemeral() as nfs:
|
|
97
|
+
assert await nfs.listdir("/") == []
|
|
98
|
+
```
|
|
99
|
+
"""
|
|
100
|
+
...
|
|
101
|
+
|
|
36
102
|
@staticmethod
|
|
37
103
|
async def create_deployed(
|
|
38
104
|
deployment_name: str,
|
|
39
|
-
namespace=
|
|
105
|
+
namespace=None,
|
|
40
106
|
client: typing.Optional[modal.client._Client] = None,
|
|
41
107
|
environment_name: typing.Optional[str] = None,
|
|
42
|
-
) -> str:
|
|
108
|
+
) -> str:
|
|
109
|
+
"""mdmd:hidden"""
|
|
110
|
+
...
|
|
111
|
+
|
|
43
112
|
@staticmethod
|
|
44
113
|
async def delete(
|
|
45
114
|
name: str, client: typing.Optional[modal.client._Client] = None, environment_name: typing.Optional[str] = None
|
|
@@ -49,9 +118,30 @@ class _NetworkFileSystem(modal._object._Object):
|
|
|
49
118
|
remote_path: str,
|
|
50
119
|
fp: typing.BinaryIO,
|
|
51
120
|
progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
|
|
52
|
-
) -> int:
|
|
53
|
-
|
|
54
|
-
|
|
121
|
+
) -> int:
|
|
122
|
+
"""Write from a file object to a path on the network file system, atomically.
|
|
123
|
+
|
|
124
|
+
Will create any needed parent directories automatically.
|
|
125
|
+
|
|
126
|
+
If remote_path ends with `/` it's assumed to be a directory and the
|
|
127
|
+
file will be uploaded with its current name to that directory.
|
|
128
|
+
"""
|
|
129
|
+
...
|
|
130
|
+
|
|
131
|
+
def read_file(self, path: str) -> collections.abc.AsyncIterator[bytes]:
|
|
132
|
+
"""Read a file from the network file system"""
|
|
133
|
+
...
|
|
134
|
+
|
|
135
|
+
def iterdir(self, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]:
|
|
136
|
+
"""Iterate over all files in a directory in the network file system.
|
|
137
|
+
|
|
138
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
139
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
140
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
141
|
+
that glob path (using absolute paths)
|
|
142
|
+
"""
|
|
143
|
+
...
|
|
144
|
+
|
|
55
145
|
async def add_local_file(
|
|
56
146
|
self,
|
|
57
147
|
local_path: typing.Union[pathlib.Path, str],
|
|
@@ -64,64 +154,134 @@ class _NetworkFileSystem(modal._object._Object):
|
|
|
64
154
|
remote_path: typing.Union[str, pathlib.PurePosixPath, None] = None,
|
|
65
155
|
progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
|
|
66
156
|
): ...
|
|
67
|
-
async def listdir(self, path: str) -> list[modal.volume.FileEntry]:
|
|
68
|
-
|
|
157
|
+
async def listdir(self, path: str) -> list[modal.volume.FileEntry]:
|
|
158
|
+
"""List all files in a directory in the network file system.
|
|
159
|
+
|
|
160
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
161
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
162
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
163
|
+
that glob path (using absolute paths)
|
|
164
|
+
"""
|
|
165
|
+
...
|
|
166
|
+
|
|
167
|
+
async def remove_file(self, path: str, recursive=False):
|
|
168
|
+
"""Remove a file in a network file system."""
|
|
169
|
+
...
|
|
69
170
|
|
|
70
171
|
SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
|
|
71
172
|
|
|
72
173
|
class NetworkFileSystem(modal.object.Object):
|
|
73
|
-
|
|
174
|
+
"""A shared, writable file system accessible by one or more Modal functions.
|
|
175
|
+
|
|
176
|
+
By attaching this file system as a mount to one or more functions, they can
|
|
177
|
+
share and persist data with each other.
|
|
178
|
+
|
|
179
|
+
**Note: `NetworkFileSystem` has been deprecated and will be removed.**
|
|
180
|
+
|
|
181
|
+
**Usage**
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
import modal
|
|
185
|
+
|
|
186
|
+
nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
|
|
187
|
+
app = modal.App()
|
|
188
|
+
|
|
189
|
+
@app.function(network_file_systems={"/root/foo": nfs})
|
|
190
|
+
def f():
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
@app.function(network_file_systems={"/root/goo": nfs})
|
|
194
|
+
def g():
|
|
195
|
+
pass
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Also see the CLI methods for accessing network file systems:
|
|
199
|
+
|
|
200
|
+
```
|
|
201
|
+
modal nfs --help
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
A `NetworkFileSystem` can also be useful for some local scripting scenarios, e.g.:
|
|
205
|
+
|
|
206
|
+
```python notest
|
|
207
|
+
nfs = modal.NetworkFileSystem.from_name("my-network-file-system")
|
|
208
|
+
for chunk in nfs.read_file("my_db_dump.csv"):
|
|
209
|
+
...
|
|
210
|
+
```
|
|
211
|
+
"""
|
|
212
|
+
def __init__(self, *args, **kwargs):
|
|
213
|
+
"""mdmd:hidden"""
|
|
214
|
+
...
|
|
215
|
+
|
|
74
216
|
@staticmethod
|
|
75
217
|
def from_name(
|
|
76
|
-
name: str,
|
|
77
|
-
|
|
218
|
+
name: str,
|
|
219
|
+
*,
|
|
220
|
+
namespace=None,
|
|
221
|
+
environment_name: typing.Optional[str] = None,
|
|
222
|
+
create_if_missing: bool = False,
|
|
223
|
+
client: typing.Optional[modal.client.Client] = None,
|
|
224
|
+
) -> NetworkFileSystem:
|
|
225
|
+
"""Reference a NetworkFileSystem by its name, creating if necessary.
|
|
226
|
+
|
|
227
|
+
This is a lazy method that defers hydrating the local object with
|
|
228
|
+
metadata from Modal servers until the first time it is actually
|
|
229
|
+
used.
|
|
230
|
+
|
|
231
|
+
```python notest
|
|
232
|
+
nfs = NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
|
|
233
|
+
|
|
234
|
+
@app.function(network_file_systems={"/data": nfs})
|
|
235
|
+
def f():
|
|
236
|
+
pass
|
|
237
|
+
```
|
|
238
|
+
"""
|
|
239
|
+
...
|
|
240
|
+
|
|
78
241
|
@classmethod
|
|
79
242
|
def ephemeral(
|
|
80
243
|
cls: type[NetworkFileSystem],
|
|
81
244
|
client: typing.Optional[modal.client.Client] = None,
|
|
82
245
|
environment_name: typing.Optional[str] = None,
|
|
83
246
|
_heartbeat_sleep: float = 300,
|
|
84
|
-
) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]:
|
|
247
|
+
) -> synchronicity.combined_types.AsyncAndBlockingContextManager[NetworkFileSystem]:
|
|
248
|
+
"""Creates a new ephemeral network filesystem within a context manager:
|
|
85
249
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
namespace=1,
|
|
92
|
-
client: typing.Optional[modal.client.Client] = None,
|
|
93
|
-
environment_name: typing.Optional[str] = None,
|
|
94
|
-
create_if_missing: bool = False,
|
|
95
|
-
) -> NetworkFileSystem: ...
|
|
96
|
-
async def aio(
|
|
97
|
-
self,
|
|
98
|
-
/,
|
|
99
|
-
name: str,
|
|
100
|
-
namespace=1,
|
|
101
|
-
client: typing.Optional[modal.client.Client] = None,
|
|
102
|
-
environment_name: typing.Optional[str] = None,
|
|
103
|
-
create_if_missing: bool = False,
|
|
104
|
-
) -> NetworkFileSystem: ...
|
|
250
|
+
Usage:
|
|
251
|
+
```python
|
|
252
|
+
with modal.NetworkFileSystem.ephemeral() as nfs:
|
|
253
|
+
assert nfs.listdir("/") == []
|
|
254
|
+
```
|
|
105
255
|
|
|
106
|
-
|
|
256
|
+
```python notest
|
|
257
|
+
async with modal.NetworkFileSystem.ephemeral() as nfs:
|
|
258
|
+
assert await nfs.listdir("/") == []
|
|
259
|
+
```
|
|
260
|
+
"""
|
|
261
|
+
...
|
|
107
262
|
|
|
108
263
|
class __create_deployed_spec(typing_extensions.Protocol):
|
|
109
264
|
def __call__(
|
|
110
265
|
self,
|
|
111
266
|
/,
|
|
112
267
|
deployment_name: str,
|
|
113
|
-
namespace=
|
|
268
|
+
namespace=None,
|
|
114
269
|
client: typing.Optional[modal.client.Client] = None,
|
|
115
270
|
environment_name: typing.Optional[str] = None,
|
|
116
|
-
) -> str:
|
|
271
|
+
) -> str:
|
|
272
|
+
"""mdmd:hidden"""
|
|
273
|
+
...
|
|
274
|
+
|
|
117
275
|
async def aio(
|
|
118
276
|
self,
|
|
119
277
|
/,
|
|
120
278
|
deployment_name: str,
|
|
121
|
-
namespace=
|
|
279
|
+
namespace=None,
|
|
122
280
|
client: typing.Optional[modal.client.Client] = None,
|
|
123
281
|
environment_name: typing.Optional[str] = None,
|
|
124
|
-
) -> str:
|
|
282
|
+
) -> str:
|
|
283
|
+
"""mdmd:hidden"""
|
|
284
|
+
...
|
|
125
285
|
|
|
126
286
|
create_deployed: __create_deployed_spec
|
|
127
287
|
|
|
@@ -150,26 +310,65 @@ class NetworkFileSystem(modal.object.Object):
|
|
|
150
310
|
remote_path: str,
|
|
151
311
|
fp: typing.BinaryIO,
|
|
152
312
|
progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
|
|
153
|
-
) -> int:
|
|
313
|
+
) -> int:
|
|
314
|
+
"""Write from a file object to a path on the network file system, atomically.
|
|
315
|
+
|
|
316
|
+
Will create any needed parent directories automatically.
|
|
317
|
+
|
|
318
|
+
If remote_path ends with `/` it's assumed to be a directory and the
|
|
319
|
+
file will be uploaded with its current name to that directory.
|
|
320
|
+
"""
|
|
321
|
+
...
|
|
322
|
+
|
|
154
323
|
async def aio(
|
|
155
324
|
self,
|
|
156
325
|
/,
|
|
157
326
|
remote_path: str,
|
|
158
327
|
fp: typing.BinaryIO,
|
|
159
328
|
progress_cb: typing.Optional[collections.abc.Callable[..., typing.Any]] = None,
|
|
160
|
-
) -> int:
|
|
329
|
+
) -> int:
|
|
330
|
+
"""Write from a file object to a path on the network file system, atomically.
|
|
331
|
+
|
|
332
|
+
Will create any needed parent directories automatically.
|
|
333
|
+
|
|
334
|
+
If remote_path ends with `/` it's assumed to be a directory and the
|
|
335
|
+
file will be uploaded with its current name to that directory.
|
|
336
|
+
"""
|
|
337
|
+
...
|
|
161
338
|
|
|
162
339
|
write_file: __write_file_spec[typing_extensions.Self]
|
|
163
340
|
|
|
164
341
|
class __read_file_spec(typing_extensions.Protocol[SUPERSELF]):
|
|
165
|
-
def __call__(self, /, path: str) -> typing.Iterator[bytes]:
|
|
166
|
-
|
|
342
|
+
def __call__(self, /, path: str) -> typing.Iterator[bytes]:
|
|
343
|
+
"""Read a file from the network file system"""
|
|
344
|
+
...
|
|
345
|
+
|
|
346
|
+
def aio(self, /, path: str) -> collections.abc.AsyncIterator[bytes]:
|
|
347
|
+
"""Read a file from the network file system"""
|
|
348
|
+
...
|
|
167
349
|
|
|
168
350
|
read_file: __read_file_spec[typing_extensions.Self]
|
|
169
351
|
|
|
170
352
|
class __iterdir_spec(typing_extensions.Protocol[SUPERSELF]):
|
|
171
|
-
def __call__(self, /, path: str) -> typing.Iterator[modal.volume.FileEntry]:
|
|
172
|
-
|
|
353
|
+
def __call__(self, /, path: str) -> typing.Iterator[modal.volume.FileEntry]:
|
|
354
|
+
"""Iterate over all files in a directory in the network file system.
|
|
355
|
+
|
|
356
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
357
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
358
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
359
|
+
that glob path (using absolute paths)
|
|
360
|
+
"""
|
|
361
|
+
...
|
|
362
|
+
|
|
363
|
+
def aio(self, /, path: str) -> collections.abc.AsyncIterator[modal.volume.FileEntry]:
|
|
364
|
+
"""Iterate over all files in a directory in the network file system.
|
|
365
|
+
|
|
366
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
367
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
368
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
369
|
+
that glob path (using absolute paths)
|
|
370
|
+
"""
|
|
371
|
+
...
|
|
173
372
|
|
|
174
373
|
iterdir: __iterdir_spec[typing_extensions.Self]
|
|
175
374
|
|
|
@@ -210,13 +409,35 @@ class NetworkFileSystem(modal.object.Object):
|
|
|
210
409
|
add_local_dir: __add_local_dir_spec[typing_extensions.Self]
|
|
211
410
|
|
|
212
411
|
class __listdir_spec(typing_extensions.Protocol[SUPERSELF]):
|
|
213
|
-
def __call__(self, /, path: str) -> list[modal.volume.FileEntry]:
|
|
214
|
-
|
|
412
|
+
def __call__(self, /, path: str) -> list[modal.volume.FileEntry]:
|
|
413
|
+
"""List all files in a directory in the network file system.
|
|
414
|
+
|
|
415
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
416
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
417
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
418
|
+
that glob path (using absolute paths)
|
|
419
|
+
"""
|
|
420
|
+
...
|
|
421
|
+
|
|
422
|
+
async def aio(self, /, path: str) -> list[modal.volume.FileEntry]:
|
|
423
|
+
"""List all files in a directory in the network file system.
|
|
424
|
+
|
|
425
|
+
* Passing a directory path lists all files in the directory (names are relative to the directory)
|
|
426
|
+
* Passing a file path returns a list containing only that file's listing description
|
|
427
|
+
* Passing a glob path (including at least one * or ** sequence) returns all files matching
|
|
428
|
+
that glob path (using absolute paths)
|
|
429
|
+
"""
|
|
430
|
+
...
|
|
215
431
|
|
|
216
432
|
listdir: __listdir_spec[typing_extensions.Self]
|
|
217
433
|
|
|
218
434
|
class __remove_file_spec(typing_extensions.Protocol[SUPERSELF]):
|
|
219
|
-
def __call__(self, /, path: str, recursive=False):
|
|
220
|
-
|
|
435
|
+
def __call__(self, /, path: str, recursive=False):
|
|
436
|
+
"""Remove a file in a network file system."""
|
|
437
|
+
...
|
|
438
|
+
|
|
439
|
+
async def aio(self, /, path: str, recursive=False):
|
|
440
|
+
"""Remove a file in a network file system."""
|
|
441
|
+
...
|
|
221
442
|
|
|
222
443
|
remove_file: __remove_file_spec[typing_extensions.Self]
|