modal 1.1.5.dev83__py3-none-any.whl → 1.3.1.dev8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of modal might be problematic. Click here for more details.
- modal/__init__.py +4 -4
- modal/__main__.py +4 -29
- modal/_billing.py +84 -0
- modal/_clustered_functions.py +1 -3
- modal/_container_entrypoint.py +33 -208
- modal/_functions.py +146 -121
- modal/_grpc_client.py +191 -0
- modal/_ipython.py +16 -6
- modal/_load_context.py +106 -0
- modal/_object.py +72 -21
- modal/_output.py +12 -14
- modal/_partial_function.py +31 -4
- modal/_resolver.py +44 -57
- modal/_runtime/container_io_manager.py +26 -28
- modal/_runtime/container_io_manager.pyi +42 -44
- modal/_runtime/gpu_memory_snapshot.py +9 -7
- modal/_runtime/user_code_event_loop.py +80 -0
- modal/_runtime/user_code_imports.py +236 -10
- modal/_serialization.py +2 -1
- modal/_traceback.py +4 -13
- modal/_tunnel.py +16 -11
- modal/_tunnel.pyi +25 -3
- modal/_utils/async_utils.py +337 -10
- modal/_utils/auth_token_manager.py +1 -4
- modal/_utils/blob_utils.py +29 -22
- modal/_utils/function_utils.py +20 -21
- modal/_utils/grpc_testing.py +6 -3
- modal/_utils/grpc_utils.py +223 -64
- modal/_utils/mount_utils.py +26 -1
- modal/_utils/package_utils.py +0 -1
- modal/_utils/rand_pb_testing.py +8 -1
- modal/_utils/task_command_router_client.py +524 -0
- modal/_vendor/cloudpickle.py +144 -48
- modal/app.py +215 -96
- modal/app.pyi +78 -37
- modal/billing.py +5 -0
- modal/builder/2025.06.txt +6 -3
- modal/builder/PREVIEW.txt +2 -1
- modal/builder/base-images.json +4 -2
- modal/cli/_download.py +19 -3
- modal/cli/cluster.py +4 -2
- modal/cli/config.py +3 -1
- modal/cli/container.py +5 -4
- modal/cli/dict.py +5 -2
- modal/cli/entry_point.py +26 -2
- modal/cli/environment.py +2 -16
- modal/cli/launch.py +1 -76
- modal/cli/network_file_system.py +5 -20
- modal/cli/queues.py +5 -4
- modal/cli/run.py +24 -204
- modal/cli/secret.py +1 -2
- modal/cli/shell.py +375 -0
- modal/cli/utils.py +1 -13
- modal/cli/volume.py +11 -17
- modal/client.py +16 -125
- modal/client.pyi +94 -144
- modal/cloud_bucket_mount.py +3 -1
- modal/cloud_bucket_mount.pyi +4 -0
- modal/cls.py +101 -64
- modal/cls.pyi +9 -8
- modal/config.py +21 -1
- modal/container_process.py +288 -12
- modal/container_process.pyi +99 -38
- modal/dict.py +72 -33
- modal/dict.pyi +88 -57
- modal/environments.py +16 -8
- modal/environments.pyi +6 -2
- modal/exception.py +154 -16
- modal/experimental/__init__.py +23 -5
- modal/experimental/flash.py +161 -74
- modal/experimental/flash.pyi +97 -49
- modal/file_io.py +50 -92
- modal/file_io.pyi +117 -89
- modal/functions.pyi +70 -87
- modal/image.py +73 -47
- modal/image.pyi +33 -30
- modal/io_streams.py +500 -149
- modal/io_streams.pyi +279 -189
- modal/mount.py +60 -45
- modal/mount.pyi +41 -17
- modal/network_file_system.py +19 -11
- modal/network_file_system.pyi +72 -39
- modal/object.pyi +114 -22
- modal/parallel_map.py +42 -44
- modal/parallel_map.pyi +9 -17
- modal/partial_function.pyi +4 -2
- modal/proxy.py +14 -6
- modal/proxy.pyi +10 -2
- modal/queue.py +45 -38
- modal/queue.pyi +88 -52
- modal/runner.py +96 -96
- modal/runner.pyi +44 -27
- modal/sandbox.py +225 -108
- modal/sandbox.pyi +226 -63
- modal/secret.py +58 -56
- modal/secret.pyi +28 -13
- modal/serving.py +7 -11
- modal/serving.pyi +7 -8
- modal/snapshot.py +29 -15
- modal/snapshot.pyi +18 -10
- modal/token_flow.py +1 -1
- modal/token_flow.pyi +4 -6
- modal/volume.py +102 -55
- modal/volume.pyi +125 -66
- {modal-1.1.5.dev83.dist-info → modal-1.3.1.dev8.dist-info}/METADATA +10 -9
- modal-1.3.1.dev8.dist-info/RECORD +189 -0
- modal_proto/api.proto +86 -30
- modal_proto/api_grpc.py +10 -25
- modal_proto/api_pb2.py +1080 -1047
- modal_proto/api_pb2.pyi +253 -79
- modal_proto/api_pb2_grpc.py +14 -48
- modal_proto/api_pb2_grpc.pyi +6 -18
- modal_proto/modal_api_grpc.py +175 -176
- modal_proto/{sandbox_router.proto → task_command_router.proto} +62 -45
- modal_proto/task_command_router_grpc.py +138 -0
- modal_proto/task_command_router_pb2.py +180 -0
- modal_proto/{sandbox_router_pb2.pyi → task_command_router_pb2.pyi} +110 -63
- modal_proto/task_command_router_pb2_grpc.py +272 -0
- modal_proto/task_command_router_pb2_grpc.pyi +100 -0
- modal_version/__init__.py +1 -1
- modal_version/__main__.py +1 -1
- modal/cli/programs/launch_instance_ssh.py +0 -94
- modal/cli/programs/run_marimo.py +0 -95
- modal-1.1.5.dev83.dist-info/RECORD +0 -191
- modal_proto/modal_options_grpc.py +0 -3
- modal_proto/options.proto +0 -19
- modal_proto/options_grpc.py +0 -3
- modal_proto/options_pb2.py +0 -35
- modal_proto/options_pb2.pyi +0 -20
- modal_proto/options_pb2_grpc.py +0 -4
- modal_proto/options_pb2_grpc.pyi +0 -7
- modal_proto/sandbox_router_grpc.py +0 -105
- modal_proto/sandbox_router_pb2.py +0 -148
- modal_proto/sandbox_router_pb2_grpc.py +0 -203
- modal_proto/sandbox_router_pb2_grpc.pyi +0 -75
- {modal-1.1.5.dev83.dist-info → modal-1.3.1.dev8.dist-info}/WHEEL +0 -0
- {modal-1.1.5.dev83.dist-info → modal-1.3.1.dev8.dist-info}/entry_points.txt +0 -0
- {modal-1.1.5.dev83.dist-info → modal-1.3.1.dev8.dist-info}/licenses/LICENSE +0 -0
- {modal-1.1.5.dev83.dist-info → modal-1.3.1.dev8.dist-info}/top_level.txt +0 -0
modal/mount.py
CHANGED
|
@@ -13,18 +13,18 @@ from pathlib import Path, PurePosixPath
|
|
|
13
13
|
from typing import Callable, Optional, Sequence, Union
|
|
14
14
|
|
|
15
15
|
from google.protobuf.message import Message
|
|
16
|
-
from grpclib import GRPCError
|
|
17
16
|
|
|
18
17
|
import modal.exception
|
|
19
18
|
import modal.file_pattern_matcher
|
|
20
19
|
from modal_proto import api_pb2
|
|
21
20
|
from modal_version import __version__
|
|
22
21
|
|
|
23
|
-
from .
|
|
22
|
+
from ._load_context import LoadContext
|
|
23
|
+
from ._object import _Object
|
|
24
24
|
from ._resolver import Resolver
|
|
25
25
|
from ._utils.async_utils import TaskContext, aclosing, async_map, synchronize_api
|
|
26
26
|
from ._utils.blob_utils import FileUploadSpec, blob_upload_file, get_file_upload_spec_from_path
|
|
27
|
-
from ._utils.grpc_utils import
|
|
27
|
+
from ._utils.grpc_utils import Retry
|
|
28
28
|
from ._utils.name_utils import check_object_name
|
|
29
29
|
from ._utils.package_utils import get_module_mount_info
|
|
30
30
|
from .client import _Client
|
|
@@ -40,11 +40,11 @@ MOUNT_PUT_FILE_CLIENT_TIMEOUT = 10 * 60 # 10 min max for transferring files
|
|
|
40
40
|
# These can be updated safely, but changes will trigger a rebuild for all images
|
|
41
41
|
# that rely on `add_python()` in their constructor.
|
|
42
42
|
PYTHON_STANDALONE_VERSIONS: dict[str, tuple[str, str]] = {
|
|
43
|
-
"3.9": ("20230826", "3.9.18"),
|
|
44
43
|
"3.10": ("20230826", "3.10.13"),
|
|
45
44
|
"3.11": ("20230826", "3.11.5"),
|
|
46
45
|
"3.12": ("20240107", "3.12.1"),
|
|
47
46
|
"3.13": ("20241008", "3.13.0"),
|
|
47
|
+
"3.14": ("20251205", "3.14.2"),
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
MOUNT_DEPRECATION_MESSAGE_PATTERN = """modal.Mount usage will soon be deprecated.
|
|
@@ -310,7 +310,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
310
310
|
_entries: Optional[list[_MountEntry]] = None
|
|
311
311
|
_deployment_name: Optional[str] = None
|
|
312
312
|
_namespace: Optional[int] = None
|
|
313
|
-
|
|
313
|
+
|
|
314
314
|
_allow_overwrite: bool = False
|
|
315
315
|
_content_checksum_sha256_hex: Optional[str] = None
|
|
316
316
|
|
|
@@ -325,7 +325,12 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
325
325
|
return None
|
|
326
326
|
return (_Mount._type_prefix, "local", frozenset(included_files))
|
|
327
327
|
|
|
328
|
-
obj = _Mount._from_loader(
|
|
328
|
+
obj = _Mount._from_loader(
|
|
329
|
+
_Mount._load_mount,
|
|
330
|
+
rep,
|
|
331
|
+
deduplication_key=mount_content_deduplication_key,
|
|
332
|
+
load_context_overrides=LoadContext.empty(),
|
|
333
|
+
)
|
|
329
334
|
obj._entries = entries
|
|
330
335
|
obj._is_local = True
|
|
331
336
|
return obj
|
|
@@ -460,16 +465,18 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
460
465
|
loop = asyncio.get_event_loop()
|
|
461
466
|
with concurrent.futures.ThreadPoolExecutor() as exe:
|
|
462
467
|
all_files = await loop.run_in_executor(exe, _select_files, entries)
|
|
468
|
+
logger.debug(f"Computing checksums for {len(all_files)} files using {exe._max_workers} worker threads")
|
|
463
469
|
|
|
464
|
-
|
|
470
|
+
# Yield FileUploadSpec objects lazily as they're consumed by async_map downstream.
|
|
471
|
+
# async_map's concurrency limit provides natural backpressure, so we don't need
|
|
472
|
+
# a separate semaphore here. This keeps memory bounded without creating all tasks upfront.
|
|
465
473
|
for local_filename, remote_filename in all_files:
|
|
466
|
-
logger.debug(f"Mounting {local_filename} as {remote_filename}")
|
|
467
|
-
futs.append(loop.run_in_executor(exe, get_file_upload_spec_from_path, local_filename, remote_filename))
|
|
468
|
-
|
|
469
|
-
logger.debug(f"Computing checksums for {len(futs)} files using {exe._max_workers} worker threads")
|
|
470
|
-
for fut in asyncio.as_completed(futs):
|
|
471
474
|
try:
|
|
472
|
-
|
|
475
|
+
logger.debug(f"Mounting {local_filename} as {remote_filename}")
|
|
476
|
+
file_spec = await loop.run_in_executor(
|
|
477
|
+
exe, get_file_upload_spec_from_path, local_filename, remote_filename
|
|
478
|
+
)
|
|
479
|
+
yield file_spec
|
|
473
480
|
except FileNotFoundError as exc:
|
|
474
481
|
# Can happen with temporary files (e.g. emacs will write temp files and delete them quickly)
|
|
475
482
|
logger.info(f"Ignoring file not found: {exc}")
|
|
@@ -477,6 +484,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
477
484
|
async def _load_mount(
|
|
478
485
|
self: "_Mount",
|
|
479
486
|
resolver: Resolver,
|
|
487
|
+
load_context: LoadContext,
|
|
480
488
|
existing_object_id: Optional[str],
|
|
481
489
|
):
|
|
482
490
|
t0 = time.monotonic()
|
|
@@ -518,7 +526,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
518
526
|
|
|
519
527
|
request = api_pb2.MountPutFileRequest(sha256_hex=file_spec.sha256_hex)
|
|
520
528
|
accounted_hashes.add(file_spec.sha256_hex)
|
|
521
|
-
response = await
|
|
529
|
+
response = await load_context.client.stub.MountPutFile(request, retry=Retry(base_delay=1))
|
|
522
530
|
|
|
523
531
|
if response.exists:
|
|
524
532
|
n_finished += 1
|
|
@@ -532,7 +540,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
532
540
|
async with blob_upload_concurrency:
|
|
533
541
|
with file_spec.source() as fp:
|
|
534
542
|
blob_id = await blob_upload_file(
|
|
535
|
-
fp,
|
|
543
|
+
fp, load_context.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
|
|
536
544
|
)
|
|
537
545
|
logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
|
|
538
546
|
request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
|
|
@@ -540,11 +548,15 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
540
548
|
logger.debug(
|
|
541
549
|
f"Uploading file {file_spec.source_description} to {remote_filename} ({file_spec.size} bytes)"
|
|
542
550
|
)
|
|
543
|
-
|
|
551
|
+
if file_spec.content is None:
|
|
552
|
+
content = await asyncio.to_thread(file_spec.read_content)
|
|
553
|
+
else:
|
|
554
|
+
content = file_spec.content
|
|
555
|
+
request2 = api_pb2.MountPutFileRequest(data=content, sha256_hex=file_spec.sha256_hex)
|
|
544
556
|
|
|
545
557
|
start_time = time.monotonic()
|
|
546
558
|
while time.monotonic() - start_time < MOUNT_PUT_FILE_CLIENT_TIMEOUT:
|
|
547
|
-
response = await
|
|
559
|
+
response = await load_context.client.stub.MountPutFile(request2, retry=Retry(base_delay=1))
|
|
548
560
|
if response.exists:
|
|
549
561
|
n_finished += 1
|
|
550
562
|
return mount_file
|
|
@@ -552,7 +564,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
552
564
|
raise modal.exception.MountUploadTimeoutError(f"Mounting of {file_spec.source_description} timed out")
|
|
553
565
|
|
|
554
566
|
# Upload files, or check if they already exist.
|
|
555
|
-
n_concurrent_uploads =
|
|
567
|
+
n_concurrent_uploads = 64
|
|
556
568
|
files: list[api_pb2.MountFile] = []
|
|
557
569
|
async with aclosing(
|
|
558
570
|
async_map(_Mount._get_files(self._entries), _put_file, concurrency=n_concurrent_uploads)
|
|
@@ -574,28 +586,28 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
574
586
|
req = api_pb2.MountGetOrCreateRequest(
|
|
575
587
|
deployment_name=self._deployment_name,
|
|
576
588
|
namespace=self._namespace,
|
|
577
|
-
environment_name=
|
|
589
|
+
environment_name=load_context.environment_name,
|
|
578
590
|
object_creation_type=creation_type,
|
|
579
591
|
files=files,
|
|
580
592
|
)
|
|
581
|
-
elif
|
|
593
|
+
elif load_context.app_id is not None:
|
|
582
594
|
req = api_pb2.MountGetOrCreateRequest(
|
|
583
595
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_ANONYMOUS_OWNED_BY_APP,
|
|
584
596
|
files=files,
|
|
585
|
-
app_id=
|
|
597
|
+
app_id=load_context.app_id,
|
|
586
598
|
)
|
|
587
599
|
else:
|
|
588
600
|
req = api_pb2.MountGetOrCreateRequest(
|
|
589
601
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_EPHEMERAL,
|
|
590
602
|
files=files,
|
|
591
|
-
environment_name=
|
|
603
|
+
environment_name=load_context.environment_name,
|
|
592
604
|
)
|
|
593
605
|
|
|
594
|
-
resp = await
|
|
606
|
+
resp = await load_context.client.stub.MountGetOrCreate(req, retry=Retry(base_delay=1))
|
|
595
607
|
status_row.finish(f"Created mount {message_label}")
|
|
596
608
|
|
|
597
609
|
logger.debug(f"Uploaded {total_uploads} new files and {total_bytes} bytes in {time.monotonic() - t0}s")
|
|
598
|
-
self._hydrate(resp.mount_id,
|
|
610
|
+
self._hydrate(resp.mount_id, load_context.client, resp.handle_metadata)
|
|
599
611
|
|
|
600
612
|
@staticmethod
|
|
601
613
|
def _from_local_python_packages(
|
|
@@ -628,19 +640,25 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
628
640
|
*,
|
|
629
641
|
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
|
|
630
642
|
environment_name: Optional[str] = None,
|
|
643
|
+
client: Optional[_Client] = None,
|
|
631
644
|
) -> "_Mount":
|
|
632
645
|
"""mdmd:hidden"""
|
|
633
646
|
|
|
634
|
-
async def _load(provider: _Mount, resolver: Resolver, existing_object_id: Optional[str]):
|
|
647
|
+
async def _load(provider: _Mount, resolver: Resolver, load_context, existing_object_id: Optional[str]):
|
|
635
648
|
req = api_pb2.MountGetOrCreateRequest(
|
|
636
649
|
deployment_name=name,
|
|
637
650
|
namespace=namespace,
|
|
638
|
-
environment_name=
|
|
651
|
+
environment_name=load_context.environment_name,
|
|
639
652
|
)
|
|
640
|
-
response = await
|
|
641
|
-
provider._hydrate(response.mount_id,
|
|
642
|
-
|
|
643
|
-
return _Mount._from_loader(
|
|
653
|
+
response = await load_context.client.stub.MountGetOrCreate(req)
|
|
654
|
+
provider._hydrate(response.mount_id, load_context.client, response.handle_metadata)
|
|
655
|
+
|
|
656
|
+
return _Mount._from_loader(
|
|
657
|
+
_load,
|
|
658
|
+
"Mount()",
|
|
659
|
+
hydrate_lazily=True,
|
|
660
|
+
load_context_overrides=LoadContext(environment_name=environment_name, client=client),
|
|
661
|
+
)
|
|
644
662
|
|
|
645
663
|
async def _deploy(
|
|
646
664
|
self: "_Mount",
|
|
@@ -652,15 +670,12 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
652
670
|
client: Optional[_Client] = None,
|
|
653
671
|
) -> None:
|
|
654
672
|
check_object_name(deployment_name, "Mount")
|
|
655
|
-
environment_name = _get_environment_name(environment_name, resolver=None)
|
|
656
673
|
self._deployment_name = deployment_name
|
|
657
674
|
self._namespace = namespace
|
|
658
|
-
self._environment_name = environment_name
|
|
659
675
|
self._allow_overwrite = allow_overwrite
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
resolver
|
|
663
|
-
await resolver.load(self)
|
|
676
|
+
resolver = Resolver()
|
|
677
|
+
root_metadata = LoadContext(client=client, environment_name=environment_name)
|
|
678
|
+
await resolver.load(self, root_metadata)
|
|
664
679
|
|
|
665
680
|
def _get_metadata(self) -> api_pb2.MountHandleMetadata:
|
|
666
681
|
if self._content_checksum_sha256_hex is None:
|
|
@@ -759,13 +774,13 @@ async def _create_single_client_dependency_mount(
|
|
|
759
774
|
if check_if_exists:
|
|
760
775
|
try:
|
|
761
776
|
await Mount.from_name(mount_name, namespace=api_pb2.DEPLOYMENT_NAMESPACE_GLOBAL).hydrate.aio(client)
|
|
762
|
-
print(f"➖ Found existing mount {mount_name} in global namespace.")
|
|
777
|
+
print(f"➖ Found existing mount {mount_name} in global namespace.") # noqa: T201
|
|
763
778
|
return
|
|
764
779
|
except modal.exception.NotFoundError:
|
|
765
780
|
pass
|
|
766
781
|
|
|
767
782
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpd:
|
|
768
|
-
print(f"📦 Building {mount_name}.")
|
|
783
|
+
print(f"📦 Building {mount_name}.") # noqa: T201
|
|
769
784
|
requirements = os.path.join(os.path.dirname(__file__), f"builder/{builder_version}.txt")
|
|
770
785
|
cmd = " ".join(
|
|
771
786
|
[
|
|
@@ -794,11 +809,11 @@ async def _create_single_client_dependency_mount(
|
|
|
794
809
|
await proc.wait()
|
|
795
810
|
if proc.returncode:
|
|
796
811
|
stdout, stderr = await proc.communicate()
|
|
797
|
-
print(stdout.decode("utf-8"))
|
|
798
|
-
print(stderr.decode("utf-8"))
|
|
812
|
+
print(stdout.decode("utf-8")) # noqa: T201
|
|
813
|
+
print(stderr.decode("utf-8")) # noqa: T201
|
|
799
814
|
raise RuntimeError(f"Subprocess failed with {proc.returncode}")
|
|
800
815
|
|
|
801
|
-
print(f"🌐 Downloaded and unpacked {mount_name} packages to {tmpd}.")
|
|
816
|
+
print(f"🌐 Downloaded and unpacked {mount_name} packages to {tmpd}.") # noqa: T201
|
|
802
817
|
|
|
803
818
|
python_mount = Mount._from_local_dir(tmpd, remote_path=REMOTE_PACKAGES_PATH)
|
|
804
819
|
|
|
@@ -822,11 +837,11 @@ async def _create_single_client_dependency_mount(
|
|
|
822
837
|
allow_overwrite=allow_overwrite,
|
|
823
838
|
client=client,
|
|
824
839
|
)
|
|
825
|
-
print(f"✅ Deployed mount {mount_name} to global namespace.")
|
|
826
|
-
except
|
|
827
|
-
print(f"⚠️ Mount creation failed with {e.
|
|
840
|
+
print(f"✅ Deployed mount {mount_name} to global namespace.") # noqa: T201
|
|
841
|
+
except modal.exception.Error as e:
|
|
842
|
+
print(f"⚠️ Mount creation failed with {type(e).__name__}: {e}") # noqa: T201
|
|
828
843
|
else:
|
|
829
|
-
print(f"Dry run - skipping deployment of mount {mount_name}")
|
|
844
|
+
print(f"Dry run - skipping deployment of mount {mount_name}") # noqa: T201
|
|
830
845
|
|
|
831
846
|
|
|
832
847
|
async def _create_client_dependency_mounts(
|
modal/mount.pyi
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import collections.abc
|
|
2
2
|
import google.protobuf.message
|
|
3
|
+
import modal._load_context
|
|
3
4
|
import modal._object
|
|
4
5
|
import modal._resolver
|
|
5
6
|
import modal._utils.blob_utils
|
|
@@ -154,7 +155,6 @@ class _Mount(modal._object._Object):
|
|
|
154
155
|
_entries: typing.Optional[list[_MountEntry]]
|
|
155
156
|
_deployment_name: typing.Optional[str]
|
|
156
157
|
_namespace: typing.Optional[int]
|
|
157
|
-
_environment_name: typing.Optional[str]
|
|
158
158
|
_allow_overwrite: bool
|
|
159
159
|
_content_checksum_sha256_hex: typing.Optional[str]
|
|
160
160
|
|
|
@@ -216,7 +216,10 @@ class _Mount(modal._object._Object):
|
|
|
216
216
|
entries: list[_MountEntry],
|
|
217
217
|
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
|
218
218
|
async def _load_mount(
|
|
219
|
-
self: _Mount,
|
|
219
|
+
self: _Mount,
|
|
220
|
+
resolver: modal._resolver.Resolver,
|
|
221
|
+
load_context: modal._load_context.LoadContext,
|
|
222
|
+
existing_object_id: typing.Optional[str],
|
|
220
223
|
): ...
|
|
221
224
|
@staticmethod
|
|
222
225
|
def _from_local_python_packages(
|
|
@@ -226,7 +229,13 @@ class _Mount(modal._object._Object):
|
|
|
226
229
|
ignore: typing.Union[typing.Sequence[str], collections.abc.Callable[[pathlib.Path], bool], None] = None,
|
|
227
230
|
) -> _Mount: ...
|
|
228
231
|
@staticmethod
|
|
229
|
-
def from_name(
|
|
232
|
+
def from_name(
|
|
233
|
+
name: str,
|
|
234
|
+
*,
|
|
235
|
+
namespace=1,
|
|
236
|
+
environment_name: typing.Optional[str] = None,
|
|
237
|
+
client: typing.Optional[modal.client._Client] = None,
|
|
238
|
+
) -> _Mount:
|
|
230
239
|
"""mdmd:hidden"""
|
|
231
240
|
...
|
|
232
241
|
|
|
@@ -241,8 +250,6 @@ class _Mount(modal._object._Object):
|
|
|
241
250
|
) -> None: ...
|
|
242
251
|
def _get_metadata(self) -> modal_proto.api_pb2.MountHandleMetadata: ...
|
|
243
252
|
|
|
244
|
-
SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
|
|
245
|
-
|
|
246
253
|
class Mount(modal.object.Object):
|
|
247
254
|
"""**Deprecated**: Mounts should not be used explicitly anymore, use `Image.add_local_*` commands instead.
|
|
248
255
|
|
|
@@ -269,7 +276,6 @@ class Mount(modal.object.Object):
|
|
|
269
276
|
_entries: typing.Optional[list[_MountEntry]]
|
|
270
277
|
_deployment_name: typing.Optional[str]
|
|
271
278
|
_namespace: typing.Optional[int]
|
|
272
|
-
_environment_name: typing.Optional[str]
|
|
273
279
|
_allow_overwrite: bool
|
|
274
280
|
_content_checksum_sha256_hex: typing.Optional[str]
|
|
275
281
|
|
|
@@ -339,13 +345,25 @@ class Mount(modal.object.Object):
|
|
|
339
345
|
self, /, entries: list[_MountEntry]
|
|
340
346
|
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
|
341
347
|
|
|
342
|
-
_get_files: ___get_files_spec
|
|
348
|
+
_get_files: typing.ClassVar[___get_files_spec]
|
|
343
349
|
|
|
344
|
-
class ___load_mount_spec(typing_extensions.Protocol
|
|
345
|
-
def __call__(
|
|
346
|
-
|
|
350
|
+
class ___load_mount_spec(typing_extensions.Protocol):
|
|
351
|
+
def __call__(
|
|
352
|
+
self,
|
|
353
|
+
/,
|
|
354
|
+
resolver: modal._resolver.Resolver,
|
|
355
|
+
load_context: modal._load_context.LoadContext,
|
|
356
|
+
existing_object_id: typing.Optional[str],
|
|
357
|
+
): ...
|
|
358
|
+
async def aio(
|
|
359
|
+
self,
|
|
360
|
+
/,
|
|
361
|
+
resolver: modal._resolver.Resolver,
|
|
362
|
+
load_context: modal._load_context.LoadContext,
|
|
363
|
+
existing_object_id: typing.Optional[str],
|
|
364
|
+
): ...
|
|
347
365
|
|
|
348
|
-
_load_mount: ___load_mount_spec
|
|
366
|
+
_load_mount: ___load_mount_spec
|
|
349
367
|
|
|
350
368
|
@staticmethod
|
|
351
369
|
def _from_local_python_packages(
|
|
@@ -355,11 +373,17 @@ class Mount(modal.object.Object):
|
|
|
355
373
|
ignore: typing.Union[typing.Sequence[str], collections.abc.Callable[[pathlib.Path], bool], None] = None,
|
|
356
374
|
) -> Mount: ...
|
|
357
375
|
@staticmethod
|
|
358
|
-
def from_name(
|
|
376
|
+
def from_name(
|
|
377
|
+
name: str,
|
|
378
|
+
*,
|
|
379
|
+
namespace=1,
|
|
380
|
+
environment_name: typing.Optional[str] = None,
|
|
381
|
+
client: typing.Optional[modal.client.Client] = None,
|
|
382
|
+
) -> Mount:
|
|
359
383
|
"""mdmd:hidden"""
|
|
360
384
|
...
|
|
361
385
|
|
|
362
|
-
class ___deploy_spec(typing_extensions.Protocol
|
|
386
|
+
class ___deploy_spec(typing_extensions.Protocol):
|
|
363
387
|
def __call__(
|
|
364
388
|
self,
|
|
365
389
|
/,
|
|
@@ -381,7 +405,7 @@ class Mount(modal.object.Object):
|
|
|
381
405
|
client: typing.Optional[modal.client.Client] = None,
|
|
382
406
|
) -> None: ...
|
|
383
407
|
|
|
384
|
-
_deploy: ___deploy_spec
|
|
408
|
+
_deploy: ___deploy_spec
|
|
385
409
|
|
|
386
410
|
def _get_metadata(self) -> modal_proto.api_pb2.MountHandleMetadata: ...
|
|
387
411
|
|
|
@@ -402,7 +426,7 @@ async def _create_single_client_dependency_mount(
|
|
|
402
426
|
): ...
|
|
403
427
|
async def _create_client_dependency_mounts(
|
|
404
428
|
client=None,
|
|
405
|
-
python_versions: list[str] = ["3.
|
|
429
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
406
430
|
builder_versions: list[str] = ["2025.06"],
|
|
407
431
|
check_if_exists=True,
|
|
408
432
|
dry_run=False,
|
|
@@ -413,7 +437,7 @@ class __create_client_dependency_mounts_spec(typing_extensions.Protocol):
|
|
|
413
437
|
self,
|
|
414
438
|
/,
|
|
415
439
|
client=None,
|
|
416
|
-
python_versions: list[str] = ["3.
|
|
440
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
417
441
|
builder_versions: list[str] = ["2025.06"],
|
|
418
442
|
check_if_exists=True,
|
|
419
443
|
dry_run=False,
|
|
@@ -422,7 +446,7 @@ class __create_client_dependency_mounts_spec(typing_extensions.Protocol):
|
|
|
422
446
|
self,
|
|
423
447
|
/,
|
|
424
448
|
client=None,
|
|
425
|
-
python_versions: list[str] = ["3.
|
|
449
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
426
450
|
builder_versions: list[str] = ["2025.06"],
|
|
427
451
|
check_if_exists=True,
|
|
428
452
|
dry_run=False,
|
modal/network_file_system.py
CHANGED
|
@@ -11,6 +11,7 @@ from synchronicity.async_wrap import asynccontextmanager
|
|
|
11
11
|
import modal
|
|
12
12
|
from modal_proto import api_pb2
|
|
13
13
|
|
|
14
|
+
from ._load_context import LoadContext
|
|
14
15
|
from ._object import (
|
|
15
16
|
EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
|
16
17
|
_get_environment_name,
|
|
@@ -22,7 +23,6 @@ from ._resolver import Resolver
|
|
|
22
23
|
from ._utils.async_utils import TaskContext, aclosing, async_map, sync_or_async_iter, synchronize_api
|
|
23
24
|
from ._utils.blob_utils import LARGE_FILE_LIMIT, blob_iter, blob_upload_file
|
|
24
25
|
from ._utils.deprecation import warn_if_passing_namespace
|
|
25
|
-
from ._utils.grpc_utils import retry_transient_errors
|
|
26
26
|
from ._utils.hash_utils import get_sha256_hex
|
|
27
27
|
from ._utils.name_utils import check_object_name
|
|
28
28
|
from .client import _Client
|
|
@@ -96,6 +96,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
96
96
|
namespace=None, # mdmd:line-hidden
|
|
97
97
|
environment_name: Optional[str] = None,
|
|
98
98
|
create_if_missing: bool = False,
|
|
99
|
+
client: Optional[_Client] = None,
|
|
99
100
|
) -> "_NetworkFileSystem":
|
|
100
101
|
"""Reference a NetworkFileSystem by its name, creating if necessary.
|
|
101
102
|
|
|
@@ -114,15 +115,17 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
114
115
|
check_object_name(name, "NetworkFileSystem")
|
|
115
116
|
warn_if_passing_namespace(namespace, "modal.NetworkFileSystem.from_name")
|
|
116
117
|
|
|
117
|
-
async def _load(
|
|
118
|
+
async def _load(
|
|
119
|
+
self: _NetworkFileSystem, resolver: Resolver, load_context: LoadContext, existing_object_id: Optional[str]
|
|
120
|
+
):
|
|
118
121
|
req = api_pb2.SharedVolumeGetOrCreateRequest(
|
|
119
122
|
deployment_name=name,
|
|
120
|
-
environment_name=
|
|
123
|
+
environment_name=load_context.environment_name,
|
|
121
124
|
object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
|
|
122
125
|
)
|
|
123
126
|
try:
|
|
124
|
-
response = await
|
|
125
|
-
self._hydrate(response.shared_volume_id,
|
|
127
|
+
response = await load_context.client.stub.SharedVolumeGetOrCreate(req)
|
|
128
|
+
self._hydrate(response.shared_volume_id, load_context.client, None)
|
|
126
129
|
except modal.exception.NotFoundError as exc:
|
|
127
130
|
if exc.args[0] == "App has wrong entity vo":
|
|
128
131
|
raise InvalidError(
|
|
@@ -130,7 +133,12 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
130
133
|
)
|
|
131
134
|
raise
|
|
132
135
|
|
|
133
|
-
return _NetworkFileSystem._from_loader(
|
|
136
|
+
return _NetworkFileSystem._from_loader(
|
|
137
|
+
_load,
|
|
138
|
+
"NetworkFileSystem()",
|
|
139
|
+
hydrate_lazily=True,
|
|
140
|
+
load_context_overrides=LoadContext(environment_name=environment_name, client=client),
|
|
141
|
+
)
|
|
134
142
|
|
|
135
143
|
@classmethod
|
|
136
144
|
@asynccontextmanager
|
|
@@ -188,14 +196,14 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
188
196
|
environment_name=_get_environment_name(environment_name),
|
|
189
197
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
|
|
190
198
|
)
|
|
191
|
-
resp = await
|
|
199
|
+
resp = await client.stub.SharedVolumeGetOrCreate(request)
|
|
192
200
|
return resp.shared_volume_id
|
|
193
201
|
|
|
194
202
|
@staticmethod
|
|
195
203
|
async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
|
|
196
204
|
obj = await _NetworkFileSystem.from_name(name, environment_name=environment_name).hydrate(client)
|
|
197
205
|
req = api_pb2.SharedVolumeDeleteRequest(shared_volume_id=obj.object_id)
|
|
198
|
-
await
|
|
206
|
+
await obj._client.stub.SharedVolumeDelete(req)
|
|
199
207
|
|
|
200
208
|
@live_method
|
|
201
209
|
async def write_file(self, remote_path: str, fp: BinaryIO, progress_cb: Optional[Callable[..., Any]] = None) -> int:
|
|
@@ -235,7 +243,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
235
243
|
|
|
236
244
|
t0 = time.monotonic()
|
|
237
245
|
while time.monotonic() - t0 < NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT:
|
|
238
|
-
response = await
|
|
246
|
+
response = await self._client.stub.SharedVolumePutFile(req)
|
|
239
247
|
if response.exists:
|
|
240
248
|
break
|
|
241
249
|
else:
|
|
@@ -248,7 +256,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
248
256
|
"""Read a file from the network file system"""
|
|
249
257
|
req = api_pb2.SharedVolumeGetFileRequest(shared_volume_id=self.object_id, path=path)
|
|
250
258
|
try:
|
|
251
|
-
response = await
|
|
259
|
+
response = await self._client.stub.SharedVolumeGetFile(req)
|
|
252
260
|
except modal.exception.NotFoundError as exc:
|
|
253
261
|
raise FileNotFoundError(exc.args[0])
|
|
254
262
|
|
|
@@ -333,7 +341,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
333
341
|
"""Remove a file in a network file system."""
|
|
334
342
|
req = api_pb2.SharedVolumeRemoveFileRequest(shared_volume_id=self.object_id, path=path, recursive=recursive)
|
|
335
343
|
try:
|
|
336
|
-
await
|
|
344
|
+
await self._client.stub.SharedVolumeRemoveFile(req)
|
|
337
345
|
except modal.exception.NotFoundError as exc:
|
|
338
346
|
raise FileNotFoundError(exc.args[0])
|
|
339
347
|
|