modal 1.1.5.dev66__py3-none-any.whl → 1.3.1.dev8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of modal might be problematic. Click here for more details.
- modal/__init__.py +4 -4
- modal/__main__.py +4 -29
- modal/_billing.py +84 -0
- modal/_clustered_functions.py +1 -3
- modal/_container_entrypoint.py +33 -208
- modal/_functions.py +171 -138
- modal/_grpc_client.py +191 -0
- modal/_ipython.py +16 -6
- modal/_load_context.py +106 -0
- modal/_object.py +72 -21
- modal/_output.py +12 -14
- modal/_partial_function.py +31 -4
- modal/_resolver.py +44 -57
- modal/_runtime/container_io_manager.py +30 -28
- modal/_runtime/container_io_manager.pyi +42 -44
- modal/_runtime/gpu_memory_snapshot.py +9 -7
- modal/_runtime/user_code_event_loop.py +80 -0
- modal/_runtime/user_code_imports.py +236 -10
- modal/_serialization.py +2 -1
- modal/_traceback.py +4 -13
- modal/_tunnel.py +16 -11
- modal/_tunnel.pyi +25 -3
- modal/_utils/async_utils.py +337 -10
- modal/_utils/auth_token_manager.py +1 -4
- modal/_utils/blob_utils.py +29 -22
- modal/_utils/function_utils.py +20 -21
- modal/_utils/grpc_testing.py +6 -3
- modal/_utils/grpc_utils.py +223 -64
- modal/_utils/mount_utils.py +26 -1
- modal/_utils/name_utils.py +2 -3
- modal/_utils/package_utils.py +0 -1
- modal/_utils/rand_pb_testing.py +8 -1
- modal/_utils/task_command_router_client.py +524 -0
- modal/_vendor/cloudpickle.py +144 -48
- modal/app.py +285 -105
- modal/app.pyi +216 -53
- modal/billing.py +5 -0
- modal/builder/2025.06.txt +6 -3
- modal/builder/PREVIEW.txt +2 -1
- modal/builder/base-images.json +4 -2
- modal/cli/_download.py +19 -3
- modal/cli/cluster.py +4 -2
- modal/cli/config.py +3 -1
- modal/cli/container.py +5 -4
- modal/cli/dict.py +5 -2
- modal/cli/entry_point.py +26 -2
- modal/cli/environment.py +2 -16
- modal/cli/launch.py +1 -76
- modal/cli/network_file_system.py +5 -20
- modal/cli/programs/run_jupyter.py +1 -1
- modal/cli/programs/vscode.py +1 -1
- modal/cli/queues.py +5 -4
- modal/cli/run.py +24 -204
- modal/cli/secret.py +1 -2
- modal/cli/shell.py +375 -0
- modal/cli/utils.py +1 -13
- modal/cli/volume.py +11 -17
- modal/client.py +16 -125
- modal/client.pyi +94 -144
- modal/cloud_bucket_mount.py +3 -1
- modal/cloud_bucket_mount.pyi +4 -0
- modal/cls.py +101 -64
- modal/cls.pyi +9 -8
- modal/config.py +21 -1
- modal/container_process.py +288 -12
- modal/container_process.pyi +99 -38
- modal/dict.py +72 -33
- modal/dict.pyi +88 -57
- modal/environments.py +16 -8
- modal/environments.pyi +6 -2
- modal/exception.py +154 -16
- modal/experimental/__init__.py +24 -53
- modal/experimental/flash.py +161 -74
- modal/experimental/flash.pyi +97 -49
- modal/file_io.py +50 -92
- modal/file_io.pyi +117 -89
- modal/functions.pyi +70 -87
- modal/image.py +82 -47
- modal/image.pyi +51 -30
- modal/io_streams.py +500 -149
- modal/io_streams.pyi +279 -189
- modal/mount.py +60 -46
- modal/mount.pyi +41 -17
- modal/network_file_system.py +19 -11
- modal/network_file_system.pyi +72 -39
- modal/object.pyi +114 -22
- modal/parallel_map.py +42 -44
- modal/parallel_map.pyi +9 -17
- modal/partial_function.pyi +4 -2
- modal/proxy.py +14 -6
- modal/proxy.pyi +10 -2
- modal/queue.py +45 -38
- modal/queue.pyi +88 -52
- modal/runner.py +96 -96
- modal/runner.pyi +44 -27
- modal/sandbox.py +225 -107
- modal/sandbox.pyi +226 -60
- modal/secret.py +58 -56
- modal/secret.pyi +28 -13
- modal/serving.py +7 -11
- modal/serving.pyi +7 -8
- modal/snapshot.py +29 -15
- modal/snapshot.pyi +18 -10
- modal/token_flow.py +1 -1
- modal/token_flow.pyi +4 -6
- modal/volume.py +102 -55
- modal/volume.pyi +125 -66
- {modal-1.1.5.dev66.dist-info → modal-1.3.1.dev8.dist-info}/METADATA +10 -9
- modal-1.3.1.dev8.dist-info/RECORD +189 -0
- modal_proto/api.proto +141 -70
- modal_proto/api_grpc.py +42 -26
- modal_proto/api_pb2.py +1123 -1103
- modal_proto/api_pb2.pyi +331 -83
- modal_proto/api_pb2_grpc.py +80 -48
- modal_proto/api_pb2_grpc.pyi +26 -18
- modal_proto/modal_api_grpc.py +175 -174
- modal_proto/task_command_router.proto +164 -0
- modal_proto/task_command_router_grpc.py +138 -0
- modal_proto/task_command_router_pb2.py +180 -0
- modal_proto/{sandbox_router_pb2.pyi → task_command_router_pb2.pyi} +148 -57
- modal_proto/task_command_router_pb2_grpc.py +272 -0
- modal_proto/task_command_router_pb2_grpc.pyi +100 -0
- modal_version/__init__.py +1 -1
- modal_version/__main__.py +1 -1
- modal/cli/programs/launch_instance_ssh.py +0 -94
- modal/cli/programs/run_marimo.py +0 -95
- modal-1.1.5.dev66.dist-info/RECORD +0 -191
- modal_proto/modal_options_grpc.py +0 -3
- modal_proto/options.proto +0 -19
- modal_proto/options_grpc.py +0 -3
- modal_proto/options_pb2.py +0 -35
- modal_proto/options_pb2.pyi +0 -20
- modal_proto/options_pb2_grpc.py +0 -4
- modal_proto/options_pb2_grpc.pyi +0 -7
- modal_proto/sandbox_router.proto +0 -125
- modal_proto/sandbox_router_grpc.py +0 -89
- modal_proto/sandbox_router_pb2.py +0 -128
- modal_proto/sandbox_router_pb2_grpc.py +0 -169
- modal_proto/sandbox_router_pb2_grpc.pyi +0 -63
- {modal-1.1.5.dev66.dist-info → modal-1.3.1.dev8.dist-info}/WHEEL +0 -0
- {modal-1.1.5.dev66.dist-info → modal-1.3.1.dev8.dist-info}/entry_points.txt +0 -0
- {modal-1.1.5.dev66.dist-info → modal-1.3.1.dev8.dist-info}/licenses/LICENSE +0 -0
- {modal-1.1.5.dev66.dist-info → modal-1.3.1.dev8.dist-info}/top_level.txt +0 -0
modal/mount.py
CHANGED
|
@@ -13,19 +13,18 @@ from pathlib import Path, PurePosixPath
|
|
|
13
13
|
from typing import Callable, Optional, Sequence, Union
|
|
14
14
|
|
|
15
15
|
from google.protobuf.message import Message
|
|
16
|
-
from grpclib import GRPCError
|
|
17
16
|
|
|
18
17
|
import modal.exception
|
|
19
18
|
import modal.file_pattern_matcher
|
|
20
19
|
from modal_proto import api_pb2
|
|
21
20
|
from modal_version import __version__
|
|
22
21
|
|
|
23
|
-
from .
|
|
22
|
+
from ._load_context import LoadContext
|
|
23
|
+
from ._object import _Object
|
|
24
24
|
from ._resolver import Resolver
|
|
25
25
|
from ._utils.async_utils import TaskContext, aclosing, async_map, synchronize_api
|
|
26
26
|
from ._utils.blob_utils import FileUploadSpec, blob_upload_file, get_file_upload_spec_from_path
|
|
27
|
-
from ._utils.
|
|
28
|
-
from ._utils.grpc_utils import retry_transient_errors
|
|
27
|
+
from ._utils.grpc_utils import Retry
|
|
29
28
|
from ._utils.name_utils import check_object_name
|
|
30
29
|
from ._utils.package_utils import get_module_mount_info
|
|
31
30
|
from .client import _Client
|
|
@@ -41,11 +40,11 @@ MOUNT_PUT_FILE_CLIENT_TIMEOUT = 10 * 60 # 10 min max for transferring files
|
|
|
41
40
|
# These can be updated safely, but changes will trigger a rebuild for all images
|
|
42
41
|
# that rely on `add_python()` in their constructor.
|
|
43
42
|
PYTHON_STANDALONE_VERSIONS: dict[str, tuple[str, str]] = {
|
|
44
|
-
"3.9": ("20230826", "3.9.18"),
|
|
45
43
|
"3.10": ("20230826", "3.10.13"),
|
|
46
44
|
"3.11": ("20230826", "3.11.5"),
|
|
47
45
|
"3.12": ("20240107", "3.12.1"),
|
|
48
46
|
"3.13": ("20241008", "3.13.0"),
|
|
47
|
+
"3.14": ("20251205", "3.14.2"),
|
|
49
48
|
}
|
|
50
49
|
|
|
51
50
|
MOUNT_DEPRECATION_MESSAGE_PATTERN = """modal.Mount usage will soon be deprecated.
|
|
@@ -311,7 +310,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
311
310
|
_entries: Optional[list[_MountEntry]] = None
|
|
312
311
|
_deployment_name: Optional[str] = None
|
|
313
312
|
_namespace: Optional[int] = None
|
|
314
|
-
|
|
313
|
+
|
|
315
314
|
_allow_overwrite: bool = False
|
|
316
315
|
_content_checksum_sha256_hex: Optional[str] = None
|
|
317
316
|
|
|
@@ -326,7 +325,12 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
326
325
|
return None
|
|
327
326
|
return (_Mount._type_prefix, "local", frozenset(included_files))
|
|
328
327
|
|
|
329
|
-
obj = _Mount._from_loader(
|
|
328
|
+
obj = _Mount._from_loader(
|
|
329
|
+
_Mount._load_mount,
|
|
330
|
+
rep,
|
|
331
|
+
deduplication_key=mount_content_deduplication_key,
|
|
332
|
+
load_context_overrides=LoadContext.empty(),
|
|
333
|
+
)
|
|
330
334
|
obj._entries = entries
|
|
331
335
|
obj._is_local = True
|
|
332
336
|
return obj
|
|
@@ -461,16 +465,18 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
461
465
|
loop = asyncio.get_event_loop()
|
|
462
466
|
with concurrent.futures.ThreadPoolExecutor() as exe:
|
|
463
467
|
all_files = await loop.run_in_executor(exe, _select_files, entries)
|
|
468
|
+
logger.debug(f"Computing checksums for {len(all_files)} files using {exe._max_workers} worker threads")
|
|
464
469
|
|
|
465
|
-
|
|
470
|
+
# Yield FileUploadSpec objects lazily as they're consumed by async_map downstream.
|
|
471
|
+
# async_map's concurrency limit provides natural backpressure, so we don't need
|
|
472
|
+
# a separate semaphore here. This keeps memory bounded without creating all tasks upfront.
|
|
466
473
|
for local_filename, remote_filename in all_files:
|
|
467
|
-
logger.debug(f"Mounting {local_filename} as {remote_filename}")
|
|
468
|
-
futs.append(loop.run_in_executor(exe, get_file_upload_spec_from_path, local_filename, remote_filename))
|
|
469
|
-
|
|
470
|
-
logger.debug(f"Computing checksums for {len(futs)} files using {exe._max_workers} worker threads")
|
|
471
|
-
for fut in asyncio.as_completed(futs):
|
|
472
474
|
try:
|
|
473
|
-
|
|
475
|
+
logger.debug(f"Mounting {local_filename} as {remote_filename}")
|
|
476
|
+
file_spec = await loop.run_in_executor(
|
|
477
|
+
exe, get_file_upload_spec_from_path, local_filename, remote_filename
|
|
478
|
+
)
|
|
479
|
+
yield file_spec
|
|
474
480
|
except FileNotFoundError as exc:
|
|
475
481
|
# Can happen with temporary files (e.g. emacs will write temp files and delete them quickly)
|
|
476
482
|
logger.info(f"Ignoring file not found: {exc}")
|
|
@@ -478,6 +484,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
478
484
|
async def _load_mount(
|
|
479
485
|
self: "_Mount",
|
|
480
486
|
resolver: Resolver,
|
|
487
|
+
load_context: LoadContext,
|
|
481
488
|
existing_object_id: Optional[str],
|
|
482
489
|
):
|
|
483
490
|
t0 = time.monotonic()
|
|
@@ -519,7 +526,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
519
526
|
|
|
520
527
|
request = api_pb2.MountPutFileRequest(sha256_hex=file_spec.sha256_hex)
|
|
521
528
|
accounted_hashes.add(file_spec.sha256_hex)
|
|
522
|
-
response = await
|
|
529
|
+
response = await load_context.client.stub.MountPutFile(request, retry=Retry(base_delay=1))
|
|
523
530
|
|
|
524
531
|
if response.exists:
|
|
525
532
|
n_finished += 1
|
|
@@ -533,7 +540,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
533
540
|
async with blob_upload_concurrency:
|
|
534
541
|
with file_spec.source() as fp:
|
|
535
542
|
blob_id = await blob_upload_file(
|
|
536
|
-
fp,
|
|
543
|
+
fp, load_context.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
|
|
537
544
|
)
|
|
538
545
|
logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
|
|
539
546
|
request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
|
|
@@ -541,11 +548,15 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
541
548
|
logger.debug(
|
|
542
549
|
f"Uploading file {file_spec.source_description} to {remote_filename} ({file_spec.size} bytes)"
|
|
543
550
|
)
|
|
544
|
-
|
|
551
|
+
if file_spec.content is None:
|
|
552
|
+
content = await asyncio.to_thread(file_spec.read_content)
|
|
553
|
+
else:
|
|
554
|
+
content = file_spec.content
|
|
555
|
+
request2 = api_pb2.MountPutFileRequest(data=content, sha256_hex=file_spec.sha256_hex)
|
|
545
556
|
|
|
546
557
|
start_time = time.monotonic()
|
|
547
558
|
while time.monotonic() - start_time < MOUNT_PUT_FILE_CLIENT_TIMEOUT:
|
|
548
|
-
response = await
|
|
559
|
+
response = await load_context.client.stub.MountPutFile(request2, retry=Retry(base_delay=1))
|
|
549
560
|
if response.exists:
|
|
550
561
|
n_finished += 1
|
|
551
562
|
return mount_file
|
|
@@ -553,7 +564,7 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
553
564
|
raise modal.exception.MountUploadTimeoutError(f"Mounting of {file_spec.source_description} timed out")
|
|
554
565
|
|
|
555
566
|
# Upload files, or check if they already exist.
|
|
556
|
-
n_concurrent_uploads =
|
|
567
|
+
n_concurrent_uploads = 64
|
|
557
568
|
files: list[api_pb2.MountFile] = []
|
|
558
569
|
async with aclosing(
|
|
559
570
|
async_map(_Mount._get_files(self._entries), _put_file, concurrency=n_concurrent_uploads)
|
|
@@ -575,28 +586,28 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
575
586
|
req = api_pb2.MountGetOrCreateRequest(
|
|
576
587
|
deployment_name=self._deployment_name,
|
|
577
588
|
namespace=self._namespace,
|
|
578
|
-
environment_name=
|
|
589
|
+
environment_name=load_context.environment_name,
|
|
579
590
|
object_creation_type=creation_type,
|
|
580
591
|
files=files,
|
|
581
592
|
)
|
|
582
|
-
elif
|
|
593
|
+
elif load_context.app_id is not None:
|
|
583
594
|
req = api_pb2.MountGetOrCreateRequest(
|
|
584
595
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_ANONYMOUS_OWNED_BY_APP,
|
|
585
596
|
files=files,
|
|
586
|
-
app_id=
|
|
597
|
+
app_id=load_context.app_id,
|
|
587
598
|
)
|
|
588
599
|
else:
|
|
589
600
|
req = api_pb2.MountGetOrCreateRequest(
|
|
590
601
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_EPHEMERAL,
|
|
591
602
|
files=files,
|
|
592
|
-
environment_name=
|
|
603
|
+
environment_name=load_context.environment_name,
|
|
593
604
|
)
|
|
594
605
|
|
|
595
|
-
resp = await
|
|
606
|
+
resp = await load_context.client.stub.MountGetOrCreate(req, retry=Retry(base_delay=1))
|
|
596
607
|
status_row.finish(f"Created mount {message_label}")
|
|
597
608
|
|
|
598
609
|
logger.debug(f"Uploaded {total_uploads} new files and {total_bytes} bytes in {time.monotonic() - t0}s")
|
|
599
|
-
self._hydrate(resp.mount_id,
|
|
610
|
+
self._hydrate(resp.mount_id, load_context.client, resp.handle_metadata)
|
|
600
611
|
|
|
601
612
|
@staticmethod
|
|
602
613
|
def _from_local_python_packages(
|
|
@@ -629,19 +640,25 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
629
640
|
*,
|
|
630
641
|
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
|
|
631
642
|
environment_name: Optional[str] = None,
|
|
643
|
+
client: Optional[_Client] = None,
|
|
632
644
|
) -> "_Mount":
|
|
633
645
|
"""mdmd:hidden"""
|
|
634
646
|
|
|
635
|
-
async def _load(provider: _Mount, resolver: Resolver, existing_object_id: Optional[str]):
|
|
647
|
+
async def _load(provider: _Mount, resolver: Resolver, load_context, existing_object_id: Optional[str]):
|
|
636
648
|
req = api_pb2.MountGetOrCreateRequest(
|
|
637
649
|
deployment_name=name,
|
|
638
650
|
namespace=namespace,
|
|
639
|
-
environment_name=
|
|
651
|
+
environment_name=load_context.environment_name,
|
|
640
652
|
)
|
|
641
|
-
response = await
|
|
642
|
-
provider._hydrate(response.mount_id,
|
|
643
|
-
|
|
644
|
-
return _Mount._from_loader(
|
|
653
|
+
response = await load_context.client.stub.MountGetOrCreate(req)
|
|
654
|
+
provider._hydrate(response.mount_id, load_context.client, response.handle_metadata)
|
|
655
|
+
|
|
656
|
+
return _Mount._from_loader(
|
|
657
|
+
_load,
|
|
658
|
+
"Mount()",
|
|
659
|
+
hydrate_lazily=True,
|
|
660
|
+
load_context_overrides=LoadContext(environment_name=environment_name, client=client),
|
|
661
|
+
)
|
|
645
662
|
|
|
646
663
|
async def _deploy(
|
|
647
664
|
self: "_Mount",
|
|
@@ -653,15 +670,12 @@ class _Mount(_Object, type_prefix="mo"):
|
|
|
653
670
|
client: Optional[_Client] = None,
|
|
654
671
|
) -> None:
|
|
655
672
|
check_object_name(deployment_name, "Mount")
|
|
656
|
-
environment_name = _get_environment_name(environment_name, resolver=None)
|
|
657
673
|
self._deployment_name = deployment_name
|
|
658
674
|
self._namespace = namespace
|
|
659
|
-
self._environment_name = environment_name
|
|
660
675
|
self._allow_overwrite = allow_overwrite
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
resolver
|
|
664
|
-
await resolver.load(self)
|
|
676
|
+
resolver = Resolver()
|
|
677
|
+
root_metadata = LoadContext(client=client, environment_name=environment_name)
|
|
678
|
+
await resolver.load(self, root_metadata)
|
|
665
679
|
|
|
666
680
|
def _get_metadata(self) -> api_pb2.MountHandleMetadata:
|
|
667
681
|
if self._content_checksum_sha256_hex is None:
|
|
@@ -760,13 +774,13 @@ async def _create_single_client_dependency_mount(
|
|
|
760
774
|
if check_if_exists:
|
|
761
775
|
try:
|
|
762
776
|
await Mount.from_name(mount_name, namespace=api_pb2.DEPLOYMENT_NAMESPACE_GLOBAL).hydrate.aio(client)
|
|
763
|
-
print(f"➖ Found existing mount {mount_name} in global namespace.")
|
|
777
|
+
print(f"➖ Found existing mount {mount_name} in global namespace.") # noqa: T201
|
|
764
778
|
return
|
|
765
779
|
except modal.exception.NotFoundError:
|
|
766
780
|
pass
|
|
767
781
|
|
|
768
782
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpd:
|
|
769
|
-
print(f"📦 Building {mount_name}.")
|
|
783
|
+
print(f"📦 Building {mount_name}.") # noqa: T201
|
|
770
784
|
requirements = os.path.join(os.path.dirname(__file__), f"builder/{builder_version}.txt")
|
|
771
785
|
cmd = " ".join(
|
|
772
786
|
[
|
|
@@ -795,11 +809,11 @@ async def _create_single_client_dependency_mount(
|
|
|
795
809
|
await proc.wait()
|
|
796
810
|
if proc.returncode:
|
|
797
811
|
stdout, stderr = await proc.communicate()
|
|
798
|
-
print(stdout.decode("utf-8"))
|
|
799
|
-
print(stderr.decode("utf-8"))
|
|
812
|
+
print(stdout.decode("utf-8")) # noqa: T201
|
|
813
|
+
print(stderr.decode("utf-8")) # noqa: T201
|
|
800
814
|
raise RuntimeError(f"Subprocess failed with {proc.returncode}")
|
|
801
815
|
|
|
802
|
-
print(f"🌐 Downloaded and unpacked {mount_name} packages to {tmpd}.")
|
|
816
|
+
print(f"🌐 Downloaded and unpacked {mount_name} packages to {tmpd}.") # noqa: T201
|
|
803
817
|
|
|
804
818
|
python_mount = Mount._from_local_dir(tmpd, remote_path=REMOTE_PACKAGES_PATH)
|
|
805
819
|
|
|
@@ -823,11 +837,11 @@ async def _create_single_client_dependency_mount(
|
|
|
823
837
|
allow_overwrite=allow_overwrite,
|
|
824
838
|
client=client,
|
|
825
839
|
)
|
|
826
|
-
print(f"✅ Deployed mount {mount_name} to global namespace.")
|
|
827
|
-
except
|
|
828
|
-
print(f"⚠️ Mount creation failed with {e.
|
|
840
|
+
print(f"✅ Deployed mount {mount_name} to global namespace.") # noqa: T201
|
|
841
|
+
except modal.exception.Error as e:
|
|
842
|
+
print(f"⚠️ Mount creation failed with {type(e).__name__}: {e}") # noqa: T201
|
|
829
843
|
else:
|
|
830
|
-
print(f"Dry run - skipping deployment of mount {mount_name}")
|
|
844
|
+
print(f"Dry run - skipping deployment of mount {mount_name}") # noqa: T201
|
|
831
845
|
|
|
832
846
|
|
|
833
847
|
async def _create_client_dependency_mounts(
|
modal/mount.pyi
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import collections.abc
|
|
2
2
|
import google.protobuf.message
|
|
3
|
+
import modal._load_context
|
|
3
4
|
import modal._object
|
|
4
5
|
import modal._resolver
|
|
5
6
|
import modal._utils.blob_utils
|
|
@@ -154,7 +155,6 @@ class _Mount(modal._object._Object):
|
|
|
154
155
|
_entries: typing.Optional[list[_MountEntry]]
|
|
155
156
|
_deployment_name: typing.Optional[str]
|
|
156
157
|
_namespace: typing.Optional[int]
|
|
157
|
-
_environment_name: typing.Optional[str]
|
|
158
158
|
_allow_overwrite: bool
|
|
159
159
|
_content_checksum_sha256_hex: typing.Optional[str]
|
|
160
160
|
|
|
@@ -216,7 +216,10 @@ class _Mount(modal._object._Object):
|
|
|
216
216
|
entries: list[_MountEntry],
|
|
217
217
|
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
|
218
218
|
async def _load_mount(
|
|
219
|
-
self: _Mount,
|
|
219
|
+
self: _Mount,
|
|
220
|
+
resolver: modal._resolver.Resolver,
|
|
221
|
+
load_context: modal._load_context.LoadContext,
|
|
222
|
+
existing_object_id: typing.Optional[str],
|
|
220
223
|
): ...
|
|
221
224
|
@staticmethod
|
|
222
225
|
def _from_local_python_packages(
|
|
@@ -226,7 +229,13 @@ class _Mount(modal._object._Object):
|
|
|
226
229
|
ignore: typing.Union[typing.Sequence[str], collections.abc.Callable[[pathlib.Path], bool], None] = None,
|
|
227
230
|
) -> _Mount: ...
|
|
228
231
|
@staticmethod
|
|
229
|
-
def from_name(
|
|
232
|
+
def from_name(
|
|
233
|
+
name: str,
|
|
234
|
+
*,
|
|
235
|
+
namespace=1,
|
|
236
|
+
environment_name: typing.Optional[str] = None,
|
|
237
|
+
client: typing.Optional[modal.client._Client] = None,
|
|
238
|
+
) -> _Mount:
|
|
230
239
|
"""mdmd:hidden"""
|
|
231
240
|
...
|
|
232
241
|
|
|
@@ -241,8 +250,6 @@ class _Mount(modal._object._Object):
|
|
|
241
250
|
) -> None: ...
|
|
242
251
|
def _get_metadata(self) -> modal_proto.api_pb2.MountHandleMetadata: ...
|
|
243
252
|
|
|
244
|
-
SUPERSELF = typing.TypeVar("SUPERSELF", covariant=True)
|
|
245
|
-
|
|
246
253
|
class Mount(modal.object.Object):
|
|
247
254
|
"""**Deprecated**: Mounts should not be used explicitly anymore, use `Image.add_local_*` commands instead.
|
|
248
255
|
|
|
@@ -269,7 +276,6 @@ class Mount(modal.object.Object):
|
|
|
269
276
|
_entries: typing.Optional[list[_MountEntry]]
|
|
270
277
|
_deployment_name: typing.Optional[str]
|
|
271
278
|
_namespace: typing.Optional[int]
|
|
272
|
-
_environment_name: typing.Optional[str]
|
|
273
279
|
_allow_overwrite: bool
|
|
274
280
|
_content_checksum_sha256_hex: typing.Optional[str]
|
|
275
281
|
|
|
@@ -339,13 +345,25 @@ class Mount(modal.object.Object):
|
|
|
339
345
|
self, /, entries: list[_MountEntry]
|
|
340
346
|
) -> collections.abc.AsyncGenerator[modal._utils.blob_utils.FileUploadSpec, None]: ...
|
|
341
347
|
|
|
342
|
-
_get_files: ___get_files_spec
|
|
348
|
+
_get_files: typing.ClassVar[___get_files_spec]
|
|
343
349
|
|
|
344
|
-
class ___load_mount_spec(typing_extensions.Protocol
|
|
345
|
-
def __call__(
|
|
346
|
-
|
|
350
|
+
class ___load_mount_spec(typing_extensions.Protocol):
|
|
351
|
+
def __call__(
|
|
352
|
+
self,
|
|
353
|
+
/,
|
|
354
|
+
resolver: modal._resolver.Resolver,
|
|
355
|
+
load_context: modal._load_context.LoadContext,
|
|
356
|
+
existing_object_id: typing.Optional[str],
|
|
357
|
+
): ...
|
|
358
|
+
async def aio(
|
|
359
|
+
self,
|
|
360
|
+
/,
|
|
361
|
+
resolver: modal._resolver.Resolver,
|
|
362
|
+
load_context: modal._load_context.LoadContext,
|
|
363
|
+
existing_object_id: typing.Optional[str],
|
|
364
|
+
): ...
|
|
347
365
|
|
|
348
|
-
_load_mount: ___load_mount_spec
|
|
366
|
+
_load_mount: ___load_mount_spec
|
|
349
367
|
|
|
350
368
|
@staticmethod
|
|
351
369
|
def _from_local_python_packages(
|
|
@@ -355,11 +373,17 @@ class Mount(modal.object.Object):
|
|
|
355
373
|
ignore: typing.Union[typing.Sequence[str], collections.abc.Callable[[pathlib.Path], bool], None] = None,
|
|
356
374
|
) -> Mount: ...
|
|
357
375
|
@staticmethod
|
|
358
|
-
def from_name(
|
|
376
|
+
def from_name(
|
|
377
|
+
name: str,
|
|
378
|
+
*,
|
|
379
|
+
namespace=1,
|
|
380
|
+
environment_name: typing.Optional[str] = None,
|
|
381
|
+
client: typing.Optional[modal.client.Client] = None,
|
|
382
|
+
) -> Mount:
|
|
359
383
|
"""mdmd:hidden"""
|
|
360
384
|
...
|
|
361
385
|
|
|
362
|
-
class ___deploy_spec(typing_extensions.Protocol
|
|
386
|
+
class ___deploy_spec(typing_extensions.Protocol):
|
|
363
387
|
def __call__(
|
|
364
388
|
self,
|
|
365
389
|
/,
|
|
@@ -381,7 +405,7 @@ class Mount(modal.object.Object):
|
|
|
381
405
|
client: typing.Optional[modal.client.Client] = None,
|
|
382
406
|
) -> None: ...
|
|
383
407
|
|
|
384
|
-
_deploy: ___deploy_spec
|
|
408
|
+
_deploy: ___deploy_spec
|
|
385
409
|
|
|
386
410
|
def _get_metadata(self) -> modal_proto.api_pb2.MountHandleMetadata: ...
|
|
387
411
|
|
|
@@ -402,7 +426,7 @@ async def _create_single_client_dependency_mount(
|
|
|
402
426
|
): ...
|
|
403
427
|
async def _create_client_dependency_mounts(
|
|
404
428
|
client=None,
|
|
405
|
-
python_versions: list[str] = ["3.
|
|
429
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
406
430
|
builder_versions: list[str] = ["2025.06"],
|
|
407
431
|
check_if_exists=True,
|
|
408
432
|
dry_run=False,
|
|
@@ -413,7 +437,7 @@ class __create_client_dependency_mounts_spec(typing_extensions.Protocol):
|
|
|
413
437
|
self,
|
|
414
438
|
/,
|
|
415
439
|
client=None,
|
|
416
|
-
python_versions: list[str] = ["3.
|
|
440
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
417
441
|
builder_versions: list[str] = ["2025.06"],
|
|
418
442
|
check_if_exists=True,
|
|
419
443
|
dry_run=False,
|
|
@@ -422,7 +446,7 @@ class __create_client_dependency_mounts_spec(typing_extensions.Protocol):
|
|
|
422
446
|
self,
|
|
423
447
|
/,
|
|
424
448
|
client=None,
|
|
425
|
-
python_versions: list[str] = ["3.
|
|
449
|
+
python_versions: list[str] = ["3.10", "3.11", "3.12", "3.13", "3.14"],
|
|
426
450
|
builder_versions: list[str] = ["2025.06"],
|
|
427
451
|
check_if_exists=True,
|
|
428
452
|
dry_run=False,
|
modal/network_file_system.py
CHANGED
|
@@ -11,6 +11,7 @@ from synchronicity.async_wrap import asynccontextmanager
|
|
|
11
11
|
import modal
|
|
12
12
|
from modal_proto import api_pb2
|
|
13
13
|
|
|
14
|
+
from ._load_context import LoadContext
|
|
14
15
|
from ._object import (
|
|
15
16
|
EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
|
|
16
17
|
_get_environment_name,
|
|
@@ -22,7 +23,6 @@ from ._resolver import Resolver
|
|
|
22
23
|
from ._utils.async_utils import TaskContext, aclosing, async_map, sync_or_async_iter, synchronize_api
|
|
23
24
|
from ._utils.blob_utils import LARGE_FILE_LIMIT, blob_iter, blob_upload_file
|
|
24
25
|
from ._utils.deprecation import warn_if_passing_namespace
|
|
25
|
-
from ._utils.grpc_utils import retry_transient_errors
|
|
26
26
|
from ._utils.hash_utils import get_sha256_hex
|
|
27
27
|
from ._utils.name_utils import check_object_name
|
|
28
28
|
from .client import _Client
|
|
@@ -96,6 +96,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
96
96
|
namespace=None, # mdmd:line-hidden
|
|
97
97
|
environment_name: Optional[str] = None,
|
|
98
98
|
create_if_missing: bool = False,
|
|
99
|
+
client: Optional[_Client] = None,
|
|
99
100
|
) -> "_NetworkFileSystem":
|
|
100
101
|
"""Reference a NetworkFileSystem by its name, creating if necessary.
|
|
101
102
|
|
|
@@ -114,15 +115,17 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
114
115
|
check_object_name(name, "NetworkFileSystem")
|
|
115
116
|
warn_if_passing_namespace(namespace, "modal.NetworkFileSystem.from_name")
|
|
116
117
|
|
|
117
|
-
async def _load(
|
|
118
|
+
async def _load(
|
|
119
|
+
self: _NetworkFileSystem, resolver: Resolver, load_context: LoadContext, existing_object_id: Optional[str]
|
|
120
|
+
):
|
|
118
121
|
req = api_pb2.SharedVolumeGetOrCreateRequest(
|
|
119
122
|
deployment_name=name,
|
|
120
|
-
environment_name=
|
|
123
|
+
environment_name=load_context.environment_name,
|
|
121
124
|
object_creation_type=(api_pb2.OBJECT_CREATION_TYPE_CREATE_IF_MISSING if create_if_missing else None),
|
|
122
125
|
)
|
|
123
126
|
try:
|
|
124
|
-
response = await
|
|
125
|
-
self._hydrate(response.shared_volume_id,
|
|
127
|
+
response = await load_context.client.stub.SharedVolumeGetOrCreate(req)
|
|
128
|
+
self._hydrate(response.shared_volume_id, load_context.client, None)
|
|
126
129
|
except modal.exception.NotFoundError as exc:
|
|
127
130
|
if exc.args[0] == "App has wrong entity vo":
|
|
128
131
|
raise InvalidError(
|
|
@@ -130,7 +133,12 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
130
133
|
)
|
|
131
134
|
raise
|
|
132
135
|
|
|
133
|
-
return _NetworkFileSystem._from_loader(
|
|
136
|
+
return _NetworkFileSystem._from_loader(
|
|
137
|
+
_load,
|
|
138
|
+
"NetworkFileSystem()",
|
|
139
|
+
hydrate_lazily=True,
|
|
140
|
+
load_context_overrides=LoadContext(environment_name=environment_name, client=client),
|
|
141
|
+
)
|
|
134
142
|
|
|
135
143
|
@classmethod
|
|
136
144
|
@asynccontextmanager
|
|
@@ -188,14 +196,14 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
188
196
|
environment_name=_get_environment_name(environment_name),
|
|
189
197
|
object_creation_type=api_pb2.OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTS,
|
|
190
198
|
)
|
|
191
|
-
resp = await
|
|
199
|
+
resp = await client.stub.SharedVolumeGetOrCreate(request)
|
|
192
200
|
return resp.shared_volume_id
|
|
193
201
|
|
|
194
202
|
@staticmethod
|
|
195
203
|
async def delete(name: str, client: Optional[_Client] = None, environment_name: Optional[str] = None):
|
|
196
204
|
obj = await _NetworkFileSystem.from_name(name, environment_name=environment_name).hydrate(client)
|
|
197
205
|
req = api_pb2.SharedVolumeDeleteRequest(shared_volume_id=obj.object_id)
|
|
198
|
-
await
|
|
206
|
+
await obj._client.stub.SharedVolumeDelete(req)
|
|
199
207
|
|
|
200
208
|
@live_method
|
|
201
209
|
async def write_file(self, remote_path: str, fp: BinaryIO, progress_cb: Optional[Callable[..., Any]] = None) -> int:
|
|
@@ -235,7 +243,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
235
243
|
|
|
236
244
|
t0 = time.monotonic()
|
|
237
245
|
while time.monotonic() - t0 < NETWORK_FILE_SYSTEM_PUT_FILE_CLIENT_TIMEOUT:
|
|
238
|
-
response = await
|
|
246
|
+
response = await self._client.stub.SharedVolumePutFile(req)
|
|
239
247
|
if response.exists:
|
|
240
248
|
break
|
|
241
249
|
else:
|
|
@@ -248,7 +256,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
248
256
|
"""Read a file from the network file system"""
|
|
249
257
|
req = api_pb2.SharedVolumeGetFileRequest(shared_volume_id=self.object_id, path=path)
|
|
250
258
|
try:
|
|
251
|
-
response = await
|
|
259
|
+
response = await self._client.stub.SharedVolumeGetFile(req)
|
|
252
260
|
except modal.exception.NotFoundError as exc:
|
|
253
261
|
raise FileNotFoundError(exc.args[0])
|
|
254
262
|
|
|
@@ -333,7 +341,7 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
|
|
|
333
341
|
"""Remove a file in a network file system."""
|
|
334
342
|
req = api_pb2.SharedVolumeRemoveFileRequest(shared_volume_id=self.object_id, path=path, recursive=recursive)
|
|
335
343
|
try:
|
|
336
|
-
await
|
|
344
|
+
await self._client.stub.SharedVolumeRemoveFile(req)
|
|
337
345
|
except modal.exception.NotFoundError as exc:
|
|
338
346
|
raise FileNotFoundError(exc.args[0])
|
|
339
347
|
|