modal 0.68.11__py3-none-any.whl → 0.68.14__py3-none-any.whl

modal/_ipython.py

@@ -1,21 +1,11 @@
 # Copyright Modal Labs 2022
 import sys
-import warnings
-
-ipy_outstream = None
-try:
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        import ipykernel.iostream
-
-    ipy_outstream = ipykernel.iostream.OutStream
-except ImportError:
-    pass
 
 
 def is_notebook(stdout=None):
-    if ipy_outstream is None:
+    ipykernel_iostream = sys.modules.get("ipykernel.iostream")
+    if ipykernel_iostream is None:
         return False
     if stdout is None:
         stdout = sys.stdout
-    return isinstance(stdout, ipy_outstream)
+    return isinstance(stdout, ipykernel_iostream.OutStream)

modal/_utils/blob_utils.py

@@ -21,7 +21,7 @@ from modal_proto.modal_api_grpc import ModalClientModal
 from ..exception import ExecutionError
 from .async_utils import TaskContext, retry
 from .grpc_utils import retry_transient_errors
-from .hash_utils import UploadHashes, get_sha256_hex, get_upload_hashes
+from .hash_utils import UploadHashes, get_upload_hashes
 from .http_utils import ClientSessionRegistry
 from .logger import logger
 
@@ -38,6 +38,11 @@ BLOB_MAX_PARALLELISM = 10
 # read ~16MiB chunks by default
 DEFAULT_SEGMENT_CHUNK_SIZE = 2**24
 
+# Files larger than this will be multipart uploaded. The server might request multipart upload for smaller files as
+# well, but the limit will never be raised.
+# TODO(dano): remove this once we stop requiring md5 for blobs
+MULTIPART_UPLOAD_THRESHOLD = 1024**3
+
 
 class BytesIOSegmentPayload(BytesIOPayload):
     """Modified bytes payload for concurrent sends of chunks from the same file.
@@ -305,9 +310,13 @@ async def blob_upload(payload: bytes, stub: ModalClientModal) -> str:
 
 
 async def blob_upload_file(
-    file_obj: BinaryIO, stub: ModalClientModal, progress_report_cb: Optional[Callable] = None
+    file_obj: BinaryIO,
+    stub: ModalClientModal,
+    progress_report_cb: Optional[Callable] = None,
+    sha256_hex: Optional[str] = None,
+    md5_hex: Optional[str] = None,
 ) -> str:
-    upload_hashes = get_upload_hashes(file_obj)
+    upload_hashes = get_upload_hashes(file_obj, sha256_hex=sha256_hex, md5_hex=md5_hex)
     return await _blob_upload(upload_hashes, file_obj, stub, progress_report_cb)
 
 
@@ -366,6 +375,7 @@ class FileUploadSpec:
     use_blob: bool
     content: Optional[bytes]  # typically None if using blob, required otherwise
     sha256_hex: str
+    md5_hex: str
     mode: int  # file permission bits (last 12 bits of st_mode)
     size: int
 
@@ -383,13 +393,15 @@ def _get_file_upload_spec(
         fp.seek(0)
 
         if size >= LARGE_FILE_LIMIT:
+            # TODO(dano): remove the placeholder md5 once we stop requiring md5 for blobs
+            md5_hex = "baadbaadbaadbaadbaadbaadbaadbaad" if size > MULTIPART_UPLOAD_THRESHOLD else None
             use_blob = True
             content = None
-            sha256_hex = get_sha256_hex(fp)
+            hashes = get_upload_hashes(fp, md5_hex=md5_hex)
         else:
             use_blob = False
             content = fp.read()
-            sha256_hex = get_sha256_hex(content)
+            hashes = get_upload_hashes(content)
 
     return FileUploadSpec(
         source=source,
@@ -397,7 +409,8 @@ def _get_file_upload_spec(
         mount_filename=mount_filename.as_posix(),
         use_blob=use_blob,
         content=content,
-        sha256_hex=sha256_hex,
+        sha256_hex=hashes.sha256_hex(),
+        md5_hex=hashes.md5_hex(),
         mode=mode & 0o7777,
         size=size,
     )

modal/_utils/hash_utils.py

@@ -3,14 +3,14 @@ import base64
 import dataclasses
 import hashlib
 import time
-from typing import BinaryIO, Callable, Union
+from typing import BinaryIO, Callable, Optional, Sequence, Union
 
 from modal.config import logger
 
 HASH_CHUNK_SIZE = 65536
 
 
-def _update(hashers: list[Callable[[bytes], None]], data: Union[bytes, BinaryIO]) -> None:
+def _update(hashers: Sequence[Callable[[bytes], None]], data: Union[bytes, BinaryIO]) -> None:
     if isinstance(data, bytes):
         for hasher in hashers:
             hasher(data)
@@ -57,15 +57,44 @@ class UploadHashes:
     md5_base64: str
     sha256_base64: str
 
+    def md5_hex(self) -> str:
+        return base64.b64decode(self.md5_base64).hex()
 
-def get_upload_hashes(data: Union[bytes, BinaryIO]) -> UploadHashes:
+    def sha256_hex(self) -> str:
+        return base64.b64decode(self.sha256_base64).hex()
+
+
+def get_upload_hashes(
+    data: Union[bytes, BinaryIO], sha256_hex: Optional[str] = None, md5_hex: Optional[str] = None
+) -> UploadHashes:
     t0 = time.monotonic()
-    md5 = hashlib.md5()
-    sha256 = hashlib.sha256()
-    _update([md5.update, sha256.update], data)
+    hashers = {}
+
+    if not sha256_hex:
+        sha256 = hashlib.sha256()
+        hashers["sha256"] = sha256
+    if not md5_hex:
+        md5 = hashlib.md5()
+        hashers["md5"] = md5
+
+    if hashers:
+        updaters = [h.update for h in hashers.values()]
+        _update(updaters, data)
+
+    if sha256_hex:
+        sha256_base64 = base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii")
+    else:
+        sha256_base64 = base64.b64encode(hashers["sha256"].digest()).decode("ascii")
+
+    if md5_hex:
+        md5_base64 = base64.b64encode(bytes.fromhex(md5_hex)).decode("ascii")
+    else:
+        md5_base64 = base64.b64encode(hashers["md5"].digest()).decode("ascii")
+
     hashes = UploadHashes(
-        md5_base64=base64.b64encode(md5.digest()).decode("ascii"),
-        sha256_base64=base64.b64encode(sha256.digest()).decode("ascii"),
+        md5_base64=md5_base64,
+        sha256_base64=sha256_base64,
     )
-    logger.debug("get_upload_hashes took %.3fs", time.monotonic() - t0)
+
+    logger.debug("get_upload_hashes took %.3fs (%s)", time.monotonic() - t0, hashers.keys())
     return hashes

modal/client.pyi

@@ -26,7 +26,7 @@ class _Client:
     _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
 
     def __init__(
-        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.11"
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.14"
     ): ...
     def is_closed(self) -> bool: ...
     @property
@@ -81,7 +81,7 @@ class Client:
     _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
 
     def __init__(
-        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.11"
+        self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.14"
     ): ...
     def is_closed(self) -> bool: ...
     @property

modal/mount.py

@@ -483,7 +483,9 @@ class _Mount(_Object, type_prefix="mo"):
                 logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
                 async with blob_upload_concurrency:
                     with file_spec.source() as fp:
-                        blob_id = await blob_upload_file(fp, resolver.client.stub)
+                        blob_id = await blob_upload_file(
+                            fp, resolver.client.stub, sha256_hex=file_spec.sha256_hex, md5_hex=file_spec.md5_hex
+                        )
                 logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
                 request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)
             else:

modal/network_file_system.py

@@ -245,7 +245,10 @@ class _NetworkFileSystem(_Object, type_prefix="sv"):
         if data_size > LARGE_FILE_LIMIT:
             progress_task_id = progress_cb(name=remote_path, size=data_size)
             blob_id = await blob_upload_file(
-                fp, self._client.stub, progress_report_cb=functools.partial(progress_cb, progress_task_id)
+                fp,
+                self._client.stub,
+                progress_report_cb=functools.partial(progress_cb, progress_task_id),
+                sha256_hex=sha_hash,
             )
             req = api_pb2.SharedVolumePutFileRequest(
                 shared_volume_id=self.object_id,

modal/volume.py

@@ -632,7 +632,11 @@ class _VolumeUploadContextManager:
                 logger.debug(f"Creating blob file for {file_spec.source_description} ({file_spec.size} bytes)")
                 with file_spec.source() as fp:
                     blob_id = await blob_upload_file(
-                        fp, self._client.stub, functools.partial(self._progress_cb, progress_task_id)
+                        fp,
+                        self._client.stub,
+                        functools.partial(self._progress_cb, progress_task_id),
+                        sha256_hex=file_spec.sha256_hex,
+                        md5_hex=file_spec.md5_hex,
                     )
                 logger.debug(f"Uploading blob file {file_spec.source_description} as {remote_filename}")
                 request2 = api_pb2.MountPutFileRequest(data_blob_id=blob_id, sha256_hex=file_spec.sha256_hex)

{modal-0.68.11.dist-info → modal-0.68.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: modal
-Version: 0.68.11
+Version: 0.68.14
 Summary: Python client library for Modal
 Author: Modal Labs
 Author-email: support@modal.com

{modal-0.68.11.dist-info → modal-0.68.14.dist-info}/RECORD

@@ -3,7 +3,7 @@ modal/__main__.py,sha256=scYhGFqh8OJcVDo-VOxIT6CCwxOgzgflYWMnIZiMRqE,2871
 modal/_clustered_functions.py,sha256=kTf-9YBXY88NutC1akI-gCbvf01RhMPCw-zoOI_YIUE,2700
 modal/_clustered_functions.pyi,sha256=vllkegc99A0jrUOWa8mdlSbdp6uz36TsHhGxysAOpaQ,771
 modal/_container_entrypoint.py,sha256=wk10vA5vRZZsVwQ0yINOLd0i-NwH7x6XbhTslumvGjo,28910
-modal/_ipython.py,sha256=HF_DYy0e0qM9WnGDmTY30s1RxzGya9GeORCauCEpRaE,450
+modal/_ipython.py,sha256=TW1fkVOmZL3YYqdS2YlM1hqpf654Yf8ZyybHdBnlhSw,301
 modal/_location.py,sha256=S3lSxIU3h9HkWpkJ3Pwo0pqjIOSB1fjeSgUsY3x7eec,1202
 modal/_output.py,sha256=0fWX_KQwhER--U81ys16CL-pA5A-LN20C0EZjElKGJQ,25410
 modal/_proxy_tunnel.py,sha256=gnKyCfmVB7x2d1A6c-JDysNIP3kEFxmXzhcXhPrzPn0,1906
@@ -19,7 +19,7 @@ modal/app.py,sha256=EJ7FUN6rWnSwLJoYJh8nmKg_t-8hdN8_rt0OrkP7JvQ,46084
 modal/app.pyi,sha256=BE5SlR5tRECuc6-e2lUuOknDdov3zxgZ4N0AsLb5ZVQ,25270
 modal/call_graph.py,sha256=1g2DGcMIJvRy-xKicuf63IVE98gJSnQsr8R_NVMptNc,2581
 modal/client.py,sha256=nyPjfromWBeOyurexpFP2QLQNk822RPggMCLyX9j1jA,15247
-modal/client.pyi,sha256=wnmsE-wrmK3qkFH1a5PoT45B11cFC5T3gLLO9KceJNU,7280
+modal/client.pyi,sha256=4x5j0yBlE_sZf8cS2eChs-pYuxr-_SsRiKrf1fSJmfA,7280
 modal/cloud_bucket_mount.py,sha256=G7T7jWLD0QkmrfKR75mSTwdUZ2xNfj7pkVqb4ipmxmI,5735
 modal/cloud_bucket_mount.pyi,sha256=CEi7vrH3kDUF4LAy4qP6tfImy2UJuFRcRbsgRNM1wo8,1403
 modal/cls.py,sha256=ONnrfZ2vPcaY2JuKypPiBA9eTiyg8Qfg-Ull40nn9zs,30956
@@ -42,9 +42,9 @@ modal/image.py,sha256=cQ6WP1xHXZT_nY8z3aEFiGwKzrTV0yxi3Ab8JzF91eo,79653
 modal/image.pyi,sha256=PIKH6JBA4L5TfdJrQu3pm2ykyIITmiP920TpP8cdyQA,24585
 modal/io_streams.py,sha256=QkQiizKRzd5bnbKQsap31LJgBYlAnj4-XkV_50xPYX0,15079
 modal/io_streams.pyi,sha256=bCCVSxkMcosYd8O3PQDDwJw7TQ8JEcnYonLJ5t27TQs,4804
-modal/mount.py,sha256=liaid5p42o0OKnzoocJJ_oCovDVderk3-JuCTa5pqtA,27656
+modal/mount.py,sha256=7FJrS-QkRJGndKuvRnMz452wfUcbLpd_UEnmFgQCKQQ,27770
 modal/mount.pyi,sha256=3e4nkXUeeVmUmOyK8Tiyk_EQlHeWruN3yGJVnmDUVrI,9761
-modal/network_file_system.py,sha256=NKZgh_p8MyJyyJgP92lhRgTmwA3kOPw7m8AbYlchhCE,14530
+modal/network_file_system.py,sha256=kwwQLCJVO086FTiAWSF_jz9BkqijZLpSbEYXpFvS0Ik,14600
 modal/network_file_system.pyi,sha256=8mHKXuRkxHPazF6ljIW7g4M5aVqLSl6eKUPLgDCug5c,7901
 modal/object.py,sha256=HZs3N59C6JxlMuPQWJYvrWV1FEEkH9txUovVDorVUbs,9763
 modal/object.pyi,sha256=MO78H9yFSE5i1gExPEwyyQzLdlshkcGHN1aQ0ylyvq0,8802
@@ -73,7 +73,7 @@ modal/serving.pyi,sha256=ncV-9jY_vZYFnGs5ZnMb3ffrX8LmcLdIMHBC56xRbtE,1711
 modal/stream_type.py,sha256=A6320qoAAWhEfwOCZfGtymQTu5AfLfJXXgARqooTPvY,417
 modal/token_flow.py,sha256=LcgSce_MSQ2p7j55DPwpVRpiAtCDe8GRSEwzO7muNR8,6774
 modal/token_flow.pyi,sha256=gOYtYujrWt_JFZeiI8EmfahXPx5GCR5Na-VaPQcWgEY,1937
-modal/volume.py,sha256=IISuMeXq9MoSkhXg8Q6JG0F-2n9NTkWk0xGuJB8l3d8,29159
+modal/volume.py,sha256=PGzbninvRU-IhSwJgM2jZKzD8llRhZhadsOxZ-YNwaM,29316
 modal/volume.pyi,sha256=St0mDiaojfep6Bs4sBbkRJmeacYHF6lh6FKOWGmheHA,11182
 modal/_runtime/__init__.py,sha256=MIEP8jhXUeGq_eCjYFcqN5b1bxBM4fdk0VESpjWR0fc,28
 modal/_runtime/asgi.py,sha256=GvuxZqWnIHMIR-Bx5f7toCQlkERaJO8CHjTPNM9IFIw,21537
@@ -84,11 +84,11 @@ modal/_runtime/user_code_imports.py,sha256=4fI0F9OIaNOcO_S4Tx2JcnYwZwZq6JdhMAkUz
 modal/_utils/__init__.py,sha256=waLjl5c6IPDhSsdWAm9Bji4e2PVxamYABKAze6CHVXY,28
 modal/_utils/app_utils.py,sha256=88BT4TPLWfYAQwKTHcyzNQRHg8n9B-QE2UyJs96iV-0,108
 modal/_utils/async_utils.py,sha256=9ubwMkwiDB4gzOYG2jL9j7Fs-5dxHjcifZe3r7JRg-k,25091
-modal/_utils/blob_utils.py,sha256=0k_qUpO5GHnz538wjRhyRw4NdJ5O322N7QSilIu32jw,16601
+modal/_utils/blob_utils.py,sha256=1D_dXspFdsVxkW3gsYH-wJUUHiCpYvlwhmCgYZZgN9k,17237
 modal/_utils/function_utils.py,sha256=LgcveUUb4XU_dWxtqgK_3ujZBvS3cGVzcDOkljyFZ2w,25066
 modal/_utils/grpc_testing.py,sha256=H1zHqthv19eGPJz2HKXDyWXWGSqO4BRsxah3L5Xaa8A,8619
 modal/_utils/grpc_utils.py,sha256=PPB5ay-vXencXNIWPVw5modr3EH7gfq2QPcO5YJ1lMU,7737
-modal/_utils/hash_utils.py,sha256=LOWJ9U5Eaye2ZIGQOdEk8RN8ExPFovGQisyuGv1PXU0,2236
+modal/_utils/hash_utils.py,sha256=zg3J6OGxTFGSFri1qQ12giDz90lWk8bzaxCTUCRtiX4,3034
 modal/_utils/http_utils.py,sha256=VKXYNPJtrSwZ1ttcXVGQUWmn8cLAXiOTv05g2ac3GbU,2179
 modal/_utils/logger.py,sha256=ePzdudrtx9jJCjuO6-bcL_kwUJfi4AwloUmIiNtqkY0,1330
 modal/_utils/mount_utils.py,sha256=J-FRZbPQv1i__Tob-FIpbB1oXWpFLAwZiB4OCiJpFG0,3206
@@ -161,10 +161,10 @@ modal_proto/options_pb2_grpc.pyi,sha256=CImmhxHsYnF09iENPoe8S4J-n93jtgUYD2JPAc0y
 modal_proto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 modal_version/__init__.py,sha256=RT6zPoOdFO99u5Wcxxaoir4ZCuPTbQ22cvzFAXl3vUY,470
 modal_version/__main__.py,sha256=2FO0yYQQwDTh6udt1h-cBnGd1c4ZyHnHSI4BksxzVac,105
-modal_version/_version_generated.py,sha256=iMmZstd6JhPFScB-dfN5XinMS8GWtT-kQD4yTA3FQr4,149
-modal-0.68.11.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-modal-0.68.11.dist-info/METADATA,sha256=xrxk7D7aRQurLJimfndYZrKyYvfhEFqBVT_o7q7Mxgk,2329
-modal-0.68.11.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-modal-0.68.11.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
-modal-0.68.11.dist-info/top_level.txt,sha256=1nvYbOSIKcmU50fNrpnQnrrOpj269ei3LzgB6j9xGqg,64
-modal-0.68.11.dist-info/RECORD,,
+modal_version/_version_generated.py,sha256=SWqJPFBaGp3gDChTb6Zx02oI_iQyIRtKpGgcw69Yegs,149
+modal-0.68.14.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+modal-0.68.14.dist-info/METADATA,sha256=gHSGyvKb5gQiL1YbO3mRis3T8En0t19ug9mfOY4HYos,2329
+modal-0.68.14.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+modal-0.68.14.dist-info/entry_points.txt,sha256=An-wYgeEUnm6xzrAP9_NTSTSciYvvEWsMZILtYrvpAI,46
+modal-0.68.14.dist-info/top_level.txt,sha256=1nvYbOSIKcmU50fNrpnQnrrOpj269ei3LzgB6j9xGqg,64
+modal-0.68.14.dist-info/RECORD,,

modal_version/_version_generated.py

@@ -1,4 +1,4 @@
 # Copyright Modal Labs 2024
 
 # Note: Reset this value to -1 whenever you make a minor `0.X` release of the client.
-build_number = 11  # git: 51e786f
+build_number = 14  # git: 45ed247