ygg 0.1.38__py3-none-any.whl → 0.1.41__py3-none-any.whl

{ygg-0.1.38.dist-info → ygg-0.1.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.38
+Version: 0.1.41
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.38.dist-info → ygg-0.1.41.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ygg-0.1.38.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+ygg-0.1.41.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 yggdrasil/__init__.py,sha256=PfH7Xwt6uue6oqe6S5V8NhDJcVQClkKrBE1KXhdelZc,117
-yggdrasil/version.py,sha256=pPo3-5ffVm5NbQms9PN259Sjq7R-Q9UY4HYJ8M2iom4,22
+yggdrasil/version.py,sha256=MW_v7YHPKwbcPR_PtCE3ALJ0csFwgMewFF071yAYGCE,22
 yggdrasil/databricks/__init__.py,sha256=skctY2c8W-hI81upx9F_PWRe5ishL3hrdiTuizgDjdw,152
 yggdrasil/databricks/compute/__init__.py,sha256=NvdzmaJSNYY1uJthv1hHdBuNu3bD_-Z65DWnaJt9yXg,289
-yggdrasil/databricks/compute/cluster.py,sha256=mnNzjCx7X3iK22oZ7K3pqot0AXq9JTdg97kT61j2_UU,40729
+yggdrasil/databricks/compute/cluster.py,sha256=Ejc3VlS56QzyjFX1-wNXHC4BNc2MpDRfFFafVxPF8rE,40968
 yggdrasil/databricks/compute/execution_context.py,sha256=nxrNXoarq_JAB-Cpj0udHhq2jx-DmMbRWJdAezLrPis,22347
 yggdrasil/databricks/compute/remote.py,sha256=nEN_Fr1Ouul_iKOf4B5QjEGscYAcl7nHjGsl2toRzrU,2874
 yggdrasil/databricks/jobs/__init__.py,sha256=snxGSJb0M5I39v0y3IR-uEeSlZR248cQ_4DJ1sYs-h8,154
@@ -11,7 +11,7 @@ yggdrasil/databricks/jobs/config.py,sha256=9LGeHD04hbfy0xt8_6oobC4moKJh4_DTjZiK4
 yggdrasil/databricks/sql/__init__.py,sha256=y1n5yg-drZ8QVZbEgznsRG24kdJSnFis9l2YfYCsaCM,234
 yggdrasil/databricks/sql/engine.py,sha256=Azx3gKtWOMy3D9I2FhkLmpthZPWAJZ9iZkaDivmt_0s,41002
 yggdrasil/databricks/sql/exceptions.py,sha256=Jqd_gT_VyPL8klJEHYEzpv5eHtmdY43WiQ7HZBaEqSk,53
-yggdrasil/databricks/sql/statement_result.py,sha256=tik53oB7Eq6nA4Unv8qH6fe9RJJO9HcumI5BAkW3omA,16684
+yggdrasil/databricks/sql/statement_result.py,sha256=_mBolHae0AASfe1Tlld1KTXs-K4-oy9dniHDyR2ILYc,16736
 yggdrasil/databricks/sql/types.py,sha256=5G-BM9_eOsRKEMzeDTWUsWW5g4Idvs-czVCpOCrMhdA,6412
 yggdrasil/databricks/workspaces/__init__.py,sha256=Ti1I99JTC3koYJaCy8WYvkAox4KdcuMRk8b2rHroWCY,133
 yggdrasil/databricks/workspaces/filesytem.py,sha256=Z8JXU7_XUEbw9fpTQT1avRQKi-IAP2KemXBMPkUoY4w,9805
@@ -30,7 +30,7 @@ yggdrasil/libs/extensions/__init__.py,sha256=mcXW5Li3Cbprbs4Ci-b5A0Ju0wmLcfvEiFu
 yggdrasil/libs/extensions/polars_extensions.py,sha256=RTkGi8llhPJjX7x9egix7-yXWo2X24zIAPSKXV37SSA,12397
 yggdrasil/libs/extensions/spark_extensions.py,sha256=E64n-3SFTDgMuXwWitX6vOYP9ln2lpGKb0htoBLEZgc,16745
 yggdrasil/pyutils/__init__.py,sha256=tl-LapAc71TV7RMgf2ftKwrzr8iiLOGHeJgA3RvO93w,293
-yggdrasil/pyutils/callable_serde.py,sha256=euY7Kiy04i1tpWKuB0b2qQ1FokLC3nq0cv7PObWYUBE,21809
+yggdrasil/pyutils/callable_serde.py,sha256=FtbY2PUBHwDe2IyX28gLJwQYjtrvhvqtTSIIBc5x-dk,23538
 yggdrasil/pyutils/equality.py,sha256=Xyf8D1dLUCm3spDEir8Zyj7O4US_fBJwEylJCfJ9slI,3080
 yggdrasil/pyutils/exceptions.py,sha256=ssKNm-rjhavHUOZmGA7_1Gq9tSHDrb2EFI-cnBuWgng,3388
 yggdrasil/pyutils/expiring_dict.py,sha256=q9gb09-2EUN-jQZumUw5BXOQGYcj1wb85qKtQlciSxg,5825
@@ -55,8 +55,8 @@ yggdrasil/types/cast/registry.py,sha256=_zdFGmUBB7P-e_LIcJlOxMcxAkXoA-UXB6HqLMgT
 yggdrasil/types/cast/spark_cast.py,sha256=_KAsl1DqmKMSfWxqhVE7gosjYdgiL1C5bDQv6eP3HtA,24926
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=BuTiWrdCANZCdD_p2MAytqm74eq-rdRXd-LGojBRrfU,5023
 yggdrasil/types/cast/spark_polars_cast.py,sha256=btmZNHXn2NSt3fUuB4xg7coaE0RezIBdZD92H8NK0Jw,9073
-ygg-0.1.38.dist-info/METADATA,sha256=2G6ZcCvdN5TLyP9Z7z3GHQio7xjzBCd-ZEOO2qnL2Yc,19204
-ygg-0.1.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.38.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
-ygg-0.1.38.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.38.dist-info/RECORD,,
+ygg-0.1.41.dist-info/METADATA,sha256=EYXc8625quuP-JkcUD5FIK-5AmgYMddSWj2t4mnN2bg,19204
+ygg-0.1.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.41.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
+ygg-0.1.41.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.41.dist-info/RECORD,,

yggdrasil/databricks/compute/cluster.py

@@ -667,6 +667,8 @@ class Cluster(WorkspaceService):
         Returns:
             The updated Cluster instance.
         """
+        self.install_libraries(libraries=libraries, wait_timeout=None, raise_error=False)
+
         existing_details = {
             k: v
             for k, v in self.details.as_shallow_dict().items()
@@ -699,16 +701,20 @@ class Cluster(WorkspaceService):
             )
 
             self.wait_for_status()
-            self.details = self.clusters_client().edit(**update_details)
-            self.wait_for_status()
+            try:
+                self.details = self.clusters_client().edit_and_wait(**update_details)
+            except Exception as e:
+                if self.state == State.TERMINATED:
+                    self.start()
+                    self.details = self.clusters_client().edit_and_wait(**update_details)
+                else:
+                    raise e
 
             logger.info(
                 "Updated %s",
                 self
             )
 
-        self.install_libraries(libraries=libraries, wait_timeout=None, raise_error=False)
-
         return self
 
     def list_clusters(self) -> Iterator["Cluster"]:
@@ -816,7 +822,6 @@ class Cluster(WorkspaceService):
 
         return self
 
-    @retry(tries=4)
     def restart(
         self,
     ):

yggdrasil/databricks/sql/statement_result.py

@@ -195,6 +195,7 @@ class StatementResult:
         Returns:
             The result manifest or None for Spark SQL results.
         """
+        self.wait()
         return self.response.manifest
 
     @property
@@ -204,6 +205,7 @@ class StatementResult:
         Returns:
             The statement result payload from the API.
         """
+        self.wait()
         return self.response.result
 
     @property
@@ -253,7 +255,6 @@ class StatementResult:
             self, self.disposition, Disposition.EXTERNAL_LINKS
         )
 
-        self.wait()
         result_data = self.result
         wsdk = self.workspace.sdk()
 
@@ -320,6 +321,8 @@ class StatementResult:
                 f"Statement {self.statement_id} {self.state}: " + " | ".join(parts)
             )
 
+        return self
+
     def wait(
         self,
         timeout: Optional[int] = None,
@@ -339,20 +342,21 @@ class StatementResult:
 
         start = time.time()
         poll_interval = poll_interval or 1
-        current = self
 
         while not self.done:
             # still running / queued / pending
             if timeout is not None and (time.time() - start) > timeout:
                 raise TimeoutError(
-                    f"Statement {current.statement_id} did not finish within {timeout} seconds "
-                    f"(last state={current})"
+                    f"Statement {self.statement_id} did not finish within {timeout} seconds "
+                    f"(last state={self.state})"
                 )
 
             poll_interval = max(10, poll_interval * 1.2)
             time.sleep(poll_interval)
 
-        return current
+        self.raise_for_status()
+
+        return self
 
     def arrow_schema(self):
         """Return the Arrow schema for the result.

yggdrasil/pyutils/callable_serde.py

@@ -546,7 +546,15 @@ class CallableSerde:
         Returns Python code string to execute in another interpreter.
         Prints one line: "{result_tag}:{base64(blob)}"
         where blob is raw dill bytes or framed+zlib.
+
+        Also compresses the input call payload (args/kwargs) using the same framing
+        scheme when it exceeds byte_limit.
         """
+        import base64
+        import json
+        import struct
+        import zlib
+
         args = args or ()
         kwargs = kwargs or {}
 
@@ -555,13 +563,32 @@ class CallableSerde:
             dump_env=dump_env,
             filter_used_globals=filter_used_globals,
             env_keys=env_keys,
-            env_variables=env_variables
+            env_variables=env_variables,
         )
         serde_json = json.dumps(serde_dict, ensure_ascii=False)
 
-        call_payload_b64 = base64.b64encode(
-            dill.dumps((args, kwargs), recurse=True)
-        ).decode("ascii")
+        # --- input payload compression (args/kwargs) ---
+        MAGIC = b"CS1"
+        FLAG_COMPRESSED = 1
+
+        def _pick_level(n: int, limit: int) -> int:
+            ratio = n / max(1, limit)
+            x = min(1.0, max(0.0, (ratio - 1.0) / 3.0))
+            return max(1, min(9, int(round(1 + 8 * x))))
+
+        def _encode_blob(raw: bytes, limit: int) -> bytes:
+            if len(raw) <= limit:
+                return raw
+            level = _pick_level(len(raw), limit)
+            compressed = zlib.compress(raw, level)
+            if len(compressed) >= len(raw):
+                return raw
+            header = MAGIC + struct.pack(">BIB", FLAG_COMPRESSED, len(raw), level)
+            return header + compressed
+
+        call_raw = dill.dumps((args, kwargs), recurse=True)
+        call_blob = _encode_blob(call_raw, int(byte_limit))
+        call_payload_b64 = base64.b64encode(call_blob).decode("ascii")
 
         # NOTE: plain string template + replace. No f-string. No brace escaping.
         template = r"""
@@ -595,6 +622,19 @@ def _encode_result(raw: bytes, byte_limit: int) -> bytes:
     header = MAGIC + struct.pack(">BIB", FLAG_COMPRESSED, len(raw), level)
     return header + compressed
 
+def _decode_blob(blob: bytes) -> bytes:
+    # If it's framed (MAGIC + header), decompress; else return as-is.
+    if isinstance(blob, (bytes, bytearray)) and len(blob) >= 3 and blob[:3] == MAGIC:
+        if len(blob) >= 3 + 6:
+            flag, orig_len, level = struct.unpack(">BIB", blob[3:3+6])
+            if flag & FLAG_COMPRESSED:
+                raw = zlib.decompress(blob[3+6:])
+                # best-effort sanity check; don't hard-fail on mismatch
+                if isinstance(orig_len, int) and orig_len > 0 and len(raw) != orig_len:
+                    return raw
+                return raw
+    return blob
+
 def _needed_globals(fn) -> set[str]:
     names = set()
     try:
@@ -657,7 +697,9 @@ if env_b64:
     meta = serde.get("env_meta") or {}
     _apply_env(fn, env, bool(meta.get("filter_used_globals", True)))
 
-args, kwargs = dill.loads(base64.b64decode(__CALL_PAYLOAD_B64__))
+call_blob = base64.b64decode(__CALL_PAYLOAD_B64__)
+call_raw = _decode_blob(call_blob)
+args, kwargs = dill.loads(call_raw)
 
 res = fn(*args, **kwargs)
 raw = dill.dumps(res, recurse=True)

yggdrasil/version.py

@@ -1 +1 @@
-__version__ = "0.1.38"
+__version__ = "0.1.41"