ygg 0.1.19__py3-none-any.whl → 0.1.20__py3-none-any.whl

{ygg-0.1.19.dist-info → ygg-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.19
+Version: 0.1.20
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 Project-URL: Homepage, https://github.com/Platob/Yggdrasil

{ygg-0.1.19.dist-info → ygg-0.1.20.dist-info}/RECORD

@@ -12,7 +12,7 @@ yggdrasil/databricks/sql/exceptions.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 yggdrasil/databricks/sql/statement_result.py,sha256=L-hrK5MVnH3XG57BpGmaETtRzYjAtYGQthgAPGVj618,12610
 yggdrasil/databricks/sql/types.py,sha256=YgasSyq8sygk1h6ZOTcRwXAZWNKSuk-9g9VqlR8kJl4,5324
 yggdrasil/databricks/workspaces/__init__.py,sha256=tNNS3A_Pl9FYkQ8nGERhr4VF-hwKrvh8k1W8vTaR0uo,58
-yggdrasil/databricks/workspaces/databricks_path.py,sha256=1jGLqAxSUvhBiSs78W2rS31ENEz4HcAQ0U_HqgqlJMM,29551
+yggdrasil/databricks/workspaces/databricks_path.py,sha256=ieaAEPo2lBmgqrdFwtXdy9uIDFyhFT8hjbO_KP5ayec,29547
 yggdrasil/databricks/workspaces/workspace.py,sha256=8T-d0DTq-s2zTkmMbaEeS_7AiZwcUDxorch9IbZACko,33333
 yggdrasil/dataclasses/__init__.py,sha256=QVAvZnNl7gFYTLOVTfMkdQZf6o_WL8_UuDV1uTZ7Aeg,67
 yggdrasil/dataclasses/dataclass.py,sha256=ln-D1-bbiCLBd2khRMGs4dFoxzJEEGzHTKekWCnF2uk,5436
@@ -33,7 +33,7 @@ yggdrasil/requests/__init__.py,sha256=THJz1IoZYQccwmXcQR3N8D-uWxCkfMtgeXDhONdERR
 yggdrasil/requests/msal.py,sha256=ucnN45iZZpbXkByw212PX4shH4g0EeyrW8JEmfimWtY,5861
 yggdrasil/requests/session.py,sha256=YomLcDf8O_mc8BUnf9fr5wrupDnxEzaGw-guhV91NsE,830
 yggdrasil/ser/__init__.py,sha256=sS66Bxu8aiLb-8N2aNayquamfi7FobEH51JyV5ULDFI,31
-yggdrasil/ser/callable_serde.py,sha256=GlCHYayy1MbrtRL9A4JQ4H91HhPpBDSZPEeXTGcimB0,20751
+yggdrasil/ser/callable_serde.py,sha256=hgJilSWYcAivFCfwRJJ-eKzGBtDfUE1eNppX4ac7Sd0,21473
 yggdrasil/types/__init__.py,sha256=p0Qu_69RkePPgQGM9nSue_bcbEIAM2u9eo3zsEplHJ8,82
 yggdrasil/types/libs.py,sha256=7-p0M4C6TnEWpUGf2nY4XshhJxtXOE_-bsYmJWU6jtk,227
 yggdrasil/types/python_arrow.py,sha256=1Ac1ZnEgmH4nLNBfrfbM6E9EfANGWtdANqlwMENQkTw,21408
@@ -48,7 +48,7 @@ yggdrasil/types/cast/registry.py,sha256=-88mq-U1pDSGbEC9PRY0zJCzloyBodXgeSRBPb6h
 yggdrasil/types/cast/spark_cast.py,sha256=IHthM78dugabGXxNNW9sSHn-olDwzXcFdIFcPo9IiXU,23021
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=8PgJItF_XbyBcNuBnXkMQU3PBy3sAPEXZT9SXL2WbU4,4200
 yggdrasil/types/cast/spark_polars_cast.py,sha256=ba1UOvY1ouGCro1Np9slXmJ4TEyWnUtwVEAwxGvPLlk,8336
-ygg-0.1.19.dist-info/METADATA,sha256=__YFsy3OqdkKedGn-zOh6Y1VFpTWVl_rZMS4Aa8AtZY,5981
-ygg-0.1.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.19.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.19.dist-info/RECORD,,
+ygg-0.1.20.dist-info/METADATA,sha256=MlGCV96KVsKcbu92-pX4CKQjA9KlOE1lafq2KhG4DdQ,5981
+ygg-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.20.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.20.dist-info/RECORD,,

yggdrasil/databricks/workspaces/databricks_path.py

@@ -44,6 +44,8 @@ class DatabricksPathKind(str, Enum):
 
     @classmethod
     def parse(cls, path: str, workspace: Optional["Workspace"] = None) -> Tuple["DatabricksPathKind", Optional["Workspace"], str]:
+        from .workspace import Workspace
+
         if path.startswith("/Workspace") or path.startswith("/Users") or path.startswith("/Shared"):
             if path.startswith("/Users/me"):
                 workspace = Workspace() if workspace is None else workspace
@@ -54,8 +56,6 @@ class DatabricksPathKind(str, Enum):
             return cls.VOLUME, workspace, path
 
         if path.startswith("dbfs://"):
-            from .workspace import Workspace
-
             parsed = urlparse.urlparse(path)
             kind, _, inner_path = cls.parse(parsed.path)
             workspace = Workspace(host=parsed.hostname) if workspace is None else workspace

yggdrasil/ser/callable_serde.py

@@ -174,6 +174,17 @@ def _capture_module_imports(fn: Callable[..., Any]) -> str:
         return ""
 
 
+def _pack_payload(payload: Dict[str, Any], *, level: int = 6) -> str:
+    lvl = int(level)
+    if lvl < 1:
+        lvl = 1
+    elif lvl > 9:
+        lvl = 9
+    raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    comp = zlib.compress(raw, level=lvl)
+    return _b64e(comp)
+
+
 def parse_tagged_result(stdout_text: str, result_tag: str) -> Dict[str, Any]:
     """
     Extract the last tagged JSON payload printed by a cluster command.
@@ -359,6 +370,8 @@ class CallableSerdeMixin:
         use_dill: bool,
         byte_limit: int = 0,
         result_tag: str = "<<<RESULT>>>",
+        compress_input_payload: bool = True,  # NEW
+        payload_compression_level: int = 6,  # NEW (1..9)
     ) -> str:
         if kwargs is None:
             kwargs = {}
@@ -390,7 +403,7 @@ class CallableSerdeMixin:
             "name": getattr(self.fn, "__name__", None),
             "qualname": getattr(self.fn, "__qualname__", None),
             "module": getattr(self.fn, "__module__", None),
-            "imports": imports,  # NEW
+            "imports": imports,
             "source": src,
             "dill_b64": None,
             "env": _capture_exec_env(self.fn),
@@ -418,16 +431,32 @@ class CallableSerdeMixin:
             "kwargs": kwargs_pack,
             "env": client_env,
             "result_tag": result_tag,
-            "byte_limit": byte_limit
+            "byte_limit": byte_limit,
         }
 
+        # NEW: compress the embedded input payload to keep command size small
+        if compress_input_payload:
+            payload_b64 = _pack_payload(payload, level=payload_compression_level)
+            payload_bootstrap = f"""
+_payload_b64 = {payload_b64!r}
+
+def _load_payload(b64: str) -> dict:
+    raw = base64.b64decode(b64.encode("ascii"))
+    raw = zlib.decompress(raw)
+    return json.loads(raw.decode("utf-8"))
+
+_payload = _load_payload(_payload_b64)
+""".rstrip()
+        else:
+            payload_bootstrap = f"_payload = {payload!r}"
+
         return f"""
 # --- generated by yggdrasil.ser.CallableSerdeMixin.to_command ---
 import base64, json, os, traceback, zlib
 import dill
 from yggdrasil.databricks import *
 
-_payload = {payload!r}
+{payload_bootstrap}
 
 def _b64d(s: str) -> bytes:
     return base64.b64decode(s.encode("ascii"))
@@ -472,7 +501,6 @@ def _load_callable(cpack, use_dill: bool):
     ns = {{}}
     ns.update(_hydrate_env(cpack.get("env") or {{}}))
 
-    # NEW: exec module imports first
     if imports.strip():
         exec(imports, ns, ns)
 
@@ -490,34 +518,22 @@ def _load_callable(cpack, use_dill: bool):
 def _emit(tag: str, obj: dict):
     print(tag + json.dumps(obj, ensure_ascii=False, separators=(",", ":")))
 
-
 def _zlib_level(n: int, limit: int) -> int:
-    # ratio of size to limit
     r = n / max(1, limit)
-
-    # fast for huge payloads, stronger only when slightly over
-    if r >= 32:
-        return 1
-    if r >= 16:
-        return 2
-    if r >= 8:
-        return 3
-    if r >= 4:
-        return 4
-    if r >= 2:
-        return 5
-    if r >= 1.25:
-        return 6
-    # barely over: squeeze a bit more
+    if r >= 32: return 1
+    if r >= 16: return 2
+    if r >= 8:  return 3
+    if r >= 4:  return 4
+    if r >= 2:  return 5
+    if r >= 1.25: return 6
     return 7
 
-# apply env vars
 for k, v in (_payload.get("env") or {{}}).items():
     if v is not None:
         os.environ[str(k)] = str(v)
 
 tag = _payload.get("result_tag", "<<<RESULT>>>")
-byte_limit = int((_payload.get("byte_limit") or 2_000_000))  # ~2MB serialized bytes
+byte_limit = int((_payload.get("byte_limit") or 2_000_000))
 
 try:
     use_dill = bool(_payload.get("use_dill", False))