ygg 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

{ygg-0.1.18.dist-info → ygg-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.18
+Version: 0.1.20
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 Project-URL: Homepage, https://github.com/Platob/Yggdrasil

{ygg-0.1.18.dist-info → ygg-0.1.20.dist-info}/RECORD

@@ -2,7 +2,7 @@ yggdrasil/__init__.py,sha256=6OPibApplA5TF4TeixkQO_qewpaAidYX-fSDvvKYcTI,91
 yggdrasil/databricks/__init__.py,sha256=aGVve5mpoQtxSK2nfzrexjRPoutCIyaOnKZijkG4_QE,92
 yggdrasil/databricks/compute/__init__.py,sha256=TVDwPmW2SOmHmnhzZhsvrWbrxZ_lEcgqe3l9BeB-oxM,218
 yggdrasil/databricks/compute/cluster.py,sha256=xElDioObG6exkUS08K-Ccs_EFNbWD69Z15fjvnHwOx8,26958
-yggdrasil/databricks/compute/execution_context.py,sha256=pnzA_itZiYW4LpjgWnKZlovABEjHToDAp2ahVMnZmRQ,18625
+yggdrasil/databricks/compute/execution_context.py,sha256=0W8GiOJH8iWsonXxKy7KLsyX7d8SMLvV5IfM5s8KTGs,18481
 yggdrasil/databricks/compute/remote.py,sha256=DzPVPk-4bt5bOL52Onur3xLNh3UzS2K70DU5HglhGSg,1216
 yggdrasil/databricks/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 yggdrasil/databricks/jobs/config.py,sha256=8Slfw4Wl7vu0kIlaUUqVqjjOgPwuULoo0rroENCbC20,11494
@@ -11,8 +11,8 @@ yggdrasil/databricks/sql/engine.py,sha256=WRDrY-FpXMegF3qNmMkCsfIitJB9rY_lW8Pnk7
 yggdrasil/databricks/sql/exceptions.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 yggdrasil/databricks/sql/statement_result.py,sha256=L-hrK5MVnH3XG57BpGmaETtRzYjAtYGQthgAPGVj618,12610
 yggdrasil/databricks/sql/types.py,sha256=YgasSyq8sygk1h6ZOTcRwXAZWNKSuk-9g9VqlR8kJl4,5324
-yggdrasil/databricks/workspaces/__init__.py,sha256=ffKl4XrLPjBicXa7Ygda_VcbhSGw8IPJVm5NqrMzHks,26
-yggdrasil/databricks/workspaces/databricks_path.py,sha256=1jGLqAxSUvhBiSs78W2rS31ENEz4HcAQ0U_HqgqlJMM,29551
+yggdrasil/databricks/workspaces/__init__.py,sha256=tNNS3A_Pl9FYkQ8nGERhr4VF-hwKrvh8k1W8vTaR0uo,58
+yggdrasil/databricks/workspaces/databricks_path.py,sha256=ieaAEPo2lBmgqrdFwtXdy9uIDFyhFT8hjbO_KP5ayec,29547
 yggdrasil/databricks/workspaces/workspace.py,sha256=8T-d0DTq-s2zTkmMbaEeS_7AiZwcUDxorch9IbZACko,33333
 yggdrasil/dataclasses/__init__.py,sha256=QVAvZnNl7gFYTLOVTfMkdQZf6o_WL8_UuDV1uTZ7Aeg,67
 yggdrasil/dataclasses/dataclass.py,sha256=ln-D1-bbiCLBd2khRMGs4dFoxzJEEGzHTKekWCnF2uk,5436
@@ -33,7 +33,7 @@ yggdrasil/requests/__init__.py,sha256=THJz1IoZYQccwmXcQR3N8D-uWxCkfMtgeXDhONdERR
 yggdrasil/requests/msal.py,sha256=ucnN45iZZpbXkByw212PX4shH4g0EeyrW8JEmfimWtY,5861
 yggdrasil/requests/session.py,sha256=YomLcDf8O_mc8BUnf9fr5wrupDnxEzaGw-guhV91NsE,830
 yggdrasil/ser/__init__.py,sha256=sS66Bxu8aiLb-8N2aNayquamfi7FobEH51JyV5ULDFI,31
-yggdrasil/ser/callable_serde.py,sha256=1pDgrzAceoFQ7JS7qIeSxC4hCz2oAcQOkeWyYYeT7iY,21206
+yggdrasil/ser/callable_serde.py,sha256=hgJilSWYcAivFCfwRJJ-eKzGBtDfUE1eNppX4ac7Sd0,21473
 yggdrasil/types/__init__.py,sha256=p0Qu_69RkePPgQGM9nSue_bcbEIAM2u9eo3zsEplHJ8,82
 yggdrasil/types/libs.py,sha256=7-p0M4C6TnEWpUGf2nY4XshhJxtXOE_-bsYmJWU6jtk,227
 yggdrasil/types/python_arrow.py,sha256=1Ac1ZnEgmH4nLNBfrfbM6E9EfANGWtdANqlwMENQkTw,21408
@@ -48,7 +48,7 @@ yggdrasil/types/cast/registry.py,sha256=-88mq-U1pDSGbEC9PRY0zJCzloyBodXgeSRBPb6h
 yggdrasil/types/cast/spark_cast.py,sha256=IHthM78dugabGXxNNW9sSHn-olDwzXcFdIFcPo9IiXU,23021
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=8PgJItF_XbyBcNuBnXkMQU3PBy3sAPEXZT9SXL2WbU4,4200
 yggdrasil/types/cast/spark_polars_cast.py,sha256=ba1UOvY1ouGCro1Np9slXmJ4TEyWnUtwVEAwxGvPLlk,8336
-ygg-0.1.18.dist-info/METADATA,sha256=d_adw4j5tP0rLrz-9sgI6fL5vWp5s6KgICs7lyLctgI,5981
-ygg-0.1.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.18.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.18.dist-info/RECORD,,
+ygg-0.1.20.dist-info/METADATA,sha256=MlGCV96KVsKcbu92-pX4CKQjA9KlOE1lafq2KhG4DdQ,5981
+ygg-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.20.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.20.dist-info/RECORD,,

yggdrasil/databricks/compute/execution_context.py

@@ -294,8 +294,6 @@ print(json.dumps(meta))
         if use_dill is None:
             if current_version == self.cluster.python_version:
                 use_dill = True
-            elif current_version in ((3, 12), (3, 13)) and self.cluster.python_version in ((3, 12), (3, 13)):
-                use_dill = True
             else:
                 use_dill = False
 

yggdrasil/databricks/workspaces/__init__.py

@@ -1 +1,2 @@
 from .workspace import *
+from .databricks_path import *

yggdrasil/databricks/workspaces/databricks_path.py

@@ -44,6 +44,8 @@ class DatabricksPathKind(str, Enum):
 
     @classmethod
     def parse(cls, path: str, workspace: Optional["Workspace"] = None) -> Tuple["DatabricksPathKind", Optional["Workspace"], str]:
+        from .workspace import Workspace
+
         if path.startswith("/Workspace") or path.startswith("/Users") or path.startswith("/Shared"):
             if path.startswith("/Users/me"):
                 workspace = Workspace() if workspace is None else workspace
@@ -54,8 +56,6 @@ class DatabricksPathKind(str, Enum):
             return cls.VOLUME, workspace, path
 
         if path.startswith("dbfs://"):
-            from .workspace import Workspace
-
             parsed = urlparse.urlparse(path)
             kind, _, inner_path = cls.parse(parsed.path)
             workspace = Workspace(host=parsed.hostname) if workspace is None else workspace

yggdrasil/ser/callable_serde.py

@@ -174,6 +174,17 @@ def _capture_module_imports(fn: Callable[..., Any]) -> str:
         return ""
 
 
+def _pack_payload(payload: Dict[str, Any], *, level: int = 6) -> str:
+    lvl = int(level)
+    if lvl < 1:
+        lvl = 1
+    elif lvl > 9:
+        lvl = 9
+    raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    comp = zlib.compress(raw, level=lvl)
+    return _b64e(comp)
+
+
 def parse_tagged_result(stdout_text: str, result_tag: str) -> Dict[str, Any]:
     """
     Extract the last tagged JSON payload printed by a cluster command.
@@ -359,6 +370,8 @@ class CallableSerdeMixin:
         use_dill: bool,
         byte_limit: int = 0,
         result_tag: str = "<<<RESULT>>>",
+        compress_input_payload: bool = True,  # NEW
+        payload_compression_level: int = 6,  # NEW (1..9)
     ) -> str:
         if kwargs is None:
             kwargs = {}
@@ -390,7 +403,7 @@ class CallableSerdeMixin:
             "name": getattr(self.fn, "__name__", None),
             "qualname": getattr(self.fn, "__qualname__", None),
             "module": getattr(self.fn, "__module__", None),
-            "imports": imports,  # NEW
+            "imports": imports,
             "source": src,
             "dill_b64": None,
             "env": _capture_exec_env(self.fn),
@@ -400,22 +413,13 @@ class CallableSerdeMixin:
             callable_payload["dill_b64"] = _b64e(dumped_fn)
 
         # args/kwargs transport
-        if use_dill:
-            dumped_args = _safe_dill_dumps(args)
-            dumped_kwargs = _safe_dill_dumps(kwargs)
-            if dumped_args is None or dumped_kwargs is None:
-                raise ValueError("Failed to dill-serialize args/kwargs")
-            args_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_args)}
-            kwargs_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_kwargs)}
-        else:
-            try:
-                args_pack = {"kind": "json", "text": json.dumps(args)}
-                kwargs_pack = {"kind": "json", "text": json.dumps(kwargs)}
-            except TypeError as e:
-                raise ValueError(
-                    "use_dill=False requires JSON-serializable args/kwargs. "
-                    "If you need complex args, run same Python minor and use_dill=True."
-                ) from e
+        dumped_args = _safe_dill_dumps(args)
+        dumped_kwargs = _safe_dill_dumps(kwargs)
+        if dumped_args is None or dumped_kwargs is None:
+            raise ValueError("Failed to dill-serialize args/kwargs")
+
+        args_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_args)}
+        kwargs_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_kwargs)}
 
         if not byte_limit:
             byte_limit = 512 * 1024
@@ -427,15 +431,32 @@ class CallableSerdeMixin:
             "kwargs": kwargs_pack,
             "env": client_env,
             "result_tag": result_tag,
-            "byte_limit": byte_limit
+            "byte_limit": byte_limit,
         }
 
+        # NEW: compress the embedded input payload to keep command size small
+        if compress_input_payload:
+            payload_b64 = _pack_payload(payload, level=payload_compression_level)
+            payload_bootstrap = f"""
+_payload_b64 = {payload_b64!r}
+
+def _load_payload(b64: str) -> dict:
+    raw = base64.b64decode(b64.encode("ascii"))
+    raw = zlib.decompress(raw)
+    return json.loads(raw.decode("utf-8"))
+
+_payload = _load_payload(_payload_b64)
+""".rstrip()
+        else:
+            payload_bootstrap = f"_payload = {payload!r}"
+
         return f"""
 # --- generated by yggdrasil.ser.CallableSerdeMixin.to_command ---
 import base64, json, os, traceback, zlib
 import dill
+from yggdrasil.databricks import *
 
-_payload = {payload!r}
+{payload_bootstrap}
 
 def _b64d(s: str) -> bytes:
     return base64.b64decode(s.encode("ascii"))
@@ -480,7 +501,6 @@ def _load_callable(cpack, use_dill: bool):
     ns = {{}}
     ns.update(_hydrate_env(cpack.get("env") or {{}}))
 
-    # NEW: exec module imports first
     if imports.strip():
         exec(imports, ns, ns)
 
@@ -498,34 +518,22 @@ def _load_callable(cpack, use_dill: bool):
 def _emit(tag: str, obj: dict):
     print(tag + json.dumps(obj, ensure_ascii=False, separators=(",", ":")))
 
-
 def _zlib_level(n: int, limit: int) -> int:
-    # ratio of size to limit
     r = n / max(1, limit)
-
-    # fast for huge payloads, stronger only when slightly over
-    if r >= 32:
-        return 1
-    if r >= 16:
-        return 2
-    if r >= 8:
-        return 3
-    if r >= 4:
-        return 4
-    if r >= 2:
-        return 5
-    if r >= 1.25:
-        return 6
-    # barely over: squeeze a bit more
+    if r >= 32: return 1
+    if r >= 16: return 2
+    if r >= 8:  return 3
+    if r >= 4:  return 4
+    if r >= 2:  return 5
+    if r >= 1.25: return 6
     return 7
 
-# apply env vars
 for k, v in (_payload.get("env") or {{}}).items():
     if v is not None:
         os.environ[str(k)] = str(v)
 
 tag = _payload.get("result_tag", "<<<RESULT>>>")
-byte_limit = int((_payload.get("byte_limit") or 2_000_000))  # ~2MB serialized bytes
+byte_limit = int((_payload.get("byte_limit") or 2_000_000))
 
 try:
     use_dill = bool(_payload.get("use_dill", False))