ygg 0.1.18__tar.gz → 0.1.20__tar.gz

{ygg-0.1.18 → ygg-0.1.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.18
+Version: 0.1.20
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 Project-URL: Homepage, https://github.com/Platob/Yggdrasil

{ygg-0.1.18 → ygg-0.1.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.18"
+version = "0.1.20"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{ygg-0.1.18 → ygg-0.1.20}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.18
+Version: 0.1.20
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 Project-URL: Homepage, https://github.com/Platob/Yggdrasil

{ygg-0.1.18 → ygg-0.1.20}/src/yggdrasil/databricks/compute/execution_context.py

@@ -294,8 +294,6 @@ print(json.dumps(meta))
         if use_dill is None:
             if current_version == self.cluster.python_version:
                 use_dill = True
-            elif current_version in ((3, 12), (3, 13)) and self.cluster.python_version in ((3, 12), (3, 13)):
-                use_dill = True
             else:
                 use_dill = False
 

ygg-0.1.20/src/yggdrasil/databricks/workspaces/__init__.py

@@ -0,0 +1,2 @@
+from .workspace import *
+from .databricks_path import *

{ygg-0.1.18 → ygg-0.1.20}/src/yggdrasil/databricks/workspaces/databricks_path.py

@@ -44,6 +44,8 @@ class DatabricksPathKind(str, Enum):
 
     @classmethod
     def parse(cls, path: str, workspace: Optional["Workspace"] = None) -> Tuple["DatabricksPathKind", Optional["Workspace"], str]:
+        from .workspace import Workspace
+
         if path.startswith("/Workspace") or path.startswith("/Users") or path.startswith("/Shared"):
             if path.startswith("/Users/me"):
                 workspace = Workspace() if workspace is None else workspace
@@ -54,8 +56,6 @@ class DatabricksPathKind(str, Enum):
             return cls.VOLUME, workspace, path
 
         if path.startswith("dbfs://"):
-            from .workspace import Workspace
-
             parsed = urlparse.urlparse(path)
             kind, _, inner_path = cls.parse(parsed.path)
             workspace = Workspace(host=parsed.hostname) if workspace is None else workspace

{ygg-0.1.18 → ygg-0.1.20}/src/yggdrasil/ser/callable_serde.py

@@ -174,6 +174,17 @@ def _capture_module_imports(fn: Callable[..., Any]) -> str:
         return ""
 
 
+def _pack_payload(payload: Dict[str, Any], *, level: int = 6) -> str:
+    lvl = int(level)
+    if lvl < 1:
+        lvl = 1
+    elif lvl > 9:
+        lvl = 9
+    raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    comp = zlib.compress(raw, level=lvl)
+    return _b64e(comp)
+
+
 def parse_tagged_result(stdout_text: str, result_tag: str) -> Dict[str, Any]:
     """
     Extract the last tagged JSON payload printed by a cluster command.
@@ -359,6 +370,8 @@ class CallableSerdeMixin:
         use_dill: bool,
         byte_limit: int = 0,
         result_tag: str = "<<<RESULT>>>",
+        compress_input_payload: bool = True,  # NEW
+        payload_compression_level: int = 6,  # NEW (1..9)
     ) -> str:
         if kwargs is None:
             kwargs = {}
@@ -390,7 +403,7 @@ class CallableSerdeMixin:
             "name": getattr(self.fn, "__name__", None),
             "qualname": getattr(self.fn, "__qualname__", None),
             "module": getattr(self.fn, "__module__", None),
-            "imports": imports,  # NEW
+            "imports": imports,
             "source": src,
             "dill_b64": None,
             "env": _capture_exec_env(self.fn),
@@ -400,22 +413,13 @@ class CallableSerdeMixin:
             callable_payload["dill_b64"] = _b64e(dumped_fn)
 
         # args/kwargs transport
-        if use_dill:
-            dumped_args = _safe_dill_dumps(args)
-            dumped_kwargs = _safe_dill_dumps(kwargs)
-            if dumped_args is None or dumped_kwargs is None:
-                raise ValueError("Failed to dill-serialize args/kwargs")
-            args_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_args)}
-            kwargs_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_kwargs)}
-        else:
-            try:
-                args_pack = {"kind": "json", "text": json.dumps(args)}
-                kwargs_pack = {"kind": "json", "text": json.dumps(kwargs)}
-            except TypeError as e:
-                raise ValueError(
-                    "use_dill=False requires JSON-serializable args/kwargs. "
-                    "If you need complex args, run same Python minor and use_dill=True."
-                ) from e
+        dumped_args = _safe_dill_dumps(args)
+        dumped_kwargs = _safe_dill_dumps(kwargs)
+        if dumped_args is None or dumped_kwargs is None:
+            raise ValueError("Failed to dill-serialize args/kwargs")
+
+        args_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_args)}
+        kwargs_pack: Dict[str, Any] = {"kind": "dill", "b64": _b64e(dumped_kwargs)}
 
         if not byte_limit:
             byte_limit = 512 * 1024
@@ -427,15 +431,32 @@ class CallableSerdeMixin:
             "kwargs": kwargs_pack,
             "env": client_env,
             "result_tag": result_tag,
-            "byte_limit": byte_limit
+            "byte_limit": byte_limit,
         }
 
+        # NEW: compress the embedded input payload to keep command size small
+        if compress_input_payload:
+            payload_b64 = _pack_payload(payload, level=payload_compression_level)
+            payload_bootstrap = f"""
+_payload_b64 = {payload_b64!r}
+
+def _load_payload(b64: str) -> dict:
+    raw = base64.b64decode(b64.encode("ascii"))
+    raw = zlib.decompress(raw)
+    return json.loads(raw.decode("utf-8"))
+
+_payload = _load_payload(_payload_b64)
+""".rstrip()
+        else:
+            payload_bootstrap = f"_payload = {payload!r}"
+
         return f"""
 # --- generated by yggdrasil.ser.CallableSerdeMixin.to_command ---
 import base64, json, os, traceback, zlib
 import dill
+from yggdrasil.databricks import *
 
-_payload = {payload!r}
+{payload_bootstrap}
 
 def _b64d(s: str) -> bytes:
     return base64.b64decode(s.encode("ascii"))
@@ -480,7 +501,6 @@ def _load_callable(cpack, use_dill: bool):
     ns = {{}}
     ns.update(_hydrate_env(cpack.get("env") or {{}}))
 
-    # NEW: exec module imports first
     if imports.strip():
         exec(imports, ns, ns)
 
@@ -498,34 +518,22 @@ def _load_callable(cpack, use_dill: bool):
 def _emit(tag: str, obj: dict):
     print(tag + json.dumps(obj, ensure_ascii=False, separators=(",", ":")))
 
-
 def _zlib_level(n: int, limit: int) -> int:
-    # ratio of size to limit
     r = n / max(1, limit)
-
-    # fast for huge payloads, stronger only when slightly over
-    if r >= 32:
-        return 1
-    if r >= 16:
-        return 2
-    if r >= 8:
-        return 3
-    if r >= 4:
-        return 4
-    if r >= 2:
-        return 5
-    if r >= 1.25:
-        return 6
-    # barely over: squeeze a bit more
+    if r >= 32: return 1
+    if r >= 16: return 2
+    if r >= 8:  return 3
+    if r >= 4:  return 4
+    if r >= 2:  return 5
+    if r >= 1.25: return 6
     return 7
 
-# apply env vars
 for k, v in (_payload.get("env") or {{}}).items():
     if v is not None:
         os.environ[str(k)] = str(v)
 
 tag = _payload.get("result_tag", "<<<RESULT>>>")
-byte_limit = int((_payload.get("byte_limit") or 2_000_000))  # ~2MB serialized bytes
+byte_limit = int((_payload.get("byte_limit") or 2_000_000))
 
 try:
     use_dill = bool(_payload.get("use_dill", False))

ygg-0.1.18/src/yggdrasil/databricks/workspaces/__init__.py

@@ -1 +0,0 @@
-from .workspace import *