ygg 0.1.47__py3-none-any.whl → 0.1.49__py3-none-any.whl

{ygg-0.1.47.dist-info → ygg-0.1.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.47
+Version: 0.1.49
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License
@@ -235,6 +235,8 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
 Dynamic: license-file
 
 # Yggdrasil (Python)
@@ -270,34 +272,6 @@ Extras are grouped by engine:
 - `.[polars]`, `.[pandas]`, `.[spark]`, `.[databricks]` – install only the integrations you need.
 - `.[dev]` – adds testing, linting, and typing tools (`pytest`, `ruff`, `black`, `mypy`).
 
-## Quickstart
-Define an Arrow-aware dataclass, coerce inputs, and cast across containers:
-
-```python
-from yggdrasil import yggdataclass
-from yggdrasil.types.cast import convert
-from yggdrasil.types import arrow_field_from_hint
-
-@yggdataclass
-class User:
-    id: int
-    email: str
-    active: bool = True
-
-user = User.__safe_init__("123", email="alice@example.com")
-assert user.id == 123 and user.active is True
-
-payload = {"id": "45", "email": "bob@example.com", "active": "false"}
-clean = User.from_dict(payload)
-print(clean.to_dict())
-
-field = arrow_field_from_hint(User, name="user")
-print(field)  # user: struct<id: int64, email: string, active: bool>
-
-numbers = convert(["1", "2", "3"], list[int])
-print(numbers)
-```
-
 ### Databricks example
 Install the `databricks` extra and run SQL with typed results:
 

{ygg-0.1.47.dist-info → ygg-0.1.49.dist-info}/RECORD

@@ -1,28 +1,28 @@
-ygg-0.1.47.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-yggdrasil/__init__.py,sha256=PfH7Xwt6uue6oqe6S5V8NhDJcVQClkKrBE1KXhdelZc,117
-yggdrasil/version.py,sha256=TPABAGTrArQ_wApv_9aI4NXV4UtQ6uCrBrlNiCowSWI,22
+ygg-0.1.49.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+yggdrasil/__init__.py,sha256=4-ghPak2S6zfMqmnlxW2GCgPb5s79znpKa2hGEGXcE4,24
+yggdrasil/version.py,sha256=pnii9XXudF0U50FobVvNgNzGy9lA9q_DntGQAvyqaFA,22
 yggdrasil/databricks/__init__.py,sha256=skctY2c8W-hI81upx9F_PWRe5ishL3hrdiTuizgDjdw,152
 yggdrasil/databricks/compute/__init__.py,sha256=NvdzmaJSNYY1uJthv1hHdBuNu3bD_-Z65DWnaJt9yXg,289
-yggdrasil/databricks/compute/cluster.py,sha256=HI9811oBCpWeo4V921FVAlRUXKXM4XO7HS9DQVOuzpM,41340
-yggdrasil/databricks/compute/execution_context.py,sha256=anOxfNms83dZ5FTknbfT8uj889LjheMqEx9W5NtJC9E,23094
-yggdrasil/databricks/compute/remote.py,sha256=nEN_Fr1Ouul_iKOf4B5QjEGscYAcl7nHjGsl2toRzrU,2874
+yggdrasil/databricks/compute/cluster.py,sha256=YomLfvB0oxbgl6WDgBRxI1UXsxwlEbR6gq3FUbPHscY,44199
+yggdrasil/databricks/compute/execution_context.py,sha256=jIV6uru2NeX3O5lg-3KEqmXtLxxq45CFgkBQgQIIOHQ,23327
+yggdrasil/databricks/compute/remote.py,sha256=yicEhyQypssRa2ByscO36s3cBkEgORFsRME9aaq91Pc,3045
 yggdrasil/databricks/jobs/__init__.py,sha256=snxGSJb0M5I39v0y3IR-uEeSlZR248cQ_4DJ1sYs-h8,154
 yggdrasil/databricks/jobs/config.py,sha256=9LGeHD04hbfy0xt8_6oobC4moKJh4_DTjZiK4Q2Tqjk,11557
-yggdrasil/databricks/sql/__init__.py,sha256=y1n5yg-drZ8QVZbEgznsRG24kdJSnFis9l2YfYCsaCM,234
+yggdrasil/databricks/sql/__init__.py,sha256=Vp_1cFaX1l-JGzCknvkbiB8CBFX2fQbBNntIeVn3lEg,231
 yggdrasil/databricks/sql/engine.py,sha256=K5WmGKpXU78JA3UdK8dLxBD_GXKidZJFe7hytuC5UHg,41029
-yggdrasil/databricks/sql/exceptions.py,sha256=Jqd_gT_VyPL8klJEHYEzpv5eHtmdY43WiQ7HZBaEqSk,53
-yggdrasil/databricks/sql/statement_result.py,sha256=_mBolHae0AASfe1Tlld1KTXs-K4-oy9dniHDyR2ILYc,16736
+yggdrasil/databricks/sql/exceptions.py,sha256=uC-BoG0u0LtORKUS1X3iLID8nc-0TV5MQN3M8RXHsO4,1495
+yggdrasil/databricks/sql/statement_result.py,sha256=kMBvpwyRv3_JUZSvxMS0c9Vqlh6LtCRJvXsDpu9RIAs,16137
 yggdrasil/databricks/sql/types.py,sha256=5G-BM9_eOsRKEMzeDTWUsWW5g4Idvs-czVCpOCrMhdA,6412
 yggdrasil/databricks/sql/warehouse.py,sha256=1J0dyQLJb-OS1_1xU1eAVZ4CoL2-FhFeowKSvU3RzFc,9773
-yggdrasil/databricks/workspaces/__init__.py,sha256=Ti1I99JTC3koYJaCy8WYvkAox4KdcuMRk8b2rHroWCY,133
+yggdrasil/databricks/workspaces/__init__.py,sha256=dv2zotoFVhNFlTCdRq6gwf5bEzeZkOZszoNZMs0k59g,114
 yggdrasil/databricks/workspaces/filesytem.py,sha256=Z8JXU7_XUEbw9fpTQT1avRQKi-IAP2KemXBMPkUoY4w,9805
-yggdrasil/databricks/workspaces/io.py,sha256=Tdde4LaGNJNT50R11OkEYZyNacyIW9QrOXMAicAlIr4,32208
-yggdrasil/databricks/workspaces/path.py,sha256=-XnCD9p42who3DAwnITVE1KyrZUSoXDKHA8iZi-7wk4,47743
+yggdrasil/databricks/workspaces/io.py,sha256=CDq9NsYFjlSJ1QbKFlfWvZLQPVoWyZ4b3XR_lxNPcZE,32776
+yggdrasil/databricks/workspaces/path.py,sha256=BxDwxE7q1-NLKEZQT4xLM3LeCeQKO3wUy7R-Ce-cSMk,47875
 yggdrasil/databricks/workspaces/path_kind.py,sha256=Xc319NysH8_6E9C0Q8nCxDHYG07_SnzyUVKHe0dNdDQ,305
-yggdrasil/databricks/workspaces/workspace.py,sha256=c6CBBun2BskEnsP74pbLVOe_TKXZs4L4r4gPQtIzlQE,23821
-yggdrasil/dataclasses/__init__.py,sha256=6SdfIyTsoM4AuVw5TW4Q-UWXz41EyfsMcpD30cmjbSM,125
-yggdrasil/dataclasses/dataclass.py,sha256=fKokFUnqe4CmXXGMTdF4XDWbCUl_c_-se-UD48L5s1E,6594
-yggdrasil/libs/__init__.py,sha256=ulzk-ZkFUI2Pfo93YKtO8MBsEWtRZzLos7HAxN74R0w,168
+yggdrasil/databricks/workspaces/workspace.py,sha256=zBlQdYNT_xKwUCYo3O4Q4g-8pfMvff3I26efyCfY_TY,24961
+yggdrasil/dataclasses/__init__.py,sha256=_RkhfF3KC1eSORby1dzvBXQ0-UGG3u6wyUQWX2jq1Pc,108
+yggdrasil/dataclasses/dataclass.py,sha256=LxrCjwvmBnb8yRI_N-c31RHHxB4XoJPixmKg9iBIuaI,1148
+yggdrasil/libs/__init__.py,sha256=zdC9OU0Xy36CLY9mg2drxN6S7isPR8aTLzJA6xVIeLE,91
 yggdrasil/libs/databrickslib.py,sha256=NHJeUViHhZc8LI5oDVfi1axRyUy_pDJLy4hjD0KZEBQ,980
 yggdrasil/libs/pandaslib.py,sha256=Edm3SXgvr8qe2wsojuRvD1ewNB-Sff0RWoTqaddVruI,509
 yggdrasil/libs/polarslib.py,sha256=7EWP5iS8F9cW79M6d8Yg5ysjnOY3w4_k7TW-5DCRACw,511
@@ -37,18 +37,17 @@ yggdrasil/pyutils/exceptions.py,sha256=ssKNm-rjhavHUOZmGA7_1Gq9tSHDrb2EFI-cnBuWg
 yggdrasil/pyutils/expiring_dict.py,sha256=pr2u25LGwPVbLfsLptiHGovUtYRRo0AMjaJtCtJl7nQ,8477
 yggdrasil/pyutils/modules.py,sha256=B7IP99YqUMW6-DIESFzBx8-09V1d0a8qrIJUDFhhL2g,11424
 yggdrasil/pyutils/parallel.py,sha256=ubuq2m9dJzWYUyKCga4Y_9bpaeMYUrleYxdp49CHr44,6781
-yggdrasil/pyutils/python_env.py,sha256=tuglnjdqHQjNh18qDladVoSEOjCD0RcnMEPYJ0tArOs,50985
+yggdrasil/pyutils/python_env.py,sha256=Gh5geFK9ABpyWEfyegGUfIJUoPxKwcH0pqLBiMrW9Rw,51103
 yggdrasil/pyutils/retry.py,sha256=n5sr-Zu7fYrdLbjJ4WifK2lk0gEGmHv5FYt2HaCm1Qc,11916
-yggdrasil/requests/__init__.py,sha256=wNkP5INH-SshEBZ1MTlqP2yAMLxo5BQK5B5oHEw3FkI,140
+yggdrasil/requests/__init__.py,sha256=dMesyzq97_DmI765x0TwaDPEfsxFtgGNgchk8LvEN-o,103
 yggdrasil/requests/msal.py,sha256=s2GCyzbgFdgdlJ1JqMrZ4qYVbmoG46-ZOTcaVQhZ-sQ,9220
 yggdrasil/requests/session.py,sha256=SLnrgHY0Lby7ZxclRFUjHdfM8euN_8bSQEWl7TkJY2U,1461
 yggdrasil/types/__init__.py,sha256=CrLiDeYNM9fO975sE5ufeVKcy7Ca702IsaG2Pk8T3YU,139
-yggdrasil/types/libs.py,sha256=2iRT9JDUdr9seuGz9ZR3wWdrxZ8LRnc9i-m_tkKdKgI,293
 yggdrasil/types/python_arrow.py,sha256=mOhyecAxa5u8JWsyTO26OMOWimHHgwLKWlkNSAyIVas,25636
 yggdrasil/types/python_defaults.py,sha256=GO3hZBZcwRHs9qiXes75y8l5X00kZHTfEC7el_x73uw,10184
 yggdrasil/types/cast/__init__.py,sha256=Oft3pTs2bRM5hT7YqJAuOKTYYk-SACLaMOXUVdafy_I,311
 yggdrasil/types/cast/arrow_cast.py,sha256=_OMYc4t5GlgE4ztlWaCoK8Jnba09rgDbmHVP-QXhOL0,41523
-yggdrasil/types/cast/cast_options.py,sha256=iVfZIp6XR85yxftQXp_oGVdrZ3k6Mijt0doJEBRmNTY,15527
+yggdrasil/types/cast/cast_options.py,sha256=nDaEvCCs7TBamhTWyDrYf3LVaBWzioIP2Q5_LXrChF4,15532
 yggdrasil/types/cast/pandas_cast.py,sha256=I3xu0sZ59ZbK3NDcQ2dslzdeKzhpFV5zR02ZEixd5hI,8713
 yggdrasil/types/cast/polars_cast.py,sha256=K2nnQ7bexArneYEhUPgV_6er4JNq6N5RmbMUhw-2_Xw,28766
 yggdrasil/types/cast/polars_pandas_cast.py,sha256=CS0P7teVv15IdX5g7v40RfkH1VMg6b-HM0V_gOfacm8,5071
@@ -56,8 +55,8 @@ yggdrasil/types/cast/registry.py,sha256=_zdFGmUBB7P-e_LIcJlOxMcxAkXoA-UXB6HqLMgT
 yggdrasil/types/cast/spark_cast.py,sha256=_KAsl1DqmKMSfWxqhVE7gosjYdgiL1C5bDQv6eP3HtA,24926
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=BuTiWrdCANZCdD_p2MAytqm74eq-rdRXd-LGojBRrfU,5023
 yggdrasil/types/cast/spark_polars_cast.py,sha256=btmZNHXn2NSt3fUuB4xg7coaE0RezIBdZD92H8NK0Jw,9073
-ygg-0.1.47.dist-info/METADATA,sha256=gX37uRIIu4qh1VRItyOxOZK1_WNVvfxX6gBbb5DCN0Q,19204
-ygg-0.1.47.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.47.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
-ygg-0.1.47.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.47.dist-info/RECORD,,
+ygg-0.1.49.dist-info/METADATA,sha256=CHTqeVyiYa1868ZDwISDHKyXYxPeUH0mHhvHLYYoDbg,18528
+ygg-0.1.49.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.49.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
+ygg-0.1.49.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.49.dist-info/RECORD,,

yggdrasil/__init__.py

@@ -1,5 +1 @@
 from .version import *
-from .dataclasses import yggdataclass
-from .types import convert
-from .pyutils import *
-

yggdrasil/databricks/compute/cluster.py

@@ -22,8 +22,8 @@ from typing import Any, Iterator, Optional, Union, List, Callable, Dict, ClassVa
 
 from .execution_context import ExecutionContext
 from ..workspaces.workspace import WorkspaceService, Workspace
-from ... import CallableSerde
 from ...libs.databrickslib import databricks_sdk
+from ...pyutils.callable_serde import CallableSerde
 from ...pyutils.equality import dicts_equal, dict_diff
 from ...pyutils.expiring_dict import ExpiringDict
 from ...pyutils.modules import PipIndexSettings
@@ -36,7 +36,8 @@ else:  # pragma: no cover - runtime fallback when SDK is missing
     from databricks.sdk.errors import DatabricksError
     from databricks.sdk.errors.platform import ResourceDoesNotExist
     from databricks.sdk.service.compute import (
-        ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus
+        ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus,
+        ClusterAccessControlRequest, ClusterPermissionLevel
     )
     from databricks.sdk.service.compute import SparkVersion, RuntimeEngine
 
@@ -143,6 +144,7 @@ class Cluster(WorkspaceService):
         single_user_name: Optional[str] = None,
         runtime_engine: Optional["RuntimeEngine"] = None,
         libraries: Optional[list[str]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **kwargs
     ) -> "Cluster":
         """Create or reuse a cluster that mirrors the current Python environment.
@@ -151,9 +153,10 @@ class Cluster(WorkspaceService):
             workspace: Workspace to use for the cluster.
             cluster_id: Optional cluster id to reuse.
             cluster_name: Optional cluster name to reuse.
-            single_user_name: Optional user name for single-user clusters.
+            single_user_name: Optional username for single-user clusters.
             runtime_engine: Optional Databricks runtime engine.
             libraries: Optional list of libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **kwargs: Additional cluster specification overrides.
 
         Returns:
@@ -175,6 +178,7 @@ class Cluster(WorkspaceService):
                 single_user_name=single_user_name,
                 runtime_engine=runtime_engine,
                 libraries=libraries,
+                update_timeout=update_timeout,
                 **kwargs
             )
         )
@@ -189,6 +193,7 @@ class Cluster(WorkspaceService):
         single_user_name: Optional[str] = "current",
         runtime_engine: Optional["RuntimeEngine"] = None,
         libraries: Optional[list[str]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **kwargs
     ) -> "Cluster":
         """Create/update a cluster to match the local Python environment.
@@ -197,9 +202,10 @@ class Cluster(WorkspaceService):
             source: Optional PythonEnv to mirror (defaults to current).
             cluster_id: Optional cluster id to update.
             cluster_name: Optional cluster name to update.
-            single_user_name: Optional single user name for the cluster.
+            single_user_name: Optional single username for the cluster.
             runtime_engine: Optional runtime engine selection.
             libraries: Optional list of libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **kwargs: Additional cluster specification overrides.
 
         Returns:
@@ -241,6 +247,7 @@ class Cluster(WorkspaceService):
             single_user_name=single_user_name,
             runtime_engine=runtime_engine or RuntimeEngine.PHOTON,
             libraries=libraries,
+            update_timeout=update_timeout,
             **kwargs
         )
 
@@ -379,7 +386,9 @@ class Cluster(WorkspaceService):
         start = time.time()
         sleep_time = tick
 
-        if isinstance(timeout, dt.timedelta):
+        if not timeout:
+            timeout = 20 * 60.0
+        elif isinstance(timeout, dt.timedelta):
             timeout = timeout.total_seconds()
 
         while self.is_pending:
@@ -411,12 +420,14 @@ class Cluster(WorkspaceService):
         # Extract "major.minor" from strings like "17.3.x-scala2.13-ml-gpu"
         v = self.spark_version
 
-        if v is None:
+        if not v:
             return None
 
         parts = v.split(".")
+
         if len(parts) < 2:
             return None
+
         return ".".join(parts[:2])  # e.g. "17.3"
 
     @property
@@ -427,8 +438,10 @@ class Cluster(WorkspaceService):
         When the runtime can't be mapped, returns ``None``.
         """
         v = self.runtime_version
-        if v is None:
+
+        if not v:
             return None
+
         return _PYTHON_BY_DBR.get(v)
 
     # ------------------------------------------------------------------ #
@@ -585,6 +598,7 @@ class Cluster(WorkspaceService):
         cluster_id: Optional[str] = None,
         cluster_name: Optional[str] = None,
         libraries: Optional[List[Union[str, "Library"]]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ):
         """Create a new cluster or update an existing one.
@@ -593,6 +607,7 @@ class Cluster(WorkspaceService):
             cluster_id: Optional cluster id to update.
             cluster_name: Optional cluster name to update or create.
             libraries: Optional libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **cluster_spec: Cluster specification overrides.
 
         Returns:
@@ -608,24 +623,28 @@ class Cluster(WorkspaceService):
             return found.update(
                 cluster_name=cluster_name,
                 libraries=libraries,
+                wait_timeout=update_timeout,
                 **cluster_spec
             )
 
         return self.create(
             cluster_name=cluster_name,
             libraries=libraries,
+            wait_timeout=update_timeout,
             **cluster_spec
         )
 
     def create(
         self,
         libraries: Optional[List[Union[str, "Library"]]] = None,
+        wait_timeout: Union[float, dt.timedelta] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ) -> str:
         """Create a new cluster and optionally install libraries.
 
         Args:
             libraries: Optional list of libraries to install after creation.
+            wait_timeout: wait timeout, if None it will not wait completion
             **cluster_spec: Cluster specification overrides.
 
         Returns:
@@ -645,27 +664,32 @@ class Cluster(WorkspaceService):
             update_details,
         )
 
-        self.details = self.clusters_client().create_and_wait(**update_details)
+        self.details = self.clusters_client().create(**update_details)
 
         LOGGER.info(
             "Created %s",
             self
         )
 
-        self.install_libraries(libraries=libraries, raise_error=False)
+        self.install_libraries(libraries=libraries, raise_error=False, wait_timeout=None)
+
+        if wait_timeout:
+            self.wait_for_status(timeout=wait_timeout)
 
         return self
 
     def update(
         self,
         libraries: Optional[List[Union[str, "Library"]]] = None,
-        wait_timeout: Union[float, dt.timedelta] = dt.timedelta(minutes=20),
+        access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
+        wait_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ) -> "Cluster":
         """Update cluster configuration and optionally install libraries.
 
         Args:
             libraries: Optional libraries to install.
+            access_control_list: List of permissions
             wait_timeout: waiting timeout until done, if None it does not wait
             **cluster_spec: Cluster specification overrides.
 
@@ -705,8 +729,9 @@ class Cluster(WorkspaceService):
                 self, diff
             )
 
-            self.wait_for_status()
+            self.wait_for_status(timeout=wait_timeout)
             self.clusters_client().edit(**update_details)
+            self.update_permissions(access_control_list=access_control_list)
 
             LOGGER.info(
                 "Updated %s",
@@ -718,6 +743,56 @@ class Cluster(WorkspaceService):
 
         return self
 
+    def update_permissions(
+        self,
+        access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
+    ):
+        if not access_control_list:
+            return self
+
+        access_control_list = self._check_permission(access_control_list)
+
+        self.clusters_client().update_permissions(
+            cluster_id=self.cluster_id,
+            access_control_list=access_control_list
+        )
+
+    def default_permissions(self):
+        current_groups = self.current_user.groups or []
+
+        return [
+            ClusterAccessControlRequest(
+                group_name=name,
+                permission_level=ClusterPermissionLevel.CAN_MANAGE
+            )
+            for name in current_groups
+            if name not in {"users"}
+        ]
+
+    def _check_permission(
+        self,
+        permission: Union[str, "ClusterAccessControlRequest", List[Union[str, "ClusterAccessControlRequest"]]],
+    ):
+        if isinstance(permission, ClusterAccessControlRequest):
+            return permission
+
+        if isinstance(permission, str):
+            if "@" in permission:
+                group_name, user_name = None, permission
+            else:
+                group_name, user_name = permission, None
+
+            return ClusterAccessControlRequest(
+                group_name=group_name,
+                user_name=user_name,
+                permission_level=ClusterPermissionLevel.CAN_MANAGE
+            )
+
+        return [
+            self._check_permission(_)
+            for _ in permission
+        ]
+
     def list_clusters(self) -> Iterator["Cluster"]:
         """Iterate clusters, yielding helpers annotated with metadata.
 
@@ -809,18 +884,22 @@ class Cluster(WorkspaceService):
         Returns:
             The current Cluster instance.
         """
+        if self.is_running:
+            return self
+
         self.wait_for_status()
 
-        if not self.is_running:
-            LOGGER.debug("Starting %s", self)
+        if self.is_running:
+            return self
 
-            if wait_timeout:
-                self.clusters_client().start(cluster_id=self.cluster_id)
-                self.wait_for_status(timeout=wait_timeout.total_seconds())
-            else:
-                self.clusters_client().start(cluster_id=self.cluster_id)
+        LOGGER.debug("Starting %s", self)
+
+        self.clusters_client().start(cluster_id=self.cluster_id)
 
-            LOGGER.info("Started %s", self)
+        LOGGER.info("Started %s", self)
+
+        if wait_timeout:
+            self.wait_for_status(timeout=wait_timeout.total_seconds())
 
         return self
 
@@ -836,7 +915,7 @@ class Cluster(WorkspaceService):
 
         if self.is_running:
             self.details = self.clusters_client().restart_and_wait(cluster_id=self.cluster_id)
-            return self.wait_for_status()
+            return self
 
         return self.start()
 

yggdrasil/databricks/compute/execution_context.py

@@ -180,7 +180,7 @@ print(json.dumps(meta))"""
         """
         return self.cluster.workspace.sdk()
 
-    def _create_command(
+    def create_command(
         self,
         language: "Language",
     ) -> any:
@@ -192,17 +192,29 @@ print(json.dumps(meta))"""
         Returns:
             The created command execution context response.
         """
-        self.cluster.ensure_running()
-
         LOGGER.debug(
             "Creating Databricks command execution context for %s",
             self.cluster
         )
 
-        created = self._workspace_client().command_execution.create_and_wait(
-            cluster_id=self.cluster.cluster_id,
-            language=language,
+        try:
+            created = self._workspace_client().command_execution.create_and_wait(
+                cluster_id=self.cluster.cluster_id,
+                language=language,
+            )
+        except:
+            self.cluster.ensure_running()
+
+            created = self._workspace_client().command_execution.create_and_wait(
+                cluster_id=self.cluster.cluster_id,
+                language=language,
+            )
+
+        LOGGER.info(
+            "Created Databricks command execution context %s",
+            self
         )
+
         created = getattr(created, "response", created)
 
         return created
@@ -220,10 +232,6 @@ print(json.dumps(meta))"""
             The connected ExecutionContext instance.
         """
         if self.context_id is not None:
-            LOGGER.debug(
-                "Execution context already open for %s",
-                self
-            )
             return self
 
         self.language = language or self.language
@@ -231,7 +239,7 @@ print(json.dumps(meta))"""
         if self.language is None:
             self.language = Language.PYTHON
 
-        ctx = self._create_command(language=self.language)
+        ctx = self.create_command(language=self.language)
 
         context_id = ctx.id
         if not context_id:

yggdrasil/databricks/compute/remote.py

@@ -39,6 +39,7 @@ def databricks_remote_compute(
     timeout: Optional[dt.timedelta] = None,
     env_keys: Optional[List[str]] = None,
     force_local: bool = False,
+    update_timeout: Optional[Union[float, dt.timedelta]] = None,
     **options
 ) -> Callable[[Callable[..., ReturnType]], Callable[..., ReturnType]]:
     """Return a decorator that executes functions on a remote cluster.
@@ -52,6 +53,7 @@ def databricks_remote_compute(
         timeout: Optional execution timeout for remote calls.
         env_keys: Optional environment variable names to forward.
         force_local: Force local execution
+        update_timeout: creation or update wait timeout
         **options: Extra options forwarded to the execution decorator.
 
     Returns:
@@ -82,7 +84,8 @@ def databricks_remote_compute(
             cluster = workspace.clusters().replicated_current_environment(
                 workspace=workspace,
                 cluster_name=cluster_name,
-                single_user_name=workspace.current_user.user_name
+                single_user_name=workspace.current_user.user_name,
+                update_timeout=update_timeout
             )
 
     cluster.ensure_running(wait_timeout=None)

yggdrasil/databricks/sql/__init__.py

@@ -1,9 +1,8 @@
 """Databricks SQL helpers and engine wrappers."""
 
 from .engine import SQLEngine, StatementResult
+from .exceptions import SqlStatementError
 
 # Backwards compatibility
 DBXSQL = SQLEngine
 DBXStatementResult = StatementResult
-
-__all__ = ["SQLEngine", "StatementResult"]

yggdrasil/databricks/sql/exceptions.py

@@ -1 +1,45 @@
 """Custom exceptions for Databricks SQL helpers."""
+from dataclasses import dataclass
+from typing import Optional, Any
+
+__all__ = [
+    "SqlStatementError"
+]
+
+
+@dataclass(frozen=True)
+class SqlStatementError(RuntimeError):
+    statement_id: str
+    state: str
+    message: str
+    error_code: Optional[str] = None
+    sql_state: Optional[str] = None
+
+    def __str__(self) -> str:
+        meta = []
+        if self.error_code:
+            meta.append(f"code={self.error_code}")
+        if self.sql_state:
+            meta.append(f"state={self.sql_state}")
+
+        meta_str = f" ({', '.join(meta)})" if meta else ""
+        return f"SQL statement {self.statement_id} failed [{self.state}]: {self.message}{meta_str}"
+
+    @classmethod
+    def from_statement(cls, stmt: Any) -> "SqlStatementError":
+        statement_id = getattr(stmt, "statement_id", "<unknown>")
+        state = getattr(stmt, "state", "<unknown>")
+
+        err = getattr(getattr(stmt, "status", None), "error", None)
+
+        message = getattr(err, "message", None) or "Unknown SQL error"
+        error_code = getattr(err, "error_code", None)
+        sql_state = getattr(err, "sql_state", None)
+
+        return cls(
+            statement_id=str(statement_id),
+            state=str(state),
+            message=str(message),
+            error_code=str(error_code) if error_code is not None else None,
+            sql_state=str(sql_state) if sql_state is not None else None,
+        )

yggdrasil/databricks/sql/statement_result.py

@@ -9,6 +9,7 @@ from typing import Optional, Iterator, TYPE_CHECKING
 import pyarrow as pa
 import pyarrow.ipc as pipc
 
+from .exceptions import SqlStatementError
 from .types import column_info_to_arrow_field
 from ...libs.databrickslib import databricks_sdk
 from ...libs.pandaslib import pandas
@@ -32,9 +33,7 @@ except ImportError:
 if databricks_sdk is not None:
     from databricks.sdk.service.sql import (
         StatementState, StatementResponse, Disposition, StatementStatus
-)
-
-    StatementResponse = StatementResponse
+    )
 else:
     class StatementResponse:
         pass
@@ -299,28 +298,8 @@ class StatementResult:
                 )
 
     def raise_for_status(self):
-        """Raise a ValueError if the statement failed.
-
-        Returns:
-            None.
-        """
         if self.failed:
-            # grab error info if present
-            err = self.status.error
-            message = err.message or "Unknown SQL error"
-            error_code = err.error_code
-            sql_state = getattr(err, "sql_state", None)
-
-            parts = [message]
-            if error_code:
-                parts.append(f"error_code={error_code}")
-            if sql_state:
-                parts.append(f"sql_state={sql_state}")
-
-            raise ValueError(
-                f"Statement {self.statement_id} {self.state}: " + " | ".join(parts)
-            )
-
+            raise SqlStatementError.from_statement(self)
         return self
 
     def wait(
@@ -337,22 +316,20 @@ class StatementResult:
         Returns:
             The current StatementResult instance.
         """
-        if self.done:
-            return self
-
-        start = time.time()
-        poll_interval = poll_interval or 1
-
-        while not self.done:
-            # still running / queued / pending
-            if timeout is not None and (time.time() - start) > timeout:
-                raise TimeoutError(
-                    f"Statement {self.statement_id} did not finish within {timeout} seconds "
-                    f"(last state={self.state})"
-                )
-
-            poll_interval = max(10, poll_interval * 1.2)
-            time.sleep(poll_interval)
+        if not self.done:
+            start = time.time()
+            poll_interval = poll_interval or 1
+
+            while not self.done:
+                # still running / queued / pending
+                if timeout is not None and (time.time() - start) > timeout:
+                    raise TimeoutError(
+                        f"Statement {self.statement_id} did not finish within {timeout} seconds "
+                        f"(last state={self.state})"
+                    )
+
+                poll_interval = max(10, poll_interval * 1.2)
+                time.sleep(poll_interval)
 
         self.raise_for_status()
 

yggdrasil/databricks/workspaces/__init__.py

@@ -2,4 +2,3 @@
 
 from .workspace import *
 from .path import *
-from .io import *

yggdrasil/databricks/workspaces/io.py

@@ -975,28 +975,40 @@ class DatabricksVolumeIO(DatabricksIO):
         """Read bytes from a volume file.
 
         Args:
-            start: Starting byte offset.
+            start: Starting byte offset (0-based).
             length: Number of bytes to read.
             allow_not_found: Whether to suppress missing-path errors.
 
         Returns:
             Bytes read from the file.
         """
-        if length == 0:
+        if length <= 0:
             return b""
+        if start < 0:
+            raise ValueError(f"start must be >= 0, got {start}")
+        if length < 0:
+            raise ValueError(f"length must be >= 0, got {length}")
 
         sdk = self.workspace.sdk()
         client = sdk.files
         full_path = self.path.files_full_path()
 
-        resp = client.download(full_path)
-        result = (
-            resp.contents
-            .seek(start, io.SEEK_SET)
-            .read(length)
-        )
+        try:
+            resp = client.download(full_path)
+        except Exception as e:
+            # Databricks SDK exceptions vary a bit by version; keep it pragmatic.
+            if allow_not_found and any(s in str(e).lower() for s in ("not found", "does not exist", "404")):
+                return b""
+            raise
+
+        data = resp.contents.read()
 
-        return result
+        # If start is past EOF, return empty (common file-like behavior).
+        if start >= len(data):
+            return b""
+
+        end = start + length
+        return data[start:end]
 
     def write_all_bytes(self, data: bytes):
         """Write bytes to a volume file.

yggdrasil/databricks/workspaces/path.py

@@ -494,11 +494,15 @@ class DatabricksPath:
 
         try:
             info = sdk.files.get_directory_metadata(full_path)
-            mtime = (
-                dt.datetime.strptime(info.last_modified, "%a, %d %b %Y %H:%M:%S %Z").replace(tzinfo=dt.timezone.utc)
-                if info.last_modified
-                else None
-            )
+
+            if info is None:
+                mtime = dt.datetime.now(tz=dt.timezone.utc)
+            else:
+                mtime = (
+                    dt.datetime.strptime(info.last_modified, "%a, %d %b %Y %H:%M:%S %Z").replace(tzinfo=dt.timezone.utc)
+                    if info.last_modified
+                    else None
+                )
 
             return self.reset_metadata(is_file=False, is_dir=True, size=info, mtime=mtime)
         except (NotFound, ResourceDoesNotExist, BadRequest, PermissionDenied):

yggdrasil/databricks/workspaces/workspace.py

@@ -8,7 +8,6 @@ from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
 from typing import (
-    Any,
     BinaryIO,
     Iterator,
     Optional,
@@ -55,7 +54,9 @@ def _get_env_product_version():
     v = os.getenv("DATABRICKS_PRODUCT_VERSION")
 
     if not v:
-        return YGGDRASIL_VERSION
+        if _get_env_product() == "yggdrasil":
+            return YGGDRASIL_VERSION
+        return None
     return v.strip().lower()
 
 
@@ -106,11 +107,12 @@ class Workspace:
     product: Optional[str] = dataclasses.field(default_factory=_get_env_product, repr=False)
     product_version: Optional[str] = dataclasses.field(default_factory=_get_env_product_version, repr=False)
     product_tag: Optional[str] = dataclasses.field(default_factory=_get_env_product_tag, repr=False)
+    custom_tags: Optional[dict] = dataclasses.field(default=None, repr=False)
 
     # Runtime cache (never serialized)
-    _sdk: Any = dataclasses.field(init=False, default=None, repr=False, compare=False, hash=False)
-    _was_connected: bool = dataclasses.field(init=False, default=False, repr=False, compare=False)
-    _cached_token: Optional[str] = dataclasses.field(init=False, default=None, repr=False, compare=False)
+    _sdk: Optional["WorkspaceClient"] = dataclasses.field(default=None, repr=False, compare=False, hash=False)
+    _was_connected: bool = dataclasses.field(default=None, repr=False, compare=False, hash=False)
+    _cached_token: Optional[str] = dataclasses.field(default=None, repr=False, compare=False, hash=False)
 
     # -------------------------
     # Pickle support
@@ -175,19 +177,43 @@ class Workspace:
     # -------------------------
     def clone_instance(
         self,
-        **kwargs
     ) -> "Workspace":
         """Clone the workspace config with overrides.
 
-        Args:
-            **kwargs: Field overrides for the clone.
-
         Returns:
             A new Workspace instance with updated fields.
         """
-        state = self.__getstate__()
-        state.update(kwargs)
-        return Workspace().__setstate__(state)
+        return Workspace(
+            host = self.host,
+            account_id = self.account_id,
+            token = self.token,
+            client_id = self.client_id,
+            client_secret = self.client_secret,
+            token_audience = self.token_audience,
+            azure_workspace_resource_id = self.azure_workspace_resource_id,
+            azure_use_msi = self.azure_use_msi,
+            azure_client_secret = self.azure_client_secret,
+            azure_client_id = self.azure_client_id,
+            azure_tenant_id = self.azure_tenant_id,
+            azure_environment = self.azure_environment,
+            google_credentials = self.google_credentials,
+            google_service_account = self.google_service_account,
+            profile = self.profile,
+            config_file = self.config_file,
+            auth_type = self.auth_type,
+            http_timeout_seconds = self.http_timeout_seconds,
+            retry_timeout_seconds = self.retry_timeout_seconds,
+            debug_truncate_bytes = self.debug_truncate_bytes,
+            debug_headers = self.debug_headers,
+            rate_limit = self.rate_limit,
+            product = self.product,
+            product_version = self.product_version,
+            product_tag = self.product_tag,
+            custom_tags = self.custom_tags,
+            _sdk = self._sdk,
+            _was_connected = self._was_connected,
+            _cached_token = self._cached_token,
+        )
 
     # -------------------------
     # SDK connection
@@ -300,8 +326,9 @@ class Workspace:
         Drop the cached WorkspaceClient (no actual close needed, but this
         avoids reusing stale config).
         """
-        self._sdk = None
-        self._was_connected = False
+        if self._sdk is not None:
+            self._sdk = None
+            self._was_connected = False
 
     # ------------------------------------------------------------------ #
     # Properties
@@ -561,28 +588,19 @@ class Workspace:
         Returns:
             A dict of default tags.
         """
-        return {
+        base = {
             k: v
             for k, v in (
                 ("Product", self.product),
-                ("ProductVersion", self.product_version),
                 ("ProductTag", self.product_tag),
-                ("ProductUser", self.current_user.user_name)
             )
             if v
         }
 
-    def merge_tags(self, existing: dict | None = None):
-        """Merge default tags with an existing set.
-
-        Args:
-            existing: Optional existing tags.
+        if self.custom_tags:
+            base.update(self.custom_tags)
 
-        Returns:
-            A dict of merged tags.
-        """
-        if existing:
-            return self.default_tags()
+        return base
 
     def sql(
         self,

yggdrasil/dataclasses/__init__.py

@@ -1,5 +1,3 @@
 """Enhanced dataclass helpers with Arrow awareness."""
 
-from .dataclass import yggdataclass
-
-__all__ = ["yggdataclass"]
+from .dataclass import get_dataclass_arrow_field

yggdrasil/dataclasses/dataclass.py

@@ -2,32 +2,17 @@
 
 import dataclasses
 from inspect import isclass
-from typing import Any, Iterable, Mapping, Tuple
+from typing import Any
 
 import pyarrow as pa
 
 __all__ = [
-    "yggdataclass",
-    "is_yggdataclass",
     "get_dataclass_arrow_field"
 ]
 
 DATACLASS_ARROW_FIELD_CACHE: dict[type, pa.Field] = {}
 
 
-def is_yggdataclass(cls_or_instance: Any) -> bool:
-    """Check if a class or instance is a yggdrasil dataclass.
-
-    Args:
-        cls_or_instance: The class or instance to check.
-
-    Returns:
-        True if the class or instance
-        is a yggdrasil dataclass, False otherwise.
-    """
-    return hasattr(cls_or_instance, "__arrow_field__")
-
-
 def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
     """Return a cached Arrow Field describing the dataclass type.
 
@@ -37,9 +22,6 @@ def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
     Returns:
         Arrow field describing the dataclass schema.
     """
-    if is_yggdataclass(cls_or_instance):
-        return cls_or_instance.__arrow_field__()
-
     if dataclasses.is_dataclass(cls_or_instance):
         cls = cls_or_instance
         if not isclass(cls_or_instance):
@@ -56,151 +38,3 @@ def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
         return built
 
     raise ValueError(f"{cls_or_instance!r} is not a dataclass or yggdrasil dataclass")
-
-
-def yggdataclass(
-    cls=None, /,
-    *,
-    init=True,
-    repr=True,
-    eq=True,
-    order=False,
-    unsafe_hash=False, frozen=False, match_args=True,
-    kw_only=False, slots=False,
-    weakref_slot=False
-):
-    """Decorate a class with dataclass behavior plus Arrow helpers.
-
-    Examines PEP 526 __annotations__ to determine fields.
-
-    If init is true, an __init__() method is added to the class. If repr
-    is true, a __repr__() method is added. If order is true, rich
-    comparison dunder methods are added. If unsafe_hash is true, a
-    __hash__() method is added. If frozen is true, fields may not be
-    assigned to after instance creation. If match_args is true, the
-    __match_args__ tuple is added. If kw_only is true, then by default
-    all fields are keyword-only. If slots is true, a new class with a
-    __slots__ attribute is returned.
-    """
-
-    def wrap(c):
-        """Wrap a class with yggdrasil dataclass enhancements.
-
-        Args:
-            c: Class to decorate.
-
-        Returns:
-            Decorated dataclass type.
-        """
-
-        def _init_public_fields(cls):
-            """Return init-enabled, public dataclass fields.
-
-            Args:
-                cls: Dataclass type.
-
-            Returns:
-                List of dataclasses.Field objects.
-            """
-            return [
-                field
-                for field in dataclasses.fields(cls)
-                if field.init and not field.name.startswith("_")
-            ]
-
-        if not hasattr(c, "default_instance"):
-            @classmethod
-            def default_instance(cls):
-                """Return a default instance built from type defaults.
-
-                Returns:
-                    Default instance of the dataclass.
-                """
-                from yggdrasil.types import default_scalar
-
-                if not hasattr(cls, "__default_instance__"):
-                    cls.__default_instance__ = default_scalar(cls)
-
-                return dataclasses.replace(cls.__default_instance__)
-
-            c.default_instance = default_instance
-
-        if not hasattr(c, "__safe_init__"):
-            @classmethod
-            def __safe_init__(cls, *args, **kwargs):
-                """Safely initialize a dataclass using type conversion and defaults."""
-
-                fields = _init_public_fields(cls)
-                field_names = [field.name for field in fields]
-
-                if len(args) > len(field_names):
-                    raise TypeError(
-                        f"Expected at most {len(field_names)} positional arguments, got {len(args)}"
-                    )
-
-                provided = {name: value for name, value in zip(field_names, args)}
-
-                for key, value in kwargs.items():
-                    if key in provided:
-                        raise TypeError(f"Got multiple values for argument '{key}'")
-                    if key not in field_names:
-                        raise TypeError(
-                            f"{key!r} is an invalid field for {cls.__name__}"
-                        )
-
-                    provided[key] = value
-
-                from yggdrasil.types.cast import convert
-
-                defaults = cls.default_instance()
-                init_kwargs = {}
-
-                for field in fields:
-                    if field.name in provided:
-                        init_kwargs[field.name] = convert(provided[field.name], field.type)
-                    else:
-                        init_kwargs[field.name] = getattr(defaults, field.name, None)
-
-                return cls(**init_kwargs)
-
-            c.__safe_init__ = __safe_init__
-
-        if not hasattr(c, "__arrow_field__"):
-            @classmethod
-            def __arrow_field__(cls, name: str | None = None):
-                """Return an Arrow field representing the dataclass schema.
-
-                Args:
-                    name: Optional override for the field name.
-
-                Returns:
-                    Arrow field describing the dataclass schema.
-                """
-                from yggdrasil.types.python_arrow import arrow_field_from_hint
-
-                return arrow_field_from_hint(cls, name=name)
-
-            c.__arrow_field__ = __arrow_field__
-
-        base = dataclasses.dataclass(
-            c,
-            init=init,
-            repr=repr,
-            eq=eq,
-            order=order,
-            unsafe_hash=unsafe_hash,
-            frozen=frozen,
-            match_args=match_args,
-            kw_only=kw_only,
-            slots=slots,
-        )
-
-        return base
-
-    # See if we're being called as @dataclass or @dataclass().
-    if cls is None:
-        # We're called with parens.
-        return wrap
-
-    # We're called as @dataclass without parens.
-    return wrap(cls)

yggdrasil/libs/__init__.py

@@ -1,6 +1,3 @@
 """Helper utilities for optional dependency integrations."""
 
-from .sparklib import *
-from .polarslib import *
-from .pandaslib import *
 from .extensions import *

yggdrasil/pyutils/python_env.py

@@ -16,7 +16,7 @@ import sys
 import tempfile
 import threading
 from contextlib import contextmanager
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Optional, Union, List, Tuple
 
@@ -415,11 +415,13 @@ def _locked_env(root: Path):
 # PythonEnv
 # -----------------------
 
-@dataclass(frozen=True)
+@dataclass
 class PythonEnv:
     """Represent a managed Python environment rooted at a filesystem path."""
     root: Path
 
+    _version: Optional[str] = field(default=None, repr=False)
+
     def __post_init__(self) -> None:
         """Normalize the root path after dataclass initialization.
 
@@ -862,8 +864,9 @@ class PythonEnv:
         Returns:
             Version string.
         """
-        out = self.exec_code("import sys; print(sys.version.split()[0])", check=True)
-        return out.strip()
+        if self._version is None:
+            self._version = self.exec_code("import sys; print(sys.version.split()[0])", check=True).strip()
+        return self._version
 
     @property
     def version_info(self) -> tuple[int, int, int]:

yggdrasil/requests/__init__.py

@@ -1,5 +1,4 @@
 """Convenience imports for request session helpers."""
 
-from .msal import MSALSession, MSALAuth
-
-__all__ = ["MSALSession", "MSALAuth"]
+from .msal import *
+from .session import *

yggdrasil/types/cast/cast_options.py

@@ -7,16 +7,15 @@ import pyarrow as pa
 
 from .registry import convert
 from ..python_arrow import is_arrow_type_list_like
-from ...dataclasses import yggdataclass
+from ...libs.polarslib import polars
+from ...libs.sparklib import pyspark
 
 __all__ = [
     "CastOptions",
 ]
 
-from ...libs import pyspark, polars
 
-
-@yggdataclass
+@dataclasses.dataclass
 class CastOptions:
     """
     Options controlling Arrow casting behavior.

yggdrasil/version.py

@@ -1 +1 @@
-__version__ = "0.1.47"
+__version__ = "0.1.49"

yggdrasil/types/libs.py

@@ -1,12 +0,0 @@
-"""Re-export optional dependency helpers for types modules."""
-
-from ..libs import pandas, polars, pyspark, require_pandas, require_polars, require_pyspark
-
-__all__ = [
-    "pandas",
-    "polars",
-    "pyspark",
-    "require_pandas",
-    "require_polars",
-    "require_pyspark",
-]