ygg 0.1.47__tar.gz → 0.1.49__tar.gz

{ygg-0.1.47 → ygg-0.1.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.47
+Version: 0.1.49
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License
@@ -235,6 +235,8 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
 Dynamic: license-file
 
 # Yggdrasil (Python)
@@ -270,34 +272,6 @@ Extras are grouped by engine:
 - `.[polars]`, `.[pandas]`, `.[spark]`, `.[databricks]` – install only the integrations you need.
 - `.[dev]` – adds testing, linting, and typing tools (`pytest`, `ruff`, `black`, `mypy`).
 
-## Quickstart
-Define an Arrow-aware dataclass, coerce inputs, and cast across containers:
-
-```python
-from yggdrasil import yggdataclass
-from yggdrasil.types.cast import convert
-from yggdrasil.types import arrow_field_from_hint
-
-@yggdataclass
-class User:
-    id: int
-    email: str
-    active: bool = True
-
-user = User.__safe_init__("123", email="alice@example.com")
-assert user.id == 123 and user.active is True
-
-payload = {"id": "45", "email": "bob@example.com", "active": "false"}
-clean = User.from_dict(payload)
-print(clean.to_dict())
-
-field = arrow_field_from_hint(User, name="user")
-print(field)  # user: struct<id: int64, email: string, active: bool>
-
-numbers = convert(["1", "2", "3"], list[int])
-print(numbers)
-```
-
 ### Databricks example
 Install the `databricks` extra and run SQL with typed results:
 

{ygg-0.1.47 → ygg-0.1.49}/README.md

@@ -31,34 +31,6 @@ Extras are grouped by engine:
 - `.[polars]`, `.[pandas]`, `.[spark]`, `.[databricks]` – install only the integrations you need.
 - `.[dev]` – adds testing, linting, and typing tools (`pytest`, `ruff`, `black`, `mypy`).
 
-## Quickstart
-Define an Arrow-aware dataclass, coerce inputs, and cast across containers:
-
-```python
-from yggdrasil import yggdataclass
-from yggdrasil.types.cast import convert
-from yggdrasil.types import arrow_field_from_hint
-
-@yggdataclass
-class User:
-    id: int
-    email: str
-    active: bool = True
-
-user = User.__safe_init__("123", email="alice@example.com")
-assert user.id == 123 and user.active is True
-
-payload = {"id": "45", "email": "bob@example.com", "active": "false"}
-clean = User.from_dict(payload)
-print(clean.to_dict())
-
-field = arrow_field_from_hint(User, name="user")
-print(field)  # user: struct<id: int64, email: string, active: bool>
-
-numbers = convert(["1", "2", "3"], list[int])
-print(numbers)
-```
-
 ### Databricks example
 Install the `databricks` extra and run SQL with typed results:
 

{ygg-0.1.47 → ygg-0.1.49}/pyproject.toml

@@ -1,17 +1,16 @@
 [build-system]
-requires = ["setuptools>=61", "wheel"]
+# bump setuptools so type-info files are handled sanely
+requires = ["setuptools>=69", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.47"
+version = "0.1.49"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }
 requires-python = ">=3.10"
-authors = [
-  { name = "Yggdrasil contributors" },
-]
+authors = [{ name = "Yggdrasil contributors" }]
 keywords = ["arrow", "polars", "pandas", "spark", "databricks", "typing", "dataclass", "serialization"]
 classifiers = [
   "Development Status :: 3 - Alpha",
@@ -42,6 +41,8 @@ dev = [
   "black",
   "ruff",
   "mypy",
+  "build",
+  "twine",
 ]
 
 [project.scripts]
@@ -55,9 +56,15 @@ Documentation = "https://github.com/Platob/Yggdrasil/tree/main/python/docs"
 [tool.setuptools]
 package-dir = { "" = "src" }
 license-files = ["LICENSE"]
-
-[tool.uv]
-native-tls = true
+include-package-data = true
 
 [tool.setuptools.packages.find]
 where = ["src"]
+
+# If your import package is yggdrasil (seems likely from yggenv entrypoint),
+# ship the PEP 561 marker file. Put `py.typed` inside src/yggdrasil/
+[tool.setuptools.package-data]
+yggdrasil = ["py.typed"]
+
+[tool.uv]
+native-tls = true

{ygg-0.1.47 → ygg-0.1.49}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.47
+Version: 0.1.49
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License
@@ -235,6 +235,8 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
 Dynamic: license-file
 
 # Yggdrasil (Python)
@@ -270,34 +272,6 @@ Extras are grouped by engine:
 - `.[polars]`, `.[pandas]`, `.[spark]`, `.[databricks]` – install only the integrations you need.
 - `.[dev]` – adds testing, linting, and typing tools (`pytest`, `ruff`, `black`, `mypy`).
 
-## Quickstart
-Define an Arrow-aware dataclass, coerce inputs, and cast across containers:
-
-```python
-from yggdrasil import yggdataclass
-from yggdrasil.types.cast import convert
-from yggdrasil.types import arrow_field_from_hint
-
-@yggdataclass
-class User:
-    id: int
-    email: str
-    active: bool = True
-
-user = User.__safe_init__("123", email="alice@example.com")
-assert user.id == 123 and user.active is True
-
-payload = {"id": "45", "email": "bob@example.com", "active": "false"}
-clean = User.from_dict(payload)
-print(clean.to_dict())
-
-field = arrow_field_from_hint(User, name="user")
-print(field)  # user: struct<id: int64, email: string, active: bool>
-
-numbers = convert(["1", "2", "3"], list[int])
-print(numbers)
-```
-
 ### Databricks example
 Install the `databricks` extra and run SQL with typed results:
 

{ygg-0.1.47 → ygg-0.1.49}/src/ygg.egg-info/SOURCES.txt

@@ -51,7 +51,6 @@ src/yggdrasil/requests/__init__.py
 src/yggdrasil/requests/msal.py
 src/yggdrasil/requests/session.py
 src/yggdrasil/types/__init__.py
-src/yggdrasil/types/libs.py
 src/yggdrasil/types/python_arrow.py
 src/yggdrasil/types/python_defaults.py
 src/yggdrasil/types/cast/__init__.py

{ygg-0.1.47 → ygg-0.1.49}/src/ygg.egg-info/requires.txt

@@ -11,3 +11,5 @@ pytest-asyncio
 black
 ruff
 mypy
+build
+twine

ygg-0.1.49/src/yggdrasil/__init__.py

@@ -0,0 +1 @@
+from .version import *

{ygg-0.1.47 → ygg-0.1.49}/src/yggdrasil/databricks/compute/cluster.py

@@ -22,8 +22,8 @@ from typing import Any, Iterator, Optional, Union, List, Callable, Dict, ClassVa
 
 from .execution_context import ExecutionContext
 from ..workspaces.workspace import WorkspaceService, Workspace
-from ... import CallableSerde
 from ...libs.databrickslib import databricks_sdk
+from ...pyutils.callable_serde import CallableSerde
 from ...pyutils.equality import dicts_equal, dict_diff
 from ...pyutils.expiring_dict import ExpiringDict
 from ...pyutils.modules import PipIndexSettings
@@ -36,7 +36,8 @@ else:  # pragma: no cover - runtime fallback when SDK is missing
     from databricks.sdk.errors import DatabricksError
     from databricks.sdk.errors.platform import ResourceDoesNotExist
     from databricks.sdk.service.compute import (
-        ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus
+        ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus,
+        ClusterAccessControlRequest, ClusterPermissionLevel
     )
     from databricks.sdk.service.compute import SparkVersion, RuntimeEngine
 
@@ -143,6 +144,7 @@ class Cluster(WorkspaceService):
         single_user_name: Optional[str] = None,
         runtime_engine: Optional["RuntimeEngine"] = None,
         libraries: Optional[list[str]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **kwargs
     ) -> "Cluster":
         """Create or reuse a cluster that mirrors the current Python environment.
@@ -151,9 +153,10 @@ class Cluster(WorkspaceService):
             workspace: Workspace to use for the cluster.
             cluster_id: Optional cluster id to reuse.
             cluster_name: Optional cluster name to reuse.
-            single_user_name: Optional user name for single-user clusters.
+            single_user_name: Optional username for single-user clusters.
             runtime_engine: Optional Databricks runtime engine.
             libraries: Optional list of libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **kwargs: Additional cluster specification overrides.
 
         Returns:
@@ -175,6 +178,7 @@ class Cluster(WorkspaceService):
                 single_user_name=single_user_name,
                 runtime_engine=runtime_engine,
                 libraries=libraries,
+                update_timeout=update_timeout,
                 **kwargs
             )
         )
@@ -189,6 +193,7 @@ class Cluster(WorkspaceService):
         single_user_name: Optional[str] = "current",
         runtime_engine: Optional["RuntimeEngine"] = None,
         libraries: Optional[list[str]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **kwargs
     ) -> "Cluster":
         """Create/update a cluster to match the local Python environment.
@@ -197,9 +202,10 @@ class Cluster(WorkspaceService):
             source: Optional PythonEnv to mirror (defaults to current).
             cluster_id: Optional cluster id to update.
             cluster_name: Optional cluster name to update.
-            single_user_name: Optional single user name for the cluster.
+            single_user_name: Optional single username for the cluster.
             runtime_engine: Optional runtime engine selection.
             libraries: Optional list of libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **kwargs: Additional cluster specification overrides.
 
         Returns:
@@ -241,6 +247,7 @@ class Cluster(WorkspaceService):
             single_user_name=single_user_name,
             runtime_engine=runtime_engine or RuntimeEngine.PHOTON,
             libraries=libraries,
+            update_timeout=update_timeout,
             **kwargs
         )
 
@@ -379,7 +386,9 @@ class Cluster(WorkspaceService):
         start = time.time()
         sleep_time = tick
 
-        if isinstance(timeout, dt.timedelta):
+        if not timeout:
+            timeout = 20 * 60.0
+        elif isinstance(timeout, dt.timedelta):
             timeout = timeout.total_seconds()
 
         while self.is_pending:
@@ -411,12 +420,14 @@ class Cluster(WorkspaceService):
         # Extract "major.minor" from strings like "17.3.x-scala2.13-ml-gpu"
         v = self.spark_version
 
-        if v is None:
+        if not v:
             return None
 
         parts = v.split(".")
+
         if len(parts) < 2:
             return None
+
         return ".".join(parts[:2])  # e.g. "17.3"
 
     @property
@@ -427,8 +438,10 @@ class Cluster(WorkspaceService):
         When the runtime can't be mapped, returns ``None``.
         """
         v = self.runtime_version
-        if v is None:
+
+        if not v:
             return None
+
         return _PYTHON_BY_DBR.get(v)
 
     # ------------------------------------------------------------------ #
@@ -585,6 +598,7 @@ class Cluster(WorkspaceService):
         cluster_id: Optional[str] = None,
         cluster_name: Optional[str] = None,
         libraries: Optional[List[Union[str, "Library"]]] = None,
+        update_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ):
         """Create a new cluster or update an existing one.
@@ -593,6 +607,7 @@ class Cluster(WorkspaceService):
             cluster_id: Optional cluster id to update.
             cluster_name: Optional cluster name to update or create.
             libraries: Optional libraries to install.
+            update_timeout: wait timeout, if None it will not wait completion
             **cluster_spec: Cluster specification overrides.
 
         Returns:
@@ -608,24 +623,28 @@ class Cluster(WorkspaceService):
             return found.update(
                 cluster_name=cluster_name,
                 libraries=libraries,
+                wait_timeout=update_timeout,
                 **cluster_spec
             )
 
         return self.create(
             cluster_name=cluster_name,
             libraries=libraries,
+            wait_timeout=update_timeout,
             **cluster_spec
         )
 
     def create(
         self,
         libraries: Optional[List[Union[str, "Library"]]] = None,
+        wait_timeout: Union[float, dt.timedelta] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ) -> str:
         """Create a new cluster and optionally install libraries.
 
         Args:
             libraries: Optional list of libraries to install after creation.
+            wait_timeout: wait timeout, if None it will not wait completion
             **cluster_spec: Cluster specification overrides.
 
         Returns:
@@ -645,27 +664,32 @@ class Cluster(WorkspaceService):
             update_details,
         )
 
-        self.details = self.clusters_client().create_and_wait(**update_details)
+        self.details = self.clusters_client().create(**update_details)
 
         LOGGER.info(
             "Created %s",
             self
         )
 
-        self.install_libraries(libraries=libraries, raise_error=False)
+        self.install_libraries(libraries=libraries, raise_error=False, wait_timeout=None)
+
+        if wait_timeout:
+            self.wait_for_status(timeout=wait_timeout)
 
         return self
 
     def update(
         self,
         libraries: Optional[List[Union[str, "Library"]]] = None,
-        wait_timeout: Union[float, dt.timedelta] = dt.timedelta(minutes=20),
+        access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
+        wait_timeout: Optional[Union[float, dt.timedelta]] = dt.timedelta(minutes=20),
         **cluster_spec: Any
     ) -> "Cluster":
         """Update cluster configuration and optionally install libraries.
 
         Args:
             libraries: Optional libraries to install.
+            access_control_list: List of permissions
             wait_timeout: waiting timeout until done, if None it does not wait
             **cluster_spec: Cluster specification overrides.
 
@@ -705,8 +729,9 @@ class Cluster(WorkspaceService):
                 self, diff
             )
 
-            self.wait_for_status()
+            self.wait_for_status(timeout=wait_timeout)
             self.clusters_client().edit(**update_details)
+            self.update_permissions(access_control_list=access_control_list)
 
             LOGGER.info(
                 "Updated %s",
@@ -718,6 +743,56 @@ class Cluster(WorkspaceService):
 
         return self
 
+    def update_permissions(
+        self,
+        access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
+    ):
+        if not access_control_list:
+            return self
+
+        access_control_list = self._check_permission(access_control_list)
+
+        self.clusters_client().update_permissions(
+            cluster_id=self.cluster_id,
+            access_control_list=access_control_list
+        )
+
+    def default_permissions(self):
+        current_groups = self.current_user.groups or []
+
+        return [
+            ClusterAccessControlRequest(
+                group_name=name,
+                permission_level=ClusterPermissionLevel.CAN_MANAGE
+            )
+            for name in current_groups
+            if name not in {"users"}
+        ]
+
+    def _check_permission(
+        self,
+        permission: Union[str, "ClusterAccessControlRequest", List[Union[str, "ClusterAccessControlRequest"]]],
+    ):
+        if isinstance(permission, ClusterAccessControlRequest):
+            return permission
+
+        if isinstance(permission, str):
+            if "@" in permission:
+                group_name, user_name = None, permission
+            else:
+                group_name, user_name = permission, None
+
+            return ClusterAccessControlRequest(
+                group_name=group_name,
+                user_name=user_name,
+                permission_level=ClusterPermissionLevel.CAN_MANAGE
+            )
+
+        return [
+            self._check_permission(_)
+            for _ in permission
+        ]
+
     def list_clusters(self) -> Iterator["Cluster"]:
         """Iterate clusters, yielding helpers annotated with metadata.
 
@@ -809,18 +884,22 @@ class Cluster(WorkspaceService):
         Returns:
             The current Cluster instance.
         """
+        if self.is_running:
+            return self
+
         self.wait_for_status()
 
-        if not self.is_running:
-            LOGGER.debug("Starting %s", self)
+        if self.is_running:
+            return self
 
-            if wait_timeout:
-                self.clusters_client().start(cluster_id=self.cluster_id)
-                self.wait_for_status(timeout=wait_timeout.total_seconds())
-            else:
-                self.clusters_client().start(cluster_id=self.cluster_id)
+        LOGGER.debug("Starting %s", self)
+
+        self.clusters_client().start(cluster_id=self.cluster_id)
 
-            LOGGER.info("Started %s", self)
+        LOGGER.info("Started %s", self)
+
+        if wait_timeout:
+            self.wait_for_status(timeout=wait_timeout.total_seconds())
 
         return self
 
@@ -836,7 +915,7 @@ class Cluster(WorkspaceService):
 
         if self.is_running:
             self.details = self.clusters_client().restart_and_wait(cluster_id=self.cluster_id)
-            return self.wait_for_status()
+            return self
 
         return self.start()
 

{ygg-0.1.47 → ygg-0.1.49}/src/yggdrasil/databricks/compute/execution_context.py

@@ -180,7 +180,7 @@ print(json.dumps(meta))"""
         """
         return self.cluster.workspace.sdk()
 
-    def _create_command(
+    def create_command(
         self,
         language: "Language",
     ) -> any:
@@ -192,17 +192,29 @@ print(json.dumps(meta))"""
         Returns:
             The created command execution context response.
         """
-        self.cluster.ensure_running()
-
         LOGGER.debug(
             "Creating Databricks command execution context for %s",
             self.cluster
         )
 
-        created = self._workspace_client().command_execution.create_and_wait(
-            cluster_id=self.cluster.cluster_id,
-            language=language,
+        try:
+            created = self._workspace_client().command_execution.create_and_wait(
+                cluster_id=self.cluster.cluster_id,
+                language=language,
+            )
+        except:
+            self.cluster.ensure_running()
+
+            created = self._workspace_client().command_execution.create_and_wait(
+                cluster_id=self.cluster.cluster_id,
+                language=language,
+            )
+
+        LOGGER.info(
+            "Created Databricks command execution context %s",
+            self
         )
+
         created = getattr(created, "response", created)
 
         return created
@@ -220,10 +232,6 @@ print(json.dumps(meta))"""
             The connected ExecutionContext instance.
         """
         if self.context_id is not None:
-            LOGGER.debug(
-                "Execution context already open for %s",
-                self
-            )
             return self
 
         self.language = language or self.language
@@ -231,7 +239,7 @@ print(json.dumps(meta))"""
         if self.language is None:
             self.language = Language.PYTHON
 
-        ctx = self._create_command(language=self.language)
+        ctx = self.create_command(language=self.language)
 
         context_id = ctx.id
         if not context_id:

{ygg-0.1.47 → ygg-0.1.49}/src/yggdrasil/databricks/compute/remote.py

@@ -39,6 +39,7 @@ def databricks_remote_compute(
     timeout: Optional[dt.timedelta] = None,
     env_keys: Optional[List[str]] = None,
     force_local: bool = False,
+    update_timeout: Optional[Union[float, dt.timedelta]] = None,
     **options
 ) -> Callable[[Callable[..., ReturnType]], Callable[..., ReturnType]]:
     """Return a decorator that executes functions on a remote cluster.
@@ -52,6 +53,7 @@ def databricks_remote_compute(
         timeout: Optional execution timeout for remote calls.
         env_keys: Optional environment variable names to forward.
         force_local: Force local execution
+        update_timeout: creation or update wait timeout
         **options: Extra options forwarded to the execution decorator.
 
     Returns:
@@ -82,7 +84,8 @@ def databricks_remote_compute(
             cluster = workspace.clusters().replicated_current_environment(
                 workspace=workspace,
                 cluster_name=cluster_name,
-                single_user_name=workspace.current_user.user_name
+                single_user_name=workspace.current_user.user_name,
+                update_timeout=update_timeout
             )
 
     cluster.ensure_running(wait_timeout=None)

{ygg-0.1.47 → ygg-0.1.49}/src/yggdrasil/databricks/sql/__init__.py

@@ -1,9 +1,8 @@
 """Databricks SQL helpers and engine wrappers."""
 
 from .engine import SQLEngine, StatementResult
+from .exceptions import SqlStatementError
 
 # Backwards compatibility
 DBXSQL = SQLEngine
 DBXStatementResult = StatementResult
-
-__all__ = ["SQLEngine", "StatementResult"]

ygg-0.1.49/src/yggdrasil/databricks/sql/exceptions.py

@@ -0,0 +1,45 @@
+"""Custom exceptions for Databricks SQL helpers."""
+from dataclasses import dataclass
+from typing import Optional, Any
+
+__all__ = [
+    "SqlStatementError"
+]
+
+
+@dataclass(frozen=True)
+class SqlStatementError(RuntimeError):
+    statement_id: str
+    state: str
+    message: str
+    error_code: Optional[str] = None
+    sql_state: Optional[str] = None
+
+    def __str__(self) -> str:
+        meta = []
+        if self.error_code:
+            meta.append(f"code={self.error_code}")
+        if self.sql_state:
+            meta.append(f"state={self.sql_state}")
+
+        meta_str = f" ({', '.join(meta)})" if meta else ""
+        return f"SQL statement {self.statement_id} failed [{self.state}]: {self.message}{meta_str}"
+
+    @classmethod
+    def from_statement(cls, stmt: Any) -> "SqlStatementError":
+        statement_id = getattr(stmt, "statement_id", "<unknown>")
+        state = getattr(stmt, "state", "<unknown>")
+
+        err = getattr(getattr(stmt, "status", None), "error", None)
+
+        message = getattr(err, "message", None) or "Unknown SQL error"
+        error_code = getattr(err, "error_code", None)
+        sql_state = getattr(err, "sql_state", None)
+
+        return cls(
+            statement_id=str(statement_id),
+            state=str(state),
+            message=str(message),
+            error_code=str(error_code) if error_code is not None else None,
+            sql_state=str(sql_state) if sql_state is not None else None,
+        )