ygg 0.1.47__py3-none-any.whl → 0.1.48__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/METADATA +1 -29
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/RECORD +18 -19
- yggdrasil/__init__.py +0 -4
- yggdrasil/databricks/compute/cluster.py +55 -2
- yggdrasil/databricks/sql/__init__.py +1 -2
- yggdrasil/databricks/sql/exceptions.py +44 -0
- yggdrasil/databricks/sql/statement_result.py +17 -40
- yggdrasil/databricks/workspaces/__init__.py +0 -1
- yggdrasil/dataclasses/__init__.py +1 -3
- yggdrasil/dataclasses/dataclass.py +1 -167
- yggdrasil/libs/__init__.py +0 -3
- yggdrasil/requests/__init__.py +2 -3
- yggdrasil/types/cast/cast_options.py +3 -4
- yggdrasil/version.py +1 -1
- yggdrasil/types/libs.py +0 -12
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/WHEEL +0 -0
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/entry_points.txt +0 -0
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/licenses/LICENSE +0 -0
- {ygg-0.1.47.dist-info → ygg-0.1.48.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ygg
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.48
|
|
4
4
|
Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
|
|
5
5
|
Author: Yggdrasil contributors
|
|
6
6
|
License: Apache License
|
|
@@ -270,34 +270,6 @@ Extras are grouped by engine:
|
|
|
270
270
|
- `.[polars]`, `.[pandas]`, `.[spark]`, `.[databricks]` – install only the integrations you need.
|
|
271
271
|
- `.[dev]` – adds testing, linting, and typing tools (`pytest`, `ruff`, `black`, `mypy`).
|
|
272
272
|
|
|
273
|
-
## Quickstart
|
|
274
|
-
Define an Arrow-aware dataclass, coerce inputs, and cast across containers:
|
|
275
|
-
|
|
276
|
-
```python
|
|
277
|
-
from yggdrasil import yggdataclass
|
|
278
|
-
from yggdrasil.types.cast import convert
|
|
279
|
-
from yggdrasil.types import arrow_field_from_hint
|
|
280
|
-
|
|
281
|
-
@yggdataclass
|
|
282
|
-
class User:
|
|
283
|
-
id: int
|
|
284
|
-
email: str
|
|
285
|
-
active: bool = True
|
|
286
|
-
|
|
287
|
-
user = User.__safe_init__("123", email="alice@example.com")
|
|
288
|
-
assert user.id == 123 and user.active is True
|
|
289
|
-
|
|
290
|
-
payload = {"id": "45", "email": "bob@example.com", "active": "false"}
|
|
291
|
-
clean = User.from_dict(payload)
|
|
292
|
-
print(clean.to_dict())
|
|
293
|
-
|
|
294
|
-
field = arrow_field_from_hint(User, name="user")
|
|
295
|
-
print(field) # user: struct<id: int64, email: string, active: bool>
|
|
296
|
-
|
|
297
|
-
numbers = convert(["1", "2", "3"], list[int])
|
|
298
|
-
print(numbers)
|
|
299
|
-
```
|
|
300
|
-
|
|
301
273
|
### Databricks example
|
|
302
274
|
Install the `databricks` extra and run SQL with typed results:
|
|
303
275
|
|
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
ygg-0.1.
|
|
2
|
-
yggdrasil/__init__.py,sha256=
|
|
3
|
-
yggdrasil/version.py,sha256=
|
|
1
|
+
ygg-0.1.48.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
|
|
2
|
+
yggdrasil/__init__.py,sha256=4-ghPak2S6zfMqmnlxW2GCgPb5s79znpKa2hGEGXcE4,24
|
|
3
|
+
yggdrasil/version.py,sha256=GL56LdSW6fsXlq5LHiGjsIVgyhxVQeeDdO3Sd6nzZYc,22
|
|
4
4
|
yggdrasil/databricks/__init__.py,sha256=skctY2c8W-hI81upx9F_PWRe5ishL3hrdiTuizgDjdw,152
|
|
5
5
|
yggdrasil/databricks/compute/__init__.py,sha256=NvdzmaJSNYY1uJthv1hHdBuNu3bD_-Z65DWnaJt9yXg,289
|
|
6
|
-
yggdrasil/databricks/compute/cluster.py,sha256=
|
|
6
|
+
yggdrasil/databricks/compute/cluster.py,sha256=0QjYHlaXSMgYqzMRy1Jypm2j7xoGRkPdwURZsQn_73U,43228
|
|
7
7
|
yggdrasil/databricks/compute/execution_context.py,sha256=anOxfNms83dZ5FTknbfT8uj889LjheMqEx9W5NtJC9E,23094
|
|
8
8
|
yggdrasil/databricks/compute/remote.py,sha256=nEN_Fr1Ouul_iKOf4B5QjEGscYAcl7nHjGsl2toRzrU,2874
|
|
9
9
|
yggdrasil/databricks/jobs/__init__.py,sha256=snxGSJb0M5I39v0y3IR-uEeSlZR248cQ_4DJ1sYs-h8,154
|
|
10
10
|
yggdrasil/databricks/jobs/config.py,sha256=9LGeHD04hbfy0xt8_6oobC4moKJh4_DTjZiK4Q2Tqjk,11557
|
|
11
|
-
yggdrasil/databricks/sql/__init__.py,sha256=
|
|
11
|
+
yggdrasil/databricks/sql/__init__.py,sha256=Vp_1cFaX1l-JGzCknvkbiB8CBFX2fQbBNntIeVn3lEg,231
|
|
12
12
|
yggdrasil/databricks/sql/engine.py,sha256=K5WmGKpXU78JA3UdK8dLxBD_GXKidZJFe7hytuC5UHg,41029
|
|
13
|
-
yggdrasil/databricks/sql/exceptions.py,sha256=
|
|
14
|
-
yggdrasil/databricks/sql/statement_result.py,sha256=
|
|
13
|
+
yggdrasil/databricks/sql/exceptions.py,sha256=uC-BoG0u0LtORKUS1X3iLID8nc-0TV5MQN3M8RXHsO4,1495
|
|
14
|
+
yggdrasil/databricks/sql/statement_result.py,sha256=kMBvpwyRv3_JUZSvxMS0c9Vqlh6LtCRJvXsDpu9RIAs,16137
|
|
15
15
|
yggdrasil/databricks/sql/types.py,sha256=5G-BM9_eOsRKEMzeDTWUsWW5g4Idvs-czVCpOCrMhdA,6412
|
|
16
16
|
yggdrasil/databricks/sql/warehouse.py,sha256=1J0dyQLJb-OS1_1xU1eAVZ4CoL2-FhFeowKSvU3RzFc,9773
|
|
17
|
-
yggdrasil/databricks/workspaces/__init__.py,sha256=
|
|
17
|
+
yggdrasil/databricks/workspaces/__init__.py,sha256=dv2zotoFVhNFlTCdRq6gwf5bEzeZkOZszoNZMs0k59g,114
|
|
18
18
|
yggdrasil/databricks/workspaces/filesytem.py,sha256=Z8JXU7_XUEbw9fpTQT1avRQKi-IAP2KemXBMPkUoY4w,9805
|
|
19
19
|
yggdrasil/databricks/workspaces/io.py,sha256=Tdde4LaGNJNT50R11OkEYZyNacyIW9QrOXMAicAlIr4,32208
|
|
20
20
|
yggdrasil/databricks/workspaces/path.py,sha256=-XnCD9p42who3DAwnITVE1KyrZUSoXDKHA8iZi-7wk4,47743
|
|
21
21
|
yggdrasil/databricks/workspaces/path_kind.py,sha256=Xc319NysH8_6E9C0Q8nCxDHYG07_SnzyUVKHe0dNdDQ,305
|
|
22
22
|
yggdrasil/databricks/workspaces/workspace.py,sha256=c6CBBun2BskEnsP74pbLVOe_TKXZs4L4r4gPQtIzlQE,23821
|
|
23
|
-
yggdrasil/dataclasses/__init__.py,sha256=
|
|
24
|
-
yggdrasil/dataclasses/dataclass.py,sha256=
|
|
25
|
-
yggdrasil/libs/__init__.py,sha256=
|
|
23
|
+
yggdrasil/dataclasses/__init__.py,sha256=_RkhfF3KC1eSORby1dzvBXQ0-UGG3u6wyUQWX2jq1Pc,108
|
|
24
|
+
yggdrasil/dataclasses/dataclass.py,sha256=LxrCjwvmBnb8yRI_N-c31RHHxB4XoJPixmKg9iBIuaI,1148
|
|
25
|
+
yggdrasil/libs/__init__.py,sha256=zdC9OU0Xy36CLY9mg2drxN6S7isPR8aTLzJA6xVIeLE,91
|
|
26
26
|
yggdrasil/libs/databrickslib.py,sha256=NHJeUViHhZc8LI5oDVfi1axRyUy_pDJLy4hjD0KZEBQ,980
|
|
27
27
|
yggdrasil/libs/pandaslib.py,sha256=Edm3SXgvr8qe2wsojuRvD1ewNB-Sff0RWoTqaddVruI,509
|
|
28
28
|
yggdrasil/libs/polarslib.py,sha256=7EWP5iS8F9cW79M6d8Yg5ysjnOY3w4_k7TW-5DCRACw,511
|
|
@@ -39,16 +39,15 @@ yggdrasil/pyutils/modules.py,sha256=B7IP99YqUMW6-DIESFzBx8-09V1d0a8qrIJUDFhhL2g,
|
|
|
39
39
|
yggdrasil/pyutils/parallel.py,sha256=ubuq2m9dJzWYUyKCga4Y_9bpaeMYUrleYxdp49CHr44,6781
|
|
40
40
|
yggdrasil/pyutils/python_env.py,sha256=tuglnjdqHQjNh18qDladVoSEOjCD0RcnMEPYJ0tArOs,50985
|
|
41
41
|
yggdrasil/pyutils/retry.py,sha256=n5sr-Zu7fYrdLbjJ4WifK2lk0gEGmHv5FYt2HaCm1Qc,11916
|
|
42
|
-
yggdrasil/requests/__init__.py,sha256=
|
|
42
|
+
yggdrasil/requests/__init__.py,sha256=dMesyzq97_DmI765x0TwaDPEfsxFtgGNgchk8LvEN-o,103
|
|
43
43
|
yggdrasil/requests/msal.py,sha256=s2GCyzbgFdgdlJ1JqMrZ4qYVbmoG46-ZOTcaVQhZ-sQ,9220
|
|
44
44
|
yggdrasil/requests/session.py,sha256=SLnrgHY0Lby7ZxclRFUjHdfM8euN_8bSQEWl7TkJY2U,1461
|
|
45
45
|
yggdrasil/types/__init__.py,sha256=CrLiDeYNM9fO975sE5ufeVKcy7Ca702IsaG2Pk8T3YU,139
|
|
46
|
-
yggdrasil/types/libs.py,sha256=2iRT9JDUdr9seuGz9ZR3wWdrxZ8LRnc9i-m_tkKdKgI,293
|
|
47
46
|
yggdrasil/types/python_arrow.py,sha256=mOhyecAxa5u8JWsyTO26OMOWimHHgwLKWlkNSAyIVas,25636
|
|
48
47
|
yggdrasil/types/python_defaults.py,sha256=GO3hZBZcwRHs9qiXes75y8l5X00kZHTfEC7el_x73uw,10184
|
|
49
48
|
yggdrasil/types/cast/__init__.py,sha256=Oft3pTs2bRM5hT7YqJAuOKTYYk-SACLaMOXUVdafy_I,311
|
|
50
49
|
yggdrasil/types/cast/arrow_cast.py,sha256=_OMYc4t5GlgE4ztlWaCoK8Jnba09rgDbmHVP-QXhOL0,41523
|
|
51
|
-
yggdrasil/types/cast/cast_options.py,sha256=
|
|
50
|
+
yggdrasil/types/cast/cast_options.py,sha256=nDaEvCCs7TBamhTWyDrYf3LVaBWzioIP2Q5_LXrChF4,15532
|
|
52
51
|
yggdrasil/types/cast/pandas_cast.py,sha256=I3xu0sZ59ZbK3NDcQ2dslzdeKzhpFV5zR02ZEixd5hI,8713
|
|
53
52
|
yggdrasil/types/cast/polars_cast.py,sha256=K2nnQ7bexArneYEhUPgV_6er4JNq6N5RmbMUhw-2_Xw,28766
|
|
54
53
|
yggdrasil/types/cast/polars_pandas_cast.py,sha256=CS0P7teVv15IdX5g7v40RfkH1VMg6b-HM0V_gOfacm8,5071
|
|
@@ -56,8 +55,8 @@ yggdrasil/types/cast/registry.py,sha256=_zdFGmUBB7P-e_LIcJlOxMcxAkXoA-UXB6HqLMgT
|
|
|
56
55
|
yggdrasil/types/cast/spark_cast.py,sha256=_KAsl1DqmKMSfWxqhVE7gosjYdgiL1C5bDQv6eP3HtA,24926
|
|
57
56
|
yggdrasil/types/cast/spark_pandas_cast.py,sha256=BuTiWrdCANZCdD_p2MAytqm74eq-rdRXd-LGojBRrfU,5023
|
|
58
57
|
yggdrasil/types/cast/spark_polars_cast.py,sha256=btmZNHXn2NSt3fUuB4xg7coaE0RezIBdZD92H8NK0Jw,9073
|
|
59
|
-
ygg-0.1.
|
|
60
|
-
ygg-0.1.
|
|
61
|
-
ygg-0.1.
|
|
62
|
-
ygg-0.1.
|
|
63
|
-
ygg-0.1.
|
|
58
|
+
ygg-0.1.48.dist-info/METADATA,sha256=gpScM9WWu0y7C5ebXB6gsJBe9VbehZEU__E7HfWp8hk,18452
|
|
59
|
+
ygg-0.1.48.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
60
|
+
ygg-0.1.48.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
|
|
61
|
+
ygg-0.1.48.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
|
|
62
|
+
ygg-0.1.48.dist-info/RECORD,,
|
yggdrasil/__init__.py
CHANGED
|
@@ -22,8 +22,8 @@ from typing import Any, Iterator, Optional, Union, List, Callable, Dict, ClassVa
|
|
|
22
22
|
|
|
23
23
|
from .execution_context import ExecutionContext
|
|
24
24
|
from ..workspaces.workspace import WorkspaceService, Workspace
|
|
25
|
-
from ... import CallableSerde
|
|
26
25
|
from ...libs.databrickslib import databricks_sdk
|
|
26
|
+
from ...pyutils.callable_serde import CallableSerde
|
|
27
27
|
from ...pyutils.equality import dicts_equal, dict_diff
|
|
28
28
|
from ...pyutils.expiring_dict import ExpiringDict
|
|
29
29
|
from ...pyutils.modules import PipIndexSettings
|
|
@@ -36,7 +36,8 @@ else: # pragma: no cover - runtime fallback when SDK is missing
|
|
|
36
36
|
from databricks.sdk.errors import DatabricksError
|
|
37
37
|
from databricks.sdk.errors.platform import ResourceDoesNotExist
|
|
38
38
|
from databricks.sdk.service.compute import (
|
|
39
|
-
ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus
|
|
39
|
+
ClusterDetails, Language, Kind, State, DataSecurityMode, Library, PythonPyPiLibrary, LibraryInstallStatus,
|
|
40
|
+
ClusterAccessControlRequest, ClusterPermissionLevel
|
|
40
41
|
)
|
|
41
42
|
from databricks.sdk.service.compute import SparkVersion, RuntimeEngine
|
|
42
43
|
|
|
@@ -659,6 +660,7 @@ class Cluster(WorkspaceService):
|
|
|
659
660
|
def update(
|
|
660
661
|
self,
|
|
661
662
|
libraries: Optional[List[Union[str, "Library"]]] = None,
|
|
663
|
+
access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
|
|
662
664
|
wait_timeout: Union[float, dt.timedelta] = dt.timedelta(minutes=20),
|
|
663
665
|
**cluster_spec: Any
|
|
664
666
|
) -> "Cluster":
|
|
@@ -666,6 +668,7 @@ class Cluster(WorkspaceService):
|
|
|
666
668
|
|
|
667
669
|
Args:
|
|
668
670
|
libraries: Optional libraries to install.
|
|
671
|
+
access_control_list: List of permissions
|
|
669
672
|
wait_timeout: waiting timeout until done, if None it does not wait
|
|
670
673
|
**cluster_spec: Cluster specification overrides.
|
|
671
674
|
|
|
@@ -707,6 +710,7 @@ class Cluster(WorkspaceService):
|
|
|
707
710
|
|
|
708
711
|
self.wait_for_status()
|
|
709
712
|
self.clusters_client().edit(**update_details)
|
|
713
|
+
self.update_permissions(access_control_list=access_control_list)
|
|
710
714
|
|
|
711
715
|
LOGGER.info(
|
|
712
716
|
"Updated %s",
|
|
@@ -718,6 +722,55 @@ class Cluster(WorkspaceService):
|
|
|
718
722
|
|
|
719
723
|
return self
|
|
720
724
|
|
|
725
|
+
def update_permissions(
|
|
726
|
+
self,
|
|
727
|
+
access_control_list: Optional[List["ClusterAccessControlRequest"]] = None,
|
|
728
|
+
):
|
|
729
|
+
if not access_control_list:
|
|
730
|
+
access_control_list = self.default_permissions()
|
|
731
|
+
|
|
732
|
+
access_control_list = self._check_permission(access_control_list)
|
|
733
|
+
|
|
734
|
+
self.clusters_client().update_permissions(
|
|
735
|
+
cluster_id=self.cluster_id,
|
|
736
|
+
access_control_list=access_control_list
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
def default_permissions(self):
|
|
740
|
+
current_groups = self.current_user.groups or []
|
|
741
|
+
|
|
742
|
+
return [
|
|
743
|
+
ClusterAccessControlRequest(
|
|
744
|
+
group_name=name,
|
|
745
|
+
permission_level=ClusterPermissionLevel.CAN_MANAGE
|
|
746
|
+
)
|
|
747
|
+
for name in current_groups
|
|
748
|
+
]
|
|
749
|
+
|
|
750
|
+
def _check_permission(
|
|
751
|
+
self,
|
|
752
|
+
permission: Union[str, "ClusterAccessControlRequest", List[Union[str, "ClusterAccessControlRequest"]]],
|
|
753
|
+
):
|
|
754
|
+
if isinstance(permission, ClusterAccessControlRequest):
|
|
755
|
+
return permission
|
|
756
|
+
|
|
757
|
+
if isinstance(permission, str):
|
|
758
|
+
if "@" in permission:
|
|
759
|
+
group_name, user_name = None, permission
|
|
760
|
+
else:
|
|
761
|
+
group_name, user_name = permission, None
|
|
762
|
+
|
|
763
|
+
return ClusterAccessControlRequest(
|
|
764
|
+
group_name=group_name,
|
|
765
|
+
user_name=user_name,
|
|
766
|
+
permission_level=ClusterPermissionLevel.CAN_MANAGE
|
|
767
|
+
)
|
|
768
|
+
|
|
769
|
+
return [
|
|
770
|
+
self._check_permission(_)
|
|
771
|
+
for _ in permission
|
|
772
|
+
]
|
|
773
|
+
|
|
721
774
|
def list_clusters(self) -> Iterator["Cluster"]:
|
|
722
775
|
"""Iterate clusters, yielding helpers annotated with metadata.
|
|
723
776
|
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"""Databricks SQL helpers and engine wrappers."""
|
|
2
2
|
|
|
3
3
|
from .engine import SQLEngine, StatementResult
|
|
4
|
+
from .exceptions import SqlStatementError
|
|
4
5
|
|
|
5
6
|
# Backwards compatibility
|
|
6
7
|
DBXSQL = SQLEngine
|
|
7
8
|
DBXStatementResult = StatementResult
|
|
8
|
-
|
|
9
|
-
__all__ = ["SQLEngine", "StatementResult"]
|
|
@@ -1 +1,45 @@
|
|
|
1
1
|
"""Custom exceptions for Databricks SQL helpers."""
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Optional, Any
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"SqlStatementError"
|
|
7
|
+
]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class SqlStatementError(RuntimeError):
|
|
12
|
+
statement_id: str
|
|
13
|
+
state: str
|
|
14
|
+
message: str
|
|
15
|
+
error_code: Optional[str] = None
|
|
16
|
+
sql_state: Optional[str] = None
|
|
17
|
+
|
|
18
|
+
def __str__(self) -> str:
|
|
19
|
+
meta = []
|
|
20
|
+
if self.error_code:
|
|
21
|
+
meta.append(f"code={self.error_code}")
|
|
22
|
+
if self.sql_state:
|
|
23
|
+
meta.append(f"state={self.sql_state}")
|
|
24
|
+
|
|
25
|
+
meta_str = f" ({', '.join(meta)})" if meta else ""
|
|
26
|
+
return f"SQL statement {self.statement_id} failed [{self.state}]: {self.message}{meta_str}"
|
|
27
|
+
|
|
28
|
+
@classmethod
|
|
29
|
+
def from_statement(cls, stmt: Any) -> "SqlStatementError":
|
|
30
|
+
statement_id = getattr(stmt, "statement_id", "<unknown>")
|
|
31
|
+
state = getattr(stmt, "state", "<unknown>")
|
|
32
|
+
|
|
33
|
+
err = getattr(getattr(stmt, "status", None), "error", None)
|
|
34
|
+
|
|
35
|
+
message = getattr(err, "message", None) or "Unknown SQL error"
|
|
36
|
+
error_code = getattr(err, "error_code", None)
|
|
37
|
+
sql_state = getattr(err, "sql_state", None)
|
|
38
|
+
|
|
39
|
+
return cls(
|
|
40
|
+
statement_id=str(statement_id),
|
|
41
|
+
state=str(state),
|
|
42
|
+
message=str(message),
|
|
43
|
+
error_code=str(error_code) if error_code is not None else None,
|
|
44
|
+
sql_state=str(sql_state) if sql_state is not None else None,
|
|
45
|
+
)
|
|
@@ -9,6 +9,7 @@ from typing import Optional, Iterator, TYPE_CHECKING
|
|
|
9
9
|
import pyarrow as pa
|
|
10
10
|
import pyarrow.ipc as pipc
|
|
11
11
|
|
|
12
|
+
from .exceptions import SqlStatementError
|
|
12
13
|
from .types import column_info_to_arrow_field
|
|
13
14
|
from ...libs.databrickslib import databricks_sdk
|
|
14
15
|
from ...libs.pandaslib import pandas
|
|
@@ -32,9 +33,7 @@ except ImportError:
|
|
|
32
33
|
if databricks_sdk is not None:
|
|
33
34
|
from databricks.sdk.service.sql import (
|
|
34
35
|
StatementState, StatementResponse, Disposition, StatementStatus
|
|
35
|
-
)
|
|
36
|
-
|
|
37
|
-
StatementResponse = StatementResponse
|
|
36
|
+
)
|
|
38
37
|
else:
|
|
39
38
|
class StatementResponse:
|
|
40
39
|
pass
|
|
@@ -299,28 +298,8 @@ class StatementResult:
|
|
|
299
298
|
)
|
|
300
299
|
|
|
301
300
|
def raise_for_status(self):
|
|
302
|
-
"""Raise a ValueError if the statement failed.
|
|
303
|
-
|
|
304
|
-
Returns:
|
|
305
|
-
None.
|
|
306
|
-
"""
|
|
307
301
|
if self.failed:
|
|
308
|
-
|
|
309
|
-
err = self.status.error
|
|
310
|
-
message = err.message or "Unknown SQL error"
|
|
311
|
-
error_code = err.error_code
|
|
312
|
-
sql_state = getattr(err, "sql_state", None)
|
|
313
|
-
|
|
314
|
-
parts = [message]
|
|
315
|
-
if error_code:
|
|
316
|
-
parts.append(f"error_code={error_code}")
|
|
317
|
-
if sql_state:
|
|
318
|
-
parts.append(f"sql_state={sql_state}")
|
|
319
|
-
|
|
320
|
-
raise ValueError(
|
|
321
|
-
f"Statement {self.statement_id} {self.state}: " + " | ".join(parts)
|
|
322
|
-
)
|
|
323
|
-
|
|
302
|
+
raise SqlStatementError.from_statement(self)
|
|
324
303
|
return self
|
|
325
304
|
|
|
326
305
|
def wait(
|
|
@@ -337,22 +316,20 @@ class StatementResult:
|
|
|
337
316
|
Returns:
|
|
338
317
|
The current StatementResult instance.
|
|
339
318
|
"""
|
|
340
|
-
if self.done:
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
)
|
|
353
|
-
|
|
354
|
-
poll_interval = max(10, poll_interval * 1.2)
|
|
355
|
-
time.sleep(poll_interval)
|
|
319
|
+
if not self.done:
|
|
320
|
+
start = time.time()
|
|
321
|
+
poll_interval = poll_interval or 1
|
|
322
|
+
|
|
323
|
+
while not self.done:
|
|
324
|
+
# still running / queued / pending
|
|
325
|
+
if timeout is not None and (time.time() - start) > timeout:
|
|
326
|
+
raise TimeoutError(
|
|
327
|
+
f"Statement {self.statement_id} did not finish within {timeout} seconds "
|
|
328
|
+
f"(last state={self.state})"
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
poll_interval = max(10, poll_interval * 1.2)
|
|
332
|
+
time.sleep(poll_interval)
|
|
356
333
|
|
|
357
334
|
self.raise_for_status()
|
|
358
335
|
|
|
@@ -2,32 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
import dataclasses
|
|
4
4
|
from inspect import isclass
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
import pyarrow as pa
|
|
8
8
|
|
|
9
9
|
__all__ = [
|
|
10
|
-
"yggdataclass",
|
|
11
|
-
"is_yggdataclass",
|
|
12
10
|
"get_dataclass_arrow_field"
|
|
13
11
|
]
|
|
14
12
|
|
|
15
13
|
DATACLASS_ARROW_FIELD_CACHE: dict[type, pa.Field] = {}
|
|
16
14
|
|
|
17
15
|
|
|
18
|
-
def is_yggdataclass(cls_or_instance: Any) -> bool:
|
|
19
|
-
"""Check if a class or instance is a yggdrasil dataclass.
|
|
20
|
-
|
|
21
|
-
Args:
|
|
22
|
-
cls_or_instance: The class or instance to check.
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
True if the class or instance
|
|
26
|
-
is a yggdrasil dataclass, False otherwise.
|
|
27
|
-
"""
|
|
28
|
-
return hasattr(cls_or_instance, "__arrow_field__")
|
|
29
|
-
|
|
30
|
-
|
|
31
16
|
def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
|
|
32
17
|
"""Return a cached Arrow Field describing the dataclass type.
|
|
33
18
|
|
|
@@ -37,9 +22,6 @@ def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
|
|
|
37
22
|
Returns:
|
|
38
23
|
Arrow field describing the dataclass schema.
|
|
39
24
|
"""
|
|
40
|
-
if is_yggdataclass(cls_or_instance):
|
|
41
|
-
return cls_or_instance.__arrow_field__()
|
|
42
|
-
|
|
43
25
|
if dataclasses.is_dataclass(cls_or_instance):
|
|
44
26
|
cls = cls_or_instance
|
|
45
27
|
if not isclass(cls_or_instance):
|
|
@@ -56,151 +38,3 @@ def get_dataclass_arrow_field(cls_or_instance: Any) -> pa.Field:
|
|
|
56
38
|
return built
|
|
57
39
|
|
|
58
40
|
raise ValueError(f"{cls_or_instance!r} is not a dataclass or yggdrasil dataclass")
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def yggdataclass(
|
|
62
|
-
cls=None, /,
|
|
63
|
-
*,
|
|
64
|
-
init=True,
|
|
65
|
-
repr=True,
|
|
66
|
-
eq=True,
|
|
67
|
-
order=False,
|
|
68
|
-
unsafe_hash=False, frozen=False, match_args=True,
|
|
69
|
-
kw_only=False, slots=False,
|
|
70
|
-
weakref_slot=False
|
|
71
|
-
):
|
|
72
|
-
"""Decorate a class with dataclass behavior plus Arrow helpers.
|
|
73
|
-
|
|
74
|
-
Examines PEP 526 __annotations__ to determine fields.
|
|
75
|
-
|
|
76
|
-
If init is true, an __init__() method is added to the class. If repr
|
|
77
|
-
is true, a __repr__() method is added. If order is true, rich
|
|
78
|
-
comparison dunder methods are added. If unsafe_hash is true, a
|
|
79
|
-
__hash__() method is added. If frozen is true, fields may not be
|
|
80
|
-
assigned to after instance creation. If match_args is true, the
|
|
81
|
-
__match_args__ tuple is added. If kw_only is true, then by default
|
|
82
|
-
all fields are keyword-only. If slots is true, a new class with a
|
|
83
|
-
__slots__ attribute is returned.
|
|
84
|
-
"""
|
|
85
|
-
|
|
86
|
-
def wrap(c):
|
|
87
|
-
"""Wrap a class with yggdrasil dataclass enhancements.
|
|
88
|
-
|
|
89
|
-
Args:
|
|
90
|
-
c: Class to decorate.
|
|
91
|
-
|
|
92
|
-
Returns:
|
|
93
|
-
Decorated dataclass type.
|
|
94
|
-
"""
|
|
95
|
-
|
|
96
|
-
def _init_public_fields(cls):
|
|
97
|
-
"""Return init-enabled, public dataclass fields.
|
|
98
|
-
|
|
99
|
-
Args:
|
|
100
|
-
cls: Dataclass type.
|
|
101
|
-
|
|
102
|
-
Returns:
|
|
103
|
-
List of dataclasses.Field objects.
|
|
104
|
-
"""
|
|
105
|
-
return [
|
|
106
|
-
field
|
|
107
|
-
for field in dataclasses.fields(cls)
|
|
108
|
-
if field.init and not field.name.startswith("_")
|
|
109
|
-
]
|
|
110
|
-
|
|
111
|
-
if not hasattr(c, "default_instance"):
|
|
112
|
-
@classmethod
|
|
113
|
-
def default_instance(cls):
|
|
114
|
-
"""Return a default instance built from type defaults.
|
|
115
|
-
|
|
116
|
-
Returns:
|
|
117
|
-
Default instance of the dataclass.
|
|
118
|
-
"""
|
|
119
|
-
from yggdrasil.types import default_scalar
|
|
120
|
-
|
|
121
|
-
if not hasattr(cls, "__default_instance__"):
|
|
122
|
-
cls.__default_instance__ = default_scalar(cls)
|
|
123
|
-
|
|
124
|
-
return dataclasses.replace(cls.__default_instance__)
|
|
125
|
-
|
|
126
|
-
c.default_instance = default_instance
|
|
127
|
-
|
|
128
|
-
if not hasattr(c, "__safe_init__"):
|
|
129
|
-
@classmethod
|
|
130
|
-
def __safe_init__(cls, *args, **kwargs):
|
|
131
|
-
"""Safely initialize a dataclass using type conversion and defaults."""
|
|
132
|
-
|
|
133
|
-
fields = _init_public_fields(cls)
|
|
134
|
-
field_names = [field.name for field in fields]
|
|
135
|
-
|
|
136
|
-
if len(args) > len(field_names):
|
|
137
|
-
raise TypeError(
|
|
138
|
-
f"Expected at most {len(field_names)} positional arguments, got {len(args)}"
|
|
139
|
-
)
|
|
140
|
-
|
|
141
|
-
provided = {name: value for name, value in zip(field_names, args)}
|
|
142
|
-
|
|
143
|
-
for key, value in kwargs.items():
|
|
144
|
-
if key in provided:
|
|
145
|
-
raise TypeError(f"Got multiple values for argument '{key}'")
|
|
146
|
-
if key not in field_names:
|
|
147
|
-
raise TypeError(
|
|
148
|
-
f"{key!r} is an invalid field for {cls.__name__}"
|
|
149
|
-
)
|
|
150
|
-
|
|
151
|
-
provided[key] = value
|
|
152
|
-
|
|
153
|
-
from yggdrasil.types.cast import convert
|
|
154
|
-
|
|
155
|
-
defaults = cls.default_instance()
|
|
156
|
-
init_kwargs = {}
|
|
157
|
-
|
|
158
|
-
for field in fields:
|
|
159
|
-
if field.name in provided:
|
|
160
|
-
init_kwargs[field.name] = convert(provided[field.name], field.type)
|
|
161
|
-
else:
|
|
162
|
-
init_kwargs[field.name] = getattr(defaults, field.name, None)
|
|
163
|
-
|
|
164
|
-
return cls(**init_kwargs)
|
|
165
|
-
|
|
166
|
-
c.__safe_init__ = __safe_init__
|
|
167
|
-
|
|
168
|
-
if not hasattr(c, "__arrow_field__"):
|
|
169
|
-
@classmethod
|
|
170
|
-
def __arrow_field__(cls, name: str | None = None):
|
|
171
|
-
"""Return an Arrow field representing the dataclass schema.
|
|
172
|
-
|
|
173
|
-
Args:
|
|
174
|
-
name: Optional override for the field name.
|
|
175
|
-
|
|
176
|
-
Returns:
|
|
177
|
-
Arrow field describing the dataclass schema.
|
|
178
|
-
"""
|
|
179
|
-
from yggdrasil.types.python_arrow import arrow_field_from_hint
|
|
180
|
-
|
|
181
|
-
return arrow_field_from_hint(cls, name=name)
|
|
182
|
-
|
|
183
|
-
c.__arrow_field__ = __arrow_field__
|
|
184
|
-
|
|
185
|
-
base = dataclasses.dataclass(
|
|
186
|
-
c,
|
|
187
|
-
init=init,
|
|
188
|
-
repr=repr,
|
|
189
|
-
eq=eq,
|
|
190
|
-
order=order,
|
|
191
|
-
unsafe_hash=unsafe_hash,
|
|
192
|
-
frozen=frozen,
|
|
193
|
-
match_args=match_args,
|
|
194
|
-
kw_only=kw_only,
|
|
195
|
-
slots=slots,
|
|
196
|
-
)
|
|
197
|
-
|
|
198
|
-
return base
|
|
199
|
-
|
|
200
|
-
# See if we're being called as @dataclass or @dataclass().
|
|
201
|
-
if cls is None:
|
|
202
|
-
# We're called with parens.
|
|
203
|
-
return wrap
|
|
204
|
-
|
|
205
|
-
# We're called as @dataclass without parens.
|
|
206
|
-
return wrap(cls)
|
yggdrasil/libs/__init__.py
CHANGED
yggdrasil/requests/__init__.py
CHANGED
|
@@ -7,16 +7,15 @@ import pyarrow as pa
|
|
|
7
7
|
|
|
8
8
|
from .registry import convert
|
|
9
9
|
from ..python_arrow import is_arrow_type_list_like
|
|
10
|
-
from ...
|
|
10
|
+
from ...libs.polarslib import polars
|
|
11
|
+
from ...libs.sparklib import pyspark
|
|
11
12
|
|
|
12
13
|
__all__ = [
|
|
13
14
|
"CastOptions",
|
|
14
15
|
]
|
|
15
16
|
|
|
16
|
-
from ...libs import pyspark, polars
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
@yggdataclass
|
|
18
|
+
@dataclasses.dataclass
|
|
20
19
|
class CastOptions:
|
|
21
20
|
"""
|
|
22
21
|
Options controlling Arrow casting behavior.
|
yggdrasil/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.1.
|
|
1
|
+
__version__ = "0.1.48"
|
yggdrasil/types/libs.py
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
"""Re-export optional dependency helpers for types modules."""
|
|
2
|
-
|
|
3
|
-
from ..libs import pandas, polars, pyspark, require_pandas, require_polars, require_pyspark
|
|
4
|
-
|
|
5
|
-
__all__ = [
|
|
6
|
-
"pandas",
|
|
7
|
-
"polars",
|
|
8
|
-
"pyspark",
|
|
9
|
-
"require_pandas",
|
|
10
|
-
"require_polars",
|
|
11
|
-
"require_pyspark",
|
|
12
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|