foundry-dev-tools 2.1.21__tar.gz → 2.1.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/PKG-INFO +1 -1
  2. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/pyproject.toml +1 -1
  3. foundry_dev_tools-2.1.23/src/foundry_dev_tools/__about__.py +1 -0
  4. foundry_dev_tools-2.1.23/src/foundry_dev_tools/clients/data_health.py +60 -0
  5. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/data_proxy.py +25 -5
  6. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/foundry_sql_server.py +30 -12
  7. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/config/context.py +6 -0
  8. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/foundry_api_client.py +15 -4
  9. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/resources/dataset.py +67 -15
  10. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/api_types.py +5 -1
  11. foundry_dev_tools-2.1.21/src/foundry_dev_tools/__about__.py +0 -1
  12. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/README.md +0 -0
  13. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/__init__.py +0 -0
  14. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/__init__.py +0 -0
  15. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/foundry_platform_sdk.py +0 -0
  16. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/pandas.py +0 -0
  17. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/polars.py +0 -0
  18. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/pyarrow.py +0 -0
  19. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/_optional/pyspark.py +0 -0
  20. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  21. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/__init__.py +0 -0
  22. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/build.py +0 -0
  23. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/config.py +0 -0
  24. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/git.py +0 -0
  25. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/git_credential_foundry.py +0 -0
  26. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/info.py +0 -0
  27. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/main.py +0 -0
  28. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/s3.py +0 -0
  29. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/cli/stubs.py +0 -0
  30. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/__init__.py +0 -0
  31. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/api_client.py +0 -0
  32. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/build2.py +0 -0
  33. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/catalog.py +0 -0
  34. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/compass.py +0 -0
  35. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/context_client.py +0 -0
  36. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/foundry_stats.py +0 -0
  37. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/jemma.py +0 -0
  38. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/magritte_coordinator.py +0 -0
  39. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/metadata.py +0 -0
  40. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/multipass.py +0 -0
  41. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/public_ontologies_client.py +0 -0
  42. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/s3_client.py +0 -0
  43. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/scheduler.py +0 -0
  44. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/schema_inference.py +0 -0
  45. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/clients/tables.py +0 -0
  46. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/config/__init__.py +0 -0
  47. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/config/config.py +0 -0
  48. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/config/config_types.py +0 -0
  49. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/config/token_provider.py +0 -0
  50. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/__init__.py +0 -0
  51. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/compass.py +0 -0
  52. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/config.py +0 -0
  53. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/dataset.py +0 -0
  54. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/handling.py +0 -0
  55. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/meta.py +0 -0
  56. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/multipass.py +0 -0
  57. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/errors/sql.py +0 -0
  58. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/helpers/__init__.py +0 -0
  59. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/helpers/multipass/__init__.py +0 -0
  60. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/helpers/multipass/group.py +0 -0
  61. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/helpers/multipass/principal.py +0 -0
  62. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/helpers/multipass/user.py +0 -0
  63. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/public_sdk/__init__.py +0 -0
  64. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/public_sdk/auth.py +0 -0
  65. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/py.typed +0 -0
  66. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/resources/__init__.py +0 -0
  67. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/resources/folder.py +0 -0
  68. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/resources/resource.py +0 -0
  69. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/__init__.py +0 -0
  70. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  71. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  72. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  73. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  74. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/cli.py +0 -0
  75. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/clients.py +0 -0
  76. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/compat.py +0 -0
  77. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/config.py +0 -0
  78. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  79. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  80. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/misc.py +0 -0
  81. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/public_api_types.py +0 -0
  82. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/repo.py +0 -0
  83. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/s3.py +0 -0
  84. {foundry_dev_tools-2.1.21 → foundry_dev_tools-2.1.23}/src/foundry_dev_tools/utils/spark.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 2.1.21
3
+ Version: 2.1.23
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -41,7 +41,7 @@ dependencies = [
41
41
  "tomli_w",
42
42
  ]
43
43
  dynamic = []
44
- version = "2.1.21"
44
+ version = "2.1.23"
45
45
 
46
46
  [project.license]
47
47
  text = "Apache-2.0"
@@ -0,0 +1 @@
1
+ __version__ = '2.1.23'
@@ -0,0 +1,60 @@
1
+ """Implementation of the data-health API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+
9
+ if TYPE_CHECKING:
10
+ import requests
11
+
12
+ from foundry_dev_tools.utils.api_types import (
13
+ CheckRid,
14
+ DatasetRid,
15
+ )
16
+
17
+
18
+ class DataHealthClient(APIClient):
19
+ """DataHealthClient class that implements methods from the 'foundry-data-health' API."""
20
+
21
+ api_name = "data-health"
22
+
23
+ def api_get_checks_for_dataset(
24
+ self,
25
+ dataset_rid: DatasetRid,
26
+ branch: str = "master",
27
+ **kwargs,
28
+ ) -> requests.Response:
29
+ """Gets all the Check RIDs available for the given Dataset RID and branch name.
30
+
31
+ Args:
32
+ dataset_rid: The RID of the dataset
33
+ branch: The name of the branch
34
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
35
+ """
36
+ return self.api_request(
37
+ "GET",
38
+ api_path=f"checks/v2/{dataset_rid}/{branch}",
39
+ **kwargs,
40
+ )
41
+
42
+ def api_get_latest_checks_history(
43
+ self, check_rids: list[CheckRid] | CheckRid, limit: int = 5, **kwargs
44
+ ) -> requests.Response:
45
+ """Returns CheckRun metadata for the last `limit` number of check runs for the given list of Check RIDs.
46
+
47
+ Args:
48
+ check_rids: a list of CheckRIDs or a single string value.
49
+ limit: number of last check runs to go for for each Check RID. Defaults to 5.
50
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
51
+ """
52
+ check_rids = check_rids if isinstance(check_rids, list) else [check_rids]
53
+ params = {"limit": limit}
54
+ return self.api_request(
55
+ "POST",
56
+ api_path="checks/reports/v2/latest",
57
+ json=check_rids,
58
+ params=params,
59
+ **kwargs,
60
+ )
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
27
27
  from pathlib import Path
28
28
 
29
29
  import pandas as pd
30
+ import polars as pl
30
31
  import pyarrow as pa
31
32
  import pyspark
32
33
  import requests
@@ -111,7 +112,17 @@ class DataProxyClient(APIClient):
111
112
  branch: Ref = ...,
112
113
  sql_dialect: SqlDialect = ...,
113
114
  timeout: int = ...,
114
- ) -> pd.core.frame.DataFrame: ...
115
+ ) -> pd.DataFrame: ...
116
+
117
+ @overload
118
+ def query_foundry_sql_legacy(
119
+ self,
120
+ query: str,
121
+ return_type: Literal["polars"],
122
+ branch: Ref = ...,
123
+ sql_dialect: SqlDialect = ...,
124
+ timeout: int = ...,
125
+ ) -> pl.DataFrame: ...
115
126
 
116
127
  @overload
117
128
  def query_foundry_sql_legacy(
@@ -151,7 +162,7 @@ class DataProxyClient(APIClient):
151
162
  branch: Ref = ...,
152
163
  sql_dialect: SqlDialect = ...,
153
164
  timeout: int = ...,
154
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
165
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
155
166
 
156
167
  def query_foundry_sql_legacy(
157
168
  self,
@@ -160,7 +171,7 @@ class DataProxyClient(APIClient):
160
171
  branch: Ref = "master",
161
172
  sql_dialect: SqlDialect = "SPARK",
162
173
  timeout: int = 600,
163
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame:
174
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame:
164
175
  """Queries the dataproxy query API with spark SQL.
165
176
 
166
177
  Example:
@@ -206,6 +217,9 @@ class DataProxyClient(APIClient):
206
217
  response_json = response.json()
207
218
  if return_type == "raw":
208
219
  return response_json["foundrySchema"], response_json["rows"]
220
+ # return_type arrow, pandas and polars use the FakeModule implementation in
221
+ # their _optional packages. The FakeModule throws an ImportError when trying
222
+ # to access attributes of the module, so no need to explicitly catch ImportError.
209
223
  if return_type == "pandas":
210
224
  from foundry_dev_tools._optional.pandas import pd
211
225
 
@@ -213,13 +227,19 @@ class DataProxyClient(APIClient):
213
227
  data=response_json["rows"],
214
228
  columns=[e["name"] for e in response_json["foundrySchema"]["fieldSchemaList"]],
215
229
  )
216
- if return_type == "arrow":
230
+ if return_type in {"arrow", "polars"}:
217
231
  from foundry_dev_tools._optional.pyarrow import pa
218
232
 
219
- return pa.table(
233
+ table = pa.table(
220
234
  data=response_json["rows"],
221
235
  names=[e["name"] for e in response_json["foundrySchema"]["fieldSchemaList"]],
222
236
  )
237
+ if return_type == "arrow":
238
+ return table
239
+
240
+ from foundry_dev_tools._optional.polars import pl
241
+
242
+ return pl.from_arrow(table)
223
243
  if return_type == "spark":
224
244
  from foundry_dev_tools.utils.converter.foundry_spark import (
225
245
  foundry_schema_to_spark_schema,
@@ -17,6 +17,7 @@ from foundry_dev_tools.utils.api_types import Ref, SqlDialect, SQLReturnType, as
17
17
 
18
18
  if TYPE_CHECKING:
19
19
  import pandas as pd
20
+ import polars as pl
20
21
  import pyarrow as pa
21
22
  import pyspark
22
23
  import requests
@@ -35,7 +36,7 @@ class FoundrySqlServerClient(APIClient):
35
36
  branch: Ref = ...,
36
37
  sql_dialect: SqlDialect = ...,
37
38
  timeout: int = ...,
38
- ) -> pd.core.frame.DataFrame: ...
39
+ ) -> pd.DataFrame: ...
39
40
 
40
41
  @overload
41
42
  def query_foundry_sql(
@@ -57,6 +58,16 @@ class FoundrySqlServerClient(APIClient):
57
58
  timeout: int = ...,
58
59
  ) -> pa.Table: ...
59
60
 
61
+ @overload
62
+ def query_foundry_sql(
63
+ self,
64
+ query: str,
65
+ return_type: Literal["polars"],
66
+ branch: Ref = ...,
67
+ sql_dialect: SqlDialect = ...,
68
+ timeout: int = ...,
69
+ ) -> pl.DataFrame: ...
70
+
60
71
  @overload
61
72
  def query_foundry_sql(
62
73
  self,
@@ -75,16 +86,16 @@ class FoundrySqlServerClient(APIClient):
75
86
  branch: Ref = ...,
76
87
  sql_dialect: SqlDialect = ...,
77
88
  timeout: int = ...,
78
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
89
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
79
90
 
80
- def query_foundry_sql(
91
+ def query_foundry_sql( # noqa: C901
81
92
  self,
82
93
  query: str,
83
94
  return_type: SQLReturnType = "pandas",
84
95
  branch: Ref = "master",
85
96
  sql_dialect: SqlDialect = "SPARK",
86
97
  timeout: int = 600,
87
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame:
98
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame:
88
99
  """Queries the Foundry SQL server with spark SQL dialect.
89
100
 
90
101
  Uses Arrow IPC to communicate with the Foundry SQL Server Endpoint.
@@ -105,9 +116,9 @@ class FoundrySqlServerClient(APIClient):
105
116
  timeout: Query Timeout, default value is 600 seconds
106
117
 
107
118
  Returns:
108
- :external+pandas:py:class:`~pandas.DataFrame` | :external+pyarrow:py:class:`~pyarrow.Table` | :external+spark:py:class:`~pyspark.sql.DataFrame`:
119
+ :external+pandas:py:class:`~pandas.DataFrame` | :external+polars:py:class:`~polars.DataFrame` | :external+pyarrow:py:class:`~pyarrow.Table` | :external+spark:py:class:`~pyspark.sql.DataFrame`:
109
120
 
110
- A pandas DataFrame, Spark DataFrame or pyarrow.Table with the result.
121
+ A pandas, polars, Spark DataFrame or pyarrow.Table with the result.
111
122
 
112
123
  Raises:
113
124
  ValueError: Only direct read eligible queries can be returned as arrow Table.
@@ -139,6 +150,15 @@ class FoundrySqlServerClient(APIClient):
139
150
  arrow_stream_reader = self.read_fsql_query_results_arrow(query_id=query_id)
140
151
  if return_type == "pandas":
141
152
  return arrow_stream_reader.read_pandas()
153
+ if return_type == "polars":
154
+ # The FakeModule implementation used in the _optional packages
155
+ # throws an ImportError when trying to access attributes of the module.
156
+ # This ImportError is caught below to fall back to query_foundry_sql_legacy
157
+ # which will again raise an ImportError when polars is not installed.
158
+ from foundry_dev_tools._optional.polars import pl
159
+
160
+ arrow_table = arrow_stream_reader.read_all()
161
+ return pl.from_arrow(arrow_table)
142
162
 
143
163
  if return_type == "spark":
144
164
  from foundry_dev_tools.utils.converter.foundry_spark import (
@@ -147,16 +167,12 @@ class FoundrySqlServerClient(APIClient):
147
167
 
148
168
  return arrow_stream_to_spark_dataframe(arrow_stream_reader)
149
169
  return arrow_stream_reader.read_all()
150
-
151
- return self._query_fsql(
152
- query=query,
153
- branch=branch,
154
- return_type=return_type,
155
- )
156
170
  except (
157
171
  FoundrySqlSerializationFormatNotImplementedError,
158
172
  ImportError,
159
173
  ) as exc:
174
+ # Swallow exception when return_type != 'arrow'
175
+ # to fall back to query_foundry_sql_legacy
160
176
  if return_type == "arrow":
161
177
  msg = (
162
178
  "Only direct read eligible queries can be returned as arrow Table. Consider using setting"
@@ -166,6 +182,8 @@ class FoundrySqlServerClient(APIClient):
166
182
  msg,
167
183
  ) from exc
168
184
 
185
+ # this fallback is not only used if return_type is 'raw', but also when one of
186
+ # the above exceptions is caught and return_type != 'arrow'
169
187
  warnings.warn("Falling back to query_foundry_sql_legacy!")
170
188
  return self.context.data_proxy.query_foundry_sql_legacy(
171
189
  query=query,
@@ -13,6 +13,7 @@ from foundry_dev_tools.clients import (
13
13
  catalog,
14
14
  compass,
15
15
  context_client,
16
+ data_health,
16
17
  data_proxy,
17
18
  foundry_sql_server,
18
19
  foundry_stats,
@@ -126,6 +127,11 @@ class FoundryContext:
126
127
  """Returns :py:class:`foundry_dev_tools.clients.metadata.MetadataClient`."""
127
128
  return metadata.MetadataClient(self)
128
129
 
130
+ @cached_property
131
+ def data_health(self) -> data_health.DataHealthClient:
132
+ """Returns :py:class:`foundry_dev_tools.clients.data_health.DataHealthClient`."""
133
+ return data_health.DataHealthClient(self)
134
+
129
135
  @cached_property
130
136
  def data_proxy(self) -> data_proxy.DataProxyClient:
131
137
  """Returns :py:class:`foundry_dev_tools.clients.data_proxy.DataProxyClient`."""
@@ -2,7 +2,7 @@
2
2
 
3
3
  One of the gaols of this module is to be self-contained so that it can be
4
4
  dropped into any python installation with minimal dependency to 'requests'
5
- Optional dependencies for the SQL functionality to work are pandas and pyarrow.
5
+ Optional dependencies for the SQL functionality to work are pandas, polars and pyarrow.
6
6
 
7
7
  """
8
8
 
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
41
41
  from collections.abc import Iterator
42
42
 
43
43
  import pandas as pd
44
+ import polars as pl
44
45
  import pyarrow as pa
45
46
  import pyspark
46
47
  import requests
@@ -1000,7 +1001,7 @@ class FoundryRestClient:
1000
1001
  branch: api_types.Ref = ...,
1001
1002
  sql_dialect: api_types.SqlDialect = ...,
1002
1003
  timeout: int = ...,
1003
- ) -> pd.core.frame.DataFrame: ...
1004
+ ) -> pd.DataFrame: ...
1004
1005
 
1005
1006
  @overload
1006
1007
  def query_foundry_sql(
@@ -1022,6 +1023,16 @@ class FoundryRestClient:
1022
1023
  timeout: int = ...,
1023
1024
  ) -> pa.Table: ...
1024
1025
 
1026
+ @overload
1027
+ def query_foundry_sql(
1028
+ self,
1029
+ query: str,
1030
+ return_type: Literal["polars"],
1031
+ branch: api_types.Ref = ...,
1032
+ sql_dialect: api_types.SqlDialect = ...,
1033
+ timeout: int = ...,
1034
+ ) -> pl.DataFrame: ...
1035
+
1025
1036
  @overload
1026
1037
  def query_foundry_sql(
1027
1038
  self,
@@ -1040,7 +1051,7 @@ class FoundryRestClient:
1040
1051
  branch: api_types.Ref = ...,
1041
1052
  sql_dialect: api_types.SqlDialect = ...,
1042
1053
  timeout: int = ...,
1043
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
1054
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
1044
1055
 
1045
1056
  def query_foundry_sql(
1046
1057
  self,
@@ -1049,7 +1060,7 @@ class FoundryRestClient:
1049
1060
  branch: api_types.Ref = "master",
1050
1061
  sql_dialect: api_types.SqlDialect = "SPARK",
1051
1062
  timeout: int = 600,
1052
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame:
1063
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame:
1053
1064
  """Queries the Foundry SQL server with spark SQL dialect.
1054
1065
 
1055
1066
  Uses Arrow IPC to communicate with the Foundry SQL Server Endpoint.
@@ -14,6 +14,7 @@ from foundry_dev_tools.errors.compass import ResourceNotFoundError
14
14
  from foundry_dev_tools.errors.dataset import (
15
15
  BranchNotFoundError,
16
16
  DatasetHasNoOpenTransactionError,
17
+ DatasetHasNoTransactionsError,
17
18
  DatasetNotFoundError,
18
19
  TransactionTypeMismatchError,
19
20
  )
@@ -31,7 +32,7 @@ if TYPE_CHECKING:
31
32
 
32
33
  import pandas as pd
33
34
  import pandas.core.frame
34
- import polars.dataframe.frame
35
+ import polars as pl
35
36
  import pyarrow as pa
36
37
  import pyspark.sql
37
38
 
@@ -261,11 +262,16 @@ class Dataset(resource.Resource):
261
262
  ).json()["values"]
262
263
  ]
263
264
 
264
- def get_last_transaction(self) -> api_types.Transaction | None:
265
- """Returns the last transaction or None if there are no transactions."""
265
+ def get_last_transaction(self, include_open_exclusive_transaction: bool = True) -> api_types.Transaction | None:
266
+ """Returns the last transaction or None if there are no transactions.
267
+
268
+ Args:
269
+ include_open_exclusive_transaction: If True, includes open transactions
270
+ in the results. If False, only returns committed transactions.
271
+ """
266
272
  v = self.get_transactions(
267
273
  page_size=1,
268
- include_open_exclusive_transaction=True,
274
+ include_open_exclusive_transaction=include_open_exclusive_transaction,
269
275
  )
270
276
  if v is not None and len(v) > 0:
271
277
  return v[0]
@@ -539,7 +545,7 @@ class Dataset(resource.Resource):
539
545
 
540
546
  def save_dataframe(
541
547
  self,
542
- df: pandas.core.frame.DataFrame | polars.dataframe.frame.DataFrame | pyspark.sql.DataFrame,
548
+ df: pandas.core.frame.DataFrame | pl.DataFrame | pyspark.sql.DataFrame,
543
549
  transaction_type: api_types.FoundryTransaction = "SNAPSHOT",
544
550
  foundry_schema: api_types.FoundrySchema | None = None,
545
551
  ) -> Self:
@@ -697,7 +703,7 @@ class Dataset(resource.Resource):
697
703
  return_type: Literal["pandas"],
698
704
  sql_dialect: api_types.SqlDialect = ...,
699
705
  timeout: int = ...,
700
- ) -> pd.core.frame.DataFrame: ...
706
+ ) -> pd.DataFrame: ...
701
707
 
702
708
  @overload
703
709
  def query_foundry_sql(
@@ -717,6 +723,15 @@ class Dataset(resource.Resource):
717
723
  timeout: int = ...,
718
724
  ) -> pa.Table: ...
719
725
 
726
+ @overload
727
+ def query_foundry_sql(
728
+ self,
729
+ query: str,
730
+ return_type: Literal["polars"],
731
+ sql_dialect: api_types.SqlDialect = ...,
732
+ timeout: int = ...,
733
+ ) -> pl.DataFrame: ...
734
+
720
735
  @overload
721
736
  def query_foundry_sql(
722
737
  self,
@@ -733,7 +748,7 @@ class Dataset(resource.Resource):
733
748
  return_type: api_types.SQLReturnType = ...,
734
749
  sql_dialect: api_types.SqlDialect = ...,
735
750
  timeout: int = ...,
736
- ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
751
+ ) -> tuple[dict, list[list]] | pd.core.frame.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame: ...
737
752
 
738
753
  def query_foundry_sql(
739
754
  self,
@@ -741,7 +756,7 @@ class Dataset(resource.Resource):
741
756
  return_type: api_types.SQLReturnType = "pandas",
742
757
  sql_dialect: api_types.SqlDialect = "SPARK",
743
758
  timeout: int = 600,
744
- ) -> tuple[dict, list[list]] | pd.DataFrame | pa.Table | pyspark.sql.DataFrame:
759
+ ) -> tuple[dict, list[list]] | pd.DataFrame | pl.DataFrame | pa.Table | pyspark.sql.DataFrame:
745
760
  """Wrapper around :py:meth:`foundry_dev_tools.clients.foundry_sql_server.FoundrySqlServerClient.query_foundry_sql`.
746
761
 
747
762
  But it automatically prepends the dataset location, so instead of:
@@ -783,17 +798,54 @@ class Dataset(resource.Resource):
783
798
  """
784
799
  return self.query_foundry_sql("SELECT *", return_type="pandas")
785
800
 
786
- def to_polars(self) -> polars.dataframe.frame.DataFrame:
801
+ def to_polars(self) -> pl.DataFrame:
787
802
  """Get dataset as a :py:class:`polars.DataFrame`.
788
803
 
789
804
  Via :py:meth:`foundry_dev_tools.resources.dataset.Dataset.query_foundry_sql`
790
805
  """
791
- try:
792
- import polars as pl
793
- except ImportError as e:
794
- msg = "The optional 'polars' package is not installed. Please install it to use the 'to_polars' method"
795
- raise ImportError(msg) from e
796
- return pl.from_arrow(self.to_arrow())
806
+ return self.query_foundry_sql("SELECT *", return_type="polars")
807
+
808
+ def to_lazy_polars(self, transaction_rid: str | None = None) -> pl.LazyFrame:
809
+ """Get dataset as a :py:class:`polars.LazyFrame`.
810
+
811
+ Returns a lazy polars DataFrame that can be queried efficiently using
812
+ polars' lazy evaluation API. The data is accessed directly via the
813
+ S3-compatible API without going through FoundrySqlServer.
814
+
815
+ Args:
816
+ transaction_rid: The transaction RID to read from. If None, uses the
817
+ last committed transaction. Useful for reading specific historical
818
+ versions of the dataset.
819
+
820
+ Returns:
821
+ pl.LazyFrame: A lazy polars DataFrame
822
+
823
+ Example:
824
+ >>> ds = ctx.get_dataset_by_path("/path/to/dataset")
825
+ >>> lf = ds.to_lazy_polars()
826
+ >>> result = lf.filter(pl.col("age") > 25).select(["name", "age"])
827
+ >>> # Execute and collect results
828
+ >>> df = result.collect()
829
+
830
+ Note:
831
+ This method uses the S3-compatible API to directly access dataset files.
832
+ For hive-partitioned datasets, polars will automatically read
833
+ the partition structure.
834
+ """
835
+ from foundry_dev_tools._optional.polars import pl
836
+
837
+ if transaction_rid is None:
838
+ maybe_transaction = self.get_last_transaction(include_open_exclusive_transaction=False)
839
+ if maybe_transaction is None:
840
+ msg = f"Dataset has no committed transactions: {self.rid=}"
841
+ raise DatasetHasNoTransactionsError(info=msg)
842
+ transaction_rid = maybe_transaction["rid"]
843
+
844
+ return pl.scan_parquet(
845
+ f"s3://{self.rid}.{transaction_rid}/**/*.parquet",
846
+ storage_options=self._context.s3.get_polars_storage_options(),
847
+ hive_partitioning=True,
848
+ )
797
849
 
798
850
  @contextmanager
799
851
  def transaction_context(
@@ -59,6 +59,9 @@ RoleId = str
59
59
  FolderRid = str
60
60
  """A compass folder resource identifier."""
61
61
 
62
+ CheckRid = str
63
+ """A foundry data check resource identifier."""
64
+
62
65
  JobSpecRid = str
63
66
  """A jobspec resource identifier."""
64
67
 
@@ -92,10 +95,11 @@ GroupId = str
92
95
  SqlDialect = Literal["ANSI", "SPARK"]
93
96
  """The SQL Dialect for Foundry SQL queries."""
94
97
 
95
- SQLReturnType = Literal["pandas", "spark", "arrow", "raw"]
98
+ SQLReturnType = Literal["pandas", "polars", "spark", "arrow", "raw"]
96
99
  """The return_types for sql queries.
97
100
 
98
101
  pandas: :external+pandas:py:class:`pandas.DataFrame`
102
+ polars: :external+polars:py:class:`polars.DataFrame`
99
103
  arrow: :external+pyarrow:py:class:`pyarrow.Table`
100
104
  spark: :external+spark:py:class:`~pyspark.sql.DataFrame`
101
105
  raw: Tuple of (foundry_schema, data) (can only be used in legacy)
@@ -1 +0,0 @@
1
- __version__ = '2.1.21'