foundry-dev-tools-transforms 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foundry_dev_tools_transforms-2.0.0.dist-info/METADATA +125 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/RECORD +12 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/WHEEL +4 -0
- transforms/__about__.py +1 -0
- transforms/__init__.py +5 -0
- transforms/api/__init__.py +33 -0
- transforms/api/_configure.py +27 -0
- transforms/api/_dataset.py +348 -0
- transforms/api/_decorators.py +275 -0
- transforms/api/_transform.py +604 -0
- transforms/errors.py +17 -0
- transforms/py.typed +0 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
|
|
2
|
+
|
|
3
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
|
|
4
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
|
|
5
|
+
|
|
6
|
+
""" # noqa: E501
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import warnings
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from transforms.api._dataset import Input, Output
|
|
14
|
+
from transforms.api._transform import Transform
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def lightweight(
|
|
21
|
+
_maybe_transform: Transform | None = None,
|
|
22
|
+
*,
|
|
23
|
+
cpu_cores: float | None = None,
|
|
24
|
+
memory_mb: float | None = None,
|
|
25
|
+
memory_gb: float | None = None,
|
|
26
|
+
gpu_type: str | None = None,
|
|
27
|
+
container_image: str | None = None,
|
|
28
|
+
container_tag: str | None = None,
|
|
29
|
+
container_shell_command: str | None = None,
|
|
30
|
+
) -> Callable[[Transform], Transform]:
|
|
31
|
+
"""Turn a transform into a Lightweight transform.
|
|
32
|
+
|
|
33
|
+
A Lightweight transform is a transform that runs without Spark, on a single node. Lightweight transforms are faster
|
|
34
|
+
and more cost-effective for small to medium-sized datasets. Lightweight transforms also provide more methods for
|
|
35
|
+
accessing datasets; however, they only support a subset of the API of a regular transforms, including Pandas and
|
|
36
|
+
the filesystem API. For more information, see the public documentation available at
|
|
37
|
+
https://www.palantir.com/docs/foundry/transforms-python/lightweight-overview
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
cpu_cores (float, optional): The number of CPU cores to request for the transform's container,
|
|
41
|
+
can be a fraction, not implemented in Foundry DevTools.
|
|
42
|
+
memory_mb (float, optional): The amount of memory to request for the container, in MB,
|
|
43
|
+
not implemented in Foundry DevTools.
|
|
44
|
+
memory_gb (float, optional): The amount of memory to request for the container, in GB,
|
|
45
|
+
not implemented in Foundry DevTools.
|
|
46
|
+
gpu_type (str, optional): The type of GPU to allocate for the transform.
|
|
47
|
+
container_image (str, optional): not implemented in Foundry DevTools
|
|
48
|
+
container_tag (str, optional): not implemented in Foundry DevTools
|
|
49
|
+
container_shell_command (str, optional): not implemented in Foundry DevTools
|
|
50
|
+
|
|
51
|
+
Notes:
|
|
52
|
+
Either `memory_gb` or `memory_mb` can be specified, but not both.
|
|
53
|
+
|
|
54
|
+
Specifying resources has no effect when running locally.
|
|
55
|
+
|
|
56
|
+
Examples:
|
|
57
|
+
>>> @lightweight
|
|
58
|
+
... @transform(
|
|
59
|
+
... my_input=Input('/input'),
|
|
60
|
+
... my_output=Output('/output')
|
|
61
|
+
... )
|
|
62
|
+
... def compute_func(my_input, my_output):
|
|
63
|
+
... my_output.write_table(my_input.polars())
|
|
64
|
+
|
|
65
|
+
>>> @lightweight()
|
|
66
|
+
... @transform(
|
|
67
|
+
... my_input=Input('/input'),
|
|
68
|
+
... my_output=Output('/output')
|
|
69
|
+
... )
|
|
70
|
+
... def compute_func(my_input, my_output):
|
|
71
|
+
... for file in my_input.filesystem().ls():
|
|
72
|
+
... with my_input.filesystem().open(file.path) as f1:
|
|
73
|
+
... with my_output.filesystem().open(file.path, "w") as f2:
|
|
74
|
+
... f2.write(f1.read())
|
|
75
|
+
"""
|
|
76
|
+
if container_image or container_tag or container_shell_command:
|
|
77
|
+
msg = (
|
|
78
|
+
"BYOC workflows enabled through container_image, container_tag, and container_shell_command "
|
|
79
|
+
"are not implemented in Foundry DevTools"
|
|
80
|
+
)
|
|
81
|
+
raise NotImplementedError(
|
|
82
|
+
msg,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
if memory_mb is not None and memory_gb is not None:
|
|
86
|
+
msg = "Only one of memory_mb or memory_gb can be specified"
|
|
87
|
+
raise ValueError(msg)
|
|
88
|
+
|
|
89
|
+
if cpu_cores or memory_mb or memory_gb or gpu_type:
|
|
90
|
+
warnings.warn("Setting resources for @lightweight transforms will have no effect when running locally.")
|
|
91
|
+
|
|
92
|
+
def _lightweight(transform: Transform) -> Transform:
|
|
93
|
+
if not isinstance(transform, Transform):
|
|
94
|
+
msg = (
|
|
95
|
+
"lightweight decorator must be used on a Transform object. "
|
|
96
|
+
"Perhaps you didn't put @lightweight as the top-most decorator?"
|
|
97
|
+
)
|
|
98
|
+
raise TypeError(
|
|
99
|
+
msg,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
if transform._type not in {"transform", "pandas"}: # noqa: SLF001
|
|
103
|
+
msg = "You can only use @lightweight on @transform or @transform_pandas"
|
|
104
|
+
raise ValueError(msg)
|
|
105
|
+
|
|
106
|
+
return Transform(
|
|
107
|
+
transform._compute_func, # noqa: SLF001
|
|
108
|
+
outputs=transform.outputs,
|
|
109
|
+
inputs=transform.inputs,
|
|
110
|
+
decorator="lightweight-pandas" if transform._type == "pandas" else "lightweight", # noqa: SLF001
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return _lightweight if _maybe_transform is None else _lightweight(_maybe_transform)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def transform_polars(output: Output, **inputs) -> Callable[[Callable], Transform]:
|
|
117
|
+
"""Register the wrapped compute function as a Polars transform.
|
|
118
|
+
|
|
119
|
+
Note:
|
|
120
|
+
To use the Polars library, you must add ``polars`` as a **run** dependency in your ``meta.yml`` file .
|
|
121
|
+
For more information, refer to the
|
|
122
|
+
:ref:`section describing the meta.yml file <transforms-python-proj-structure-meta>`.
|
|
123
|
+
|
|
124
|
+
The ``transform_polars`` decorator is just a thin wrapper around the ``lightweight`` decorator. Using it
|
|
125
|
+
results in the creation of a lightweight transform which lacks some features of a regular transform.
|
|
126
|
+
|
|
127
|
+
This works similarly to the :func:`transform_pandas` decorator, however, instead of Pandas DataFrames, the
|
|
128
|
+
user code is given and is expected to return Polars DataFrames.
|
|
129
|
+
|
|
130
|
+
Note that spark profiles can't be used with lightweight transforms, hence, neither with @transforms_polars.
|
|
131
|
+
|
|
132
|
+
>>> @transform_polars(
|
|
133
|
+
... Output('ri.main.foundry.dataset.out'), # An unnamed Output spec
|
|
134
|
+
... first_input=Input('ri.main.foundry.dataset.in1'),
|
|
135
|
+
... second_input=Input('ri.main.foundry.dataset.in2'),
|
|
136
|
+
... )
|
|
137
|
+
... def my_compute_function(first_input, second_input):
|
|
138
|
+
... # type: (polars.DataFrame, polars.DataFrame) -> polars.DataFrame
|
|
139
|
+
... return first_input.join(second_input, on='id', how="inner")
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
output (Output): The single :class:`Output` spec for the transform.
|
|
143
|
+
**inputs (Input): kwargs comprised of named :class:`Input` specs.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
def _transform_polars(compute_function: Callable) -> Transform:
|
|
147
|
+
return Transform(
|
|
148
|
+
compute_function,
|
|
149
|
+
outputs={"output": output},
|
|
150
|
+
inputs=inputs,
|
|
151
|
+
decorator="lightweight-polars",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
return _transform_polars
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def transform_df(output: Output, **inputs) -> Callable[[Callable], Transform]:
|
|
158
|
+
"""Register the wrapped compute function as a dataframe transform.
|
|
159
|
+
|
|
160
|
+
The ``transform_df`` decorator is used to construct a :class:`Transform` object from
|
|
161
|
+
a compute function that accepts and returns :class:`pyspark.sql.DataFrame` objects. Similar
|
|
162
|
+
to the :func:`transform` decorator, the input names become the compute function's parameter
|
|
163
|
+
names. However, a ``transform_df`` accepts only a single :class:`Output` spec as a
|
|
164
|
+
positional argument. The return value of the compute function is also a
|
|
165
|
+
:class:`~pyspark.sql.DataFrame` that is automatically written out to the single output
|
|
166
|
+
dataset.
|
|
167
|
+
|
|
168
|
+
>>> from transforms.api import transform_df, Input, Output
|
|
169
|
+
>>> @transform_df(
|
|
170
|
+
... Output('/path/to/output/dataset'), # An unnamed Output spec
|
|
171
|
+
... first_input=Input('/path/to/first/input/dataset'),
|
|
172
|
+
... second_input=Input('/path/to/second/input/dataset'),
|
|
173
|
+
... )
|
|
174
|
+
... def my_compute_function(first_input, second_input):
|
|
175
|
+
... # type: (pyspark.sql.DataFrame, pyspark.sql.DataFrame) -> pyspark.sql.DataFrame
|
|
176
|
+
... return first_input.union(second_input)
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
output (Output): The single :class:`Output` spec for the transform.
|
|
180
|
+
**inputs (Input): kwargs comprised of named :class:`Input` specs.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
def _transform_df(compute_func: Callable) -> Transform:
|
|
184
|
+
return Transform(compute_func, {"output": output}, inputs=inputs, decorator="spark")
|
|
185
|
+
|
|
186
|
+
return _transform_df
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def transform_pandas(output: Output, **inputs) -> Callable[[Callable], Transform]:
|
|
190
|
+
"""Register the wrapped compute function as a Pandas transform.
|
|
191
|
+
|
|
192
|
+
The ``transform_pandas`` decorator is used to construct a :class:`Transform` object from
|
|
193
|
+
a compute function that accepts and returns :class:`pandas.DataFrame` objects. This
|
|
194
|
+
decorator is similar to the :func:`transform_df` decorator,
|
|
195
|
+
however the :class:`pyspark.sql.DataFrame`
|
|
196
|
+
objects are converted to :class:`pandas.DataFrame` object before the computation,
|
|
197
|
+
and converted back afterwards.
|
|
198
|
+
|
|
199
|
+
>>> from transforms.api import transform_pandas, Input, Output
|
|
200
|
+
>>> @transform_pandas(
|
|
201
|
+
... Output('/path/to/output/dataset'), # An unnamed Output spec
|
|
202
|
+
... first_input=Input('/path/to/first/input/dataset'),
|
|
203
|
+
... second_input=Input('/path/to/second/input/dataset'),
|
|
204
|
+
... )
|
|
205
|
+
... def my_compute_function(first_input, second_input):
|
|
206
|
+
... # type: (pandas.DataFrame, pandas.DataFrame) -> pandas.DataFrame
|
|
207
|
+
... return first_input.concat(second_input)
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
output (Output): The single :class:`Output` spec for the transform.
|
|
211
|
+
**inputs (Input): kwargs comprised of named :class:`Input` specs.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
def _transform_pandas(compute_func: Callable) -> Transform:
|
|
215
|
+
return Transform(compute_func, {"output": output}, inputs=inputs, decorator="pandas")
|
|
216
|
+
|
|
217
|
+
return _transform_pandas
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def transform(**kwargs) -> Callable[[Callable], Transform]:
|
|
221
|
+
"""Wrap up a compute function as a Transform object.
|
|
222
|
+
|
|
223
|
+
>>> from transforms.api import transform, Input, Output
|
|
224
|
+
>>> @transform(
|
|
225
|
+
... first_input=Input('/path/to/first/input/dataset'),
|
|
226
|
+
... second_input=Input('/path/to/second/input/dataset'),
|
|
227
|
+
... first_output=Output('/path/to/first/output/dataset'),
|
|
228
|
+
... second_output=Output('/path/to/second/output/dataset'),
|
|
229
|
+
... )
|
|
230
|
+
... def my_compute_function(first_input, second_input, first_output, second_output):
|
|
231
|
+
... # type: (TransformInput, TransformInput, TransformOutput, TransformOutput) -> None
|
|
232
|
+
... first_output.write_dataframe(first_input.dataframe())
|
|
233
|
+
... second_output.write_dataframe(second_input.dataframe())
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
**kwargs (Param): kwargs comprised of named :class:`Param` or subclasses.
|
|
237
|
+
|
|
238
|
+
Note:
|
|
239
|
+
The compute function is responsible for writing data to its outputs.
|
|
240
|
+
"""
|
|
241
|
+
|
|
242
|
+
def _transform(compute_func: Callable) -> Transform:
|
|
243
|
+
return Transform(
|
|
244
|
+
compute_func,
|
|
245
|
+
outputs={k: v for k, v in kwargs.items() if isinstance(v, Output)},
|
|
246
|
+
inputs={k: v for k, v in kwargs.items() if isinstance(v, Input)},
|
|
247
|
+
decorator="transform",
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
return _transform
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def incremental(
|
|
254
|
+
require_incremental: bool = False, # noqa: ARG001
|
|
255
|
+
semantic_version: int = 1, # noqa: ARG001
|
|
256
|
+
snapshot_inputs=None, # noqa: ARG001,ANN001
|
|
257
|
+
allow_retention: bool = False, # noqa: ARG001
|
|
258
|
+
) -> Callable:
|
|
259
|
+
"""Not implemented in local.
|
|
260
|
+
|
|
261
|
+
Args:
|
|
262
|
+
require_incremental: not implemented
|
|
263
|
+
semantic_version: not implemented
|
|
264
|
+
snapshot_inputs: not implemented
|
|
265
|
+
allow_retention: not implemented
|
|
266
|
+
|
|
267
|
+
Returns:
|
|
268
|
+
_transform:
|
|
269
|
+
"""
|
|
270
|
+
warnings.warn("@incremental functionality not implemented in Foundry DevTools")
|
|
271
|
+
|
|
272
|
+
def _transform(compute_func: Callable) -> Callable:
|
|
273
|
+
return compute_func
|
|
274
|
+
|
|
275
|
+
return _transform
|