foundry-dev-tools-transforms 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,275 @@
1
+ """The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
2
+
3
+ https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
4
+ https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
5
+
6
+ """ # noqa: E501
7
+
8
+ from __future__ import annotations
9
+
10
+ import warnings
11
+ from typing import TYPE_CHECKING
12
+
13
+ from transforms.api._dataset import Input, Output
14
+ from transforms.api._transform import Transform
15
+
16
+ if TYPE_CHECKING:
17
+ from collections.abc import Callable
18
+
19
+
20
+ def lightweight(
21
+ _maybe_transform: Transform | None = None,
22
+ *,
23
+ cpu_cores: float | None = None,
24
+ memory_mb: float | None = None,
25
+ memory_gb: float | None = None,
26
+ gpu_type: str | None = None,
27
+ container_image: str | None = None,
28
+ container_tag: str | None = None,
29
+ container_shell_command: str | None = None,
30
+ ) -> Callable[[Transform], Transform]:
31
+ """Turn a transform into a Lightweight transform.
32
+
33
+ A Lightweight transform is a transform that runs without Spark, on a single node. Lightweight transforms are faster
34
+ and more cost-effective for small to medium-sized datasets. Lightweight transforms also provide more methods for
35
+ accessing datasets; however, they only support a subset of the API of a regular transforms, including Pandas and
36
+ the filesystem API. For more information, see the public documentation available at
37
+ https://www.palantir.com/docs/foundry/transforms-python/lightweight-overview
38
+
39
+ Args:
40
+ cpu_cores (float, optional): The number of CPU cores to request for the transform's container,
41
+ can be a fraction, not implemented in Foundry DevTools.
42
+ memory_mb (float, optional): The amount of memory to request for the container, in MB,
43
+ not implemented in Foundry DevTools.
44
+ memory_gb (float, optional): The amount of memory to request for the container, in GB,
45
+ not implemented in Foundry DevTools.
46
+ gpu_type (str, optional): The type of GPU to allocate for the transform.
47
+ container_image (str, optional): not implemented in Foundry DevTools
48
+ container_tag (str, optional): not implemented in Foundry DevTools
49
+ container_shell_command (str, optional): not implemented in Foundry DevTools
50
+
51
+ Notes:
52
+ Either `memory_gb` or `memory_mb` can be specified, but not both.
53
+
54
+ Specifying resources has no effect when running locally.
55
+
56
+ Examples:
57
+ >>> @lightweight
58
+ ... @transform(
59
+ ... my_input=Input('/input'),
60
+ ... my_output=Output('/output')
61
+ ... )
62
+ ... def compute_func(my_input, my_output):
63
+ ... my_output.write_table(my_input.polars())
64
+
65
+ >>> @lightweight()
66
+ ... @transform(
67
+ ... my_input=Input('/input'),
68
+ ... my_output=Output('/output')
69
+ ... )
70
+ ... def compute_func(my_input, my_output):
71
+ ... for file in my_input.filesystem().ls():
72
+ ... with my_input.filesystem().open(file.path) as f1:
73
+ ... with my_output.filesystem().open(file.path, "w") as f2:
74
+ ... f2.write(f1.read())
75
+ """
76
+ if container_image or container_tag or container_shell_command:
77
+ msg = (
78
+ "BYOC workflows enabled through container_image, container_tag, and container_shell_command "
79
+ "are not implemented in Foundry DevTools"
80
+ )
81
+ raise NotImplementedError(
82
+ msg,
83
+ )
84
+
85
+ if memory_mb is not None and memory_gb is not None:
86
+ msg = "Only one of memory_mb or memory_gb can be specified"
87
+ raise ValueError(msg)
88
+
89
+ if cpu_cores or memory_mb or memory_gb or gpu_type:
90
+ warnings.warn("Setting resources for @lightweight transforms will have no effect when running locally.")
91
+
92
+ def _lightweight(transform: Transform) -> Transform:
93
+ if not isinstance(transform, Transform):
94
+ msg = (
95
+ "lightweight decorator must be used on a Transform object. "
96
+ "Perhaps you didn't put @lightweight as the top-most decorator?"
97
+ )
98
+ raise TypeError(
99
+ msg,
100
+ )
101
+
102
+ if transform._type not in {"transform", "pandas"}: # noqa: SLF001
103
+ msg = "You can only use @lightweight on @transform or @transform_pandas"
104
+ raise ValueError(msg)
105
+
106
+ return Transform(
107
+ transform._compute_func, # noqa: SLF001
108
+ outputs=transform.outputs,
109
+ inputs=transform.inputs,
110
+ decorator="lightweight-pandas" if transform._type == "pandas" else "lightweight", # noqa: SLF001
111
+ )
112
+
113
+ return _lightweight if _maybe_transform is None else _lightweight(_maybe_transform)
114
+
115
+
116
+ def transform_polars(output: Output, **inputs) -> Callable[[Callable], Transform]:
117
+ """Register the wrapped compute function as a Polars transform.
118
+
119
+ Note:
120
+ To use the Polars library, you must add ``polars`` as a **run** dependency in your ``meta.yml`` file .
121
+ For more information, refer to the
122
+ :ref:`section describing the meta.yml file <transforms-python-proj-structure-meta>`.
123
+
124
+ The ``transform_polars`` decorator is just a thin wrapper around the ``lightweight`` decorator. Using it
125
+ results in the creation of a lightweight transform which lacks some features of a regular transform.
126
+
127
+ This works similarly to the :func:`transform_pandas` decorator, however, instead of Pandas DataFrames, the
128
+ user code is given and is expected to return Polars DataFrames.
129
+
130
+ Note that spark profiles can't be used with lightweight transforms, hence, neither with @transforms_polars.
131
+
132
+ >>> @transform_polars(
133
+ ... Output('ri.main.foundry.dataset.out'), # An unnamed Output spec
134
+ ... first_input=Input('ri.main.foundry.dataset.in1'),
135
+ ... second_input=Input('ri.main.foundry.dataset.in2'),
136
+ ... )
137
+ ... def my_compute_function(first_input, second_input):
138
+ ... # type: (polars.DataFrame, polars.DataFrame) -> polars.DataFrame
139
+ ... return first_input.join(second_input, on='id', how="inner")
140
+
141
+ Args:
142
+ output (Output): The single :class:`Output` spec for the transform.
143
+ **inputs (Input): kwargs comprised of named :class:`Input` specs.
144
+ """
145
+
146
+ def _transform_polars(compute_function: Callable) -> Transform:
147
+ return Transform(
148
+ compute_function,
149
+ outputs={"output": output},
150
+ inputs=inputs,
151
+ decorator="lightweight-polars",
152
+ )
153
+
154
+ return _transform_polars
155
+
156
+
157
+ def transform_df(output: Output, **inputs) -> Callable[[Callable], Transform]:
158
+ """Register the wrapped compute function as a dataframe transform.
159
+
160
+ The ``transform_df`` decorator is used to construct a :class:`Transform` object from
161
+ a compute function that accepts and returns :class:`pyspark.sql.DataFrame` objects. Similar
162
+ to the :func:`transform` decorator, the input names become the compute function's parameter
163
+ names. However, a ``transform_df`` accepts only a single :class:`Output` spec as a
164
+ positional argument. The return value of the compute function is also a
165
+ :class:`~pyspark.sql.DataFrame` that is automatically written out to the single output
166
+ dataset.
167
+
168
+ >>> from transforms.api import transform_df, Input, Output
169
+ >>> @transform_df(
170
+ ... Output('/path/to/output/dataset'), # An unnamed Output spec
171
+ ... first_input=Input('/path/to/first/input/dataset'),
172
+ ... second_input=Input('/path/to/second/input/dataset'),
173
+ ... )
174
+ ... def my_compute_function(first_input, second_input):
175
+ ... # type: (pyspark.sql.DataFrame, pyspark.sql.DataFrame) -> pyspark.sql.DataFrame
176
+ ... return first_input.union(second_input)
177
+
178
+ Args:
179
+ output (Output): The single :class:`Output` spec for the transform.
180
+ **inputs (Input): kwargs comprised of named :class:`Input` specs.
181
+ """
182
+
183
+ def _transform_df(compute_func: Callable) -> Transform:
184
+ return Transform(compute_func, {"output": output}, inputs=inputs, decorator="spark")
185
+
186
+ return _transform_df
187
+
188
+
189
+ def transform_pandas(output: Output, **inputs) -> Callable[[Callable], Transform]:
190
+ """Register the wrapped compute function as a Pandas transform.
191
+
192
+ The ``transform_pandas`` decorator is used to construct a :class:`Transform` object from
193
+ a compute function that accepts and returns :class:`pandas.DataFrame` objects. This
194
+ decorator is similar to the :func:`transform_df` decorator,
195
+ however the :class:`pyspark.sql.DataFrame`
196
+ objects are converted to :class:`pandas.DataFrame` object before the computation,
197
+ and converted back afterwards.
198
+
199
+ >>> from transforms.api import transform_pandas, Input, Output
200
+ >>> @transform_pandas(
201
+ ... Output('/path/to/output/dataset'), # An unnamed Output spec
202
+ ... first_input=Input('/path/to/first/input/dataset'),
203
+ ... second_input=Input('/path/to/second/input/dataset'),
204
+ ... )
205
+ ... def my_compute_function(first_input, second_input):
206
+ ... # type: (pandas.DataFrame, pandas.DataFrame) -> pandas.DataFrame
207
+ ... return first_input.concat(second_input)
208
+
209
+ Args:
210
+ output (Output): The single :class:`Output` spec for the transform.
211
+ **inputs (Input): kwargs comprised of named :class:`Input` specs.
212
+ """
213
+
214
+ def _transform_pandas(compute_func: Callable) -> Transform:
215
+ return Transform(compute_func, {"output": output}, inputs=inputs, decorator="pandas")
216
+
217
+ return _transform_pandas
218
+
219
+
220
+ def transform(**kwargs) -> Callable[[Callable], Transform]:
221
+ """Wrap up a compute function as a Transform object.
222
+
223
+ >>> from transforms.api import transform, Input, Output
224
+ >>> @transform(
225
+ ... first_input=Input('/path/to/first/input/dataset'),
226
+ ... second_input=Input('/path/to/second/input/dataset'),
227
+ ... first_output=Output('/path/to/first/output/dataset'),
228
+ ... second_output=Output('/path/to/second/output/dataset'),
229
+ ... )
230
+ ... def my_compute_function(first_input, second_input, first_output, second_output):
231
+ ... # type: (TransformInput, TransformInput, TransformOutput, TransformOutput) -> None
232
+ ... first_output.write_dataframe(first_input.dataframe())
233
+ ... second_output.write_dataframe(second_input.dataframe())
234
+
235
+ Args:
236
+ **kwargs (Param): kwargs comprised of named :class:`Param` or subclasses.
237
+
238
+ Note:
239
+ The compute function is responsible for writing data to its outputs.
240
+ """
241
+
242
+ def _transform(compute_func: Callable) -> Transform:
243
+ return Transform(
244
+ compute_func,
245
+ outputs={k: v for k, v in kwargs.items() if isinstance(v, Output)},
246
+ inputs={k: v for k, v in kwargs.items() if isinstance(v, Input)},
247
+ decorator="transform",
248
+ )
249
+
250
+ return _transform
251
+
252
+
253
+ def incremental(
254
+ require_incremental: bool = False, # noqa: ARG001
255
+ semantic_version: int = 1, # noqa: ARG001
256
+ snapshot_inputs=None, # noqa: ARG001,ANN001
257
+ allow_retention: bool = False, # noqa: ARG001
258
+ ) -> Callable:
259
+ """Not implemented in local.
260
+
261
+ Args:
262
+ require_incremental: not implemented
263
+ semantic_version: not implemented
264
+ snapshot_inputs: not implemented
265
+ allow_retention: not implemented
266
+
267
+ Returns:
268
+ _transform:
269
+ """
270
+ warnings.warn("@incremental functionality not implemented in Foundry DevTools")
271
+
272
+ def _transform(compute_func: Callable) -> Callable:
273
+ return compute_func
274
+
275
+ return _transform