PySpark-Column-Selectors 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PySparkSelectors/__init__.py +240 -0
- PySparkSelectors/__main__.py +4 -0
- PySparkSelectors/cli.py +18 -0
- PySparkSelectors/models.py +1564 -0
- PySparkSelectors/py.typed +1 -0
- PySparkSelectors/spark_overrides.py +677 -0
- PySparkSelectors/user_functions.py +745 -0
- PySparkSelectors/utils.py +298 -0
- pyspark_column_selectors-0.1.3.dist-info/METADATA +674 -0
- pyspark_column_selectors-0.1.3.dist-info/RECORD +13 -0
- pyspark_column_selectors-0.1.3.dist-info/WHEEL +4 -0
- pyspark_column_selectors-0.1.3.dist-info/entry_points.txt +2 -0
- pyspark_column_selectors-0.1.3.dist-info/licenses/LICENSE +279 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Marker file for PEP 561
|
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
from collections.abc import Callable, Sequence
|
|
2
|
+
from typing import Any, Union, cast
|
|
3
|
+
|
|
4
|
+
from pyspark.errors import PySparkTypeError, PySparkValueError
|
|
5
|
+
from pyspark.sql import DataFrame
|
|
6
|
+
from pyspark.sql import functions as F
|
|
7
|
+
from pyspark.sql.column import Column
|
|
8
|
+
|
|
9
|
+
from .models import (
|
|
10
|
+
BaseSelector,
|
|
11
|
+
SelectorFilterCondition,
|
|
12
|
+
)
|
|
13
|
+
from .utils import (
|
|
14
|
+
_get_column_method,
|
|
15
|
+
_resolve_selector_exprs,
|
|
16
|
+
_resolve_selector_names,
|
|
17
|
+
_true_original,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
""" enable support for many different types of pyspark environments"""
|
|
21
|
+
# if your environment isn't supported then you can add it to this section.
|
|
22
|
+
# Spark supports multiple DataFrame implementations. Keep every available runtime
|
|
23
|
+
# class in the patch set: public/classic DataFrame and Spark Connect DataFrame.
|
|
24
|
+
# using try except as to allow passthrough if environment doesn't support these.
|
|
25
|
+
try:
|
|
26
|
+
from pyspark.sql.classic.dataframe import DataFrame as _ClassicDataFrame
|
|
27
|
+
except ImportError:
|
|
28
|
+
_ClassicDataFrame = None
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
from pyspark.sql.connect.dataframe import DataFrame as _ConnectDataFrame
|
|
32
|
+
except ImportError:
|
|
33
|
+
_ConnectDataFrame = None
|
|
34
|
+
|
|
35
|
+
_DATAFRAME_CLASSES = tuple(
|
|
36
|
+
dict.fromkeys(cls for cls in (DataFrame, _ClassicDataFrame, _ConnectDataFrame) if cls is not None)
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
from pyspark.sql.connect.column import Column as _ConnectColumn
|
|
41
|
+
except ImportError:
|
|
42
|
+
_ConnectColumn = None
|
|
43
|
+
|
|
44
|
+
_COLUMN_CLASSES = tuple(dict.fromkeys(cls for cls in (Column, _ConnectColumn) if cls is not None))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# --------------------------------------------------
|
|
48
|
+
# ensure all versions of dataframes are overriden to support column selectors
|
|
49
|
+
# --------------------------------------------------
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _patch_classes(
|
|
53
|
+
classes: Sequence[type],
|
|
54
|
+
method_name: str,
|
|
55
|
+
build_override: Callable[[Callable[..., Any]], Callable[..., Any]],
|
|
56
|
+
) -> None:
|
|
57
|
+
"""Patch `method_name` onto every class in `classes` with a fresh override.
|
|
58
|
+
|
|
59
|
+
Shared "apply this override everywhere it needs to exist" helper --
|
|
60
|
+
installs a selector-aware override of `method_name` onto each class in
|
|
61
|
+
`classes` (e.g. both classic `pyspark.sql.DataFrame` and Spark Connect's
|
|
62
|
+
`pyspark.sql.connect.dataframe.DataFrame`, when Connect is available), so
|
|
63
|
+
the same selector syntax works identically no matter which backend
|
|
64
|
+
actually produced the dataframe being operated on.
|
|
65
|
+
|
|
66
|
+
Each class gets its OWN override function, built by calling
|
|
67
|
+
`build_override` once per class with that class's own real, never-patched
|
|
68
|
+
original implementation (via `_true_original`) -- classic and Connect
|
|
69
|
+
implementations of the same method are separate functions and are not
|
|
70
|
+
interchangeable, so one shared override closing over a single captured
|
|
71
|
+
original would silently call the wrong implementation for the other
|
|
72
|
+
class. Calling `build_override` fresh for each class (rather than
|
|
73
|
+
reusing one closure variable across a loop) also avoids Python's
|
|
74
|
+
late-binding closure pitfall, where every override would otherwise end
|
|
75
|
+
up referencing whichever class's original happened to be captured last.
|
|
76
|
+
|
|
77
|
+
Parameters
|
|
78
|
+
----------
|
|
79
|
+
classes : sequence of type
|
|
80
|
+
Every class to patch `method_name` onto.
|
|
81
|
+
method_name : str
|
|
82
|
+
The name of the method being overridden.
|
|
83
|
+
build_override : callable
|
|
84
|
+
Given that class's real original implementation (an unbound
|
|
85
|
+
function called as ``original(self, *args, **kwargs)``), returns
|
|
86
|
+
the override function to install in its place.
|
|
87
|
+
|
|
88
|
+
Examples
|
|
89
|
+
--------
|
|
90
|
+
>>> _patch_classes(
|
|
91
|
+
... _DATAFRAME_CLASSES, "sort",
|
|
92
|
+
... lambda original: (lambda self, *cols, **kw: original(self, *cols, **kw)),
|
|
93
|
+
... )
|
|
94
|
+
"""
|
|
95
|
+
for cls in classes:
|
|
96
|
+
original = _true_original(cls, method_name)
|
|
97
|
+
override = cast(Any, build_override(original))
|
|
98
|
+
# override = build_override(original)
|
|
99
|
+
override._cs_original = original
|
|
100
|
+
override.__name__ = method_name
|
|
101
|
+
override.__qualname__ = f"{cls.__name__}.{method_name}"
|
|
102
|
+
setattr(cls, method_name, override)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# --------------------------------------------------------------------------------
|
|
106
|
+
# wrapper that will allow for column functions to be wrapped for use in selectors
|
|
107
|
+
# we will focus on columns that are initialized in pyspark.sql.column
|
|
108
|
+
#
|
|
109
|
+
# --------------------------------------------------------------------------------
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _make_dispatching_column_wrapper(method_name: str) -> Callable[..., Column]:
|
|
113
|
+
"""Build a wrapper that dispatches to `Column` or `pyspark.sql.functions` by argument type.
|
|
114
|
+
|
|
115
|
+
Resolves the `F`/`Column` name-collision described in the markdown above:
|
|
116
|
+
`pyspark.sql.functions.<method_name>` and `pyspark.sql.Column.<method_name>`
|
|
117
|
+
share a name but treat a plain (non-`Column`) argument differently. This
|
|
118
|
+
wrapper picks whichever one matches the caller's intent based on the type of
|
|
119
|
+
the argument actually passed, so both usages keep working.
|
|
120
|
+
|
|
121
|
+
Parameters
|
|
122
|
+
----------
|
|
123
|
+
method_name : str
|
|
124
|
+
Name shared by a `pyspark.sql.functions` function and a `pyspark.sql.Column`
|
|
125
|
+
instance method (e.g. `"contains"`), used to look up both versions.
|
|
126
|
+
|
|
127
|
+
Returns
|
|
128
|
+
-------
|
|
129
|
+
callable
|
|
130
|
+
A function `wrapper(c, *args, **kwargs)` that, when the first
|
|
131
|
+
argument in `args` (if any) is a `pyspark.sql.Column`, calls
|
|
132
|
+
`pyspark.sql.functions.<method_name>(c, *args, **kwargs)` (column-to-column
|
|
133
|
+
semantics); otherwise calls `getattr(c, method_name)(*args, **kwargs)`
|
|
134
|
+
(literal semantics via `Column`'s own bound method). Carries `Column`'s
|
|
135
|
+
docstring for the method so `help()` on the resulting chained selector
|
|
136
|
+
method shows accurate documentation.
|
|
137
|
+
"""
|
|
138
|
+
column_method = _get_column_method(method_name)
|
|
139
|
+
spark_func = getattr(F, method_name)
|
|
140
|
+
|
|
141
|
+
def wrapper(c: Column, *args: Any, **kwargs: Any) -> Column:
|
|
142
|
+
# `_COLUMN_CLASSES` (classic + Spark Connect `Column`, when Connect is
|
|
143
|
+
# available) instead of the bare `Column` name -- otherwise a Spark
|
|
144
|
+
# Connect `Column` argument would be misdetected as a plain literal
|
|
145
|
+
# and routed to the wrong (literal-semantics) branch below.
|
|
146
|
+
if args and isinstance(args[0], _COLUMN_CLASSES):
|
|
147
|
+
return spark_func(c, *args, **kwargs)
|
|
148
|
+
return getattr(c, method_name)(*args, **kwargs)
|
|
149
|
+
|
|
150
|
+
wrapper.__name__ = method_name
|
|
151
|
+
wrapper.__doc__ = column_method.__doc__
|
|
152
|
+
# I don't think I actually need _is_rename_op,
|
|
153
|
+
# might be able to delete all of these comments
|
|
154
|
+
# same rename-op marker carry-through as `_make_column_only_wrapper`
|
|
155
|
+
# (defensive -- no current rename op collides with an `F` name).
|
|
156
|
+
# wrapper._is_rename_op = getattr(column_method, "_is_rename_op", False)
|
|
157
|
+
return wrapper
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
# --------------------------------------------------
|
|
161
|
+
# used to override functions that require column expressions
|
|
162
|
+
# --------------------------------------------------
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def make_expr_based_override(
|
|
166
|
+
method_name: str,
|
|
167
|
+
alias_results: bool = True,
|
|
168
|
+
classes: Sequence[type] | None = None,
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Patch a pyspark method that wants `Column` expressions.
|
|
171
|
+
|
|
172
|
+
Factory (wrapper pattern) used by every fx override whose real pyspark
|
|
173
|
+
method wants `pyspark.sql.Column` *expressions* rather than bare names --
|
|
174
|
+
e.g. `sort`/`orderBy` need real expressions so a selector's chained
|
|
175
|
+
``.desc()`` transform actually takes effect, not just the underlying
|
|
176
|
+
column name.
|
|
177
|
+
|
|
178
|
+
Unlike earlier versions of this notebook, this function performs the
|
|
179
|
+
patch itself (via `_patch_classes`) rather than returning a single
|
|
180
|
+
override for the caller to assign -- classic `pyspark.sql.DataFrame` and
|
|
181
|
+
Spark Connect's `DataFrame` each need their own override closing over
|
|
182
|
+
their own real original implementation.
|
|
183
|
+
|
|
184
|
+
Parameters
|
|
185
|
+
----------
|
|
186
|
+
method_name : str
|
|
187
|
+
The name of the `DataFrame` method to override (e.g. ``"select"``,
|
|
188
|
+
``"sort"``, ``"orderBy"``).
|
|
189
|
+
alias_results : bool, default True
|
|
190
|
+
Whether to re-alias each resolved expression back to the original
|
|
191
|
+
column name it matched (see the generated override's docstring).
|
|
192
|
+
Should be `False` for methods like `sort`/`orderBy` that don't
|
|
193
|
+
change the output schema at all -- there, a chained
|
|
194
|
+
``.desc()``/``.asc()`` produces a non-evaluable Catalyst
|
|
195
|
+
`SortOrder` expression, which `.alias(...)` cannot be applied to.
|
|
196
|
+
classes : sequence of type, optional
|
|
197
|
+
Every class to patch. Defaults to `_DATAFRAME_CLASSES` (classic
|
|
198
|
+
`pyspark.sql.DataFrame` plus Spark Connect's `DataFrame`, when
|
|
199
|
+
Spark Connect is importable).
|
|
200
|
+
|
|
201
|
+
Examples
|
|
202
|
+
--------
|
|
203
|
+
>>> make_expr_based_override('sort', alias_results=False)
|
|
204
|
+
>>> df.sort(by_dtype([T.DoubleType]).desc())
|
|
205
|
+
DataFrame[...]
|
|
206
|
+
"""
|
|
207
|
+
if classes is None:
|
|
208
|
+
classes = _DATAFRAME_CLASSES
|
|
209
|
+
|
|
210
|
+
def build_override(original):
|
|
211
|
+
|
|
212
|
+
def override(self, *cols, **kwargs):
|
|
213
|
+
|
|
214
|
+
if alias_results:
|
|
215
|
+
resolved = [
|
|
216
|
+
expr.alias(name) if name is not None else expr for name, expr in _resolve_selector_exprs(self, cols)
|
|
217
|
+
]
|
|
218
|
+
else:
|
|
219
|
+
resolved = [expr for _, expr in _resolve_selector_exprs(self, cols)]
|
|
220
|
+
|
|
221
|
+
return original(self, *resolved, **kwargs)
|
|
222
|
+
|
|
223
|
+
override.__doc__ = (
|
|
224
|
+
f"{method_name}(*cols, **kwargs) -- selector-aware override.\n\n"
|
|
225
|
+
"Accepts a column selector (e.g. `by_dtype([T.DoubleType]).desc()`) "
|
|
226
|
+
"anywhere a plain `pyspark.sql.Column` expression is normally "
|
|
227
|
+
"accepted; the selector is resolved to `Column` expressions (with "
|
|
228
|
+
"any chained transform, e.g. `.cast(...)`/`.desc()`, applied)"
|
|
229
|
+
+ (
|
|
230
|
+
", then each is re-aliased back to the original column name it "
|
|
231
|
+
"matched, so a transform like `.upper()`/`.cast(...)` still keeps "
|
|
232
|
+
"the source column's name instead of Spark's auto-generated "
|
|
233
|
+
"expression name"
|
|
234
|
+
if alias_results
|
|
235
|
+
else ""
|
|
236
|
+
)
|
|
237
|
+
+ f". The result is handed to the real `{method_name}` "
|
|
238
|
+
"implementation. Plain columns/args pass through unchanged.\n\n"
|
|
239
|
+
f"{original.__doc__ or ''}"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return override
|
|
243
|
+
|
|
244
|
+
_patch_classes(classes, method_name, build_override)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# --------------------------------------------------
|
|
248
|
+
# override functions that only require column name strings
|
|
249
|
+
# --------------------------------------------------
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def make_name_based_override(method_name: str, classes: Sequence[type] | None = None) -> None:
|
|
253
|
+
"""Patch a pyspark method that only wants column-name strings.
|
|
254
|
+
|
|
255
|
+
Factory (wrapper pattern) used by every fx override whose real pyspark
|
|
256
|
+
method just wants a flat list of column-name strings -- resolves any
|
|
257
|
+
`BaseSelector` passed positionally down to the names it matches, then
|
|
258
|
+
calls straight through to the real pyspark method of the same name so
|
|
259
|
+
the signature/behavior is otherwise identical. This is what collapses
|
|
260
|
+
`groupBy`/`drop` into one-liners instead of each needing its own
|
|
261
|
+
hand-written override.
|
|
262
|
+
|
|
263
|
+
Unlike earlier versions of this notebook, this function performs the
|
|
264
|
+
patch itself (via `_patch_classes`) rather than returning a single
|
|
265
|
+
override for the caller to assign -- classic `pyspark.sql.DataFrame` and
|
|
266
|
+
Spark Connect's `DataFrame` each need their own override closing over
|
|
267
|
+
their own real original implementation.
|
|
268
|
+
|
|
269
|
+
Parameters
|
|
270
|
+
----------
|
|
271
|
+
method_name : str
|
|
272
|
+
The name of the `DataFrame` method to override (e.g. ``"groupBy"``,
|
|
273
|
+
``"drop"``).
|
|
274
|
+
classes : sequence of type, optional
|
|
275
|
+
Every class to patch. Defaults to `_DATAFRAME_CLASSES` (classic
|
|
276
|
+
`pyspark.sql.DataFrame` plus Spark Connect's `DataFrame`, when
|
|
277
|
+
Spark Connect is importable).
|
|
278
|
+
|
|
279
|
+
Examples
|
|
280
|
+
--------
|
|
281
|
+
>>> make_name_based_override('groupBy')
|
|
282
|
+
>>> df.groupBy(by_dtype([T.StringType]))
|
|
283
|
+
GroupedData[...]
|
|
284
|
+
"""
|
|
285
|
+
if classes is None:
|
|
286
|
+
classes = _DATAFRAME_CLASSES
|
|
287
|
+
|
|
288
|
+
def build_override(original):
|
|
289
|
+
|
|
290
|
+
def override(self, *cols, **kwargs):
|
|
291
|
+
|
|
292
|
+
resolved = _resolve_selector_names(self, cols)
|
|
293
|
+
|
|
294
|
+
return original(self, *resolved, **kwargs)
|
|
295
|
+
|
|
296
|
+
override.__doc__ = (
|
|
297
|
+
f"{method_name}(*cols, **kwargs) -- selector-aware override.\n\n"
|
|
298
|
+
"Accepts a column selector (e.g. `by_dtype([T.StringType])`) anywhere a "
|
|
299
|
+
"plain column-name string is normally accepted; the selector is "
|
|
300
|
+
f"resolved to the column names it matches, then handed to the "
|
|
301
|
+
f"real `{method_name}` implementation. Plain names/args pass "
|
|
302
|
+
"through unchanged.\n\n"
|
|
303
|
+
f"{original.__doc__ or ''}"
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
return override
|
|
307
|
+
|
|
308
|
+
_patch_classes(classes, method_name, build_override)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# --------------------------------------------------
|
|
312
|
+
# used to override all functions in pyspark.sql.columns for column selector support
|
|
313
|
+
# --------------------------------------------------
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _make_column_only_wrapper(method_name: str) -> Callable[..., Column]:
|
|
317
|
+
"""Build a wrapper that delegates to a `pyspark.sql.Column`-only bound method.
|
|
318
|
+
|
|
319
|
+
For `Column` methods with no `pyspark.sql.functions` equivalent at all (e.g.
|
|
320
|
+
`isNull`, `isin`, `between`, `alias`, `getItem`), there is no `F` version to
|
|
321
|
+
dispatch to, so this simply calls the bound method directly.
|
|
322
|
+
|
|
323
|
+
Parameters
|
|
324
|
+
----------
|
|
325
|
+
method_name : str
|
|
326
|
+
Name of the `pyspark.sql.Column` instance method to delegate to.
|
|
327
|
+
|
|
328
|
+
Returns
|
|
329
|
+
-------
|
|
330
|
+
callable
|
|
331
|
+
A function `wrapper(c, *args, **kwargs)` that returns
|
|
332
|
+
`getattr(c, method_name)(*args, **kwargs)`, carrying `Column`'s own
|
|
333
|
+
docstring for the method so `help()` on the resulting chained selector
|
|
334
|
+
method shows accurate documentation.
|
|
335
|
+
"""
|
|
336
|
+
column_method = _get_column_method(method_name)
|
|
337
|
+
|
|
338
|
+
def wrapper(c: Column, *args: Any, **kwargs: Any) -> Column:
|
|
339
|
+
return getattr(c, method_name)(*args, **kwargs)
|
|
340
|
+
|
|
341
|
+
wrapper.__name__ = method_name
|
|
342
|
+
wrapper.__doc__ = column_method.__doc__
|
|
343
|
+
# I don't think I actually need _is_rename_op,
|
|
344
|
+
# might be able to delete all of these comments
|
|
345
|
+
# carry the `@_rename_op` marker (e.g. `.prefix()`/`.suffix()`/
|
|
346
|
+
# `.map_alias()`) through to this wrapper, so `spark_wrapper` -- which
|
|
347
|
+
# actually receives this wrapper, not the raw `Column` method -- can see
|
|
348
|
+
# it and flag the selector copy as having a rename op applied.
|
|
349
|
+
# wrapper._is_rename_op = getattr(column_method, "_is_rename_op", False)
|
|
350
|
+
return wrapper
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
# --------------------------------------------------
|
|
354
|
+
# handle override for agg fx
|
|
355
|
+
# --------------------------------------------------
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _build_agg_override(original):
|
|
359
|
+
"""Build one `GroupedData.agg` override closing over `original`.
|
|
360
|
+
|
|
361
|
+
Parameters
|
|
362
|
+
----------
|
|
363
|
+
original : callable
|
|
364
|
+
That class's real, never-patched `GroupedData.agg` implementation
|
|
365
|
+
(via `_true_original`).
|
|
366
|
+
|
|
367
|
+
Returns
|
|
368
|
+
-------
|
|
369
|
+
callable
|
|
370
|
+
The override function to install in place of `original`.
|
|
371
|
+
"""
|
|
372
|
+
|
|
373
|
+
def agg_patched(self, *exprs: Any, **kwargs: Any) -> DataFrame:
|
|
374
|
+
"""Aggregate, accepting column selectors (with chained aggregation transforms).
|
|
375
|
+
|
|
376
|
+
Anywhere pyspark accepts a `Column` expression, a selector with chained
|
|
377
|
+
aggregation transforms (e.g. `by_dtype([T.DoubleType]).sum()`) works too.
|
|
378
|
+
|
|
379
|
+
Patches `GroupedData.agg` (both classic `pyspark.sql.group.GroupedData`
|
|
380
|
+
and, when Spark Connect is available,
|
|
381
|
+
`pyspark.sql.connect.group.GroupedData`).
|
|
382
|
+
`by_dtype([T.DoubleType]).sum()`'s chained ``.sum()`` transform (already
|
|
383
|
+
available via `SelectorcolumnOperations`'s spark-function wrapping
|
|
384
|
+
loop) is honored via `_resolve_selector_exprs`, resolved against
|
|
385
|
+
``self._df`` (the underlying dataframe `GroupedData` was built from).
|
|
386
|
+
Each resolved aggregation is then re-aliased back to the original
|
|
387
|
+
matched column name -- instead of pyspark's default auto-generated
|
|
388
|
+
name (`sum(column_name)`) -- matching the same "preserve the source
|
|
389
|
+
column's name" behavior `select`/`withColumn`/`withColumns` already
|
|
390
|
+
have. A selector with a `.prefix()`/`.suffix()`/`.map_alias()`
|
|
391
|
+
chained on still renames as expected, since `name` here already
|
|
392
|
+
reflects any queued rename (see `_resolve_selector_exprs`).
|
|
393
|
+
|
|
394
|
+
Parameters
|
|
395
|
+
----------
|
|
396
|
+
*exprs : Any
|
|
397
|
+
Column selectors and/or plain `pyspark.sql.Column` aggregation
|
|
398
|
+
expressions, same shape as real `GroupedData.agg`.
|
|
399
|
+
**kwargs : Any
|
|
400
|
+
Passed straight through to the real `GroupedData.agg`.
|
|
401
|
+
|
|
402
|
+
Returns
|
|
403
|
+
-------
|
|
404
|
+
pyspark.sql.DataFrame
|
|
405
|
+
One row per group, with one output column per resolved
|
|
406
|
+
aggregation expression, named after the original column it
|
|
407
|
+
aggregated (unless that column had a rename chained onto it, in
|
|
408
|
+
which case the new name is used).
|
|
409
|
+
|
|
410
|
+
Examples
|
|
411
|
+
--------
|
|
412
|
+
>>> df.groupBy('region').agg(by_dtype([T.DoubleType]).sum())
|
|
413
|
+
DataFrame[...]
|
|
414
|
+
"""
|
|
415
|
+
|
|
416
|
+
resolved = [
|
|
417
|
+
expr.alias(name) if name is not None else expr for name, expr in _resolve_selector_exprs(self._df, exprs)
|
|
418
|
+
]
|
|
419
|
+
|
|
420
|
+
return original(self, *resolved, **kwargs)
|
|
421
|
+
|
|
422
|
+
return agg_patched
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# --------------------------------------------------
|
|
426
|
+
# handle override for filter fx
|
|
427
|
+
# --------------------------------------------------
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _build_filter_override(original):
|
|
431
|
+
"""Build one `DataFrame.filter` override closing over `original`.
|
|
432
|
+
|
|
433
|
+
Parameters
|
|
434
|
+
----------
|
|
435
|
+
original : callable
|
|
436
|
+
That class's real, never-patched `DataFrame.filter` implementation
|
|
437
|
+
(via `_true_original`).
|
|
438
|
+
|
|
439
|
+
Returns
|
|
440
|
+
-------
|
|
441
|
+
callable
|
|
442
|
+
The override function to install in place of `original`.
|
|
443
|
+
"""
|
|
444
|
+
|
|
445
|
+
def filter_patched(self, condition: Union[BaseSelector, "SelectorFilterCondition", Column]) -> DataFrame:
|
|
446
|
+
"""Filter rows, accepting a column selector as the condition.
|
|
447
|
+
|
|
448
|
+
Patches `DataFrame.filter` (and, via the `where` alias,
|
|
449
|
+
`DataFrame.where`) -- both classic `pyspark.sql.DataFrame` and, when
|
|
450
|
+
Spark Connect is available, its `pyspark.sql.connect.dataframe.DataFrame`.
|
|
451
|
+
A selector matching multiple columns (e.g. ``by_dtype([T.IntegerType]) > 0``)
|
|
452
|
+
is reduced down (ANDed together) into one boolean condition. Chained
|
|
453
|
+
selector conditions and/or plain `pyspark.sql.Column` conditions
|
|
454
|
+
compose with ``&``/``|``/``~`` -- see `SelectorSelectionOperations`
|
|
455
|
+
and `SelectorFilterCondition`. Comma syntax
|
|
456
|
+
(``df.filter(cond1, cond2)``) is intentionally not needed; ``&``
|
|
457
|
+
alone composes any number of conditions together.
|
|
458
|
+
|
|
459
|
+
Parameters
|
|
460
|
+
----------
|
|
461
|
+
condition : BaseSelector, SelectorFilterCondition, or pyspark.sql.Column
|
|
462
|
+
A column selector (optionally with a comparison/arithmetic transform
|
|
463
|
+
chained onto it), a `SelectorFilterCondition` built by combining
|
|
464
|
+
selectors/columns with ``& | ~``, or a plain
|
|
465
|
+
`pyspark.sql.Column` boolean expression -- same shape as real
|
|
466
|
+
`DataFrame.filter`.
|
|
467
|
+
|
|
468
|
+
Returns
|
|
469
|
+
-------
|
|
470
|
+
pyspark.sql.DataFrame
|
|
471
|
+
The filtered dataframe.
|
|
472
|
+
|
|
473
|
+
Examples
|
|
474
|
+
--------
|
|
475
|
+
>>> df.filter(by_dtype([T.DoubleType]) > 0)
|
|
476
|
+
DataFrame[...]
|
|
477
|
+
>>> df.filter((by_dtype([T.IntegerType]) > 0) & (by_dtype([T.IntegerType]) < 9))
|
|
478
|
+
DataFrame[...]
|
|
479
|
+
"""
|
|
480
|
+
|
|
481
|
+
if isinstance(condition, (BaseSelector, SelectorFilterCondition)):
|
|
482
|
+
combined = SelectorFilterCondition._to_expr(condition, self)
|
|
483
|
+
|
|
484
|
+
return original(self, combined)
|
|
485
|
+
|
|
486
|
+
return original(self, condition)
|
|
487
|
+
|
|
488
|
+
return filter_patched
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
# --------------------------------------------------
|
|
492
|
+
# handle override for with_column fx
|
|
493
|
+
# --------------------------------------------------
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _build_with_column_override(original):
|
|
497
|
+
"""Build one `DataFrame.withColumn` override closing over `original`.
|
|
498
|
+
|
|
499
|
+
Parameters
|
|
500
|
+
----------
|
|
501
|
+
original : callable
|
|
502
|
+
That class's real, never-patched `DataFrame.withColumn`
|
|
503
|
+
implementation (via `_true_original`).
|
|
504
|
+
|
|
505
|
+
Returns
|
|
506
|
+
-------
|
|
507
|
+
callable
|
|
508
|
+
The override function to install in place of `original`.
|
|
509
|
+
"""
|
|
510
|
+
|
|
511
|
+
def withColumn_patched(self, colName: str, col: BaseSelector | Column) -> DataFrame:
|
|
512
|
+
"""Add/replace one column, accepting a column selector that resolves to exactly one column.
|
|
513
|
+
|
|
514
|
+
Patches `DataFrame.withColumn` (both classic `pyspark.sql.DataFrame`
|
|
515
|
+
and, when Spark Connect is available, its
|
|
516
|
+
`pyspark.sql.connect.dataframe.DataFrame`). Keeps its normal
|
|
517
|
+
single-name/single-expression signature -- if `col` is a selector it
|
|
518
|
+
must resolve to exactly one column; otherwise this raises, pointing
|
|
519
|
+
the caller at `withColumns` for the multi-column case.
|
|
520
|
+
|
|
521
|
+
Parameters
|
|
522
|
+
----------
|
|
523
|
+
colName : str
|
|
524
|
+
The name of the column to add or replace, same as real
|
|
525
|
+
`DataFrame.withColumn`.
|
|
526
|
+
col : BaseSelector or pyspark.sql.Column
|
|
527
|
+
A column selector resolving to exactly one column (with whatever
|
|
528
|
+
transforms are chained onto it, e.g. ``.cast(...)``/``+1``), or a
|
|
529
|
+
plain `pyspark.sql.Column` expression.
|
|
530
|
+
|
|
531
|
+
Returns
|
|
532
|
+
-------
|
|
533
|
+
pyspark.sql.DataFrame
|
|
534
|
+
The dataframe with `colName` added/replaced.
|
|
535
|
+
|
|
536
|
+
Raises
|
|
537
|
+
------
|
|
538
|
+
pyspark.errors.PySparkValueError
|
|
539
|
+
If `col` is a selector that resolves to zero or more than one
|
|
540
|
+
column.
|
|
541
|
+
|
|
542
|
+
Examples
|
|
543
|
+
--------
|
|
544
|
+
>>> df.withColumn('doubled', by_dtype([T.IntegerType]) * 2)
|
|
545
|
+
DataFrame[...]
|
|
546
|
+
"""
|
|
547
|
+
|
|
548
|
+
if isinstance(col, BaseSelector):
|
|
549
|
+
pairs = _resolve_selector_exprs(self, [col])
|
|
550
|
+
|
|
551
|
+
if len(pairs) != 1:
|
|
552
|
+
raise PySparkValueError(
|
|
553
|
+
message=(
|
|
554
|
+
f"withColumn({colName!r}, ...) needs a selector resolving to exactly "
|
|
555
|
+
f"one column but got {len(pairs)} matches; use withColumns() for many at once."
|
|
556
|
+
),
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
_, expr = pairs[0]
|
|
560
|
+
return original(self, colName, expr)
|
|
561
|
+
|
|
562
|
+
return original(self, colName, col)
|
|
563
|
+
|
|
564
|
+
return withColumn_patched
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
# --------------------------------------------------
|
|
568
|
+
# handle override for with_columns fx
|
|
569
|
+
# --------------------------------------------------
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _build_with_columns_override(original):
|
|
573
|
+
"""Build one `DataFrame.withColumns` override closing over `original`.
|
|
574
|
+
|
|
575
|
+
Parameters
|
|
576
|
+
----------
|
|
577
|
+
original : callable
|
|
578
|
+
That class's real, never-patched `DataFrame.withColumns`
|
|
579
|
+
implementation (via `_true_original`).
|
|
580
|
+
|
|
581
|
+
Returns
|
|
582
|
+
-------
|
|
583
|
+
callable
|
|
584
|
+
The override function to install in place of `original`.
|
|
585
|
+
"""
|
|
586
|
+
|
|
587
|
+
def withColumns_patched(self, *args: BaseSelector | dict[str, Column]) -> DataFrame:
|
|
588
|
+
"""Add/replace many columns in one call, accepting multiple column-selector mutations.
|
|
589
|
+
|
|
590
|
+
Patches `DataFrame.withColumns` (both classic `pyspark.sql.DataFrame`
|
|
591
|
+
and, when Spark Connect is available, its
|
|
592
|
+
`pyspark.sql.connect.dataframe.DataFrame`). Real pyspark's
|
|
593
|
+
`DataFrame.withColumns` only ever took one ``colsMap`` dict argument --
|
|
594
|
+
that exact call shape (a single dict, no selector involved) still passes
|
|
595
|
+
straight through unchanged, so base pyspark behavior is preserved. When
|
|
596
|
+
multiple positional mutations are given (selectors and/or dicts), each
|
|
597
|
+
resolves independently (its own selector match, its own chained
|
|
598
|
+
transforms) into ``{name: expr}`` pairs and gets merged; a collision
|
|
599
|
+
(two different mutations targeting the same output column name within
|
|
600
|
+
the same call) raises rather than silently letting one clobber the
|
|
601
|
+
other.
|
|
602
|
+
|
|
603
|
+
Parameters
|
|
604
|
+
----------
|
|
605
|
+
*args : BaseSelector or dict of {str: pyspark.sql.Column}
|
|
606
|
+
One or more mutations. A single ``{name: Column}`` dict (pyspark's
|
|
607
|
+
original call shape) passes straight through. Multiple positional
|
|
608
|
+
args -- each a column selector (with whatever transforms are chained
|
|
609
|
+
onto it, e.g. ``.cast(...)``/``.upper()``/``+1``) and/or a
|
|
610
|
+
``{name: Column}`` dict -- are resolved and merged into one call.
|
|
611
|
+
|
|
612
|
+
Returns
|
|
613
|
+
-------
|
|
614
|
+
pyspark.sql.DataFrame
|
|
615
|
+
The dataframe with every resolved mutation applied.
|
|
616
|
+
|
|
617
|
+
Raises
|
|
618
|
+
------
|
|
619
|
+
pyspark.errors.PySparkTypeError
|
|
620
|
+
If a positional arg (when more than one is given) is neither a
|
|
621
|
+
`BaseSelector` nor a `dict`.
|
|
622
|
+
pyspark.errors.PySparkValueError
|
|
623
|
+
If two different positional args resolve to the same output column
|
|
624
|
+
name within the same call.
|
|
625
|
+
|
|
626
|
+
Examples
|
|
627
|
+
--------
|
|
628
|
+
>>> df.withColumns(by_dtype([T.DoubleType]).cast('string'))
|
|
629
|
+
DataFrame[...]
|
|
630
|
+
>>> df.withColumns(by_dtype([T.DoubleType]).cast('string'), by_dtype([T.StringType]).upper())
|
|
631
|
+
DataFrame[...]
|
|
632
|
+
"""
|
|
633
|
+
|
|
634
|
+
# original pyspark call shape: withColumns({"name": col, ...}) -- exactly one
|
|
635
|
+
# dict arg, no selector involved. pass straight through untouched.
|
|
636
|
+
if len(args) == 1 and not isinstance(args[0], BaseSelector):
|
|
637
|
+
return original(self, args[0])
|
|
638
|
+
|
|
639
|
+
# one or more mutations, each either a column selector (with whatever
|
|
640
|
+
# transforms are chained onto it, e.g. .cast(...)/.upper()/+1) or a plain
|
|
641
|
+
# {name: Column} dict -- resolve each one independently, then merge them into a
|
|
642
|
+
# single colsMap, raising on any collision instead of silently overwriting.
|
|
643
|
+
merged = {}
|
|
644
|
+
|
|
645
|
+
for arg in args:
|
|
646
|
+
if isinstance(arg, BaseSelector):
|
|
647
|
+
pairs = _resolve_selector_exprs(self, [arg])
|
|
648
|
+
arg_map = {name: expr for name, expr in pairs}
|
|
649
|
+
|
|
650
|
+
elif isinstance(arg, dict):
|
|
651
|
+
arg_map = arg
|
|
652
|
+
|
|
653
|
+
else:
|
|
654
|
+
raise PySparkTypeError(
|
|
655
|
+
message=(
|
|
656
|
+
"withColumns() positional args must each be a column selector or a "
|
|
657
|
+
f"{{name: Column}} dict when passing multiple mutations, got {type(arg)!r}"
|
|
658
|
+
),
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
collisions = set(arg_map) & set(merged)
|
|
662
|
+
|
|
663
|
+
if collisions:
|
|
664
|
+
raise PySparkValueError(
|
|
665
|
+
message=(
|
|
666
|
+
f"withColumns() column name collision on {sorted(str(name) for name in collisions)!r} -- "
|
|
667
|
+
"more than one mutation in this call targets the same column name; "
|
|
668
|
+
"rename one of them or combine them into a single mutation instead "
|
|
669
|
+
"of letting one silently overwrite the other."
|
|
670
|
+
),
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
merged.update(arg_map)
|
|
674
|
+
|
|
675
|
+
return original(self, merged)
|
|
676
|
+
|
|
677
|
+
return withColumns_patched
|