PySpark-Column-Selectors 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ # Marker file for PEP 561
@@ -0,0 +1,677 @@
1
+ from collections.abc import Callable, Sequence
2
+ from typing import Any, Union, cast
3
+
4
+ from pyspark.errors import PySparkTypeError, PySparkValueError
5
+ from pyspark.sql import DataFrame
6
+ from pyspark.sql import functions as F
7
+ from pyspark.sql.column import Column
8
+
9
+ from .models import (
10
+ BaseSelector,
11
+ SelectorFilterCondition,
12
+ )
13
+ from .utils import (
14
+ _get_column_method,
15
+ _resolve_selector_exprs,
16
+ _resolve_selector_names,
17
+ _true_original,
18
+ )
19
+
20
+ """ enable support for many different types of pyspark environments"""
21
+ # if your environment isn't supported then you can add it to this section.
22
+ # Spark supports multiple DataFrame implementations. Keep every available runtime
23
+ # class in the patch set: public/classic DataFrame and Spark Connect DataFrame.
24
+ # using try except as to allow passthrough if environment doesn't support these.
25
+ try:
26
+ from pyspark.sql.classic.dataframe import DataFrame as _ClassicDataFrame
27
+ except ImportError:
28
+ _ClassicDataFrame = None
29
+
30
+ try:
31
+ from pyspark.sql.connect.dataframe import DataFrame as _ConnectDataFrame
32
+ except ImportError:
33
+ _ConnectDataFrame = None
34
+
35
+ _DATAFRAME_CLASSES = tuple(
36
+ dict.fromkeys(cls for cls in (DataFrame, _ClassicDataFrame, _ConnectDataFrame) if cls is not None)
37
+ )
38
+
39
+ try:
40
+ from pyspark.sql.connect.column import Column as _ConnectColumn
41
+ except ImportError:
42
+ _ConnectColumn = None
43
+
44
+ _COLUMN_CLASSES = tuple(dict.fromkeys(cls for cls in (Column, _ConnectColumn) if cls is not None))
45
+
46
+
47
+ # --------------------------------------------------
48
+ # ensure all versions of dataframes are overriden to support column selectors
49
+ # --------------------------------------------------
50
+
51
+
52
+ def _patch_classes(
53
+ classes: Sequence[type],
54
+ method_name: str,
55
+ build_override: Callable[[Callable[..., Any]], Callable[..., Any]],
56
+ ) -> None:
57
+ """Patch `method_name` onto every class in `classes` with a fresh override.
58
+
59
+ Shared "apply this override everywhere it needs to exist" helper --
60
+ installs a selector-aware override of `method_name` onto each class in
61
+ `classes` (e.g. both classic `pyspark.sql.DataFrame` and Spark Connect's
62
+ `pyspark.sql.connect.dataframe.DataFrame`, when Connect is available), so
63
+ the same selector syntax works identically no matter which backend
64
+ actually produced the dataframe being operated on.
65
+
66
+ Each class gets its OWN override function, built by calling
67
+ `build_override` once per class with that class's own real, never-patched
68
+ original implementation (via `_true_original`) -- classic and Connect
69
+ implementations of the same method are separate functions and are not
70
+ interchangeable, so one shared override closing over a single captured
71
+ original would silently call the wrong implementation for the other
72
+ class. Calling `build_override` fresh for each class (rather than
73
+ reusing one closure variable across a loop) also avoids Python's
74
+ late-binding closure pitfall, where every override would otherwise end
75
+ up referencing whichever class's original happened to be captured last.
76
+
77
+ Parameters
78
+ ----------
79
+ classes : sequence of type
80
+ Every class to patch `method_name` onto.
81
+ method_name : str
82
+ The name of the method being overridden.
83
+ build_override : callable
84
+ Given that class's real original implementation (an unbound
85
+ function called as ``original(self, *args, **kwargs)``), returns
86
+ the override function to install in its place.
87
+
88
+ Examples
89
+ --------
90
+ >>> _patch_classes(
91
+ ... _DATAFRAME_CLASSES, "sort",
92
+ ... lambda original: (lambda self, *cols, **kw: original(self, *cols, **kw)),
93
+ ... )
94
+ """
95
+ for cls in classes:
96
+ original = _true_original(cls, method_name)
97
+ override = cast(Any, build_override(original))
98
+ # override = build_override(original)
99
+ override._cs_original = original
100
+ override.__name__ = method_name
101
+ override.__qualname__ = f"{cls.__name__}.{method_name}"
102
+ setattr(cls, method_name, override)
103
+
104
+
105
+ # --------------------------------------------------------------------------------
106
+ # wrapper that will allow for column functions to be wrapped for use in selectors
107
+ # we will focus on columns that are initialized in pyspark.sql.column
108
+ #
109
+ # --------------------------------------------------------------------------------
110
+
111
+
112
+ def _make_dispatching_column_wrapper(method_name: str) -> Callable[..., Column]:
113
+ """Build a wrapper that dispatches to `Column` or `pyspark.sql.functions` by argument type.
114
+
115
+ Resolves the `F`/`Column` name-collision described in the markdown above:
116
+ `pyspark.sql.functions.<method_name>` and `pyspark.sql.Column.<method_name>`
117
+ share a name but treat a plain (non-`Column`) argument differently. This
118
+ wrapper picks whichever one matches the caller's intent based on the type of
119
+ the argument actually passed, so both usages keep working.
120
+
121
+ Parameters
122
+ ----------
123
+ method_name : str
124
+ Name shared by a `pyspark.sql.functions` function and a `pyspark.sql.Column`
125
+ instance method (e.g. `"contains"`), used to look up both versions.
126
+
127
+ Returns
128
+ -------
129
+ callable
130
+ A function `wrapper(c, *args, **kwargs)` that, when the first
131
+ argument in `args` (if any) is a `pyspark.sql.Column`, calls
132
+ `pyspark.sql.functions.<method_name>(c, *args, **kwargs)` (column-to-column
133
+ semantics); otherwise calls `getattr(c, method_name)(*args, **kwargs)`
134
+ (literal semantics via `Column`'s own bound method). Carries `Column`'s
135
+ docstring for the method so `help()` on the resulting chained selector
136
+ method shows accurate documentation.
137
+ """
138
+ column_method = _get_column_method(method_name)
139
+ spark_func = getattr(F, method_name)
140
+
141
+ def wrapper(c: Column, *args: Any, **kwargs: Any) -> Column:
142
+ # `_COLUMN_CLASSES` (classic + Spark Connect `Column`, when Connect is
143
+ # available) instead of the bare `Column` name -- otherwise a Spark
144
+ # Connect `Column` argument would be misdetected as a plain literal
145
+ # and routed to the wrong (literal-semantics) branch below.
146
+ if args and isinstance(args[0], _COLUMN_CLASSES):
147
+ return spark_func(c, *args, **kwargs)
148
+ return getattr(c, method_name)(*args, **kwargs)
149
+
150
+ wrapper.__name__ = method_name
151
+ wrapper.__doc__ = column_method.__doc__
152
+ # I don't think I actually need _is_rename_op,
153
+ # might be able to delete all of these comments
154
+ # same rename-op marker carry-through as `_make_column_only_wrapper`
155
+ # (defensive -- no current rename op collides with an `F` name).
156
+ # wrapper._is_rename_op = getattr(column_method, "_is_rename_op", False)
157
+ return wrapper
158
+
159
+
160
+ # --------------------------------------------------
161
+ # used to override functions that require column expressions
162
+ # --------------------------------------------------
163
+
164
+
165
+ def make_expr_based_override(
166
+ method_name: str,
167
+ alias_results: bool = True,
168
+ classes: Sequence[type] | None = None,
169
+ ) -> None:
170
+ """Patch a pyspark method that wants `Column` expressions.
171
+
172
+ Factory (wrapper pattern) used by every fx override whose real pyspark
173
+ method wants `pyspark.sql.Column` *expressions* rather than bare names --
174
+ e.g. `sort`/`orderBy` need real expressions so a selector's chained
175
+ ``.desc()`` transform actually takes effect, not just the underlying
176
+ column name.
177
+
178
+ Unlike earlier versions of this notebook, this function performs the
179
+ patch itself (via `_patch_classes`) rather than returning a single
180
+ override for the caller to assign -- classic `pyspark.sql.DataFrame` and
181
+ Spark Connect's `DataFrame` each need their own override closing over
182
+ their own real original implementation.
183
+
184
+ Parameters
185
+ ----------
186
+ method_name : str
187
+ The name of the `DataFrame` method to override (e.g. ``"select"``,
188
+ ``"sort"``, ``"orderBy"``).
189
+ alias_results : bool, default True
190
+ Whether to re-alias each resolved expression back to the original
191
+ column name it matched (see the generated override's docstring).
192
+ Should be `False` for methods like `sort`/`orderBy` that don't
193
+ change the output schema at all -- there, a chained
194
+ ``.desc()``/``.asc()`` produces a non-evaluable Catalyst
195
+ `SortOrder` expression, which `.alias(...)` cannot be applied to.
196
+ classes : sequence of type, optional
197
+ Every class to patch. Defaults to `_DATAFRAME_CLASSES` (classic
198
+ `pyspark.sql.DataFrame` plus Spark Connect's `DataFrame`, when
199
+ Spark Connect is importable).
200
+
201
+ Examples
202
+ --------
203
+ >>> make_expr_based_override('sort', alias_results=False)
204
+ >>> df.sort(by_dtype([T.DoubleType]).desc())
205
+ DataFrame[...]
206
+ """
207
+ if classes is None:
208
+ classes = _DATAFRAME_CLASSES
209
+
210
+ def build_override(original):
211
+
212
+ def override(self, *cols, **kwargs):
213
+
214
+ if alias_results:
215
+ resolved = [
216
+ expr.alias(name) if name is not None else expr for name, expr in _resolve_selector_exprs(self, cols)
217
+ ]
218
+ else:
219
+ resolved = [expr for _, expr in _resolve_selector_exprs(self, cols)]
220
+
221
+ return original(self, *resolved, **kwargs)
222
+
223
+ override.__doc__ = (
224
+ f"{method_name}(*cols, **kwargs) -- selector-aware override.\n\n"
225
+ "Accepts a column selector (e.g. `by_dtype([T.DoubleType]).desc()`) "
226
+ "anywhere a plain `pyspark.sql.Column` expression is normally "
227
+ "accepted; the selector is resolved to `Column` expressions (with "
228
+ "any chained transform, e.g. `.cast(...)`/`.desc()`, applied)"
229
+ + (
230
+ ", then each is re-aliased back to the original column name it "
231
+ "matched, so a transform like `.upper()`/`.cast(...)` still keeps "
232
+ "the source column's name instead of Spark's auto-generated "
233
+ "expression name"
234
+ if alias_results
235
+ else ""
236
+ )
237
+ + f". The result is handed to the real `{method_name}` "
238
+ "implementation. Plain columns/args pass through unchanged.\n\n"
239
+ f"{original.__doc__ or ''}"
240
+ )
241
+
242
+ return override
243
+
244
+ _patch_classes(classes, method_name, build_override)
245
+
246
+
247
+ # --------------------------------------------------
248
+ # override functions that only require column name strings
249
+ # --------------------------------------------------
250
+
251
+
252
+ def make_name_based_override(method_name: str, classes: Sequence[type] | None = None) -> None:
253
+ """Patch a pyspark method that only wants column-name strings.
254
+
255
+ Factory (wrapper pattern) used by every fx override whose real pyspark
256
+ method just wants a flat list of column-name strings -- resolves any
257
+ `BaseSelector` passed positionally down to the names it matches, then
258
+ calls straight through to the real pyspark method of the same name so
259
+ the signature/behavior is otherwise identical. This is what collapses
260
+ `groupBy`/`drop` into one-liners instead of each needing its own
261
+ hand-written override.
262
+
263
+ Unlike earlier versions of this notebook, this function performs the
264
+ patch itself (via `_patch_classes`) rather than returning a single
265
+ override for the caller to assign -- classic `pyspark.sql.DataFrame` and
266
+ Spark Connect's `DataFrame` each need their own override closing over
267
+ their own real original implementation.
268
+
269
+ Parameters
270
+ ----------
271
+ method_name : str
272
+ The name of the `DataFrame` method to override (e.g. ``"groupBy"``,
273
+ ``"drop"``).
274
+ classes : sequence of type, optional
275
+ Every class to patch. Defaults to `_DATAFRAME_CLASSES` (classic
276
+ `pyspark.sql.DataFrame` plus Spark Connect's `DataFrame`, when
277
+ Spark Connect is importable).
278
+
279
+ Examples
280
+ --------
281
+ >>> make_name_based_override('groupBy')
282
+ >>> df.groupBy(by_dtype([T.StringType]))
283
+ GroupedData[...]
284
+ """
285
+ if classes is None:
286
+ classes = _DATAFRAME_CLASSES
287
+
288
+ def build_override(original):
289
+
290
+ def override(self, *cols, **kwargs):
291
+
292
+ resolved = _resolve_selector_names(self, cols)
293
+
294
+ return original(self, *resolved, **kwargs)
295
+
296
+ override.__doc__ = (
297
+ f"{method_name}(*cols, **kwargs) -- selector-aware override.\n\n"
298
+ "Accepts a column selector (e.g. `by_dtype([T.StringType])`) anywhere a "
299
+ "plain column-name string is normally accepted; the selector is "
300
+ f"resolved to the column names it matches, then handed to the "
301
+ f"real `{method_name}` implementation. Plain names/args pass "
302
+ "through unchanged.\n\n"
303
+ f"{original.__doc__ or ''}"
304
+ )
305
+
306
+ return override
307
+
308
+ _patch_classes(classes, method_name, build_override)
309
+
310
+
311
+ # --------------------------------------------------
312
+ # used to override all functions in pyspark.sql.columns for column selector support
313
+ # --------------------------------------------------
314
+
315
+
316
+ def _make_column_only_wrapper(method_name: str) -> Callable[..., Column]:
317
+ """Build a wrapper that delegates to a `pyspark.sql.Column`-only bound method.
318
+
319
+ For `Column` methods with no `pyspark.sql.functions` equivalent at all (e.g.
320
+ `isNull`, `isin`, `between`, `alias`, `getItem`), there is no `F` version to
321
+ dispatch to, so this simply calls the bound method directly.
322
+
323
+ Parameters
324
+ ----------
325
+ method_name : str
326
+ Name of the `pyspark.sql.Column` instance method to delegate to.
327
+
328
+ Returns
329
+ -------
330
+ callable
331
+ A function `wrapper(c, *args, **kwargs)` that returns
332
+ `getattr(c, method_name)(*args, **kwargs)`, carrying `Column`'s own
333
+ docstring for the method so `help()` on the resulting chained selector
334
+ method shows accurate documentation.
335
+ """
336
+ column_method = _get_column_method(method_name)
337
+
338
+ def wrapper(c: Column, *args: Any, **kwargs: Any) -> Column:
339
+ return getattr(c, method_name)(*args, **kwargs)
340
+
341
+ wrapper.__name__ = method_name
342
+ wrapper.__doc__ = column_method.__doc__
343
+ # I don't think I actually need _is_rename_op,
344
+ # might be able to delete all of these comments
345
+ # carry the `@_rename_op` marker (e.g. `.prefix()`/`.suffix()`/
346
+ # `.map_alias()`) through to this wrapper, so `spark_wrapper` -- which
347
+ # actually receives this wrapper, not the raw `Column` method -- can see
348
+ # it and flag the selector copy as having a rename op applied.
349
+ # wrapper._is_rename_op = getattr(column_method, "_is_rename_op", False)
350
+ return wrapper
351
+
352
+
353
+ # --------------------------------------------------
354
+ # handle override for agg fx
355
+ # --------------------------------------------------
356
+
357
+
358
+ def _build_agg_override(original):
359
+ """Build one `GroupedData.agg` override closing over `original`.
360
+
361
+ Parameters
362
+ ----------
363
+ original : callable
364
+ That class's real, never-patched `GroupedData.agg` implementation
365
+ (via `_true_original`).
366
+
367
+ Returns
368
+ -------
369
+ callable
370
+ The override function to install in place of `original`.
371
+ """
372
+
373
+ def agg_patched(self, *exprs: Any, **kwargs: Any) -> DataFrame:
374
+ """Aggregate, accepting column selectors (with chained aggregation transforms).
375
+
376
+ Anywhere pyspark accepts a `Column` expression, a selector with chained
377
+ aggregation transforms (e.g. `by_dtype([T.DoubleType]).sum()`) works too.
378
+
379
+ Patches `GroupedData.agg` (both classic `pyspark.sql.group.GroupedData`
380
+ and, when Spark Connect is available,
381
+ `pyspark.sql.connect.group.GroupedData`).
382
+ `by_dtype([T.DoubleType]).sum()`'s chained ``.sum()`` transform (already
383
+ available via `SelectorcolumnOperations`'s spark-function wrapping
384
+ loop) is honored via `_resolve_selector_exprs`, resolved against
385
+ ``self._df`` (the underlying dataframe `GroupedData` was built from).
386
+ Each resolved aggregation is then re-aliased back to the original
387
+ matched column name -- instead of pyspark's default auto-generated
388
+ name (`sum(column_name)`) -- matching the same "preserve the source
389
+ column's name" behavior `select`/`withColumn`/`withColumns` already
390
+ have. A selector with a `.prefix()`/`.suffix()`/`.map_alias()`
391
+ chained on still renames as expected, since `name` here already
392
+ reflects any queued rename (see `_resolve_selector_exprs`).
393
+
394
+ Parameters
395
+ ----------
396
+ *exprs : Any
397
+ Column selectors and/or plain `pyspark.sql.Column` aggregation
398
+ expressions, same shape as real `GroupedData.agg`.
399
+ **kwargs : Any
400
+ Passed straight through to the real `GroupedData.agg`.
401
+
402
+ Returns
403
+ -------
404
+ pyspark.sql.DataFrame
405
+ One row per group, with one output column per resolved
406
+ aggregation expression, named after the original column it
407
+ aggregated (unless that column had a rename chained onto it, in
408
+ which case the new name is used).
409
+
410
+ Examples
411
+ --------
412
+ >>> df.groupBy('region').agg(by_dtype([T.DoubleType]).sum())
413
+ DataFrame[...]
414
+ """
415
+
416
+ resolved = [
417
+ expr.alias(name) if name is not None else expr for name, expr in _resolve_selector_exprs(self._df, exprs)
418
+ ]
419
+
420
+ return original(self, *resolved, **kwargs)
421
+
422
+ return agg_patched
423
+
424
+
425
+ # --------------------------------------------------
426
+ # handle override for filter fx
427
+ # --------------------------------------------------
428
+
429
+
430
+ def _build_filter_override(original):
431
+ """Build one `DataFrame.filter` override closing over `original`.
432
+
433
+ Parameters
434
+ ----------
435
+ original : callable
436
+ That class's real, never-patched `DataFrame.filter` implementation
437
+ (via `_true_original`).
438
+
439
+ Returns
440
+ -------
441
+ callable
442
+ The override function to install in place of `original`.
443
+ """
444
+
445
+ def filter_patched(self, condition: Union[BaseSelector, "SelectorFilterCondition", Column]) -> DataFrame:
446
+ """Filter rows, accepting a column selector as the condition.
447
+
448
+ Patches `DataFrame.filter` (and, via the `where` alias,
449
+ `DataFrame.where`) -- both classic `pyspark.sql.DataFrame` and, when
450
+ Spark Connect is available, its `pyspark.sql.connect.dataframe.DataFrame`.
451
+ A selector matching multiple columns (e.g. ``by_dtype([T.IntegerType]) > 0``)
452
+ is reduced down (ANDed together) into one boolean condition. Chained
453
+ selector conditions and/or plain `pyspark.sql.Column` conditions
454
+ compose with ``&``/``|``/``~`` -- see `SelectorSelectionOperations`
455
+ and `SelectorFilterCondition`. Comma syntax
456
+ (``df.filter(cond1, cond2)``) is intentionally not needed; ``&``
457
+ alone composes any number of conditions together.
458
+
459
+ Parameters
460
+ ----------
461
+ condition : BaseSelector, SelectorFilterCondition, or pyspark.sql.Column
462
+ A column selector (optionally with a comparison/arithmetic transform
463
+ chained onto it), a `SelectorFilterCondition` built by combining
464
+ selectors/columns with ``& | ~``, or a plain
465
+ `pyspark.sql.Column` boolean expression -- same shape as real
466
+ `DataFrame.filter`.
467
+
468
+ Returns
469
+ -------
470
+ pyspark.sql.DataFrame
471
+ The filtered dataframe.
472
+
473
+ Examples
474
+ --------
475
+ >>> df.filter(by_dtype([T.DoubleType]) > 0)
476
+ DataFrame[...]
477
+ >>> df.filter((by_dtype([T.IntegerType]) > 0) & (by_dtype([T.IntegerType]) < 9))
478
+ DataFrame[...]
479
+ """
480
+
481
+ if isinstance(condition, (BaseSelector, SelectorFilterCondition)):
482
+ combined = SelectorFilterCondition._to_expr(condition, self)
483
+
484
+ return original(self, combined)
485
+
486
+ return original(self, condition)
487
+
488
+ return filter_patched
489
+
490
+
491
+ # --------------------------------------------------
492
+ # handle override for with_column fx
493
+ # --------------------------------------------------
494
+
495
+
496
+ def _build_with_column_override(original):
497
+ """Build one `DataFrame.withColumn` override closing over `original`.
498
+
499
+ Parameters
500
+ ----------
501
+ original : callable
502
+ That class's real, never-patched `DataFrame.withColumn`
503
+ implementation (via `_true_original`).
504
+
505
+ Returns
506
+ -------
507
+ callable
508
+ The override function to install in place of `original`.
509
+ """
510
+
511
+ def withColumn_patched(self, colName: str, col: BaseSelector | Column) -> DataFrame:
512
+ """Add/replace one column, accepting a column selector that resolves to exactly one column.
513
+
514
+ Patches `DataFrame.withColumn` (both classic `pyspark.sql.DataFrame`
515
+ and, when Spark Connect is available, its
516
+ `pyspark.sql.connect.dataframe.DataFrame`). Keeps its normal
517
+ single-name/single-expression signature -- if `col` is a selector it
518
+ must resolve to exactly one column; otherwise this raises, pointing
519
+ the caller at `withColumns` for the multi-column case.
520
+
521
+ Parameters
522
+ ----------
523
+ colName : str
524
+ The name of the column to add or replace, same as real
525
+ `DataFrame.withColumn`.
526
+ col : BaseSelector or pyspark.sql.Column
527
+ A column selector resolving to exactly one column (with whatever
528
+ transforms are chained onto it, e.g. ``.cast(...)``/``+1``), or a
529
+ plain `pyspark.sql.Column` expression.
530
+
531
+ Returns
532
+ -------
533
+ pyspark.sql.DataFrame
534
+ The dataframe with `colName` added/replaced.
535
+
536
+ Raises
537
+ ------
538
+ pyspark.errors.PySparkValueError
539
+ If `col` is a selector that resolves to zero or more than one
540
+ column.
541
+
542
+ Examples
543
+ --------
544
+ >>> df.withColumn('doubled', by_dtype([T.IntegerType]) * 2)
545
+ DataFrame[...]
546
+ """
547
+
548
+ if isinstance(col, BaseSelector):
549
+ pairs = _resolve_selector_exprs(self, [col])
550
+
551
+ if len(pairs) != 1:
552
+ raise PySparkValueError(
553
+ message=(
554
+ f"withColumn({colName!r}, ...) needs a selector resolving to exactly "
555
+ f"one column but got {len(pairs)} matches; use withColumns() for many at once."
556
+ ),
557
+ )
558
+
559
+ _, expr = pairs[0]
560
+ return original(self, colName, expr)
561
+
562
+ return original(self, colName, col)
563
+
564
+ return withColumn_patched
565
+
566
+
567
+ # --------------------------------------------------
568
+ # handle override for with_columns fx
569
+ # --------------------------------------------------
570
+
571
+
572
+ def _build_with_columns_override(original):
573
+ """Build one `DataFrame.withColumns` override closing over `original`.
574
+
575
+ Parameters
576
+ ----------
577
+ original : callable
578
+ That class's real, never-patched `DataFrame.withColumns`
579
+ implementation (via `_true_original`).
580
+
581
+ Returns
582
+ -------
583
+ callable
584
+ The override function to install in place of `original`.
585
+ """
586
+
587
+ def withColumns_patched(self, *args: BaseSelector | dict[str, Column]) -> DataFrame:
588
+ """Add/replace many columns in one call, accepting multiple column-selector mutations.
589
+
590
+ Patches `DataFrame.withColumns` (both classic `pyspark.sql.DataFrame`
591
+ and, when Spark Connect is available, its
592
+ `pyspark.sql.connect.dataframe.DataFrame`). Real pyspark's
593
+ `DataFrame.withColumns` only ever took one ``colsMap`` dict argument --
594
+ that exact call shape (a single dict, no selector involved) still passes
595
+ straight through unchanged, so base pyspark behavior is preserved. When
596
+ multiple positional mutations are given (selectors and/or dicts), each
597
+ resolves independently (its own selector match, its own chained
598
+ transforms) into ``{name: expr}`` pairs and gets merged; a collision
599
+ (two different mutations targeting the same output column name within
600
+ the same call) raises rather than silently letting one clobber the
601
+ other.
602
+
603
+ Parameters
604
+ ----------
605
+ *args : BaseSelector or dict of {str: pyspark.sql.Column}
606
+ One or more mutations. A single ``{name: Column}`` dict (pyspark's
607
+ original call shape) passes straight through. Multiple positional
608
+ args -- each a column selector (with whatever transforms are chained
609
+ onto it, e.g. ``.cast(...)``/``.upper()``/``+1``) and/or a
610
+ ``{name: Column}`` dict -- are resolved and merged into one call.
611
+
612
+ Returns
613
+ -------
614
+ pyspark.sql.DataFrame
615
+ The dataframe with every resolved mutation applied.
616
+
617
+ Raises
618
+ ------
619
+ pyspark.errors.PySparkTypeError
620
+ If a positional arg (when more than one is given) is neither a
621
+ `BaseSelector` nor a `dict`.
622
+ pyspark.errors.PySparkValueError
623
+ If two different positional args resolve to the same output column
624
+ name within the same call.
625
+
626
+ Examples
627
+ --------
628
+ >>> df.withColumns(by_dtype([T.DoubleType]).cast('string'))
629
+ DataFrame[...]
630
+ >>> df.withColumns(by_dtype([T.DoubleType]).cast('string'), by_dtype([T.StringType]).upper())
631
+ DataFrame[...]
632
+ """
633
+
634
+ # original pyspark call shape: withColumns({"name": col, ...}) -- exactly one
635
+ # dict arg, no selector involved. pass straight through untouched.
636
+ if len(args) == 1 and not isinstance(args[0], BaseSelector):
637
+ return original(self, args[0])
638
+
639
+ # one or more mutations, each either a column selector (with whatever
640
+ # transforms are chained onto it, e.g. .cast(...)/.upper()/+1) or a plain
641
+ # {name: Column} dict -- resolve each one independently, then merge them into a
642
+ # single colsMap, raising on any collision instead of silently overwriting.
643
+ merged = {}
644
+
645
+ for arg in args:
646
+ if isinstance(arg, BaseSelector):
647
+ pairs = _resolve_selector_exprs(self, [arg])
648
+ arg_map = {name: expr for name, expr in pairs}
649
+
650
+ elif isinstance(arg, dict):
651
+ arg_map = arg
652
+
653
+ else:
654
+ raise PySparkTypeError(
655
+ message=(
656
+ "withColumns() positional args must each be a column selector or a "
657
+ f"{{name: Column}} dict when passing multiple mutations, got {type(arg)!r}"
658
+ ),
659
+ )
660
+
661
+ collisions = set(arg_map) & set(merged)
662
+
663
+ if collisions:
664
+ raise PySparkValueError(
665
+ message=(
666
+ f"withColumns() column name collision on {sorted(str(name) for name in collisions)!r} -- "
667
+ "more than one mutation in this call targets the same column name; "
668
+ "rename one of them or combine them into a single mutation instead "
669
+ "of letting one silently overwrite the other."
670
+ ),
671
+ )
672
+
673
+ merged.update(arg_map)
674
+
675
+ return original(self, merged)
676
+
677
+ return withColumns_patched