typespark 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. typespark/__init__.py +80 -0
  2. typespark/_base.py +184 -0
  3. typespark/_default_types.py +45 -0
  4. typespark/_define.py +28 -0
  5. typespark/_exceptions.py +89 -0
  6. typespark/columns/__init__.py +43 -0
  7. typespark/columns/_generator.py +174 -0
  8. typespark/columns/_groups.py +106 -0
  9. typespark/columns/array.py +100 -0
  10. typespark/columns/columns.py +279 -0
  11. typespark/columns/primitives.py +512 -0
  12. typespark/columns/struct.py +66 -0
  13. typespark/columns/utils.py +16 -0
  14. typespark/data_contract/__init__.py +12 -0
  15. typespark/data_contract/cli.py +39 -0
  16. typespark/data_contract/generate.py +266 -0
  17. typespark/data_contract/mixins.py +15 -0
  18. typespark/data_contract/serialize.py +139 -0
  19. typespark/dataframe.py +659 -0
  20. typespark/field_transforms.py +229 -0
  21. typespark/functions/__init__.py +480 -0
  22. typespark/functions/_type_aliases.py +15 -0
  23. typespark/functions/aggregates.py +190 -0
  24. typespark/functions/array.py +355 -0
  25. typespark/functions/date_time.py +415 -0
  26. typespark/functions/higher_order.py +92 -0
  27. typespark/functions/math.py +461 -0
  28. typespark/functions/misc.py +358 -0
  29. typespark/functions/string.py +556 -0
  30. typespark/functions/window_functions.py +90 -0
  31. typespark/literals.py +20 -0
  32. typespark/metadata.py +636 -0
  33. typespark/py.typed +0 -0
  34. typespark/schema/__init__.py +3 -0
  35. typespark/schema/schema.py +144 -0
  36. typespark/utils.py +46 -0
  37. typespark/window.py +70 -0
  38. typespark-1.1.0.dist-info/METADATA +41 -0
  39. typespark-1.1.0.dist-info/RECORD +42 -0
  40. typespark-1.1.0.dist-info/WHEEL +4 -0
  41. typespark-1.1.0.dist-info/entry_points.txt +2 -0
  42. typespark-1.1.0.dist-info/licenses/LICENSE +21 -0
typespark/__init__.py ADDED
@@ -0,0 +1,80 @@
1
+ __all__ = [
2
+ # Core types
3
+ "TypedColumn",
4
+ "TypedArrayType",
5
+ "ArrayOf",
6
+ "Struct",
7
+ "BaseDataFrame",
8
+ "Window",
9
+ "WindowSpec",
10
+ # Type aliases
11
+ "Array",
12
+ "Bool",
13
+ "Column",
14
+ "DataFrame",
15
+ "String",
16
+ "StringOf",
17
+ "Binary",
18
+ "Short",
19
+ "Integer",
20
+ "IntOf",
21
+ "Int",
22
+ "Long",
23
+ "Float",
24
+ "Double",
25
+ "Decimal",
26
+ "Date",
27
+ "Timestamp",
28
+ "TimestampNTZ",
29
+ # Metadata
30
+ "field",
31
+ "decimal",
32
+ "foreign_key",
33
+ "primary_key",
34
+ # Literals
35
+ "bool_literal",
36
+ "int_literal",
37
+ "string_literal",
38
+ # Exceptions
39
+ "UnnamedColumnError",
40
+ "MissingColumnError",
41
+ "InvalidDefaultColumnError",
42
+ "InvalidLiteralDefaultError",
43
+ "LazyDataFrameError",
44
+ ]
45
+
46
+ from typespark.columns import ArrayOf, TypedArrayType, TypedColumn
47
+ from typespark.columns.primitives import (
48
+ Binary,
49
+ Bool,
50
+ Date,
51
+ Decimal,
52
+ Double,
53
+ Float,
54
+ Integer,
55
+ IntOf,
56
+ Long,
57
+ Short,
58
+ String,
59
+ StringOf,
60
+ Timestamp,
61
+ TimestampNTZ,
62
+ )
63
+ from typespark.columns.struct import Struct
64
+ from typespark.dataframe import BaseDataFrame
65
+ from typespark._exceptions import (
66
+ InvalidDefaultColumnError,
67
+ InvalidLiteralDefaultError,
68
+ LazyDataFrameError,
69
+ MissingColumnError,
70
+ UnnamedColumnError,
71
+ )
72
+ from typespark.metadata import decimal, field, foreign_key, primary_key
73
+ from typespark.window import Window, WindowSpec
74
+
75
+ from .literals import bool_literal, int_literal, string_literal
76
+
77
+ Array = TypedArrayType
78
+ Column = TypedColumn
79
+ DataFrame = BaseDataFrame
80
+ Int = Integer
typespark/_base.py ADDED
@@ -0,0 +1,184 @@
1
+ """
2
+ Schema base for TypeSpark.
3
+
4
+ _Base is a plain class (not attrs-decorated) that provides typed column field
5
+ introspection, schema generation, and the central _build() factory method.
6
+ It knows about attrs fields and TypedColumns but NOT about DataFrames.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import (
12
+ TYPE_CHECKING,
13
+ Any,
14
+ Self,
15
+ dataclass_transform,
16
+ )
17
+
18
+ import attrs
19
+ from pyspark.sql.types import StructType
20
+
21
+ from typespark import schema
22
+ from typespark._exceptions import InvalidDefaultColumnError, MissingColumnError
23
+ from typespark.columns import TypedColumn, is_typed_column_type
24
+ from typespark.utils import get_field_name, unwrap_type
25
+
26
+ from ._define import define
27
+ from .field_transforms import FieldTransformer
28
+
29
+ if TYPE_CHECKING:
30
+ pass
31
+
32
+
33
+ def _convert(f: attrs.Attribute, value: Any, instance: Any) -> Any:
34
+ """Apply a field's converter like attrs' generated ``__init__`` does."""
35
+ conv = f.converter
36
+ if conv is None:
37
+ return value
38
+ if isinstance(conv, attrs.Converter):
39
+ # attrs' stubs don't expose these attributes.
40
+ args: list[Any] = [value]
41
+ if getattr(conv, "takes_self"):
42
+ args.append(instance)
43
+ if getattr(conv, "takes_field"):
44
+ args.append(f)
45
+ return getattr(conv, "converter")(*args)
46
+ return conv(value)
47
+
48
+
49
+ @dataclass_transform(
50
+ frozen_default=True,
51
+ )
52
+ class _Base:
53
+ @property
54
+ def columns(self) -> list[TypedColumn]:
55
+ return [
56
+ getattr(self, field.name)
57
+ for field in attrs.fields(self.__class__)
58
+ if is_typed_column_type(field.type)
59
+ ]
60
+
61
+ @classmethod
62
+ def __init_subclass__(
63
+ cls, field_transformers: list[FieldTransformer] | None = None, **kwargs: Any
64
+ ):
65
+ super().__init_subclass__(**kwargs)
66
+ define(cls, field_transformers=field_transformers)
67
+
68
+ @classmethod
69
+ def _column_aliases(cls):
70
+ yield from [
71
+ get_field_name(field)
72
+ for field in attrs.fields(cls)
73
+ if is_typed_column_type(field.type)
74
+ ]
75
+
76
+ @classmethod
77
+ def generate_schema(cls):
78
+ return schema.generate_schema(cls)
79
+
80
+ def columndict(self) -> dict[str, TypedColumn]:
81
+ return {
82
+ f.name: getattr(self, f.name)
83
+ for f in attrs.fields(self.__class__)
84
+ if is_typed_column_type(f.type)
85
+ }
86
+
87
+ @classmethod
88
+ def _build(cls, source, *, col_ref=None) -> Self:
89
+ """Construct instance from a data source.
90
+
91
+ source: data container supporting __getitem__ (DataFrame or Column).
92
+ Columns are extracted via source[field_alias].
93
+ Not stored by _Base — callers store it as _dataframe, _col, etc.
94
+ col_ref: optional callable (field_alias: str) -> Column.
95
+ When provided, overrides source[field_alias] for column
96
+ reference creation. Used by from_df for aliased access.
97
+ """
98
+ new = cls.__new__(cls)
99
+ # DataFrame.columns returns list[str]; Column does not
100
+ raw_columns = getattr(source, "columns", None)
101
+ available = set(raw_columns) if isinstance(raw_columns, list) else None
102
+
103
+ for field_name, f in attrs.fields_dict(cls).items():
104
+ if not is_typed_column_type(f.type):
105
+ continue
106
+ field_alias = get_field_name(f)
107
+ missing = available is not None and field_alias not in available
108
+
109
+ if not missing:
110
+ struct_cls = unwrap_type(f.type)
111
+ if (
112
+ available is not None
113
+ and struct_cls is not None
114
+ and isinstance(struct_cls, type)
115
+ and issubclass(struct_cls, _Base)
116
+ and hasattr(source, "schema")
117
+ ):
118
+ try:
119
+ nested_type = source.schema[field_alias].dataType
120
+ except KeyError:
121
+ pass
122
+ else:
123
+ if isinstance(nested_type, StructType):
124
+ actual = set(nested_type.fieldNames())
125
+ for sub_alias in struct_cls._column_aliases():
126
+ if sub_alias not in actual:
127
+ raise MissingColumnError(
128
+ model=struct_cls,
129
+ field_name=sub_alias,
130
+ expected_column=sub_alias,
131
+ available_columns=sorted(actual),
132
+ parent_field=field_alias,
133
+ )
134
+ column = col_ref(field_alias) if col_ref else source[field_alias]
135
+ object.__setattr__(
136
+ new,
137
+ field_name,
138
+ unwrap_type(f.type)._set_column(column, field_alias),
139
+ )
140
+ elif f.default is not attrs.NOTHING:
141
+ default = f.default
142
+ if isinstance(default, attrs.Factory): # type: ignore[arg-type]
143
+ factory: Any = default
144
+ default = (
145
+ factory.factory(new)
146
+ if factory.takes_self
147
+ else factory.factory()
148
+ )
149
+ # As in attrs' __init__, the default goes through the converter.
150
+ # Every column field has one (set_alias), so a non-column default
151
+ # usually fails inside it; report that as an invalid default.
152
+ raw_default = default
153
+ invalid = InvalidDefaultColumnError(
154
+ model=cls,
155
+ field_name=field_name,
156
+ field_alias=field_alias,
157
+ default_value=raw_default,
158
+ )
159
+ try:
160
+ default = _convert(f, default, new)
161
+ except Exception as e:
162
+ if isinstance(raw_default, TypedColumn):
163
+ raise
164
+ raise invalid from e
165
+ if not isinstance(default, TypedColumn):
166
+ raise invalid
167
+ object.__setattr__(
168
+ new,
169
+ field_name,
170
+ unwrap_type(f.type)._set_column(
171
+ default.to_spark().alias(field_alias),
172
+ field_alias,
173
+ ),
174
+ )
175
+ else:
176
+ raise MissingColumnError(
177
+ model=cls,
178
+ field_name=field_name,
179
+ expected_column=field_alias,
180
+ available_columns=sorted(available)
181
+ if available is not None
182
+ else None,
183
+ )
184
+ return new
@@ -0,0 +1,45 @@
1
+ """Static-only return types for ``ts.field(default=<literal>)``.
2
+
3
+ A literal default is typed from the field annotation at runtime, so one literal
4
+ fits several column types (an ``int`` can default a ``Long``, ``Double``,
5
+ ``Decimal``, ...). The overloads on ``ts.field`` return one of these classes,
6
+ each a subclass of every column type the literal may default, so a type checker
7
+ accepts ``x: ts.Long = ts.field(default=5)`` and rejects
8
+ ``x: ts.String = ts.field(default=5)``.
9
+
10
+ Only imported under ``TYPE_CHECKING``; none of these classes exist at runtime.
11
+ """
12
+
13
+ from typing import Any
14
+
15
+ from typespark.columns.columns import Bool
16
+ from typespark.columns.primitives import (
17
+ Date,
18
+ Decimal,
19
+ Double,
20
+ Float,
21
+ IntOf,
22
+ Long,
23
+ Short,
24
+ StringOf,
25
+ Timestamp,
26
+ TimestampNTZ,
27
+ )
28
+
29
+ # The bases disagree on method signatures (e.g. comparison operand types);
30
+ # irrelevant here, since these only serve as assignment targets.
31
+
32
+
33
+ class BoolDefault(Bool): ...
34
+
35
+
36
+ class StrDefault(StringOf[Any]): ...
37
+
38
+
39
+ class IntDefault(IntOf[Any], Short, Long, Float, Double, Decimal): ... # type: ignore[misc] # pyright: ignore
40
+
41
+
42
+ class FloatDefault(Float, Double, Decimal): ... # type: ignore[misc] # pyright: ignore
43
+
44
+
45
+ class DatetimeDefault(Timestamp, TimestampNTZ, Date): ... # type: ignore[misc] # pyright: ignore
typespark/_define.py ADDED
@@ -0,0 +1,28 @@
1
+ from typing import dataclass_transform
2
+
3
+ import attr
4
+ import attrs
5
+
6
+ from typespark.field_transforms import (
7
+ FieldTransformer,
8
+ pipe_tranformers,
9
+ resolve_literal_defaults,
10
+ set_alias,
11
+ to_transformer,
12
+ )
13
+ from typespark.metadata import decimal, field, foreign_key, primary_key
14
+
15
+
16
+ @dataclass_transform(
17
+ field_specifiers=(attrs.field, attr.ib, decimal, foreign_key, primary_key, field),
18
+ )
19
+ def define(cls, field_transformers: list[FieldTransformer] | None = None):
20
+ ft = pipe_tranformers(
21
+ resolve_literal_defaults,
22
+ *(field_transformers or []),
23
+ to_transformer(set_alias),
24
+ )
25
+
26
+ return attrs.define(field_transformer=ft, slots=False, frozen=True, kw_only=True)(
27
+ cls
28
+ )
@@ -0,0 +1,89 @@
1
+ from typespark.columns.columns import TypedColumn
2
+
3
+
4
+ class UnnamedColumnError(ValueError):
5
+ def __init__(self, *, column_type: type, operation: str):
6
+ self.column_type = column_type
7
+ self.operation = operation
8
+
9
+ msg = (
10
+ f"Cannot call '{operation}' on an unnamed {column_type.__name__} column. "
11
+ f"Use a column from a typespark DataFrame, or call ._set_name() to assign a name."
12
+ )
13
+ super().__init__(msg)
14
+
15
+
16
+ class MissingColumnError(ValueError):
17
+ def __init__(
18
+ self,
19
+ *,
20
+ model: type,
21
+ field_name: str,
22
+ expected_column: str,
23
+ available_columns: list[str] | None = None,
24
+ parent_field: str | None = None,
25
+ ):
26
+ self.model = model
27
+ self.field_name = field_name
28
+ self.expected_column = expected_column
29
+ self.available_columns = available_columns
30
+ self.parent_field = parent_field
31
+
32
+ msg = (
33
+ f"Missing required column '{expected_column}' for field '{field_name}' "
34
+ f"in {model.__name__}"
35
+ )
36
+ if parent_field is not None:
37
+ msg += f" (nested under struct column '{parent_field}')"
38
+ msg += ". "
39
+ if available_columns is not None:
40
+ msg += f"Available columns: {available_columns}. "
41
+ msg += f"To fix: provide the column, adjust aliases, or set a default for '{field_name}'."
42
+ super().__init__(msg)
43
+
44
+
45
+ class LazyDataFrameError(RuntimeError):
46
+ def __init__(self, *, model: type, alias: str | None):
47
+ self.model = model
48
+ self.alias = alias
49
+
50
+ msg = (
51
+ f"{model.__name__}.lazy({alias!r}) holds column references only "
52
+ f"— it has no underlying DataFrame. Data operations (to_df, "
53
+ f"to_spark, filter, select, show, write, ...) are unavailable."
54
+ )
55
+ super().__init__(msg)
56
+
57
+
58
+ class InvalidDefaultColumnError(TypeError):
59
+ def __init__(
60
+ self,
61
+ *,
62
+ model: type,
63
+ field_name: str,
64
+ field_alias: str,
65
+ default_value: object,
66
+ ):
67
+ self.model = model
68
+ self.field_name = field_name
69
+ self.field_alias = field_alias
70
+ self.default_value = default_value
71
+
72
+ msg = (
73
+ f"Default for {model.__name__}.{field_name} must be a "
74
+ f"{TypedColumn.__name__}. "
75
+ f"Got {type(default_value).__name__}: {default_value!r}. "
76
+ f"Field alias: {field_alias!r}. "
77
+ f"Use factory=lambda: <TypedColumn>(...) for a lazy default."
78
+ )
79
+ super().__init__(msg)
80
+
81
+
82
+ class InvalidLiteralDefaultError(ValueError):
83
+ def __init__(self, *, model: type, field_name: str, value: object, reason: str):
84
+ self.model = model
85
+ self.field_name = field_name
86
+ self.value = value
87
+
88
+ msg = f"Invalid default for {model.__name__}.{field_name}: {value!r}. {reason}"
89
+ super().__init__(msg)
@@ -0,0 +1,43 @@
1
+ __all__ = [
2
+ "TypedColumn",
3
+ "AliasedTypedColumn",
4
+ "Bool",
5
+ "TypedArrayType",
6
+ "ArrayOf",
7
+ "Binary",
8
+ "Date",
9
+ "Short",
10
+ "Decimal",
11
+ "Double",
12
+ "Float",
13
+ "Int",
14
+ "Integer",
15
+ "IntOf",
16
+ "Long",
17
+ "String",
18
+ "StringOf",
19
+ "Timestamp",
20
+ "TimestampNTZ",
21
+ "is_typed_column_type",
22
+ ]
23
+
24
+ from .array import ArrayOf, TypedArrayType
25
+ from .columns import *
26
+ from .primitives import (
27
+ Binary,
28
+ Bool,
29
+ Date,
30
+ Decimal,
31
+ Double,
32
+ Float,
33
+ Int,
34
+ Integer,
35
+ IntOf,
36
+ Long,
37
+ Short,
38
+ String,
39
+ StringOf,
40
+ Timestamp,
41
+ TimestampNTZ,
42
+ )
43
+ from .utils import is_typed_column_type
@@ -0,0 +1,174 @@
1
+ from __future__ import annotations
2
+
3
+ from abc import ABC, abstractmethod
4
+ from typing import TYPE_CHECKING, Any, Union
5
+
6
+ import attrs
7
+ from pyspark.sql import Column
8
+ from pyspark.sql.functions import col, explode
9
+ from pyspark.sql.types import DataType
10
+
11
+ if TYPE_CHECKING:
12
+ from typespark.columns import TypedColumn
13
+
14
+
15
+ LiteralType = Union[str, int, float, bool]
16
+
17
+ # What an operator on a deferred column accepts on the right-hand side.
18
+ OperandType = Union["DeferredColumn", Column, "TypedColumn[Any]", LiteralType]
19
+
20
+
21
+ def _operand(other: OperandType) -> Column | LiteralType:
22
+ """Reduce an operand to a Spark column (or literal) usable in an expression."""
23
+ from typespark.columns.columns import TypedColumn as _TypedColumn
24
+
25
+ if isinstance(other, (DeferredColumn, _TypedColumn)):
26
+ return other.to_spark()
27
+ return other
28
+
29
+
30
+ @attrs.define(eq=False)
31
+ class DeferredColumn:
32
+ parent: "Generator[Any]"
33
+ col: "Column | TypedColumn[Any]"
34
+
35
+ def alias(self, alias: str) -> "DeferredColumn":
36
+ self.col = self.to_spark().alias(alias)
37
+ return self
38
+
39
+ def cast(self, data_type: DataType) -> "DeferredColumn":
40
+ self.col = self.to_spark().cast(data_type)
41
+ return self
42
+
43
+ def to_spark(self) -> Column:
44
+ """The wrapped column as a plain Spark column."""
45
+ from typespark.columns.columns import TypedColumn as _TypedColumn
46
+
47
+ return self.col.to_spark() if isinstance(self.col, _TypedColumn) else self.col
48
+
49
+ def _defer(self, col: "Column | TypedColumn[Any]") -> "DeferredColumn":
50
+ """Keep an operation deferred so the generator still materializes first."""
51
+ return DeferredColumn(self.parent, col)
52
+
53
+ # delegate unknown attributes/methods to the wrapped column, staying deferred
54
+ def __getattr__(self, name: str) -> Any:
55
+ from typespark.columns.columns import TypedColumn as _TypedColumn
56
+
57
+ if name.startswith("_") or name in ("col", "parent"):
58
+ raise AttributeError(name)
59
+
60
+ attr = getattr(self.col, name)
61
+ if isinstance(attr, (Column, _TypedColumn)):
62
+ return self._defer(attr)
63
+ if callable(attr):
64
+
65
+ def wrapped(*args: Any, **kwargs: Any) -> Any:
66
+ result = attr(
67
+ *[_operand(a) for a in args],
68
+ **{k: _operand(v) for k, v in kwargs.items()},
69
+ )
70
+ if isinstance(result, (Column, _TypedColumn)):
71
+ return self._defer(result)
72
+ return result
73
+
74
+ return wrapped
75
+ return attr
76
+
77
+ # Operators — mirror TypedColumn's set, wrapping results as deferred.
78
+ def __eq__(self, other: OperandType) -> "DeferredColumn": # type: ignore[override]
79
+ return self._defer(self.to_spark() == _operand(other))
80
+
81
+ def __ne__(self, other: OperandType) -> "DeferredColumn": # type: ignore[override]
82
+ return self._defer(self.to_spark() != _operand(other))
83
+
84
+ def __lt__(self, other: OperandType) -> "DeferredColumn":
85
+ return self._defer(self.to_spark() < _operand(other))
86
+
87
+ def __le__(self, other: OperandType) -> "DeferredColumn":
88
+ return self._defer(self.to_spark() <= _operand(other))
89
+
90
+ def __gt__(self, other: OperandType) -> "DeferredColumn":
91
+ return self._defer(self.to_spark() > _operand(other))
92
+
93
+ def __ge__(self, other: OperandType) -> "DeferredColumn":
94
+ return self._defer(self.to_spark() >= _operand(other))
95
+
96
+ def __add__(self, other: OperandType) -> "DeferredColumn":
97
+ return self._defer(self.to_spark() + _operand(other))
98
+
99
+ def __radd__(self, other: OperandType) -> "DeferredColumn":
100
+ return self._defer(self.to_spark().__radd__(_operand(other)))
101
+
102
+ def __sub__(self, other: OperandType) -> "DeferredColumn":
103
+ return self._defer(self.to_spark() - _operand(other))
104
+
105
+ def __rsub__(self, other: OperandType) -> "DeferredColumn":
106
+ return self._defer(self.to_spark().__rsub__(_operand(other)))
107
+
108
+ def __mul__(self, other: OperandType) -> "DeferredColumn":
109
+ return self._defer(self.to_spark() * _operand(other))
110
+
111
+ def __rmul__(self, other: OperandType) -> "DeferredColumn":
112
+ return self._defer(self.to_spark().__rmul__(_operand(other)))
113
+
114
+ def __truediv__(self, other: OperandType) -> "DeferredColumn":
115
+ return self._defer(self.to_spark() / _operand(other))
116
+
117
+ def __rtruediv__(self, other: OperandType) -> "DeferredColumn":
118
+ return self._defer(self.to_spark().__rtruediv__(_operand(other)))
119
+
120
+ def __mod__(self, other: OperandType) -> "DeferredColumn":
121
+ return self._defer(self.to_spark() % _operand(other))
122
+
123
+ def __pow__(self, other: OperandType) -> "DeferredColumn":
124
+ return self._defer(self.to_spark() ** _operand(other))
125
+
126
+ def __neg__(self) -> "DeferredColumn":
127
+ return self._defer(-self.to_spark())
128
+
129
+ def __and__(self, other: OperandType) -> "DeferredColumn":
130
+ return self._defer(self.to_spark() & _operand(other))
131
+
132
+ def __or__(self, other: OperandType) -> "DeferredColumn":
133
+ return self._defer(self.to_spark() | _operand(other))
134
+
135
+ def __invert__(self) -> "DeferredColumn":
136
+ return self._defer(~self.to_spark())
137
+
138
+
139
+ class Generator[T: TypedColumn[Any]](ABC):
140
+ """When building a dataframe plan, top level projections are separated into two steps."""
141
+
142
+ _parent: T
143
+ _alias: str
144
+ _elem_type: type[T]
145
+
146
+ def __init__(self, parent: T, alias: str, elem_type: type[T]) -> None:
147
+ self._parent = parent
148
+ self._alias = alias
149
+ self._elem_type = elem_type
150
+
151
+ def __hash__(self) -> int:
152
+ return hash(self._alias)
153
+
154
+ def _materialize(self) -> T:
155
+ """The generator's output, referenced by alias — it exists only after step 1."""
156
+ return self._elem_type._set_column(col(self._alias), self._alias)
157
+
158
+ # delegate all unknown attributes/methods to materialized Column
159
+ def __getattr__(self, name: str) -> "DeferredColumn | Any":
160
+ from typespark.columns.columns import TypedColumn as _TypedColumn
161
+
162
+ materialized = self._materialize()
163
+ attr = getattr(materialized, name)
164
+ if isinstance(attr, (Column, _TypedColumn)):
165
+ return DeferredColumn(self, attr)
166
+ return attr
167
+
168
+ @abstractmethod
169
+ def column_operation(self) -> Column: ...
170
+
171
+
172
+ class Exploded[T: TypedColumn[Any]](Generator[T]):
173
+ def column_operation(self) -> Column:
174
+ return explode(self._parent._col).alias(self._alias)