typespark 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- typespark/__init__.py +80 -0
- typespark/_base.py +184 -0
- typespark/_default_types.py +45 -0
- typespark/_define.py +28 -0
- typespark/_exceptions.py +89 -0
- typespark/columns/__init__.py +43 -0
- typespark/columns/_generator.py +174 -0
- typespark/columns/_groups.py +106 -0
- typespark/columns/array.py +100 -0
- typespark/columns/columns.py +279 -0
- typespark/columns/primitives.py +512 -0
- typespark/columns/struct.py +66 -0
- typespark/columns/utils.py +16 -0
- typespark/data_contract/__init__.py +12 -0
- typespark/data_contract/cli.py +39 -0
- typespark/data_contract/generate.py +266 -0
- typespark/data_contract/mixins.py +15 -0
- typespark/data_contract/serialize.py +139 -0
- typespark/dataframe.py +659 -0
- typespark/field_transforms.py +229 -0
- typespark/functions/__init__.py +480 -0
- typespark/functions/_type_aliases.py +15 -0
- typespark/functions/aggregates.py +190 -0
- typespark/functions/array.py +355 -0
- typespark/functions/date_time.py +415 -0
- typespark/functions/higher_order.py +92 -0
- typespark/functions/math.py +461 -0
- typespark/functions/misc.py +358 -0
- typespark/functions/string.py +556 -0
- typespark/functions/window_functions.py +90 -0
- typespark/literals.py +20 -0
- typespark/metadata.py +636 -0
- typespark/py.typed +0 -0
- typespark/schema/__init__.py +3 -0
- typespark/schema/schema.py +144 -0
- typespark/utils.py +46 -0
- typespark/window.py +70 -0
- typespark-1.1.0.dist-info/METADATA +41 -0
- typespark-1.1.0.dist-info/RECORD +42 -0
- typespark-1.1.0.dist-info/WHEEL +4 -0
- typespark-1.1.0.dist-info/entry_points.txt +2 -0
- typespark-1.1.0.dist-info/licenses/LICENSE +21 -0
typespark/__init__.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
__all__ = [
|
|
2
|
+
# Core types
|
|
3
|
+
"TypedColumn",
|
|
4
|
+
"TypedArrayType",
|
|
5
|
+
"ArrayOf",
|
|
6
|
+
"Struct",
|
|
7
|
+
"BaseDataFrame",
|
|
8
|
+
"Window",
|
|
9
|
+
"WindowSpec",
|
|
10
|
+
# Type aliases
|
|
11
|
+
"Array",
|
|
12
|
+
"Bool",
|
|
13
|
+
"Column",
|
|
14
|
+
"DataFrame",
|
|
15
|
+
"String",
|
|
16
|
+
"StringOf",
|
|
17
|
+
"Binary",
|
|
18
|
+
"Short",
|
|
19
|
+
"Integer",
|
|
20
|
+
"IntOf",
|
|
21
|
+
"Int",
|
|
22
|
+
"Long",
|
|
23
|
+
"Float",
|
|
24
|
+
"Double",
|
|
25
|
+
"Decimal",
|
|
26
|
+
"Date",
|
|
27
|
+
"Timestamp",
|
|
28
|
+
"TimestampNTZ",
|
|
29
|
+
# Metadata
|
|
30
|
+
"field",
|
|
31
|
+
"decimal",
|
|
32
|
+
"foreign_key",
|
|
33
|
+
"primary_key",
|
|
34
|
+
# Literals
|
|
35
|
+
"bool_literal",
|
|
36
|
+
"int_literal",
|
|
37
|
+
"string_literal",
|
|
38
|
+
# Exceptions
|
|
39
|
+
"UnnamedColumnError",
|
|
40
|
+
"MissingColumnError",
|
|
41
|
+
"InvalidDefaultColumnError",
|
|
42
|
+
"InvalidLiteralDefaultError",
|
|
43
|
+
"LazyDataFrameError",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
from typespark.columns import ArrayOf, TypedArrayType, TypedColumn
|
|
47
|
+
from typespark.columns.primitives import (
|
|
48
|
+
Binary,
|
|
49
|
+
Bool,
|
|
50
|
+
Date,
|
|
51
|
+
Decimal,
|
|
52
|
+
Double,
|
|
53
|
+
Float,
|
|
54
|
+
Integer,
|
|
55
|
+
IntOf,
|
|
56
|
+
Long,
|
|
57
|
+
Short,
|
|
58
|
+
String,
|
|
59
|
+
StringOf,
|
|
60
|
+
Timestamp,
|
|
61
|
+
TimestampNTZ,
|
|
62
|
+
)
|
|
63
|
+
from typespark.columns.struct import Struct
|
|
64
|
+
from typespark.dataframe import BaseDataFrame
|
|
65
|
+
from typespark._exceptions import (
|
|
66
|
+
InvalidDefaultColumnError,
|
|
67
|
+
InvalidLiteralDefaultError,
|
|
68
|
+
LazyDataFrameError,
|
|
69
|
+
MissingColumnError,
|
|
70
|
+
UnnamedColumnError,
|
|
71
|
+
)
|
|
72
|
+
from typespark.metadata import decimal, field, foreign_key, primary_key
|
|
73
|
+
from typespark.window import Window, WindowSpec
|
|
74
|
+
|
|
75
|
+
from .literals import bool_literal, int_literal, string_literal
|
|
76
|
+
|
|
77
|
+
Array = TypedArrayType
|
|
78
|
+
Column = TypedColumn
|
|
79
|
+
DataFrame = BaseDataFrame
|
|
80
|
+
Int = Integer
|
typespark/_base.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Schema base for TypeSpark.
|
|
3
|
+
|
|
4
|
+
_Base is a plain class (not attrs-decorated) that provides typed column field
|
|
5
|
+
introspection, schema generation, and the central _build() factory method.
|
|
6
|
+
It knows about attrs fields and TypedColumns but NOT about DataFrames.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import (
|
|
12
|
+
TYPE_CHECKING,
|
|
13
|
+
Any,
|
|
14
|
+
Self,
|
|
15
|
+
dataclass_transform,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
import attrs
|
|
19
|
+
from pyspark.sql.types import StructType
|
|
20
|
+
|
|
21
|
+
from typespark import schema
|
|
22
|
+
from typespark._exceptions import InvalidDefaultColumnError, MissingColumnError
|
|
23
|
+
from typespark.columns import TypedColumn, is_typed_column_type
|
|
24
|
+
from typespark.utils import get_field_name, unwrap_type
|
|
25
|
+
|
|
26
|
+
from ._define import define
|
|
27
|
+
from .field_transforms import FieldTransformer
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _convert(f: attrs.Attribute, value: Any, instance: Any) -> Any:
|
|
34
|
+
"""Apply a field's converter like attrs' generated ``__init__`` does."""
|
|
35
|
+
conv = f.converter
|
|
36
|
+
if conv is None:
|
|
37
|
+
return value
|
|
38
|
+
if isinstance(conv, attrs.Converter):
|
|
39
|
+
# attrs' stubs don't expose these attributes.
|
|
40
|
+
args: list[Any] = [value]
|
|
41
|
+
if getattr(conv, "takes_self"):
|
|
42
|
+
args.append(instance)
|
|
43
|
+
if getattr(conv, "takes_field"):
|
|
44
|
+
args.append(f)
|
|
45
|
+
return getattr(conv, "converter")(*args)
|
|
46
|
+
return conv(value)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass_transform(
|
|
50
|
+
frozen_default=True,
|
|
51
|
+
)
|
|
52
|
+
class _Base:
|
|
53
|
+
@property
|
|
54
|
+
def columns(self) -> list[TypedColumn]:
|
|
55
|
+
return [
|
|
56
|
+
getattr(self, field.name)
|
|
57
|
+
for field in attrs.fields(self.__class__)
|
|
58
|
+
if is_typed_column_type(field.type)
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def __init_subclass__(
|
|
63
|
+
cls, field_transformers: list[FieldTransformer] | None = None, **kwargs: Any
|
|
64
|
+
):
|
|
65
|
+
super().__init_subclass__(**kwargs)
|
|
66
|
+
define(cls, field_transformers=field_transformers)
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def _column_aliases(cls):
|
|
70
|
+
yield from [
|
|
71
|
+
get_field_name(field)
|
|
72
|
+
for field in attrs.fields(cls)
|
|
73
|
+
if is_typed_column_type(field.type)
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def generate_schema(cls):
|
|
78
|
+
return schema.generate_schema(cls)
|
|
79
|
+
|
|
80
|
+
def columndict(self) -> dict[str, TypedColumn]:
|
|
81
|
+
return {
|
|
82
|
+
f.name: getattr(self, f.name)
|
|
83
|
+
for f in attrs.fields(self.__class__)
|
|
84
|
+
if is_typed_column_type(f.type)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def _build(cls, source, *, col_ref=None) -> Self:
|
|
89
|
+
"""Construct instance from a data source.
|
|
90
|
+
|
|
91
|
+
source: data container supporting __getitem__ (DataFrame or Column).
|
|
92
|
+
Columns are extracted via source[field_alias].
|
|
93
|
+
Not stored by _Base — callers store it as _dataframe, _col, etc.
|
|
94
|
+
col_ref: optional callable (field_alias: str) -> Column.
|
|
95
|
+
When provided, overrides source[field_alias] for column
|
|
96
|
+
reference creation. Used by from_df for aliased access.
|
|
97
|
+
"""
|
|
98
|
+
new = cls.__new__(cls)
|
|
99
|
+
# DataFrame.columns returns list[str]; Column does not
|
|
100
|
+
raw_columns = getattr(source, "columns", None)
|
|
101
|
+
available = set(raw_columns) if isinstance(raw_columns, list) else None
|
|
102
|
+
|
|
103
|
+
for field_name, f in attrs.fields_dict(cls).items():
|
|
104
|
+
if not is_typed_column_type(f.type):
|
|
105
|
+
continue
|
|
106
|
+
field_alias = get_field_name(f)
|
|
107
|
+
missing = available is not None and field_alias not in available
|
|
108
|
+
|
|
109
|
+
if not missing:
|
|
110
|
+
struct_cls = unwrap_type(f.type)
|
|
111
|
+
if (
|
|
112
|
+
available is not None
|
|
113
|
+
and struct_cls is not None
|
|
114
|
+
and isinstance(struct_cls, type)
|
|
115
|
+
and issubclass(struct_cls, _Base)
|
|
116
|
+
and hasattr(source, "schema")
|
|
117
|
+
):
|
|
118
|
+
try:
|
|
119
|
+
nested_type = source.schema[field_alias].dataType
|
|
120
|
+
except KeyError:
|
|
121
|
+
pass
|
|
122
|
+
else:
|
|
123
|
+
if isinstance(nested_type, StructType):
|
|
124
|
+
actual = set(nested_type.fieldNames())
|
|
125
|
+
for sub_alias in struct_cls._column_aliases():
|
|
126
|
+
if sub_alias not in actual:
|
|
127
|
+
raise MissingColumnError(
|
|
128
|
+
model=struct_cls,
|
|
129
|
+
field_name=sub_alias,
|
|
130
|
+
expected_column=sub_alias,
|
|
131
|
+
available_columns=sorted(actual),
|
|
132
|
+
parent_field=field_alias,
|
|
133
|
+
)
|
|
134
|
+
column = col_ref(field_alias) if col_ref else source[field_alias]
|
|
135
|
+
object.__setattr__(
|
|
136
|
+
new,
|
|
137
|
+
field_name,
|
|
138
|
+
unwrap_type(f.type)._set_column(column, field_alias),
|
|
139
|
+
)
|
|
140
|
+
elif f.default is not attrs.NOTHING:
|
|
141
|
+
default = f.default
|
|
142
|
+
if isinstance(default, attrs.Factory): # type: ignore[arg-type]
|
|
143
|
+
factory: Any = default
|
|
144
|
+
default = (
|
|
145
|
+
factory.factory(new)
|
|
146
|
+
if factory.takes_self
|
|
147
|
+
else factory.factory()
|
|
148
|
+
)
|
|
149
|
+
# As in attrs' __init__, the default goes through the converter.
|
|
150
|
+
# Every column field has one (set_alias), so a non-column default
|
|
151
|
+
# usually fails inside it; report that as an invalid default.
|
|
152
|
+
raw_default = default
|
|
153
|
+
invalid = InvalidDefaultColumnError(
|
|
154
|
+
model=cls,
|
|
155
|
+
field_name=field_name,
|
|
156
|
+
field_alias=field_alias,
|
|
157
|
+
default_value=raw_default,
|
|
158
|
+
)
|
|
159
|
+
try:
|
|
160
|
+
default = _convert(f, default, new)
|
|
161
|
+
except Exception as e:
|
|
162
|
+
if isinstance(raw_default, TypedColumn):
|
|
163
|
+
raise
|
|
164
|
+
raise invalid from e
|
|
165
|
+
if not isinstance(default, TypedColumn):
|
|
166
|
+
raise invalid
|
|
167
|
+
object.__setattr__(
|
|
168
|
+
new,
|
|
169
|
+
field_name,
|
|
170
|
+
unwrap_type(f.type)._set_column(
|
|
171
|
+
default.to_spark().alias(field_alias),
|
|
172
|
+
field_alias,
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
raise MissingColumnError(
|
|
177
|
+
model=cls,
|
|
178
|
+
field_name=field_name,
|
|
179
|
+
expected_column=field_alias,
|
|
180
|
+
available_columns=sorted(available)
|
|
181
|
+
if available is not None
|
|
182
|
+
else None,
|
|
183
|
+
)
|
|
184
|
+
return new
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Static-only return types for ``ts.field(default=<literal>)``.
|
|
2
|
+
|
|
3
|
+
A literal default is typed from the field annotation at runtime, so one literal
|
|
4
|
+
fits several column types (an ``int`` can default a ``Long``, ``Double``,
|
|
5
|
+
``Decimal``, ...). The overloads on ``ts.field`` return one of these classes,
|
|
6
|
+
each a subclass of every column type the literal may default, so a type checker
|
|
7
|
+
accepts ``x: ts.Long = ts.field(default=5)`` and rejects
|
|
8
|
+
``x: ts.String = ts.field(default=5)``.
|
|
9
|
+
|
|
10
|
+
Only imported under ``TYPE_CHECKING``; none of these classes exist at runtime.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from typespark.columns.columns import Bool
|
|
16
|
+
from typespark.columns.primitives import (
|
|
17
|
+
Date,
|
|
18
|
+
Decimal,
|
|
19
|
+
Double,
|
|
20
|
+
Float,
|
|
21
|
+
IntOf,
|
|
22
|
+
Long,
|
|
23
|
+
Short,
|
|
24
|
+
StringOf,
|
|
25
|
+
Timestamp,
|
|
26
|
+
TimestampNTZ,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# The bases disagree on method signatures (e.g. comparison operand types);
|
|
30
|
+
# irrelevant here, since these only serve as assignment targets.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class BoolDefault(Bool): ...
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class StrDefault(StringOf[Any]): ...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class IntDefault(IntOf[Any], Short, Long, Float, Double, Decimal): ... # type: ignore[misc] # pyright: ignore
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class FloatDefault(Float, Double, Decimal): ... # type: ignore[misc] # pyright: ignore
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DatetimeDefault(Timestamp, TimestampNTZ, Date): ... # type: ignore[misc] # pyright: ignore
|
typespark/_define.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import dataclass_transform
|
|
2
|
+
|
|
3
|
+
import attr
|
|
4
|
+
import attrs
|
|
5
|
+
|
|
6
|
+
from typespark.field_transforms import (
|
|
7
|
+
FieldTransformer,
|
|
8
|
+
pipe_tranformers,
|
|
9
|
+
resolve_literal_defaults,
|
|
10
|
+
set_alias,
|
|
11
|
+
to_transformer,
|
|
12
|
+
)
|
|
13
|
+
from typespark.metadata import decimal, field, foreign_key, primary_key
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass_transform(
|
|
17
|
+
field_specifiers=(attrs.field, attr.ib, decimal, foreign_key, primary_key, field),
|
|
18
|
+
)
|
|
19
|
+
def define(cls, field_transformers: list[FieldTransformer] | None = None):
|
|
20
|
+
ft = pipe_tranformers(
|
|
21
|
+
resolve_literal_defaults,
|
|
22
|
+
*(field_transformers or []),
|
|
23
|
+
to_transformer(set_alias),
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
return attrs.define(field_transformer=ft, slots=False, frozen=True, kw_only=True)(
|
|
27
|
+
cls
|
|
28
|
+
)
|
typespark/_exceptions.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from typespark.columns.columns import TypedColumn
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UnnamedColumnError(ValueError):
|
|
5
|
+
def __init__(self, *, column_type: type, operation: str):
|
|
6
|
+
self.column_type = column_type
|
|
7
|
+
self.operation = operation
|
|
8
|
+
|
|
9
|
+
msg = (
|
|
10
|
+
f"Cannot call '{operation}' on an unnamed {column_type.__name__} column. "
|
|
11
|
+
f"Use a column from a typespark DataFrame, or call ._set_name() to assign a name."
|
|
12
|
+
)
|
|
13
|
+
super().__init__(msg)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class MissingColumnError(ValueError):
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
*,
|
|
20
|
+
model: type,
|
|
21
|
+
field_name: str,
|
|
22
|
+
expected_column: str,
|
|
23
|
+
available_columns: list[str] | None = None,
|
|
24
|
+
parent_field: str | None = None,
|
|
25
|
+
):
|
|
26
|
+
self.model = model
|
|
27
|
+
self.field_name = field_name
|
|
28
|
+
self.expected_column = expected_column
|
|
29
|
+
self.available_columns = available_columns
|
|
30
|
+
self.parent_field = parent_field
|
|
31
|
+
|
|
32
|
+
msg = (
|
|
33
|
+
f"Missing required column '{expected_column}' for field '{field_name}' "
|
|
34
|
+
f"in {model.__name__}"
|
|
35
|
+
)
|
|
36
|
+
if parent_field is not None:
|
|
37
|
+
msg += f" (nested under struct column '{parent_field}')"
|
|
38
|
+
msg += ". "
|
|
39
|
+
if available_columns is not None:
|
|
40
|
+
msg += f"Available columns: {available_columns}. "
|
|
41
|
+
msg += f"To fix: provide the column, adjust aliases, or set a default for '{field_name}'."
|
|
42
|
+
super().__init__(msg)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class LazyDataFrameError(RuntimeError):
|
|
46
|
+
def __init__(self, *, model: type, alias: str | None):
|
|
47
|
+
self.model = model
|
|
48
|
+
self.alias = alias
|
|
49
|
+
|
|
50
|
+
msg = (
|
|
51
|
+
f"{model.__name__}.lazy({alias!r}) holds column references only "
|
|
52
|
+
f"— it has no underlying DataFrame. Data operations (to_df, "
|
|
53
|
+
f"to_spark, filter, select, show, write, ...) are unavailable."
|
|
54
|
+
)
|
|
55
|
+
super().__init__(msg)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class InvalidDefaultColumnError(TypeError):
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
*,
|
|
62
|
+
model: type,
|
|
63
|
+
field_name: str,
|
|
64
|
+
field_alias: str,
|
|
65
|
+
default_value: object,
|
|
66
|
+
):
|
|
67
|
+
self.model = model
|
|
68
|
+
self.field_name = field_name
|
|
69
|
+
self.field_alias = field_alias
|
|
70
|
+
self.default_value = default_value
|
|
71
|
+
|
|
72
|
+
msg = (
|
|
73
|
+
f"Default for {model.__name__}.{field_name} must be a "
|
|
74
|
+
f"{TypedColumn.__name__}. "
|
|
75
|
+
f"Got {type(default_value).__name__}: {default_value!r}. "
|
|
76
|
+
f"Field alias: {field_alias!r}. "
|
|
77
|
+
f"Use factory=lambda: <TypedColumn>(...) for a lazy default."
|
|
78
|
+
)
|
|
79
|
+
super().__init__(msg)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class InvalidLiteralDefaultError(ValueError):
|
|
83
|
+
def __init__(self, *, model: type, field_name: str, value: object, reason: str):
|
|
84
|
+
self.model = model
|
|
85
|
+
self.field_name = field_name
|
|
86
|
+
self.value = value
|
|
87
|
+
|
|
88
|
+
msg = f"Invalid default for {model.__name__}.{field_name}: {value!r}. {reason}"
|
|
89
|
+
super().__init__(msg)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
__all__ = [
|
|
2
|
+
"TypedColumn",
|
|
3
|
+
"AliasedTypedColumn",
|
|
4
|
+
"Bool",
|
|
5
|
+
"TypedArrayType",
|
|
6
|
+
"ArrayOf",
|
|
7
|
+
"Binary",
|
|
8
|
+
"Date",
|
|
9
|
+
"Short",
|
|
10
|
+
"Decimal",
|
|
11
|
+
"Double",
|
|
12
|
+
"Float",
|
|
13
|
+
"Int",
|
|
14
|
+
"Integer",
|
|
15
|
+
"IntOf",
|
|
16
|
+
"Long",
|
|
17
|
+
"String",
|
|
18
|
+
"StringOf",
|
|
19
|
+
"Timestamp",
|
|
20
|
+
"TimestampNTZ",
|
|
21
|
+
"is_typed_column_type",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
from .array import ArrayOf, TypedArrayType
|
|
25
|
+
from .columns import *
|
|
26
|
+
from .primitives import (
|
|
27
|
+
Binary,
|
|
28
|
+
Bool,
|
|
29
|
+
Date,
|
|
30
|
+
Decimal,
|
|
31
|
+
Double,
|
|
32
|
+
Float,
|
|
33
|
+
Int,
|
|
34
|
+
Integer,
|
|
35
|
+
IntOf,
|
|
36
|
+
Long,
|
|
37
|
+
Short,
|
|
38
|
+
String,
|
|
39
|
+
StringOf,
|
|
40
|
+
Timestamp,
|
|
41
|
+
TimestampNTZ,
|
|
42
|
+
)
|
|
43
|
+
from .utils import is_typed_column_type
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Union
|
|
5
|
+
|
|
6
|
+
import attrs
|
|
7
|
+
from pyspark.sql import Column
|
|
8
|
+
from pyspark.sql.functions import col, explode
|
|
9
|
+
from pyspark.sql.types import DataType
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from typespark.columns import TypedColumn
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
LiteralType = Union[str, int, float, bool]
|
|
16
|
+
|
|
17
|
+
# What an operator on a deferred column accepts on the right-hand side.
|
|
18
|
+
OperandType = Union["DeferredColumn", Column, "TypedColumn[Any]", LiteralType]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _operand(other: OperandType) -> Column | LiteralType:
|
|
22
|
+
"""Reduce an operand to a Spark column (or literal) usable in an expression."""
|
|
23
|
+
from typespark.columns.columns import TypedColumn as _TypedColumn
|
|
24
|
+
|
|
25
|
+
if isinstance(other, (DeferredColumn, _TypedColumn)):
|
|
26
|
+
return other.to_spark()
|
|
27
|
+
return other
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@attrs.define(eq=False)
|
|
31
|
+
class DeferredColumn:
|
|
32
|
+
parent: "Generator[Any]"
|
|
33
|
+
col: "Column | TypedColumn[Any]"
|
|
34
|
+
|
|
35
|
+
def alias(self, alias: str) -> "DeferredColumn":
|
|
36
|
+
self.col = self.to_spark().alias(alias)
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def cast(self, data_type: DataType) -> "DeferredColumn":
|
|
40
|
+
self.col = self.to_spark().cast(data_type)
|
|
41
|
+
return self
|
|
42
|
+
|
|
43
|
+
def to_spark(self) -> Column:
|
|
44
|
+
"""The wrapped column as a plain Spark column."""
|
|
45
|
+
from typespark.columns.columns import TypedColumn as _TypedColumn
|
|
46
|
+
|
|
47
|
+
return self.col.to_spark() if isinstance(self.col, _TypedColumn) else self.col
|
|
48
|
+
|
|
49
|
+
def _defer(self, col: "Column | TypedColumn[Any]") -> "DeferredColumn":
|
|
50
|
+
"""Keep an operation deferred so the generator still materializes first."""
|
|
51
|
+
return DeferredColumn(self.parent, col)
|
|
52
|
+
|
|
53
|
+
# delegate unknown attributes/methods to the wrapped column, staying deferred
|
|
54
|
+
def __getattr__(self, name: str) -> Any:
|
|
55
|
+
from typespark.columns.columns import TypedColumn as _TypedColumn
|
|
56
|
+
|
|
57
|
+
if name.startswith("_") or name in ("col", "parent"):
|
|
58
|
+
raise AttributeError(name)
|
|
59
|
+
|
|
60
|
+
attr = getattr(self.col, name)
|
|
61
|
+
if isinstance(attr, (Column, _TypedColumn)):
|
|
62
|
+
return self._defer(attr)
|
|
63
|
+
if callable(attr):
|
|
64
|
+
|
|
65
|
+
def wrapped(*args: Any, **kwargs: Any) -> Any:
|
|
66
|
+
result = attr(
|
|
67
|
+
*[_operand(a) for a in args],
|
|
68
|
+
**{k: _operand(v) for k, v in kwargs.items()},
|
|
69
|
+
)
|
|
70
|
+
if isinstance(result, (Column, _TypedColumn)):
|
|
71
|
+
return self._defer(result)
|
|
72
|
+
return result
|
|
73
|
+
|
|
74
|
+
return wrapped
|
|
75
|
+
return attr
|
|
76
|
+
|
|
77
|
+
# Operators — mirror TypedColumn's set, wrapping results as deferred.
|
|
78
|
+
def __eq__(self, other: OperandType) -> "DeferredColumn": # type: ignore[override]
|
|
79
|
+
return self._defer(self.to_spark() == _operand(other))
|
|
80
|
+
|
|
81
|
+
def __ne__(self, other: OperandType) -> "DeferredColumn": # type: ignore[override]
|
|
82
|
+
return self._defer(self.to_spark() != _operand(other))
|
|
83
|
+
|
|
84
|
+
def __lt__(self, other: OperandType) -> "DeferredColumn":
|
|
85
|
+
return self._defer(self.to_spark() < _operand(other))
|
|
86
|
+
|
|
87
|
+
def __le__(self, other: OperandType) -> "DeferredColumn":
|
|
88
|
+
return self._defer(self.to_spark() <= _operand(other))
|
|
89
|
+
|
|
90
|
+
def __gt__(self, other: OperandType) -> "DeferredColumn":
|
|
91
|
+
return self._defer(self.to_spark() > _operand(other))
|
|
92
|
+
|
|
93
|
+
def __ge__(self, other: OperandType) -> "DeferredColumn":
|
|
94
|
+
return self._defer(self.to_spark() >= _operand(other))
|
|
95
|
+
|
|
96
|
+
def __add__(self, other: OperandType) -> "DeferredColumn":
|
|
97
|
+
return self._defer(self.to_spark() + _operand(other))
|
|
98
|
+
|
|
99
|
+
def __radd__(self, other: OperandType) -> "DeferredColumn":
|
|
100
|
+
return self._defer(self.to_spark().__radd__(_operand(other)))
|
|
101
|
+
|
|
102
|
+
def __sub__(self, other: OperandType) -> "DeferredColumn":
|
|
103
|
+
return self._defer(self.to_spark() - _operand(other))
|
|
104
|
+
|
|
105
|
+
def __rsub__(self, other: OperandType) -> "DeferredColumn":
|
|
106
|
+
return self._defer(self.to_spark().__rsub__(_operand(other)))
|
|
107
|
+
|
|
108
|
+
def __mul__(self, other: OperandType) -> "DeferredColumn":
|
|
109
|
+
return self._defer(self.to_spark() * _operand(other))
|
|
110
|
+
|
|
111
|
+
def __rmul__(self, other: OperandType) -> "DeferredColumn":
|
|
112
|
+
return self._defer(self.to_spark().__rmul__(_operand(other)))
|
|
113
|
+
|
|
114
|
+
def __truediv__(self, other: OperandType) -> "DeferredColumn":
|
|
115
|
+
return self._defer(self.to_spark() / _operand(other))
|
|
116
|
+
|
|
117
|
+
def __rtruediv__(self, other: OperandType) -> "DeferredColumn":
|
|
118
|
+
return self._defer(self.to_spark().__rtruediv__(_operand(other)))
|
|
119
|
+
|
|
120
|
+
def __mod__(self, other: OperandType) -> "DeferredColumn":
|
|
121
|
+
return self._defer(self.to_spark() % _operand(other))
|
|
122
|
+
|
|
123
|
+
def __pow__(self, other: OperandType) -> "DeferredColumn":
|
|
124
|
+
return self._defer(self.to_spark() ** _operand(other))
|
|
125
|
+
|
|
126
|
+
def __neg__(self) -> "DeferredColumn":
|
|
127
|
+
return self._defer(-self.to_spark())
|
|
128
|
+
|
|
129
|
+
def __and__(self, other: OperandType) -> "DeferredColumn":
|
|
130
|
+
return self._defer(self.to_spark() & _operand(other))
|
|
131
|
+
|
|
132
|
+
def __or__(self, other: OperandType) -> "DeferredColumn":
|
|
133
|
+
return self._defer(self.to_spark() | _operand(other))
|
|
134
|
+
|
|
135
|
+
def __invert__(self) -> "DeferredColumn":
|
|
136
|
+
return self._defer(~self.to_spark())
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class Generator[T: TypedColumn[Any]](ABC):
|
|
140
|
+
"""When building a dataframe plan, top level projections are separated into two steps."""
|
|
141
|
+
|
|
142
|
+
_parent: T
|
|
143
|
+
_alias: str
|
|
144
|
+
_elem_type: type[T]
|
|
145
|
+
|
|
146
|
+
def __init__(self, parent: T, alias: str, elem_type: type[T]) -> None:
|
|
147
|
+
self._parent = parent
|
|
148
|
+
self._alias = alias
|
|
149
|
+
self._elem_type = elem_type
|
|
150
|
+
|
|
151
|
+
def __hash__(self) -> int:
|
|
152
|
+
return hash(self._alias)
|
|
153
|
+
|
|
154
|
+
def _materialize(self) -> T:
|
|
155
|
+
"""The generator's output, referenced by alias — it exists only after step 1."""
|
|
156
|
+
return self._elem_type._set_column(col(self._alias), self._alias)
|
|
157
|
+
|
|
158
|
+
# delegate all unknown attributes/methods to materialized Column
|
|
159
|
+
def __getattr__(self, name: str) -> "DeferredColumn | Any":
|
|
160
|
+
from typespark.columns.columns import TypedColumn as _TypedColumn
|
|
161
|
+
|
|
162
|
+
materialized = self._materialize()
|
|
163
|
+
attr = getattr(materialized, name)
|
|
164
|
+
if isinstance(attr, (Column, _TypedColumn)):
|
|
165
|
+
return DeferredColumn(self, attr)
|
|
166
|
+
return attr
|
|
167
|
+
|
|
168
|
+
@abstractmethod
|
|
169
|
+
def column_operation(self) -> Column: ...
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class Exploded[T: TypedColumn[Any]](Generator[T]):
|
|
173
|
+
def column_operation(self) -> Column:
|
|
174
|
+
return explode(self._parent._col).alias(self._alias)
|