typespark 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. typespark-1.1.0/.gitignore +184 -0
  2. typespark-1.1.0/LICENSE +21 -0
  3. typespark-1.1.0/PKG-INFO +41 -0
  4. typespark-1.1.0/README.md +13 -0
  5. typespark-1.1.0/pyproject.toml +77 -0
  6. typespark-1.1.0/tests/__init__.py +0 -0
  7. typespark-1.1.0/tests/conftest.py +108 -0
  8. typespark-1.1.0/tests/test_array.py +252 -0
  9. typespark-1.1.0/tests/test_basic.py +345 -0
  10. typespark-1.1.0/tests/test_column.py +41 -0
  11. typespark-1.1.0/tests/test_create.py +183 -0
  12. typespark-1.1.0/tests/test_defaults.py +289 -0
  13. typespark-1.1.0/tests/test_functions_agg.py +415 -0
  14. typespark-1.1.0/tests/test_functions_array.py +463 -0
  15. typespark-1.1.0/tests/test_functions_date_time.py +628 -0
  16. typespark-1.1.0/tests/test_functions_higher_order.py +79 -0
  17. typespark-1.1.0/tests/test_functions_math.py +490 -0
  18. typespark-1.1.0/tests/test_functions_misc.py +403 -0
  19. typespark-1.1.0/tests/test_functions_string.py +640 -0
  20. typespark-1.1.0/tests/test_groups.py +127 -0
  21. typespark-1.1.0/tests/test_lazy.py +155 -0
  22. typespark-1.1.0/tests/test_literal_columns.py +222 -0
  23. typespark-1.1.0/tests/test_op_return_types.py +428 -0
  24. typespark-1.1.0/tests/test_operators.py +491 -0
  25. typespark-1.1.0/tests/test_primitive_init.py +282 -0
  26. typespark-1.1.0/tests/test_qualify.py +50 -0
  27. typespark-1.1.0/tests/test_schema.py +187 -0
  28. typespark-1.1.0/tests/test_struct.py +289 -0
  29. typespark-1.1.0/tests/test_subclassing.py +141 -0
  30. typespark-1.1.0/tests/test_typecheck.py +25 -0
  31. typespark-1.1.0/tests/test_window.py +116 -0
  32. typespark-1.1.0/tests/typecheck/enum_when_example.py +81 -0
  33. typespark-1.1.0/tests/typecheck/field_default_example.py +100 -0
  34. typespark-1.1.0/tests/typecheck/timestamp_ntz_example.py +32 -0
  35. typespark-1.1.0/tests/typecheck/window_example.py +46 -0
  36. typespark-1.1.0/tests/utils.py +22 -0
  37. typespark-1.1.0/typespark/__init__.py +80 -0
  38. typespark-1.1.0/typespark/_base.py +184 -0
  39. typespark-1.1.0/typespark/_default_types.py +45 -0
  40. typespark-1.1.0/typespark/_define.py +28 -0
  41. typespark-1.1.0/typespark/_exceptions.py +89 -0
  42. typespark-1.1.0/typespark/columns/__init__.py +43 -0
  43. typespark-1.1.0/typespark/columns/_generator.py +174 -0
  44. typespark-1.1.0/typespark/columns/_groups.py +106 -0
  45. typespark-1.1.0/typespark/columns/array.py +100 -0
  46. typespark-1.1.0/typespark/columns/columns.py +279 -0
  47. typespark-1.1.0/typespark/columns/primitives.py +512 -0
  48. typespark-1.1.0/typespark/columns/struct.py +66 -0
  49. typespark-1.1.0/typespark/columns/utils.py +16 -0
  50. typespark-1.1.0/typespark/data_contract/__init__.py +12 -0
  51. typespark-1.1.0/typespark/data_contract/cli.py +39 -0
  52. typespark-1.1.0/typespark/data_contract/generate.py +266 -0
  53. typespark-1.1.0/typespark/data_contract/mixins.py +15 -0
  54. typespark-1.1.0/typespark/data_contract/serialize.py +139 -0
  55. typespark-1.1.0/typespark/dataframe.py +659 -0
  56. typespark-1.1.0/typespark/field_transforms.py +229 -0
  57. typespark-1.1.0/typespark/functions/__init__.py +480 -0
  58. typespark-1.1.0/typespark/functions/_type_aliases.py +15 -0
  59. typespark-1.1.0/typespark/functions/aggregates.py +190 -0
  60. typespark-1.1.0/typespark/functions/array.py +355 -0
  61. typespark-1.1.0/typespark/functions/date_time.py +415 -0
  62. typespark-1.1.0/typespark/functions/higher_order.py +92 -0
  63. typespark-1.1.0/typespark/functions/math.py +461 -0
  64. typespark-1.1.0/typespark/functions/misc.py +358 -0
  65. typespark-1.1.0/typespark/functions/string.py +556 -0
  66. typespark-1.1.0/typespark/functions/window_functions.py +90 -0
  67. typespark-1.1.0/typespark/literals.py +20 -0
  68. typespark-1.1.0/typespark/metadata.py +636 -0
  69. typespark-1.1.0/typespark/py.typed +0 -0
  70. typespark-1.1.0/typespark/schema/__init__.py +3 -0
  71. typespark-1.1.0/typespark/schema/schema.py +144 -0
  72. typespark-1.1.0/typespark/utils.py +46 -0
  73. typespark-1.1.0/typespark/window.py +70 -0
@@ -0,0 +1,184 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ #uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
116
+ .pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
121
+ __pypackages__/
122
+
123
+ # Celery stuff
124
+ celerybeat-schedule
125
+ celerybeat.pid
126
+
127
+ # SageMath parsed files
128
+ *.sage.py
129
+
130
+ # Environments
131
+ .env
132
+ .venv
133
+ env/
134
+ venv/
135
+ ENV/
136
+ env.bak/
137
+ venv.bak/
138
+
139
+ # Spyder project settings
140
+ .spyderproject
141
+ .spyproject
142
+
143
+ # Rope project settings
144
+ .ropeproject
145
+
146
+ # mkdocs documentation
147
+ /site
148
+
149
+ # mypy
150
+ .mypy_cache/
151
+ .dmypy.json
152
+ dmypy.json
153
+
154
+ # Pyre type checker
155
+ .pyre/
156
+
157
+ # pytype static type analyzer
158
+ .pytype/
159
+
160
+ # Cython debug symbols
161
+ cython_debug/
162
+
163
+ # PyCharm
164
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
165
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
166
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
167
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
168
+ #.idea/
169
+
170
+ # Ruff stuff:
171
+ .ruff_cache/
172
+
173
+ # PyPI configuration file
174
+ .pypirc
175
+
176
+ # Cursor
177
+ # Cursor is an AI-powered code editor.`.cursorignore` specifies files/directories to
178
+ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
179
+ # refer to https://docs.cursor.com/context/ignore-files
180
+ .cursorignore
181
+ .cursorindexingignore
182
+ # Project scratch
183
+ DONE.md
184
+ TODO.md
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 eageby
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,41 @@
1
+ Metadata-Version: 2.5
2
+ Name: typespark
3
+ Version: 1.1.0
4
+ Summary: Static typing for PySpark DataFrames
5
+ Project-URL: Repository, https://github.com/eageby/typespark
6
+ Project-URL: Issues, https://github.com/eageby/typespark/issues
7
+ Author-email: Elias Ågeby <elias.ageby@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: dataframe,pyspark,schema,spark,type-safety,typing
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Database
18
+ Classifier: Topic :: Software Development :: Libraries
19
+ Classifier: Typing :: Typed
20
+ Requires-Python: <4,>=3.12
21
+ Requires-Dist: attrs<26,>=25.1.0
22
+ Requires-Dist: pyyaml<7,>=6.0.2
23
+ Requires-Dist: simple-parsing<0.2,>=0.1.7
24
+ Requires-Dist: typing-extensions>=4.15.0
25
+ Provides-Extra: pyspark
26
+ Requires-Dist: pyspark<4.1,>=3.5.2; extra == 'pyspark'
27
+ Description-Content-Type: text/markdown
28
+
29
+ # Functionality
30
+
31
+ ## Statically typed DataFrames
32
+
33
+ ## Aliasing
34
+
35
+ - Set alias for dataframe which is used to access the dataframe column
36
+
37
+ ## Pyspark Schema generation
38
+
39
+ - provide arguments to data types
40
+ - generate structfields
41
+ - comments as metadata from attribute "docstring" or comment argument in field
@@ -0,0 +1,13 @@
1
+ # Functionality
2
+
3
+ ## Statically typed DataFrames
4
+
5
+ ## Aliasing
6
+
7
+ - Set alias for dataframe which is used to access the dataframe column
8
+
9
+ ## Pyspark Schema generation
10
+
11
+ - provide arguments to data types
12
+ - generate structfields
13
+ - comments as metadata from attribute "docstring" or comment argument in field
@@ -0,0 +1,77 @@
1
+ [project]
2
+ name = "typespark"
3
+ dynamic = ["version"]
4
+ description = "Static typing for PySpark DataFrames"
5
+ authors = [{ name = "Elias Ågeby", email = "elias.ageby@gmail.com" }]
6
+ requires-python = ">=3.12,<4"
7
+ readme = "README.md"
8
+ license = "MIT"
9
+ license-files = ["LICENSE"]
10
+ keywords = ["pyspark", "spark", "dataframe", "typing", "type-safety", "schema"]
11
+ classifiers = [
12
+ "Development Status :: 5 - Production/Stable",
13
+ "Intended Audience :: Developers",
14
+ "Operating System :: OS Independent",
15
+ "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: 3.12",
17
+ "Programming Language :: Python :: 3.13",
18
+ "Topic :: Database",
19
+ "Topic :: Software Development :: Libraries",
20
+ "Typing :: Typed",
21
+ ]
22
+ dependencies = [
23
+ "attrs>=25.1.0,<26",
24
+ "pyyaml>=6.0.2,<7",
25
+ "simple-parsing>=0.1.7,<0.2",
26
+ "typing-extensions>=4.15.0",
27
+ ]
28
+
29
+ [project.optional-dependencies]
30
+ pyspark = ["pyspark>=3.5.2,<4.1"]
31
+
32
+ [project.urls]
33
+ Repository = "https://github.com/eageby/typespark"
34
+ Issues = "https://github.com/eageby/typespark/issues"
35
+
36
+ [project.scripts]
37
+ typespark = "typespark.data_contract.cli:main"
38
+
39
+ [dependency-groups]
40
+ dev = [
41
+ "mypy>=1.15.0,<2",
42
+ "pytest>=8.3.4,<9",
43
+ "pandas>=2.2.3,<3",
44
+ "pyarrow>=19.0.0,<20",
45
+ "prettyprinter>=0.18.0,<0.19",
46
+ "devtools>=0.12.2,<0.13",
47
+ "pyspark>=3.5.2,<4.1",
48
+ "pyright>=1.1.414",
49
+ ]
50
+ datacontract = [
51
+ "black>=25.1.0,<26",
52
+ "simple-parsing>=0.1.7,<0.2",
53
+ "datacontract-specification>=1.2.0,<2",
54
+ "click>=8.2.1,<9",
55
+ ]
56
+
57
+ [tool.uv]
58
+ default-groups = [
59
+ "dev",
60
+ "datacontract",
61
+ ]
62
+
63
+ [build-system]
64
+ requires = ["hatchling", "hatch-vcs"]
65
+ build-backend = "hatchling.build"
66
+
67
+ [tool.hatch.version]
68
+ source = "vcs"
69
+
70
+ [tool.hatch.build.targets.sdist]
71
+ include = ["typespark", "tests", "README.md", "LICENSE"]
72
+
73
+ [tool.hatch.build.targets.wheel]
74
+ packages = ["typespark"]
75
+
76
+ [tool.pytest.ini_options]
77
+ testpaths = ["tests"]
File without changes
@@ -0,0 +1,108 @@
1
+ import pytest
2
+ from pyspark.sql import DataFrame, Row, SparkSession, types
3
+
4
+ from typespark import Int, String
5
+ from typespark.dataframe import BaseDataFrame
6
+
7
+
8
+ @pytest.fixture(scope="module")
9
+ def spark():
10
+ """Creates a single Spark session for all tests."""
11
+ session = (
12
+ SparkSession.Builder().master("local").appName("pytest-pyspark").getOrCreate()
13
+ )
14
+
15
+ yield session # Provide the session to tests
16
+
17
+ session.stop() # Cleanup after tests
18
+
19
+
20
+ @pytest.fixture(scope="function")
21
+ def dataframe(spark):
22
+ """Creates a default dataframe."""
23
+ data = [Row(name="Alice", age=30), Row(name="Bob", age=25)]
24
+
25
+ yield spark.createDataFrame(data)
26
+
27
+
28
+ @pytest.fixture(scope="function")
29
+ def struct_dataframe(spark):
30
+ """Creates a default dataframe."""
31
+ data = [
32
+ Row(struct={"name": "Alice", "age": 30}),
33
+ Row(struct={"name": "Bob", "age": 25}),
34
+ ]
35
+
36
+ yield spark.createDataFrame(data)
37
+
38
+
39
+ @pytest.fixture(scope="function")
40
+ def array_dataframe(spark):
41
+ """Creates a default dataframe."""
42
+ data = [
43
+ Row(elements=[1, 2]),
44
+ Row(elements=[3, 4]),
45
+ ]
46
+
47
+ yield spark.createDataFrame(data)
48
+
49
+
50
+ @pytest.fixture(scope="function")
51
+ def array_struct_dataframe(spark):
52
+ """Creates a default dataframe."""
53
+ data = [
54
+ Row(
55
+ elements=[
56
+ {"name": "Alice", "age": 30},
57
+ {"name": "Bob", "age": 25},
58
+ ]
59
+ )
60
+ ]
61
+
62
+ yield spark.createDataFrame(
63
+ data,
64
+ schema=types.StructType(
65
+ [
66
+ types.StructField(
67
+ "elements",
68
+ types.ArrayType(
69
+ types.StructType(
70
+ [
71
+ types.StructField("name", types.StringType()),
72
+ types.StructField("age", types.IntegerType()),
73
+ ]
74
+ )
75
+ ),
76
+ )
77
+ ],
78
+ ),
79
+ )
80
+
81
+
82
+ class Person(BaseDataFrame):
83
+ name: String
84
+ age: Int
85
+
86
+
87
+ @pytest.fixture(scope="function")
88
+ def person(dataframe: DataFrame):
89
+ yield Person.from_df(dataframe)
90
+
91
+
92
+ class Id(BaseDataFrame):
93
+ value: Int
94
+
95
+
96
+ class Range(BaseDataFrame):
97
+ id: Int
98
+
99
+
100
+ @pytest.fixture(scope="function")
101
+ def id(spark: SparkSession):
102
+ yield Id.from_df(spark.createDataFrame([1, 2, 3, 3, 4], "int"))
103
+
104
+
105
+ @pytest.fixture(scope="function")
106
+ def small_range(spark: SparkSession):
107
+ rf = spark.range(3)
108
+ yield Range.from_df(rf.withColumn("id", rf["id"].cast(types.IntegerType())))
@@ -0,0 +1,252 @@
1
+ import warnings
2
+
3
+ import pytest
4
+ from pyspark.sql import Row, functions as F
5
+ from pyspark.sql import types
6
+ from pyspark.sql.types import StringType
7
+
8
+ from tests.utils import collect_values
9
+ from typespark import Integer, String
10
+ from typespark.columns.array import TypedArrayType
11
+ from typespark.columns.struct import Struct
12
+ from typespark.dataframe import BaseDataFrame
13
+ from typespark.metadata import field
14
+
15
+
16
+ def test_array(array_dataframe):
17
+ class DataClass(BaseDataFrame):
18
+ elements: TypedArrayType[Integer]
19
+
20
+ df = DataClass.from_df(array_dataframe)
21
+
22
+ array_values = collect_values(df.select(df.elements))
23
+
24
+ assert len(array_values) == 2
25
+ assert array_values[0]["elements"] == [1, 2]
26
+ assert array_values[1]["elements"] == [3, 4]
27
+
28
+
29
+ def test_array_explode(array_dataframe):
30
+ class DataClass(BaseDataFrame):
31
+ elements: TypedArrayType[Integer]
32
+
33
+ df = DataClass.from_df(array_dataframe)
34
+
35
+ result = df.select(df.elements.explode())
36
+ result_values = collect_values(result)
37
+
38
+ assert len(result_values) == 4
39
+ assert result_values[0]["elements"] == 1
40
+ assert result_values[1]["elements"] == 2
41
+ assert result_values[2]["elements"] == 3
42
+ assert result_values[3]["elements"] == 4
43
+
44
+
45
+ def test_array_struct(array_struct_dataframe):
46
+ class Nested(Struct):
47
+ name_: String = field(df_alias="name")
48
+ age: Integer
49
+
50
+ class DataClass(BaseDataFrame):
51
+ elements: TypedArrayType[Nested]
52
+
53
+ df = DataClass.from_df(array_struct_dataframe)
54
+
55
+ structs = df.elements.explode()
56
+
57
+ struct_values = collect_values(df.select(structs.age))
58
+
59
+ assert struct_values[0]["elements.age"] == 30
60
+ assert struct_values[1]["elements.age"] == 25
61
+
62
+
63
+ def test_array_struct_multiple(array_struct_dataframe):
64
+ class Nested(Struct):
65
+ name_: String = field(df_alias="name")
66
+ age: Integer
67
+
68
+ class DataClass(BaseDataFrame):
69
+ elements: TypedArrayType[Nested]
70
+
71
+ df = DataClass.from_df(array_struct_dataframe)
72
+
73
+ nested = df.elements.explode()
74
+ struct_values = collect_values(df.select(nested.age, nested.name_))
75
+
76
+ assert struct_values[0]["elements.age"] == 30
77
+ assert struct_values[0]["elements.name"] == "Alice"
78
+ assert struct_values[1]["elements.age"] == 25
79
+ assert struct_values[1]["elements.name"] == "Bob"
80
+
81
+
82
+ def test_array_struct_to_new(array_struct_dataframe):
83
+ class Nested(Struct):
84
+ name_: String = field(df_alias="name")
85
+ age: Integer
86
+
87
+ class New(BaseDataFrame):
88
+ n: String = field(df_alias="name")
89
+ age: Integer
90
+
91
+ class DataClass(BaseDataFrame):
92
+ elements: TypedArrayType[Nested]
93
+
94
+ df = DataClass.from_df(array_struct_dataframe)
95
+
96
+ nested = df.elements.explode()
97
+
98
+ result = New(df, n=nested.name_, age=nested.age)
99
+ result_values = collect_values(result)
100
+
101
+ assert result_values[0]["name"] == "Alice"
102
+ assert result_values[0]["age"] == 30
103
+ assert result_values[1]["name"] == "Bob"
104
+ assert result_values[1]["age"] == 25
105
+
106
+
107
+ def test_array_struct_to_new_with_cast(array_struct_dataframe):
108
+ class Nested(Struct):
109
+ name_: String = field(df_alias="name")
110
+ age: Integer
111
+
112
+ class New(BaseDataFrame):
113
+ n: String = field(df_alias="name")
114
+ age: String
115
+
116
+ class DataClass(BaseDataFrame):
117
+ elements: TypedArrayType[Nested]
118
+
119
+ df = DataClass.from_df(array_struct_dataframe)
120
+
121
+ nested = df.elements.explode()
122
+
123
+ result = New(df, n=nested.name_, age=nested.age.cast(StringType()))
124
+
125
+ result_values = collect_values(result)
126
+
127
+ assert result_values[0]["name"] == "Alice"
128
+ assert result_values[0]["age"] == "30" # After casting
129
+ assert result_values[1]["name"] == "Bob"
130
+ assert result_values[1]["age"] == "25" # After casting
131
+
132
+
133
+ @pytest.fixture(scope="function")
134
+ def id_array_struct_dataframe(spark):
135
+ """A scalar column alongside an array of structs, for mixed selects."""
136
+ schema = types.StructType(
137
+ [
138
+ types.StructField("id", types.IntegerType()),
139
+ types.StructField(
140
+ "elements",
141
+ types.ArrayType(
142
+ types.StructType(
143
+ [
144
+ types.StructField("name", types.StringType()),
145
+ types.StructField("age", types.IntegerType()),
146
+ ]
147
+ )
148
+ ),
149
+ ),
150
+ ]
151
+ )
152
+
153
+ data = [
154
+ Row(
155
+ id=1,
156
+ elements=[
157
+ {"name": "Alice", "age": 30},
158
+ {"name": "Bob", "age": 25},
159
+ ],
160
+ )
161
+ ]
162
+
163
+ yield spark.createDataFrame(data, schema=schema)
164
+
165
+
166
+ class _Nested(Struct):
167
+ name: String
168
+ age: Integer
169
+
170
+
171
+ class _IdArray(BaseDataFrame):
172
+ id: Integer
173
+ elements: TypedArrayType[_Nested]
174
+
175
+
176
+ def test_generator_with_plain_column(id_array_struct_dataframe):
177
+ """Generator column selected alongside an ordinary typed column."""
178
+ df = _IdArray.from_df(id_array_struct_dataframe)
179
+ nested = df.elements.explode()
180
+
181
+ values = collect_values(df.select(nested.age, df.id))
182
+
183
+ assert [v["elements.age"] for v in values] == [30, 25]
184
+ assert [v["id"] for v in values] == [1, 1]
185
+
186
+
187
+ def test_generator_with_calculated_column(id_array_struct_dataframe):
188
+ """Generator column alongside an aliased calculated column."""
189
+ df = _IdArray.from_df(id_array_struct_dataframe)
190
+ nested = df.elements.explode()
191
+
192
+ values = collect_values(df.select(nested.age, (df.id + 1).alias("id_plus")))
193
+
194
+ assert [v["elements.age"] for v in values] == [30, 25]
195
+ assert [v["id_plus"] for v in values] == [2, 2]
196
+
197
+
198
+ def test_generator_with_unaliased_calculated_column(id_array_struct_dataframe):
199
+ """Generator column alongside a calculated column with no explicit alias."""
200
+ df = _IdArray.from_df(id_array_struct_dataframe)
201
+ nested = df.elements.explode()
202
+
203
+ result = df.select(nested.age, df.id + 1)
204
+ values = collect_values(result)
205
+
206
+ assert [v["elements.age"] for v in values] == [30, 25]
207
+ calc_name = [c for c in result.to_spark().columns if c != "elements.age"][0]
208
+ assert [v[calc_name] for v in values] == [2, 2]
209
+
210
+
211
+ def test_calculation_on_generator_column(id_array_struct_dataframe):
212
+ """Arithmetic applied to the generator-derived column itself."""
213
+ df = _IdArray.from_df(id_array_struct_dataframe)
214
+ nested = df.elements.explode()
215
+
216
+ values = collect_values(df.select((nested.age + 1).alias("age_plus")))
217
+
218
+ assert [v["age_plus"] for v in values] == [31, 26]
219
+
220
+
221
+ def test_bare_generator_with_plain_column(id_array_struct_dataframe):
222
+ """Bare generator (no DeferredColumn access) alongside another column."""
223
+ df = _IdArray.from_df(id_array_struct_dataframe)
224
+
225
+ values = collect_values(df.select(df.elements.explode(), df.id))
226
+
227
+ assert [v["elements"] for v in values] == [
228
+ Row(name="Alice", age=30),
229
+ Row(name="Bob", age=25),
230
+ ]
231
+ assert [v["id"] for v in values] == [1, 1]
232
+
233
+
234
+ def test_generator_column_comparison(id_array_struct_dataframe):
235
+ """Comparison on a generator column yields a usable boolean projection."""
236
+ df = _IdArray.from_df(id_array_struct_dataframe)
237
+ nested = df.elements.explode()
238
+
239
+ values = collect_values(df.select((nested.age > 26).alias("is_older")))
240
+
241
+ assert [v["is_older"] for v in values] == [True, False]
242
+
243
+
244
+ def test_bare_typed_array_type_no_error(spark):
245
+ """Regression: bare TypedArrayType(col) must not raise AttributeError."""
246
+ col = F.array(F.lit(1), F.lit(2))
247
+ with warnings.catch_warnings(record=True) as w:
248
+ warnings.simplefilter("always")
249
+ instance = TypedArrayType(col)
250
+ assert instance is not None
251
+ assert any(issubclass(warning.category, UserWarning) for warning in w), \
252
+ "Expected a UserWarning when constructing bare TypedArrayType"