nested-pandas 0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nested_pandas/__init__.py +8 -0
- nested_pandas/_version.py +16 -0
- nested_pandas/datasets/__init__.py +1 -0
- nested_pandas/datasets/generation.py +84 -0
- nested_pandas/example_benchmarks.py +14 -0
- nested_pandas/nestedframe/__init__.py +2 -0
- nested_pandas/nestedframe/core.py +489 -0
- nested_pandas/nestedframe/io.py +77 -0
- nested_pandas/nestedframe/utils.py +35 -0
- nested_pandas/py.typed +0 -0
- nested_pandas/series/__init__.py +0 -0
- nested_pandas/series/accessor.py +286 -0
- nested_pandas/series/dtype.py +225 -0
- nested_pandas/series/ext_array.py +742 -0
- nested_pandas/series/packer.py +300 -0
- nested_pandas/series/utils.py +41 -0
- nested_pandas/utils/__init__.py +1 -0
- nested_pandas/utils/utils.py +44 -0
- nested_pandas-0.1.dist-info/LICENSE +21 -0
- nested_pandas-0.1.dist-info/METADATA +85 -0
- nested_pandas-0.1.dist-info/RECORD +23 -0
- nested_pandas-0.1.dist-info/WHEEL +5 -0
- nested_pandas-0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# typing.Self and "|" union syntax don't exist in Python 3.9
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from pandas._libs import lib
|
|
6
|
+
from pandas._typing import (
|
|
7
|
+
DtypeBackend,
|
|
8
|
+
FilePath,
|
|
9
|
+
ReadBuffer,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
from .core import NestedFrame
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_parquet(
|
|
16
|
+
data: FilePath | ReadBuffer[bytes],
|
|
17
|
+
to_pack: dict | None = None,
|
|
18
|
+
columns: list[str] | None = None,
|
|
19
|
+
pack_columns: dict | None = None,
|
|
20
|
+
dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
|
|
21
|
+
) -> NestedFrame:
|
|
22
|
+
"""
|
|
23
|
+
Load a parquet object from a file path and load a set of other
|
|
24
|
+
parquet objects to pack into the resulting NestedFrame.
|
|
25
|
+
|
|
26
|
+
Docstring based on the Pandas equivalent.
|
|
27
|
+
|
|
28
|
+
#TODO after MVP: Include full kwarg-set
|
|
29
|
+
#TODO: Switch dtype backend default?
|
|
30
|
+
|
|
31
|
+
Parameters
|
|
32
|
+
----------
|
|
33
|
+
data : str, path object or file-like object
|
|
34
|
+
String, path object (implementing ``os.PathLike[str]``), or file-like
|
|
35
|
+
object implementing a binary ``read()`` function.
|
|
36
|
+
The string could be a URL. Valid URL schemes include http, ftp, s3,
|
|
37
|
+
gs, and file. For file URLs, a host is expected. A local file could be:
|
|
38
|
+
``file://localhost/path/to/table.parquet``.
|
|
39
|
+
A file URL can also be a path to a directory that contains multiple
|
|
40
|
+
partitioned parquet files. Both pyarrow and fastparquet support
|
|
41
|
+
paths to directories as well as file URLs. A directory path could be:
|
|
42
|
+
``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``.
|
|
43
|
+
to_pack: dict, default=None
|
|
44
|
+
A dictionary of parquet data paths (same criteria as `data`), where
|
|
45
|
+
each key reflects the desired column name to pack the data into and
|
|
46
|
+
each value reflects the parquet data to pack. If None, it assumes
|
|
47
|
+
that any data to pack is already packed as a column within `data`.
|
|
48
|
+
columns : list, default=None
|
|
49
|
+
If not None, only these columns will be read from the file.
|
|
50
|
+
pack_columns: dict, default=None
|
|
51
|
+
If not None, selects a set of columns from each keyed nested parquet
|
|
52
|
+
object to read from the nested files.
|
|
53
|
+
dtype_backend : {{'numpy_nullable', 'pyarrow'}}, default 'numpy_nullable'
|
|
54
|
+
Back-end data type applied to the resultant :class:`DataFrame`
|
|
55
|
+
(still experimental). Behaviour is as follows:
|
|
56
|
+
|
|
57
|
+
* ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
|
|
58
|
+
(default).
|
|
59
|
+
* ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
|
|
60
|
+
DataFrame.
|
|
61
|
+
|
|
62
|
+
Returns
|
|
63
|
+
-------
|
|
64
|
+
NestedFrame
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
df = NestedFrame(pd.read_parquet(data, engine="pyarrow", columns=columns, dtype_backend=dtype_backend))
|
|
68
|
+
if to_pack is None:
|
|
69
|
+
return df
|
|
70
|
+
for pack_key in to_pack:
|
|
71
|
+
col_subset = pack_columns.get(pack_key, None) if pack_columns is not None else None
|
|
72
|
+
packed = pd.read_parquet(
|
|
73
|
+
to_pack[pack_key], engine="pyarrow", columns=col_subset, dtype_backend=dtype_backend
|
|
74
|
+
)
|
|
75
|
+
df = df.add_nested(packed, pack_key)
|
|
76
|
+
|
|
77
|
+
return df
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
def _ensure_spacing(expr) -> str:
|
|
2
|
+
"""Ensure that an eval string has spacing"""
|
|
3
|
+
single_val_operators = {"+", "-", "*", "/", "%", ">", "<", "|", "&", "~", "="} # omit "(" and ")"
|
|
4
|
+
check_for_doubles = {"=", "/", "*", ">", "<"}
|
|
5
|
+
double_val_operators = {"==", "//", "**", ">=", "<="}
|
|
6
|
+
expr_list = expr
|
|
7
|
+
|
|
8
|
+
i = 0
|
|
9
|
+
spaced_expr = ""
|
|
10
|
+
while i < len(expr_list):
|
|
11
|
+
if expr_list[i] not in single_val_operators:
|
|
12
|
+
spaced_expr += expr_list[i]
|
|
13
|
+
else:
|
|
14
|
+
if expr_list[i] in check_for_doubles:
|
|
15
|
+
if "".join(expr_list[i : i + 2]) in double_val_operators:
|
|
16
|
+
if spaced_expr[-1] != " ":
|
|
17
|
+
spaced_expr += " "
|
|
18
|
+
spaced_expr += expr_list[i : i + 2]
|
|
19
|
+
if expr_list[i + 2] != " ":
|
|
20
|
+
spaced_expr += " "
|
|
21
|
+
i += 1 # skip ahead an extra time
|
|
22
|
+
else:
|
|
23
|
+
if spaced_expr[-1] != " ":
|
|
24
|
+
spaced_expr += " "
|
|
25
|
+
spaced_expr += expr_list[i]
|
|
26
|
+
if expr_list[i + 1] != " ":
|
|
27
|
+
spaced_expr += " "
|
|
28
|
+
else:
|
|
29
|
+
if spaced_expr[-1] != " ":
|
|
30
|
+
spaced_expr += " "
|
|
31
|
+
spaced_expr += expr_list[i]
|
|
32
|
+
if expr_list[i + 1] != " ":
|
|
33
|
+
spaced_expr += " "
|
|
34
|
+
i += 1
|
|
35
|
+
return spaced_expr
|
nested_pandas/py.typed
ADDED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
# Python 3.9 doesn't support "|" for types
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Generator, MutableMapping
|
|
5
|
+
from typing import cast
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
from numpy.typing import ArrayLike
|
|
11
|
+
from pandas.api.extensions import register_series_accessor
|
|
12
|
+
|
|
13
|
+
from nested_pandas.series.dtype import NestedDtype
|
|
14
|
+
from nested_pandas.series.packer import pack_sorted_df_into_struct
|
|
15
|
+
|
|
16
|
+
__all__ = ["NestSeriesAccessor"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@register_series_accessor("nest")
|
|
20
|
+
class NestSeriesAccessor(MutableMapping):
|
|
21
|
+
"""Accessor for operations on Series of NestedDtype
|
|
22
|
+
|
|
23
|
+
This accessor implements `MutableMapping` interface over the fields of the
|
|
24
|
+
struct, so you can access, change and delete the fields as if it was a
|
|
25
|
+
dictionary, with `[]`, `[] =` and `del` operators.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, series):
|
|
29
|
+
self._check_series(series)
|
|
30
|
+
|
|
31
|
+
self._series = series
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def _check_series(series):
|
|
35
|
+
dtype = series.dtype
|
|
36
|
+
if not isinstance(dtype, NestedDtype):
|
|
37
|
+
raise AttributeError(f"Can only use .nest accessor with a Series of NestedDtype, got {dtype}")
|
|
38
|
+
|
|
39
|
+
def to_lists(self, fields: list[str] | None = None) -> pd.DataFrame:
|
|
40
|
+
"""Convert nested series into dataframe of list-array columns
|
|
41
|
+
|
|
42
|
+
Parameters
|
|
43
|
+
----------
|
|
44
|
+
fields : list[str] or None, optional
|
|
45
|
+
Names of the fields to include. Default is None, which means all fields.
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
pd.DataFrame
|
|
50
|
+
Dataframe of list-arrays.
|
|
51
|
+
"""
|
|
52
|
+
fields = fields if fields is not None else list(self._series.array.field_names)
|
|
53
|
+
if len(fields) == 0:
|
|
54
|
+
raise ValueError("Cannot convert a struct with no fields to lists")
|
|
55
|
+
|
|
56
|
+
struct_array = cast(pa.StructArray, pa.array(self._series))
|
|
57
|
+
|
|
58
|
+
list_series = {}
|
|
59
|
+
for field in fields:
|
|
60
|
+
list_array = cast(pa.ListArray, struct_array.field(field))
|
|
61
|
+
list_series[field] = pd.Series(
|
|
62
|
+
list_array,
|
|
63
|
+
dtype=pd.ArrowDtype(list_array.type),
|
|
64
|
+
index=self._series.index,
|
|
65
|
+
name=field,
|
|
66
|
+
copy=False,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
return pd.DataFrame(list_series)
|
|
70
|
+
|
|
71
|
+
def to_flat(self, fields: list[str] | None = None) -> pd.DataFrame:
|
|
72
|
+
"""Convert nested series into dataframe of flat arrays
|
|
73
|
+
|
|
74
|
+
Parameters
|
|
75
|
+
----------
|
|
76
|
+
fields : list[str] or None, optional
|
|
77
|
+
Names of the fields to include. Default is None, which means all fields.
|
|
78
|
+
|
|
79
|
+
Returns
|
|
80
|
+
-------
|
|
81
|
+
pd.DataFrame
|
|
82
|
+
Dataframe of flat arrays.
|
|
83
|
+
"""
|
|
84
|
+
fields = fields if fields is not None else list(self._series.array.field_names)
|
|
85
|
+
if len(fields) == 0:
|
|
86
|
+
raise ValueError("Cannot flatten a struct with no fields")
|
|
87
|
+
|
|
88
|
+
struct_array = cast(pa.StructArray, pa.array(self._series))
|
|
89
|
+
|
|
90
|
+
flat_series = {}
|
|
91
|
+
index = None
|
|
92
|
+
for field in fields:
|
|
93
|
+
list_array = cast(pa.ListArray, struct_array.field(field))
|
|
94
|
+
flat_array = list_array.flatten()
|
|
95
|
+
if index is None:
|
|
96
|
+
index = self.get_flat_index()
|
|
97
|
+
flat_series[field] = pd.Series(
|
|
98
|
+
flat_array,
|
|
99
|
+
index=pd.Series(index, name=self._series.index.name),
|
|
100
|
+
name=field,
|
|
101
|
+
copy=False,
|
|
102
|
+
dtype=pd.ArrowDtype(flat_array.type),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
return pd.DataFrame(flat_series)
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def flat_length(self) -> int:
|
|
109
|
+
"""Length of the flat arrays"""
|
|
110
|
+
return self._series.array.flat_length
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def fields(self) -> list[str]:
|
|
114
|
+
"""Names of the nested columns"""
|
|
115
|
+
return self._series.array.field_names
|
|
116
|
+
|
|
117
|
+
def set_flat_field(self, field: str, value: ArrayLike) -> None:
|
|
118
|
+
"""Set the field from flat-array of values, in-place
|
|
119
|
+
|
|
120
|
+
Parameters
|
|
121
|
+
----------
|
|
122
|
+
field : str
|
|
123
|
+
Name of the field to set. If not present, it will be added.
|
|
124
|
+
value : ArrayLike
|
|
125
|
+
Array of values to set. It must be a scalar or have the same length
|
|
126
|
+
as the flat arrays, e.g. `self.flat_length`.
|
|
127
|
+
"""
|
|
128
|
+
self._series.array.set_flat_field(field, value)
|
|
129
|
+
|
|
130
|
+
def set_list_field(self, field: str, value: ArrayLike) -> None:
|
|
131
|
+
"""Set the field from list-array, in-place
|
|
132
|
+
|
|
133
|
+
Parameters
|
|
134
|
+
----------
|
|
135
|
+
field : str
|
|
136
|
+
Name of the field to set. If not present, it will be added.
|
|
137
|
+
value : ArrayLike
|
|
138
|
+
Array of values to set. It must be a list-array of the same length
|
|
139
|
+
as the series, e.g. length of the series.
|
|
140
|
+
"""
|
|
141
|
+
self._series.array.set_list_field(field, value)
|
|
142
|
+
|
|
143
|
+
# I intentionally don't call it `drop` or `drop_field` because `pd.DataFrame.drop` is not inplace
|
|
144
|
+
# by default, and I wouldn't like to surprise the user.
|
|
145
|
+
def pop_field(self, field: str) -> pd.Series:
|
|
146
|
+
"""Delete the field from the struct and return it.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
----------
|
|
150
|
+
field : str
|
|
151
|
+
Name of the field to delete.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
pd.Series
|
|
156
|
+
The deleted field.
|
|
157
|
+
"""
|
|
158
|
+
series = self[field]
|
|
159
|
+
self._series.array.pop_field(field)
|
|
160
|
+
return series
|
|
161
|
+
|
|
162
|
+
def query_flat(self, query: str) -> pd.Series:
|
|
163
|
+
"""Query the flat arrays with a boolean expression
|
|
164
|
+
|
|
165
|
+
Currently, it will remove empty rows from the output series.
|
|
166
|
+
# TODO: preserve the index keeping empty rows
|
|
167
|
+
|
|
168
|
+
Parameters
|
|
169
|
+
----------
|
|
170
|
+
query : str
|
|
171
|
+
Boolean expression to filter the rows.
|
|
172
|
+
|
|
173
|
+
Returns
|
|
174
|
+
-------
|
|
175
|
+
pd.Series
|
|
176
|
+
The filtered series.
|
|
177
|
+
"""
|
|
178
|
+
flat = self.to_flat().query(query)
|
|
179
|
+
if len(flat) == 0:
|
|
180
|
+
return pd.Series([], dtype=self._series.dtype)
|
|
181
|
+
return pack_sorted_df_into_struct(flat)
|
|
182
|
+
|
|
183
|
+
def get_flat_index(self) -> pd.Index:
|
|
184
|
+
"""Index of the flat arrays"""
|
|
185
|
+
flat_index = np.repeat(self._series.index, np.diff(self._series.array.list_offsets))
|
|
186
|
+
# pd.Index supports np.repeat, so flat_index is the same type as self._series.index
|
|
187
|
+
flat_index = cast(pd.Index, flat_index)
|
|
188
|
+
return flat_index
|
|
189
|
+
|
|
190
|
+
def get_flat_series(self, field: str) -> pd.Series:
|
|
191
|
+
"""Get the flat-array field as a Series
|
|
192
|
+
|
|
193
|
+
Parameters
|
|
194
|
+
----------
|
|
195
|
+
field : str
|
|
196
|
+
Name of the field to get.
|
|
197
|
+
|
|
198
|
+
Returns
|
|
199
|
+
-------
|
|
200
|
+
pd.Series
|
|
201
|
+
The flat-array field.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
# TODO: we should make proper missed values handling here
|
|
205
|
+
|
|
206
|
+
struct_array = cast(pa.StructArray, pa.array(self._series))
|
|
207
|
+
list_array = cast(pa.ListArray, struct_array.field(field))
|
|
208
|
+
flat_array = list_array.flatten()
|
|
209
|
+
return pd.Series(
|
|
210
|
+
flat_array,
|
|
211
|
+
dtype=pd.ArrowDtype(flat_array.type),
|
|
212
|
+
index=self.get_flat_index(),
|
|
213
|
+
name=field,
|
|
214
|
+
copy=False,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
def get_list_series(self, field: str) -> pd.Series:
|
|
218
|
+
"""Get the list-array field as a Series
|
|
219
|
+
|
|
220
|
+
Parameters
|
|
221
|
+
----------
|
|
222
|
+
field : str
|
|
223
|
+
Name of the field to get.
|
|
224
|
+
|
|
225
|
+
Returns
|
|
226
|
+
-------
|
|
227
|
+
pd.Series
|
|
228
|
+
The list-array field.
|
|
229
|
+
"""
|
|
230
|
+
struct_array = cast(pa.StructArray, pa.array(self._series))
|
|
231
|
+
list_array = struct_array.field(field)
|
|
232
|
+
return pd.Series(
|
|
233
|
+
list_array,
|
|
234
|
+
dtype=pd.ArrowDtype(list_array.type),
|
|
235
|
+
index=self._series.index,
|
|
236
|
+
name=field,
|
|
237
|
+
copy=False,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
def __getitem__(self, key: str | list[str]) -> pd.Series:
|
|
241
|
+
if isinstance(key, list):
|
|
242
|
+
new_array = self._series.array.view_fields(key)
|
|
243
|
+
return pd.Series(new_array, index=self._series.index, name=self._series.name)
|
|
244
|
+
|
|
245
|
+
return self.get_flat_series(key)
|
|
246
|
+
|
|
247
|
+
def __setitem__(self, key: str, value: ArrayLike) -> None:
|
|
248
|
+
# TODO: we can be much-much smarter about the performance here
|
|
249
|
+
# TODO: think better about underlying pa.ChunkArray in both self._series.array and value
|
|
250
|
+
|
|
251
|
+
ndim = np.ndim(value)
|
|
252
|
+
|
|
253
|
+
# Everything is empty, do nothing
|
|
254
|
+
if len(self._series) == 0 and ndim != 0:
|
|
255
|
+
array = pa.array(value, from_pandas=True)
|
|
256
|
+
if len(array) == 0:
|
|
257
|
+
return
|
|
258
|
+
|
|
259
|
+
# Set single value for all rows
|
|
260
|
+
if ndim == 0:
|
|
261
|
+
self.set_flat_field(key, value)
|
|
262
|
+
return
|
|
263
|
+
|
|
264
|
+
if isinstance(value, pd.Series) and not self.get_flat_index().equals(value.index):
|
|
265
|
+
raise ValueError("Cannot set field with a Series of different index")
|
|
266
|
+
|
|
267
|
+
pa_array = pa.array(value, from_pandas=True)
|
|
268
|
+
|
|
269
|
+
# Input is a flat array of values
|
|
270
|
+
if len(pa_array) != self.flat_length:
|
|
271
|
+
ValueError(
|
|
272
|
+
f"Cannot set field {key} with value of length {len(pa_array)}, the value is expected to be "
|
|
273
|
+
f"either a scalar, a 'flat' array of length {self.flat_length}, or a 'list' array of length "
|
|
274
|
+
f"{len(self._series)}."
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
self.set_flat_field(key, pa_array)
|
|
278
|
+
|
|
279
|
+
def __delitem__(self, key: str) -> None:
|
|
280
|
+
self.pop_field(key)
|
|
281
|
+
|
|
282
|
+
def __iter__(self) -> Generator[str, None, None]:
|
|
283
|
+
yield from iter(self._series.array.field_names)
|
|
284
|
+
|
|
285
|
+
def __len__(self) -> int:
|
|
286
|
+
return len(self._series.array.field_names)
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Use Self, which is not available until Python 3.11
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
|
|
6
|
+
# We use Type, because we must use "type" as an attribute name
|
|
7
|
+
from typing import Type, cast # noqa: UP035
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import pyarrow as pa
|
|
11
|
+
from pandas import ArrowDtype
|
|
12
|
+
from pandas.api.extensions import register_extension_dtype
|
|
13
|
+
from pandas.core.arrays import ExtensionArray
|
|
14
|
+
from pandas.core.dtypes.base import ExtensionDtype
|
|
15
|
+
|
|
16
|
+
from nested_pandas.series.utils import is_pa_type_a_list
|
|
17
|
+
|
|
18
|
+
__all__ = ["NestedDtype"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@register_extension_dtype
|
|
22
|
+
class NestedDtype(ExtensionDtype):
|
|
23
|
+
"""Data type to handle packed time series data"""
|
|
24
|
+
|
|
25
|
+
# ExtensionDtype overrides #
|
|
26
|
+
|
|
27
|
+
_metadata = ("pyarrow_dtype",)
|
|
28
|
+
"""Attributes to use as metadata for __eq__ and __hash__"""
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def na_value(self) -> Type[pd.NA]:
|
|
32
|
+
"""The missing value for this dtype"""
|
|
33
|
+
return pd.NA
|
|
34
|
+
|
|
35
|
+
type = pd.DataFrame
|
|
36
|
+
"""The type of the array's elements, always pd.DataFrame"""
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def name(self) -> str:
|
|
40
|
+
"""The string representation of the nested type"""
|
|
41
|
+
fields = ", ".join([f"{field.name}: [{field.type.value_type!s}]" for field in self.pyarrow_dtype])
|
|
42
|
+
return f"nested<{fields}>"
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def construct_array_type(cls) -> Type[ExtensionArray]:
|
|
46
|
+
"""Corresponded array type, always NestedExtensionArray"""
|
|
47
|
+
from nested_pandas.series.ext_array import NestedExtensionArray
|
|
48
|
+
|
|
49
|
+
return NestedExtensionArray
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def construct_from_string(cls, string: str) -> Self: # type: ignore[name-defined] # noqa: F821
|
|
53
|
+
"""Construct NestedDtype from a string representation.
|
|
54
|
+
|
|
55
|
+
This works only for simple types, i.e. non-parametric pyarrow types.
|
|
56
|
+
|
|
57
|
+
Parameters
|
|
58
|
+
----------
|
|
59
|
+
string : str
|
|
60
|
+
The string representation of the nested type. For example,
|
|
61
|
+
'nested<x: [int64], y: [float64]'. It must be consistent with
|
|
62
|
+
the string representation of the dtype given by the `name`
|
|
63
|
+
attribute.
|
|
64
|
+
|
|
65
|
+
Returns
|
|
66
|
+
-------
|
|
67
|
+
NestedDtype
|
|
68
|
+
The constructed NestedDtype.
|
|
69
|
+
|
|
70
|
+
Raises
|
|
71
|
+
------
|
|
72
|
+
TypeError
|
|
73
|
+
If the string is not a valid nested type string or if the element types
|
|
74
|
+
are parametric pyarrow types.
|
|
75
|
+
"""
|
|
76
|
+
if not string.startswith("nested<") or not string.endswith(">"):
|
|
77
|
+
raise TypeError("Not a valid nested type string, expected 'nested<...>'")
|
|
78
|
+
fields_str = string.removeprefix("nested<").removesuffix(">")
|
|
79
|
+
|
|
80
|
+
field_strings = fields_str.split(", ")
|
|
81
|
+
|
|
82
|
+
fields = {}
|
|
83
|
+
for field_string in field_strings:
|
|
84
|
+
try:
|
|
85
|
+
field_name, field_type = field_string.split(": ", maxsplit=1)
|
|
86
|
+
except ValueError as e:
|
|
87
|
+
raise TypeError(
|
|
88
|
+
"Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
|
|
89
|
+
f"string '{field_string}'"
|
|
90
|
+
) from e
|
|
91
|
+
if not field_type.startswith("[") or not field_type.endswith("]"):
|
|
92
|
+
raise TypeError(
|
|
93
|
+
"Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
|
|
94
|
+
f"type string '{field_type}'"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
value_type = field_type.removeprefix("[").removesuffix("]")
|
|
98
|
+
# We follow ArrowDtype implementation heere and do not try to parse complex types
|
|
99
|
+
try:
|
|
100
|
+
pa_value_type = pa.type_for_alias(value_type)
|
|
101
|
+
except ValueError as e:
|
|
102
|
+
raise TypeError(
|
|
103
|
+
f"Parsing pyarrow specific parameters in the string is not supported yet: {value_type}. "
|
|
104
|
+
"Please use NestedDtype() or NestedDtype.from_fields() instead."
|
|
105
|
+
) from e
|
|
106
|
+
|
|
107
|
+
fields[field_name] = pa_value_type
|
|
108
|
+
|
|
109
|
+
return cls.from_fields(fields)
|
|
110
|
+
|
|
111
|
+
# ArrowDtype would return None so we do
|
|
112
|
+
def _get_common_dtype(self, dtypes: list) -> None:
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
# Optional methods #
|
|
116
|
+
|
|
117
|
+
def __from_arrow__(self, array: pa.Array | pa.ChunkedArray) -> ExtensionArray:
|
|
118
|
+
"""Construct a NestedExtensionArray from a pyarrow array.
|
|
119
|
+
|
|
120
|
+
Parameters
|
|
121
|
+
----------
|
|
122
|
+
array : pa.Array | pa.ChunkedArray
|
|
123
|
+
The input pyarrow array.
|
|
124
|
+
|
|
125
|
+
Returns
|
|
126
|
+
-------
|
|
127
|
+
NestedExtensionArray
|
|
128
|
+
The constructed NestedExtensionArray.
|
|
129
|
+
"""
|
|
130
|
+
from nested_pandas.series.ext_array import NestedExtensionArray
|
|
131
|
+
|
|
132
|
+
return NestedExtensionArray(array)
|
|
133
|
+
|
|
134
|
+
# Additional methods and attributes #
|
|
135
|
+
|
|
136
|
+
pyarrow_dtype: pa.StructType
|
|
137
|
+
|
|
138
|
+
def __init__(self, pyarrow_dtype: pa.DataType) -> None:
|
|
139
|
+
self.pyarrow_dtype = self._validate_dtype(pyarrow_dtype)
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def from_fields(cls, fields: Mapping[str, pa.DataType]) -> Self: # type: ignore[name-defined] # noqa: F821
|
|
143
|
+
"""Make NestedDtype from a mapping of field names and list item types.
|
|
144
|
+
|
|
145
|
+
Parameters
|
|
146
|
+
----------
|
|
147
|
+
fields : Mapping[str, pa.DataType]
|
|
148
|
+
A mapping of field names and their item types. Since all fields are lists, the item types are
|
|
149
|
+
inner types of the lists, not the list types themselves.
|
|
150
|
+
|
|
151
|
+
Returns
|
|
152
|
+
-------
|
|
153
|
+
NestedDtype
|
|
154
|
+
The constructed NestedDtype.
|
|
155
|
+
|
|
156
|
+
Examples
|
|
157
|
+
--------
|
|
158
|
+
>>> dtype = NestedDtype.from_fields({"a": pa.float64(), "b": pa.int64()})
|
|
159
|
+
>>> dtype
|
|
160
|
+
nested<a: [double], b: [int64]>
|
|
161
|
+
>>> assert (
|
|
162
|
+
... dtype.pyarrow_dtype
|
|
163
|
+
... == pa.struct({"a": pa.list_(pa.float64()), "b": pa.list_(pa.int64())})
|
|
164
|
+
... )
|
|
165
|
+
"""
|
|
166
|
+
pyarrow_dtype = pa.struct({field: pa.list_(pa_type) for field, pa_type in fields.items()})
|
|
167
|
+
pyarrow_dtype = cast(pa.StructType, pyarrow_dtype)
|
|
168
|
+
return cls(pyarrow_dtype=pyarrow_dtype)
|
|
169
|
+
|
|
170
|
+
@staticmethod
|
|
171
|
+
def _validate_dtype(pyarrow_dtype: pa.DataType) -> pa.StructType:
|
|
172
|
+
if not isinstance(pyarrow_dtype, pa.DataType):
|
|
173
|
+
raise TypeError(f"Expected a 'pyarrow.DataType' object, got {type(pyarrow_dtype)}")
|
|
174
|
+
if not pa.types.is_struct(pyarrow_dtype):
|
|
175
|
+
raise ValueError("NestedDtype can only be constructed with pyarrow struct type.")
|
|
176
|
+
pyarrow_dtype = cast(pa.StructType, pyarrow_dtype)
|
|
177
|
+
|
|
178
|
+
for field in pyarrow_dtype:
|
|
179
|
+
if not is_pa_type_a_list(field.type):
|
|
180
|
+
raise ValueError(
|
|
181
|
+
"NestedDtype can only be constructed with pyarrow struct type, all fields must be list "
|
|
182
|
+
f"type. Given struct has unsupported field {field}"
|
|
183
|
+
)
|
|
184
|
+
return pyarrow_dtype
|
|
185
|
+
|
|
186
|
+
@property
|
|
187
|
+
def fields(self) -> dict[str, pa.DataType]:
|
|
188
|
+
"""The mapping of field names and their item types."""
|
|
189
|
+
return {field.name: field.type.value_type for field in self.pyarrow_dtype}
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def field_names(self) -> list[str]:
|
|
193
|
+
"""The list of field names of the nested type"""
|
|
194
|
+
return [field.name for field in self.pyarrow_dtype]
|
|
195
|
+
|
|
196
|
+
@classmethod
|
|
197
|
+
def from_pandas_arrow_dtype(cls, pandas_arrow_dtype: ArrowDtype):
|
|
198
|
+
"""Construct NestedDtype from a pandas.ArrowDtype.
|
|
199
|
+
|
|
200
|
+
Parameters
|
|
201
|
+
----------
|
|
202
|
+
pandas_arrow_dtype : ArrowDtype
|
|
203
|
+
The pandas.ArrowDtype to construct NestedDtype from.
|
|
204
|
+
|
|
205
|
+
Returns
|
|
206
|
+
-------
|
|
207
|
+
NestedDtype
|
|
208
|
+
The constructed NestedDtype.
|
|
209
|
+
|
|
210
|
+
Raises
|
|
211
|
+
------
|
|
212
|
+
ValueError
|
|
213
|
+
If the given dtype is not a valid nested type.
|
|
214
|
+
"""
|
|
215
|
+
return cls(pyarrow_dtype=pandas_arrow_dtype.pyarrow_dtype)
|
|
216
|
+
|
|
217
|
+
def to_pandas_arrow_dtype(self) -> ArrowDtype:
|
|
218
|
+
"""Convert NestedDtype to a pandas.ArrowDtype.
|
|
219
|
+
|
|
220
|
+
Returns
|
|
221
|
+
-------
|
|
222
|
+
ArrowDtype
|
|
223
|
+
The corresponding pandas.ArrowDtype.
|
|
224
|
+
"""
|
|
225
|
+
return ArrowDtype(self.pyarrow_dtype)
|