nested-pandas 0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,77 @@
1
+ # typing.Self and "|" union syntax don't exist in Python 3.9
2
+ from __future__ import annotations
3
+
4
+ import pandas as pd
5
+ from pandas._libs import lib
6
+ from pandas._typing import (
7
+ DtypeBackend,
8
+ FilePath,
9
+ ReadBuffer,
10
+ )
11
+
12
+ from .core import NestedFrame
13
+
14
+
15
+ def read_parquet(
16
+ data: FilePath | ReadBuffer[bytes],
17
+ to_pack: dict | None = None,
18
+ columns: list[str] | None = None,
19
+ pack_columns: dict | None = None,
20
+ dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
21
+ ) -> NestedFrame:
22
+ """
23
+ Load a parquet object from a file path and load a set of other
24
+ parquet objects to pack into the resulting NestedFrame.
25
+
26
+ Docstring based on the Pandas equivalent.
27
+
28
+ #TODO after MVP: Include full kwarg-set
29
+ #TODO: Switch dtype backend default?
30
+
31
+ Parameters
32
+ ----------
33
+ data : str, path object or file-like object
34
+ String, path object (implementing ``os.PathLike[str]``), or file-like
35
+ object implementing a binary ``read()`` function.
36
+ The string could be a URL. Valid URL schemes include http, ftp, s3,
37
+ gs, and file. For file URLs, a host is expected. A local file could be:
38
+ ``file://localhost/path/to/table.parquet``.
39
+ A file URL can also be a path to a directory that contains multiple
40
+ partitioned parquet files. Both pyarrow and fastparquet support
41
+ paths to directories as well as file URLs. A directory path could be:
42
+ ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``.
43
+ to_pack: dict, default=None
44
+ A dictionary of parquet data paths (same criteria as `data`), where
45
+ each key reflects the desired column name to pack the data into and
46
+ each value reflects the parquet data to pack. If None, it assumes
47
+ that any data to pack is already packed as a column within `data`.
48
+ columns : list, default=None
49
+ If not None, only these columns will be read from the file.
50
+ pack_columns: dict, default=None
51
+ If not None, selects a set of columns from each keyed nested parquet
52
+ object to read from the nested files.
53
+ dtype_backend : {{'numpy_nullable', 'pyarrow'}}, default 'numpy_nullable'
54
+ Back-end data type applied to the resultant :class:`DataFrame`
55
+ (still experimental). Behaviour is as follows:
56
+
57
+ * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
58
+ (default).
59
+ * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
60
+ DataFrame.
61
+
62
+ Returns
63
+ -------
64
+ NestedFrame
65
+ """
66
+
67
+ df = NestedFrame(pd.read_parquet(data, engine="pyarrow", columns=columns, dtype_backend=dtype_backend))
68
+ if to_pack is None:
69
+ return df
70
+ for pack_key in to_pack:
71
+ col_subset = pack_columns.get(pack_key, None) if pack_columns is not None else None
72
+ packed = pd.read_parquet(
73
+ to_pack[pack_key], engine="pyarrow", columns=col_subset, dtype_backend=dtype_backend
74
+ )
75
+ df = df.add_nested(packed, pack_key)
76
+
77
+ return df
@@ -0,0 +1,35 @@
1
+ def _ensure_spacing(expr) -> str:
2
+ """Ensure that an eval string has spacing"""
3
+ single_val_operators = {"+", "-", "*", "/", "%", ">", "<", "|", "&", "~", "="} # omit "(" and ")"
4
+ check_for_doubles = {"=", "/", "*", ">", "<"}
5
+ double_val_operators = {"==", "//", "**", ">=", "<="}
6
+ expr_list = expr
7
+
8
+ i = 0
9
+ spaced_expr = ""
10
+ while i < len(expr_list):
11
+ if expr_list[i] not in single_val_operators:
12
+ spaced_expr += expr_list[i]
13
+ else:
14
+ if expr_list[i] in check_for_doubles:
15
+ if "".join(expr_list[i : i + 2]) in double_val_operators:
16
+ if spaced_expr[-1] != " ":
17
+ spaced_expr += " "
18
+ spaced_expr += expr_list[i : i + 2]
19
+ if expr_list[i + 2] != " ":
20
+ spaced_expr += " "
21
+ i += 1 # skip ahead an extra time
22
+ else:
23
+ if spaced_expr[-1] != " ":
24
+ spaced_expr += " "
25
+ spaced_expr += expr_list[i]
26
+ if expr_list[i + 1] != " ":
27
+ spaced_expr += " "
28
+ else:
29
+ if spaced_expr[-1] != " ":
30
+ spaced_expr += " "
31
+ spaced_expr += expr_list[i]
32
+ if expr_list[i + 1] != " ":
33
+ spaced_expr += " "
34
+ i += 1
35
+ return spaced_expr
nested_pandas/py.typed ADDED
File without changes
File without changes
@@ -0,0 +1,286 @@
1
+ # Python 3.9 doesn't support "|" for types
2
+ from __future__ import annotations
3
+
4
+ from collections.abc import Generator, MutableMapping
5
+ from typing import cast
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ import pyarrow as pa
10
+ from numpy.typing import ArrayLike
11
+ from pandas.api.extensions import register_series_accessor
12
+
13
+ from nested_pandas.series.dtype import NestedDtype
14
+ from nested_pandas.series.packer import pack_sorted_df_into_struct
15
+
16
+ __all__ = ["NestSeriesAccessor"]
17
+
18
+
19
+ @register_series_accessor("nest")
20
+ class NestSeriesAccessor(MutableMapping):
21
+ """Accessor for operations on Series of NestedDtype
22
+
23
+ This accessor implements `MutableMapping` interface over the fields of the
24
+ struct, so you can access, change and delete the fields as if it was a
25
+ dictionary, with `[]`, `[] =` and `del` operators.
26
+ """
27
+
28
+ def __init__(self, series):
29
+ self._check_series(series)
30
+
31
+ self._series = series
32
+
33
+ @staticmethod
34
+ def _check_series(series):
35
+ dtype = series.dtype
36
+ if not isinstance(dtype, NestedDtype):
37
+ raise AttributeError(f"Can only use .nest accessor with a Series of NestedDtype, got {dtype}")
38
+
39
+ def to_lists(self, fields: list[str] | None = None) -> pd.DataFrame:
40
+ """Convert nested series into dataframe of list-array columns
41
+
42
+ Parameters
43
+ ----------
44
+ fields : list[str] or None, optional
45
+ Names of the fields to include. Default is None, which means all fields.
46
+
47
+ Returns
48
+ -------
49
+ pd.DataFrame
50
+ Dataframe of list-arrays.
51
+ """
52
+ fields = fields if fields is not None else list(self._series.array.field_names)
53
+ if len(fields) == 0:
54
+ raise ValueError("Cannot convert a struct with no fields to lists")
55
+
56
+ struct_array = cast(pa.StructArray, pa.array(self._series))
57
+
58
+ list_series = {}
59
+ for field in fields:
60
+ list_array = cast(pa.ListArray, struct_array.field(field))
61
+ list_series[field] = pd.Series(
62
+ list_array,
63
+ dtype=pd.ArrowDtype(list_array.type),
64
+ index=self._series.index,
65
+ name=field,
66
+ copy=False,
67
+ )
68
+
69
+ return pd.DataFrame(list_series)
70
+
71
+ def to_flat(self, fields: list[str] | None = None) -> pd.DataFrame:
72
+ """Convert nested series into dataframe of flat arrays
73
+
74
+ Parameters
75
+ ----------
76
+ fields : list[str] or None, optional
77
+ Names of the fields to include. Default is None, which means all fields.
78
+
79
+ Returns
80
+ -------
81
+ pd.DataFrame
82
+ Dataframe of flat arrays.
83
+ """
84
+ fields = fields if fields is not None else list(self._series.array.field_names)
85
+ if len(fields) == 0:
86
+ raise ValueError("Cannot flatten a struct with no fields")
87
+
88
+ struct_array = cast(pa.StructArray, pa.array(self._series))
89
+
90
+ flat_series = {}
91
+ index = None
92
+ for field in fields:
93
+ list_array = cast(pa.ListArray, struct_array.field(field))
94
+ flat_array = list_array.flatten()
95
+ if index is None:
96
+ index = self.get_flat_index()
97
+ flat_series[field] = pd.Series(
98
+ flat_array,
99
+ index=pd.Series(index, name=self._series.index.name),
100
+ name=field,
101
+ copy=False,
102
+ dtype=pd.ArrowDtype(flat_array.type),
103
+ )
104
+
105
+ return pd.DataFrame(flat_series)
106
+
107
+ @property
108
+ def flat_length(self) -> int:
109
+ """Length of the flat arrays"""
110
+ return self._series.array.flat_length
111
+
112
+ @property
113
+ def fields(self) -> list[str]:
114
+ """Names of the nested columns"""
115
+ return self._series.array.field_names
116
+
117
+ def set_flat_field(self, field: str, value: ArrayLike) -> None:
118
+ """Set the field from flat-array of values, in-place
119
+
120
+ Parameters
121
+ ----------
122
+ field : str
123
+ Name of the field to set. If not present, it will be added.
124
+ value : ArrayLike
125
+ Array of values to set. It must be a scalar or have the same length
126
+ as the flat arrays, e.g. `self.flat_length`.
127
+ """
128
+ self._series.array.set_flat_field(field, value)
129
+
130
+ def set_list_field(self, field: str, value: ArrayLike) -> None:
131
+ """Set the field from list-array, in-place
132
+
133
+ Parameters
134
+ ----------
135
+ field : str
136
+ Name of the field to set. If not present, it will be added.
137
+ value : ArrayLike
138
+ Array of values to set. It must be a list-array of the same length
139
+ as the series, e.g. length of the series.
140
+ """
141
+ self._series.array.set_list_field(field, value)
142
+
143
+ # I intentionally don't call it `drop` or `drop_field` because `pd.DataFrame.drop` is not inplace
144
+ # by default, and I wouldn't like to surprise the user.
145
+ def pop_field(self, field: str) -> pd.Series:
146
+ """Delete the field from the struct and return it.
147
+
148
+ Parameters
149
+ ----------
150
+ field : str
151
+ Name of the field to delete.
152
+
153
+ Returns
154
+ -------
155
+ pd.Series
156
+ The deleted field.
157
+ """
158
+ series = self[field]
159
+ self._series.array.pop_field(field)
160
+ return series
161
+
162
+ def query_flat(self, query: str) -> pd.Series:
163
+ """Query the flat arrays with a boolean expression
164
+
165
+ Currently, it will remove empty rows from the output series.
166
+ # TODO: preserve the index keeping empty rows
167
+
168
+ Parameters
169
+ ----------
170
+ query : str
171
+ Boolean expression to filter the rows.
172
+
173
+ Returns
174
+ -------
175
+ pd.Series
176
+ The filtered series.
177
+ """
178
+ flat = self.to_flat().query(query)
179
+ if len(flat) == 0:
180
+ return pd.Series([], dtype=self._series.dtype)
181
+ return pack_sorted_df_into_struct(flat)
182
+
183
+ def get_flat_index(self) -> pd.Index:
184
+ """Index of the flat arrays"""
185
+ flat_index = np.repeat(self._series.index, np.diff(self._series.array.list_offsets))
186
+ # pd.Index supports np.repeat, so flat_index is the same type as self._series.index
187
+ flat_index = cast(pd.Index, flat_index)
188
+ return flat_index
189
+
190
+ def get_flat_series(self, field: str) -> pd.Series:
191
+ """Get the flat-array field as a Series
192
+
193
+ Parameters
194
+ ----------
195
+ field : str
196
+ Name of the field to get.
197
+
198
+ Returns
199
+ -------
200
+ pd.Series
201
+ The flat-array field.
202
+ """
203
+
204
+ # TODO: we should make proper missed values handling here
205
+
206
+ struct_array = cast(pa.StructArray, pa.array(self._series))
207
+ list_array = cast(pa.ListArray, struct_array.field(field))
208
+ flat_array = list_array.flatten()
209
+ return pd.Series(
210
+ flat_array,
211
+ dtype=pd.ArrowDtype(flat_array.type),
212
+ index=self.get_flat_index(),
213
+ name=field,
214
+ copy=False,
215
+ )
216
+
217
+ def get_list_series(self, field: str) -> pd.Series:
218
+ """Get the list-array field as a Series
219
+
220
+ Parameters
221
+ ----------
222
+ field : str
223
+ Name of the field to get.
224
+
225
+ Returns
226
+ -------
227
+ pd.Series
228
+ The list-array field.
229
+ """
230
+ struct_array = cast(pa.StructArray, pa.array(self._series))
231
+ list_array = struct_array.field(field)
232
+ return pd.Series(
233
+ list_array,
234
+ dtype=pd.ArrowDtype(list_array.type),
235
+ index=self._series.index,
236
+ name=field,
237
+ copy=False,
238
+ )
239
+
240
+ def __getitem__(self, key: str | list[str]) -> pd.Series:
241
+ if isinstance(key, list):
242
+ new_array = self._series.array.view_fields(key)
243
+ return pd.Series(new_array, index=self._series.index, name=self._series.name)
244
+
245
+ return self.get_flat_series(key)
246
+
247
+ def __setitem__(self, key: str, value: ArrayLike) -> None:
248
+ # TODO: we can be much-much smarter about the performance here
249
+ # TODO: think better about underlying pa.ChunkArray in both self._series.array and value
250
+
251
+ ndim = np.ndim(value)
252
+
253
+ # Everything is empty, do nothing
254
+ if len(self._series) == 0 and ndim != 0:
255
+ array = pa.array(value, from_pandas=True)
256
+ if len(array) == 0:
257
+ return
258
+
259
+ # Set single value for all rows
260
+ if ndim == 0:
261
+ self.set_flat_field(key, value)
262
+ return
263
+
264
+ if isinstance(value, pd.Series) and not self.get_flat_index().equals(value.index):
265
+ raise ValueError("Cannot set field with a Series of different index")
266
+
267
+ pa_array = pa.array(value, from_pandas=True)
268
+
269
+ # Input is a flat array of values
270
+ if len(pa_array) != self.flat_length:
271
+ ValueError(
272
+ f"Cannot set field {key} with value of length {len(pa_array)}, the value is expected to be "
273
+ f"either a scalar, a 'flat' array of length {self.flat_length}, or a 'list' array of length "
274
+ f"{len(self._series)}."
275
+ )
276
+
277
+ self.set_flat_field(key, pa_array)
278
+
279
+ def __delitem__(self, key: str) -> None:
280
+ self.pop_field(key)
281
+
282
+ def __iter__(self) -> Generator[str, None, None]:
283
+ yield from iter(self._series.array.field_names)
284
+
285
+ def __len__(self) -> int:
286
+ return len(self._series.array.field_names)
@@ -0,0 +1,225 @@
1
+ # Use Self, which is not available until Python 3.11
2
+ from __future__ import annotations
3
+
4
+ from collections.abc import Mapping
5
+
6
+ # We use Type, because we must use "type" as an attribute name
7
+ from typing import Type, cast # noqa: UP035
8
+
9
+ import pandas as pd
10
+ import pyarrow as pa
11
+ from pandas import ArrowDtype
12
+ from pandas.api.extensions import register_extension_dtype
13
+ from pandas.core.arrays import ExtensionArray
14
+ from pandas.core.dtypes.base import ExtensionDtype
15
+
16
+ from nested_pandas.series.utils import is_pa_type_a_list
17
+
18
+ __all__ = ["NestedDtype"]
19
+
20
+
21
+ @register_extension_dtype
22
+ class NestedDtype(ExtensionDtype):
23
+ """Data type to handle packed time series data"""
24
+
25
+ # ExtensionDtype overrides #
26
+
27
+ _metadata = ("pyarrow_dtype",)
28
+ """Attributes to use as metadata for __eq__ and __hash__"""
29
+
30
+ @property
31
+ def na_value(self) -> Type[pd.NA]:
32
+ """The missing value for this dtype"""
33
+ return pd.NA
34
+
35
+ type = pd.DataFrame
36
+ """The type of the array's elements, always pd.DataFrame"""
37
+
38
+ @property
39
+ def name(self) -> str:
40
+ """The string representation of the nested type"""
41
+ fields = ", ".join([f"{field.name}: [{field.type.value_type!s}]" for field in self.pyarrow_dtype])
42
+ return f"nested<{fields}>"
43
+
44
+ @classmethod
45
+ def construct_array_type(cls) -> Type[ExtensionArray]:
46
+ """Corresponded array type, always NestedExtensionArray"""
47
+ from nested_pandas.series.ext_array import NestedExtensionArray
48
+
49
+ return NestedExtensionArray
50
+
51
+ @classmethod
52
+ def construct_from_string(cls, string: str) -> Self: # type: ignore[name-defined] # noqa: F821
53
+ """Construct NestedDtype from a string representation.
54
+
55
+ This works only for simple types, i.e. non-parametric pyarrow types.
56
+
57
+ Parameters
58
+ ----------
59
+ string : str
60
+ The string representation of the nested type. For example,
61
+ 'nested<x: [int64], y: [float64]'. It must be consistent with
62
+ the string representation of the dtype given by the `name`
63
+ attribute.
64
+
65
+ Returns
66
+ -------
67
+ NestedDtype
68
+ The constructed NestedDtype.
69
+
70
+ Raises
71
+ ------
72
+ TypeError
73
+ If the string is not a valid nested type string or if the element types
74
+ are parametric pyarrow types.
75
+ """
76
+ if not string.startswith("nested<") or not string.endswith(">"):
77
+ raise TypeError("Not a valid nested type string, expected 'nested<...>'")
78
+ fields_str = string.removeprefix("nested<").removesuffix(">")
79
+
80
+ field_strings = fields_str.split(", ")
81
+
82
+ fields = {}
83
+ for field_string in field_strings:
84
+ try:
85
+ field_name, field_type = field_string.split(": ", maxsplit=1)
86
+ except ValueError as e:
87
+ raise TypeError(
88
+ "Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
89
+ f"string '{field_string}'"
90
+ ) from e
91
+ if not field_type.startswith("[") or not field_type.endswith("]"):
92
+ raise TypeError(
93
+ "Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
94
+ f"type string '{field_type}'"
95
+ )
96
+
97
+ value_type = field_type.removeprefix("[").removesuffix("]")
98
+ # We follow ArrowDtype implementation heere and do not try to parse complex types
99
+ try:
100
+ pa_value_type = pa.type_for_alias(value_type)
101
+ except ValueError as e:
102
+ raise TypeError(
103
+ f"Parsing pyarrow specific parameters in the string is not supported yet: {value_type}. "
104
+ "Please use NestedDtype() or NestedDtype.from_fields() instead."
105
+ ) from e
106
+
107
+ fields[field_name] = pa_value_type
108
+
109
+ return cls.from_fields(fields)
110
+
111
+ # ArrowDtype would return None so we do
112
+ def _get_common_dtype(self, dtypes: list) -> None:
113
+ return None
114
+
115
+ # Optional methods #
116
+
117
+ def __from_arrow__(self, array: pa.Array | pa.ChunkedArray) -> ExtensionArray:
118
+ """Construct a NestedExtensionArray from a pyarrow array.
119
+
120
+ Parameters
121
+ ----------
122
+ array : pa.Array | pa.ChunkedArray
123
+ The input pyarrow array.
124
+
125
+ Returns
126
+ -------
127
+ NestedExtensionArray
128
+ The constructed NestedExtensionArray.
129
+ """
130
+ from nested_pandas.series.ext_array import NestedExtensionArray
131
+
132
+ return NestedExtensionArray(array)
133
+
134
+ # Additional methods and attributes #
135
+
136
+ pyarrow_dtype: pa.StructType
137
+
138
+ def __init__(self, pyarrow_dtype: pa.DataType) -> None:
139
+ self.pyarrow_dtype = self._validate_dtype(pyarrow_dtype)
140
+
141
+ @classmethod
142
+ def from_fields(cls, fields: Mapping[str, pa.DataType]) -> Self: # type: ignore[name-defined] # noqa: F821
143
+ """Make NestedDtype from a mapping of field names and list item types.
144
+
145
+ Parameters
146
+ ----------
147
+ fields : Mapping[str, pa.DataType]
148
+ A mapping of field names and their item types. Since all fields are lists, the item types are
149
+ inner types of the lists, not the list types themselves.
150
+
151
+ Returns
152
+ -------
153
+ NestedDtype
154
+ The constructed NestedDtype.
155
+
156
+ Examples
157
+ --------
158
+ >>> dtype = NestedDtype.from_fields({"a": pa.float64(), "b": pa.int64()})
159
+ >>> dtype
160
+ nested<a: [double], b: [int64]>
161
+ >>> assert (
162
+ ... dtype.pyarrow_dtype
163
+ ... == pa.struct({"a": pa.list_(pa.float64()), "b": pa.list_(pa.int64())})
164
+ ... )
165
+ """
166
+ pyarrow_dtype = pa.struct({field: pa.list_(pa_type) for field, pa_type in fields.items()})
167
+ pyarrow_dtype = cast(pa.StructType, pyarrow_dtype)
168
+ return cls(pyarrow_dtype=pyarrow_dtype)
169
+
170
+ @staticmethod
171
+ def _validate_dtype(pyarrow_dtype: pa.DataType) -> pa.StructType:
172
+ if not isinstance(pyarrow_dtype, pa.DataType):
173
+ raise TypeError(f"Expected a 'pyarrow.DataType' object, got {type(pyarrow_dtype)}")
174
+ if not pa.types.is_struct(pyarrow_dtype):
175
+ raise ValueError("NestedDtype can only be constructed with pyarrow struct type.")
176
+ pyarrow_dtype = cast(pa.StructType, pyarrow_dtype)
177
+
178
+ for field in pyarrow_dtype:
179
+ if not is_pa_type_a_list(field.type):
180
+ raise ValueError(
181
+ "NestedDtype can only be constructed with pyarrow struct type, all fields must be list "
182
+ f"type. Given struct has unsupported field {field}"
183
+ )
184
+ return pyarrow_dtype
185
+
186
+ @property
187
+ def fields(self) -> dict[str, pa.DataType]:
188
+ """The mapping of field names and their item types."""
189
+ return {field.name: field.type.value_type for field in self.pyarrow_dtype}
190
+
191
+ @property
192
+ def field_names(self) -> list[str]:
193
+ """The list of field names of the nested type"""
194
+ return [field.name for field in self.pyarrow_dtype]
195
+
196
+ @classmethod
197
+ def from_pandas_arrow_dtype(cls, pandas_arrow_dtype: ArrowDtype):
198
+ """Construct NestedDtype from a pandas.ArrowDtype.
199
+
200
+ Parameters
201
+ ----------
202
+ pandas_arrow_dtype : ArrowDtype
203
+ The pandas.ArrowDtype to construct NestedDtype from.
204
+
205
+ Returns
206
+ -------
207
+ NestedDtype
208
+ The constructed NestedDtype.
209
+
210
+ Raises
211
+ ------
212
+ ValueError
213
+ If the given dtype is not a valid nested type.
214
+ """
215
+ return cls(pyarrow_dtype=pandas_arrow_dtype.pyarrow_dtype)
216
+
217
+ def to_pandas_arrow_dtype(self) -> ArrowDtype:
218
+ """Convert NestedDtype to a pandas.ArrowDtype.
219
+
220
+ Returns
221
+ -------
222
+ ArrowDtype
223
+ The corresponding pandas.ArrowDtype.
224
+ """
225
+ return ArrowDtype(self.pyarrow_dtype)