nd-arrow-array 0.5.0__cp39-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nd_arrow_array/__init__.py +14 -0
- nd_arrow_array/_core.pyd +0 -0
- nd_arrow_array/_core.pyi +148 -0
- nd_arrow_array/_xarray.py +220 -0
- nd_arrow_array/py.typed +0 -0
- nd_arrow_array-0.5.0.dist-info/METADATA +196 -0
- nd_arrow_array-0.5.0.dist-info/RECORD +9 -0
- nd_arrow_array-0.5.0.dist-info/WHEEL +4 -0
- nd_arrow_array-0.5.0.dist-info/sboms/nd-arrow-py.cyclonedx.json +2854 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""N-dimensional Arrow arrays."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version as _version
|
|
4
|
+
|
|
5
|
+
from ._core import NdArray, NdColumn, NdFormatError, NdRecordBatch, encode_batches
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"NdArray",
|
|
9
|
+
"NdColumn",
|
|
10
|
+
"NdFormatError",
|
|
11
|
+
"NdRecordBatch",
|
|
12
|
+
"encode_batches",
|
|
13
|
+
]
|
|
14
|
+
__version__ = _version("nd-arrow-array")
|
nd_arrow_array/_core.pyd
ADDED
|
Binary file
|
nd_arrow_array/_core.pyi
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
from typing import Any, Iterator, Mapping, Optional, Protocol, Sequence, Tuple, Union, final
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import numpy.typing as npt
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"NdArray",
|
|
8
|
+
"NdColumn",
|
|
9
|
+
"NdRecordBatch",
|
|
10
|
+
"_ArrowExport",
|
|
11
|
+
"encode_batches",
|
|
12
|
+
"NdFormatError",
|
|
13
|
+
"_roundtrip_numpy",
|
|
14
|
+
"_roundtrip_attrs",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
class _ArrowArrayExportable(Protocol):
|
|
18
|
+
def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
|
|
19
|
+
|
|
20
|
+
class _ArrowScalar(Protocol):
|
|
21
|
+
# A pyarrow.Scalar, without a dependency on pyarrow.
|
|
22
|
+
def as_py(self) -> Any: ...
|
|
23
|
+
|
|
24
|
+
class _ArrowStreamExportable(Protocol):
|
|
25
|
+
def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: ...
|
|
26
|
+
|
|
27
|
+
# Dimensions: {"time": 2, "lat": 3} or [("time", 2), ("lat", 3)].
|
|
28
|
+
_Dims = Union[Mapping[str, int], Sequence[Tuple[str, int]]]
|
|
29
|
+
_AttrValue = Union[
|
|
30
|
+
str,
|
|
31
|
+
bool,
|
|
32
|
+
int,
|
|
33
|
+
float,
|
|
34
|
+
np.generic,
|
|
35
|
+
npt.NDArray[Any],
|
|
36
|
+
Sequence[str],
|
|
37
|
+
Sequence[int],
|
|
38
|
+
Sequence[float],
|
|
39
|
+
_ArrowScalar,
|
|
40
|
+
]
|
|
41
|
+
_Attrs = Optional[Mapping[str, _AttrValue]]
|
|
42
|
+
|
|
43
|
+
class NdFormatError(ValueError):
|
|
44
|
+
"""Data does not follow the nd_arrow.array format."""
|
|
45
|
+
|
|
46
|
+
@final
|
|
47
|
+
class NdArray:
|
|
48
|
+
"""One N-dimensional array with named dimensions and attributes."""
|
|
49
|
+
|
|
50
|
+
def __new__(
|
|
51
|
+
cls, values: _ArrowArrayExportable, dims: _Dims, attrs: _Attrs = None
|
|
52
|
+
) -> NdArray: ...
|
|
53
|
+
@staticmethod
|
|
54
|
+
def from_numpy(
|
|
55
|
+
arr: npt.ArrayLike, dims: Sequence[str], attrs: _Attrs = None
|
|
56
|
+
) -> NdArray: ...
|
|
57
|
+
@property
|
|
58
|
+
def dims(self) -> Tuple[str, ...]: ...
|
|
59
|
+
@property
|
|
60
|
+
def shape(self) -> Tuple[int, ...]: ...
|
|
61
|
+
@property
|
|
62
|
+
def sizes(self) -> dict[str, int]: ...
|
|
63
|
+
@property
|
|
64
|
+
def ndim(self) -> int: ...
|
|
65
|
+
@property
|
|
66
|
+
def dtype(self) -> str: ...
|
|
67
|
+
@property
|
|
68
|
+
def attrs(self) -> dict[str, Any]: ...
|
|
69
|
+
@attrs.setter
|
|
70
|
+
def attrs(self, value: _Attrs) -> None: ...
|
|
71
|
+
@property
|
|
72
|
+
def values(self) -> Any:
|
|
73
|
+
"""The flat values as a pyarrow.Array. Needs pyarrow."""
|
|
74
|
+
def set_attr(self, name: str, value: _AttrValue) -> None: ...
|
|
75
|
+
def del_attr(self, name: str) -> None: ...
|
|
76
|
+
def with_attrs(self, attrs: _Attrs) -> NdArray: ...
|
|
77
|
+
def reshape(self, dims: _Dims) -> NdArray: ...
|
|
78
|
+
def to_numpy(self) -> npt.NDArray[Any]:
|
|
79
|
+
"""A numpy.ndarray, or a numpy.ma.MaskedArray when nulls exist."""
|
|
80
|
+
def to_xarray(self, name: Optional[str] = None) -> Any:
|
|
81
|
+
"""An xarray.DataArray. Nulls become NaN or NaT. Needs xarray."""
|
|
82
|
+
@staticmethod
|
|
83
|
+
def from_xarray(data_array: Any) -> NdArray:
|
|
84
|
+
"""An array from an xarray.DataArray, without coordinates. Needs xarray."""
|
|
85
|
+
def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
|
|
86
|
+
def __eq__(self, other: object, /) -> bool: ...
|
|
87
|
+
def __ne__(self, other: object, /) -> bool: ...
|
|
88
|
+
|
|
89
|
+
@final
|
|
90
|
+
class NdColumn:
|
|
91
|
+
"""An Arrow column of the nd_arrow.array extension type."""
|
|
92
|
+
|
|
93
|
+
@staticmethod
|
|
94
|
+
def from_arrays(items: Sequence[Optional[NdArray]]) -> NdColumn: ...
|
|
95
|
+
@staticmethod
|
|
96
|
+
def from_arrow(obj: _ArrowArrayExportable) -> NdColumn: ...
|
|
97
|
+
def to_arrow(self) -> Any:
|
|
98
|
+
"""The column as a pyarrow.Array. Needs pyarrow."""
|
|
99
|
+
@property
|
|
100
|
+
def null_count(self) -> int: ...
|
|
101
|
+
def __len__(self) -> int: ...
|
|
102
|
+
def __getitem__(self, index: int, /) -> Optional[NdArray]: ...
|
|
103
|
+
def __iter__(self) -> Iterator[Optional[NdArray]]: ...
|
|
104
|
+
def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
|
|
105
|
+
def __arrow_c_schema__(self) -> Any: ...
|
|
106
|
+
|
|
107
|
+
@final
|
|
108
|
+
class NdRecordBatch:
|
|
109
|
+
"""Named nd arrays on one shared grid, like an xarray Dataset."""
|
|
110
|
+
|
|
111
|
+
def __new__(
|
|
112
|
+
cls,
|
|
113
|
+
columns: Mapping[str, NdArray],
|
|
114
|
+
attrs: _Attrs = None,
|
|
115
|
+
target: Optional[_Dims] = None,
|
|
116
|
+
) -> NdRecordBatch: ...
|
|
117
|
+
@staticmethod
|
|
118
|
+
def from_arrow(
|
|
119
|
+
obj: Union[_ArrowArrayExportable, _ArrowStreamExportable],
|
|
120
|
+
) -> list[NdRecordBatch]: ...
|
|
121
|
+
@property
|
|
122
|
+
def target(self) -> dict[str, int]: ...
|
|
123
|
+
@property
|
|
124
|
+
def attrs(self) -> dict[str, Any]: ...
|
|
125
|
+
@attrs.setter
|
|
126
|
+
def attrs(self, value: _Attrs) -> None: ...
|
|
127
|
+
def keys(self) -> list[str]: ...
|
|
128
|
+
def __getitem__(self, name: str, /) -> NdArray: ...
|
|
129
|
+
def __len__(self) -> int: ...
|
|
130
|
+
def to_arrow(self) -> Any:
|
|
131
|
+
"""The batch as a one-row pyarrow.RecordBatch. Needs pyarrow."""
|
|
132
|
+
def to_xarray(self) -> Any:
|
|
133
|
+
"""An xarray.Dataset. Nulls become NaN or NaT. Needs xarray."""
|
|
134
|
+
@staticmethod
|
|
135
|
+
def from_xarray(dataset: Any) -> NdRecordBatch:
|
|
136
|
+
"""A batch from an xarray.Dataset or DataArray, with coordinates. Needs xarray."""
|
|
137
|
+
def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: ...
|
|
138
|
+
def __arrow_c_schema__(self) -> Any: ...
|
|
139
|
+
|
|
140
|
+
def encode_batches(batches: Sequence[NdRecordBatch]) -> Any:
|
|
141
|
+
"""Several batches as one pyarrow.RecordBatch, one row each. Needs pyarrow."""
|
|
142
|
+
|
|
143
|
+
@final
|
|
144
|
+
class _ArrowExport:
|
|
145
|
+
def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
|
|
146
|
+
|
|
147
|
+
def _roundtrip_numpy(arr: npt.ArrayLike) -> npt.NDArray[Any]: ...
|
|
148
|
+
def _roundtrip_attrs(dict: Mapping[str, _AttrValue]) -> dict[str, Any]: ...
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Conversion between nd arrays and xarray objects.
|
|
2
|
+
|
|
3
|
+
The methods ``to_xarray`` and ``from_xarray`` of ``NdArray`` and
|
|
4
|
+
``NdRecordBatch`` call the functions in this module.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from types import ModuleType
|
|
10
|
+
from typing import Any, Hashable, Iterable, Mapping
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from ._core import NdArray, NdRecordBatch
|
|
15
|
+
|
|
16
|
+
# CF attribute that names the non-index coordinates of a variable.
|
|
17
|
+
COORDINATES = "coordinates"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _xr(feature: str) -> ModuleType:
|
|
21
|
+
try:
|
|
22
|
+
import xarray
|
|
23
|
+
except ImportError as error:
|
|
24
|
+
raise ImportError(
|
|
25
|
+
f"xarray is required for {feature}: pip install nd-arrow-array[xarray]"
|
|
26
|
+
) from error
|
|
27
|
+
return xarray
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _name(key: Hashable, kind: str) -> str:
|
|
31
|
+
if not isinstance(key, str):
|
|
32
|
+
raise TypeError(f"{kind} names must be str, got {key!r}")
|
|
33
|
+
return key
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _decode(values: Any) -> np.ndarray[Any, Any]:
|
|
37
|
+
"""Nulls become NaN or NaT, as in xarray."""
|
|
38
|
+
if not isinstance(values, np.ma.MaskedArray):
|
|
39
|
+
return np.asarray(values)
|
|
40
|
+
if not values.mask.any():
|
|
41
|
+
return values.data
|
|
42
|
+
kind = values.dtype.kind
|
|
43
|
+
if kind in "fc":
|
|
44
|
+
return values.filled(np.nan)
|
|
45
|
+
if kind in "biu":
|
|
46
|
+
# Like xarray CF decoding: integers with missing values become float64.
|
|
47
|
+
return np.where(values.mask, np.nan, values.data.astype(np.float64))
|
|
48
|
+
if kind in "mM":
|
|
49
|
+
return values.filled(values.dtype.type("NaT", np.datetime_data(values.dtype)[0]))
|
|
50
|
+
out = values.data.astype(object)
|
|
51
|
+
out[values.mask] = np.nan
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _missing(item: Any) -> bool:
|
|
56
|
+
return item is None or (isinstance(item, float) and item != item)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Calendars where a cftime date is a real instant. Others are model calendars.
|
|
60
|
+
REAL_CALENDARS = {"standard", "gregorian", "proleptic_gregorian", "julian"}
|
|
61
|
+
MODEL_CALENDARS = {"noleap", "365_day", "all_leap", "366_day", "360_day"}
|
|
62
|
+
US_PER_SECOND = 1_000_000
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _cftime_type() -> type | None:
|
|
66
|
+
try:
|
|
67
|
+
import cftime # type: ignore[import-untyped, unused-ignore]
|
|
68
|
+
except ImportError:
|
|
69
|
+
return None
|
|
70
|
+
return cftime.datetime # type: ignore[no-any-return, unused-ignore]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _elapsed_us(date: Any, calendar: str) -> int:
|
|
74
|
+
"""Microseconds since the Unix epoch of the instant `date`."""
|
|
75
|
+
import cftime # type: ignore[import-untyped, unused-ignore]
|
|
76
|
+
|
|
77
|
+
# The Unix epoch, 1970-01-01 Gregorian, is 1969-12-19 in the Julian calendar.
|
|
78
|
+
ymd = (1969, 12, 19) if calendar == "julian" else (1970, 1, 1)
|
|
79
|
+
epoch = cftime.datetime(*ymd, calendar=calendar, has_year_zero=date.has_year_zero)
|
|
80
|
+
delta = date - epoch
|
|
81
|
+
return int((delta.days * 86_400 + delta.seconds) * US_PER_SECOND + delta.microseconds)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _fields_us(date: Any, calendar: str) -> int:
|
|
85
|
+
"""Microseconds since 1970-01-01 of the same date and time in the standard calendar."""
|
|
86
|
+
month = np.datetime64(date.year - 1970, "Y").astype("M8[M]") + np.timedelta64(date.month - 1, "M")
|
|
87
|
+
first = month.astype("M8[D]")
|
|
88
|
+
days = int(((month + np.timedelta64(1, "M")).astype("M8[D]") - first).astype(np.int64))
|
|
89
|
+
if not 1 <= date.day <= days:
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"{date} of calendar '{calendar}' does not exist in the standard calendar; "
|
|
92
|
+
"use xarray convert_calendar(..., align_on='date') first"
|
|
93
|
+
)
|
|
94
|
+
day_us = int((first + np.timedelta64(date.day - 1, "D")).astype("M8[us]").astype(np.int64))
|
|
95
|
+
seconds = (date.hour * 60 + date.minute) * 60 + date.second
|
|
96
|
+
return int(day_us + seconds * US_PER_SECOND + date.microsecond)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _cftime_to_timestamps(flat: np.ndarray[Any, Any], missing: np.ndarray[Any, Any]) -> Any:
|
|
100
|
+
"""cftime dates become datetime64[us] in the standard calendar."""
|
|
101
|
+
calendars = {item.calendar for item, gap in zip(flat, missing) if not gap}
|
|
102
|
+
if len(calendars) != 1:
|
|
103
|
+
raise ValueError(f"cftime dates must have one calendar, got {sorted(calendars)}")
|
|
104
|
+
(calendar,) = calendars
|
|
105
|
+
if calendar in REAL_CALENDARS:
|
|
106
|
+
convert = _elapsed_us
|
|
107
|
+
elif calendar in MODEL_CALENDARS:
|
|
108
|
+
convert = _fields_us
|
|
109
|
+
else:
|
|
110
|
+
raise ValueError(f"cftime calendar '{calendar}' is not supported")
|
|
111
|
+
raw = [0 if gap else convert(item, calendar) for item, gap in zip(flat, missing)]
|
|
112
|
+
return np.array(raw, dtype=np.int64).view("M8[us]")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _encode(values: Any) -> Any:
|
|
116
|
+
"""Object arrays of str or cftime dates become typed arrays. None and NaN become nulls."""
|
|
117
|
+
values = np.asarray(values)
|
|
118
|
+
if values.dtype != object:
|
|
119
|
+
return values
|
|
120
|
+
flat = values.ravel()
|
|
121
|
+
missing = np.array([_missing(item) for item in flat], dtype=bool)
|
|
122
|
+
present = [item for item, gap in zip(flat, missing) if not gap]
|
|
123
|
+
cftime_type = _cftime_type()
|
|
124
|
+
if cftime_type is not None and present and all(isinstance(i, cftime_type) for i in present):
|
|
125
|
+
out = _cftime_to_timestamps(flat, missing)
|
|
126
|
+
elif all(isinstance(item, str) for item in present):
|
|
127
|
+
out = np.where(missing, "", flat).astype(str)
|
|
128
|
+
else:
|
|
129
|
+
raise TypeError("object arrays must hold only str or cftime dates, with None and NaN")
|
|
130
|
+
out = out.reshape(values.shape)
|
|
131
|
+
missing = missing.reshape(values.shape)
|
|
132
|
+
return np.ma.MaskedArray(out, missing) if missing.any() else out
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _attrs(attrs: Mapping[Any, Any]) -> dict[str, Any]:
|
|
136
|
+
return {_name(key, "attribute"): value for key, value in attrs.items()}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _array(dims: Iterable[Hashable], values: Any, attrs: Mapping[Any, Any]) -> NdArray:
|
|
140
|
+
names = [_name(dim, "dimension") for dim in dims]
|
|
141
|
+
return NdArray.from_numpy(_encode(values), dims=names, attrs=_attrs(attrs))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def array_to_xarray(array: NdArray, name: str | None = None) -> Any:
|
|
145
|
+
xr = _xr("NdArray.to_xarray")
|
|
146
|
+
return xr.DataArray(_decode(array.to_numpy()), dims=array.dims, name=name, attrs=array.attrs)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def array_from_xarray(data_array: Any) -> NdArray:
|
|
150
|
+
xr = _xr("NdArray.from_xarray")
|
|
151
|
+
if not isinstance(data_array, xr.DataArray):
|
|
152
|
+
raise TypeError(f"expected an xarray.DataArray, got {type(data_array).__name__}")
|
|
153
|
+
return _array(data_array.dims, data_array.values, data_array.attrs)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _split(names: Any) -> list[str]:
|
|
157
|
+
if not isinstance(names, str):
|
|
158
|
+
raise TypeError(f"the '{COORDINATES}' attribute must be a str, got {names!r}")
|
|
159
|
+
return names.split()
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def batch_to_xarray(batch: NdRecordBatch) -> Any:
|
|
163
|
+
xr = _xr("NdRecordBatch.to_xarray")
|
|
164
|
+
attrs = batch.attrs
|
|
165
|
+
coords = set(_split(attrs.pop(COORDINATES, "")))
|
|
166
|
+
variables = {}
|
|
167
|
+
for name in batch.keys():
|
|
168
|
+
array = batch[name]
|
|
169
|
+
var_attrs = array.attrs
|
|
170
|
+
coords.update(_split(var_attrs.pop(COORDINATES, "")))
|
|
171
|
+
if array.dims == (name,):
|
|
172
|
+
coords.add(name)
|
|
173
|
+
variables[name] = xr.Variable(array.dims, _decode(array.to_numpy()), attrs=var_attrs)
|
|
174
|
+
unknown = coords - variables.keys()
|
|
175
|
+
if unknown:
|
|
176
|
+
raise ValueError(f"'{COORDINATES}' names columns that do not exist: {sorted(unknown)}")
|
|
177
|
+
data_vars = {k: v for k, v in variables.items() if k not in coords}
|
|
178
|
+
coord_vars = {k: v for k, v in variables.items() if k in coords}
|
|
179
|
+
return xr.Dataset(data_vars, coords=coord_vars, attrs=attrs)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _check_free(attrs: Mapping[str, Any], owner: str) -> None:
|
|
183
|
+
if COORDINATES in attrs:
|
|
184
|
+
raise ValueError(
|
|
185
|
+
f"{owner} has a '{COORDINATES}' attribute; move it to the xarray coordinates"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def batch_from_xarray(dataset: Any) -> NdRecordBatch:
|
|
190
|
+
xr = _xr("NdRecordBatch.from_xarray")
|
|
191
|
+
if isinstance(dataset, xr.DataArray):
|
|
192
|
+
dataset = dataset.to_dataset(name=dataset.name if dataset.name is not None else "data")
|
|
193
|
+
if not isinstance(dataset, xr.Dataset):
|
|
194
|
+
raise TypeError(f"expected an xarray.Dataset, got {type(dataset).__name__}")
|
|
195
|
+
|
|
196
|
+
# Non-index coordinates go to CF 'coordinates' attributes, as in xarray encoding.
|
|
197
|
+
extra = [
|
|
198
|
+
_name(name, "coordinate")
|
|
199
|
+
for name, coord in dataset.coords.items()
|
|
200
|
+
if coord.dims != (name,)
|
|
201
|
+
]
|
|
202
|
+
unclaimed = set(extra)
|
|
203
|
+
columns = {}
|
|
204
|
+
for key, var in dataset.variables.items():
|
|
205
|
+
name = _name(key, "variable")
|
|
206
|
+
attrs = _attrs(var.attrs)
|
|
207
|
+
_check_free(attrs, f"variable '{name}'")
|
|
208
|
+
if name in dataset.data_vars:
|
|
209
|
+
linked = [c for c in extra if set(dataset[c].dims) <= set(var.dims)]
|
|
210
|
+
if linked:
|
|
211
|
+
attrs[COORDINATES] = " ".join(linked)
|
|
212
|
+
unclaimed.difference_update(linked)
|
|
213
|
+
columns[name] = _array(var.dims, var.values, attrs)
|
|
214
|
+
|
|
215
|
+
attrs = _attrs(dataset.attrs)
|
|
216
|
+
_check_free(attrs, "the dataset")
|
|
217
|
+
if unclaimed:
|
|
218
|
+
attrs[COORDINATES] = " ".join(c for c in extra if c in unclaimed)
|
|
219
|
+
target = {_name(dim, "dimension"): size for dim, size in dataset.sizes.items()}
|
|
220
|
+
return NdRecordBatch(columns, attrs=attrs, target=target)
|
nd_arrow_array/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nd-arrow-array
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: Programming Language :: Rust
|
|
7
|
+
Classifier: Topic :: Scientific/Engineering
|
|
8
|
+
Classifier: Typing :: Typed
|
|
9
|
+
Requires-Dist: numpy>=1.21
|
|
10
|
+
Requires-Dist: pyarrow>=14 ; extra == 'arrow'
|
|
11
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
12
|
+
Requires-Dist: pyarrow>=14 ; extra == 'test'
|
|
13
|
+
Requires-Dist: xarray>=2023.1 ; extra == 'test'
|
|
14
|
+
Requires-Dist: cftime>=1.6 ; extra == 'test'
|
|
15
|
+
Requires-Dist: mypy ; extra == 'test'
|
|
16
|
+
Requires-Dist: xarray>=2023.1 ; extra == 'xarray'
|
|
17
|
+
Requires-Dist: cftime>=1.6 ; extra == 'xarray'
|
|
18
|
+
Provides-Extra: arrow
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Provides-Extra: xarray
|
|
21
|
+
Summary: N-dimensional Arrow arrays with named dimensions and attributes
|
|
22
|
+
Keywords: arrow,ndarray,xarray,netcdf,tensor
|
|
23
|
+
License: Apache-2.0
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
26
|
+
Project-URL: Changelog, https://github.com/robinskil/nd-arrow-array/blob/main/CHANGELOG.md
|
|
27
|
+
Project-URL: Format, https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md
|
|
28
|
+
Project-URL: Repository, https://github.com/robinskil/nd-arrow-array
|
|
29
|
+
|
|
30
|
+
# nd-arrow-array (Python)
|
|
31
|
+
|
|
32
|
+
Python bindings for the `nd_arrow.array` Arrow extension type.
|
|
33
|
+
The PyPI package is `nd-arrow-array`. The import name is `nd_arrow_array`.
|
|
34
|
+
The bindings move N-dimensional arrays between NumPy, pyarrow and the
|
|
35
|
+
format in [FORMAT.md](https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md).
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install nd-arrow-array # NumPy only
|
|
41
|
+
pip install nd-arrow-array[arrow] # with pyarrow
|
|
42
|
+
pip install nd-arrow-array[xarray] # with xarray and cftime
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
To build from source, run `pip install maturin` and `maturin develop` in this directory.
|
|
46
|
+
|
|
47
|
+
## Classes
|
|
48
|
+
|
|
49
|
+
| Class | Content |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `NdArray` | One array: values, named dimensions, attributes. |
|
|
52
|
+
| `NdColumn` | An Arrow column. Each row is one `NdArray` or `None`. |
|
|
53
|
+
| `NdRecordBatch` | Named arrays on one shared grid, with global attributes. |
|
|
54
|
+
|
|
55
|
+
## Example
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import numpy as np
|
|
59
|
+
import nd_arrow_array as nda
|
|
60
|
+
|
|
61
|
+
a = nda.NdArray.from_numpy(
|
|
62
|
+
np.arange(6.0).reshape(2, 3),
|
|
63
|
+
dims=["time", "lat"],
|
|
64
|
+
attrs={"units": "K", "valid_range": np.array([0, 400], dtype="float32")},
|
|
65
|
+
)
|
|
66
|
+
a.sizes # {'time': 2, 'lat': 3}
|
|
67
|
+
a.set_attr("long_name", "sea surface temperature")
|
|
68
|
+
a.to_numpy()
|
|
69
|
+
|
|
70
|
+
col = nda.NdColumn.from_arrays([a, None])
|
|
71
|
+
arr = col.to_arrow() # pyarrow array
|
|
72
|
+
nda.NdColumn.from_arrow(arr)[0] == a # True
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## pyarrow
|
|
76
|
+
|
|
77
|
+
`pip install nd-arrow-array[arrow]` adds pyarrow. pyarrow sees an nd column as a
|
|
78
|
+
`struct` column. The field metadata `ARROW:extension:name = nd_arrow.array`
|
|
79
|
+
marks it as an nd column. See [FORMAT.md](https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md).
|
|
80
|
+
|
|
81
|
+
### Arrays and columns
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
import pyarrow as pa
|
|
85
|
+
|
|
86
|
+
# An array from flat pyarrow values in C order.
|
|
87
|
+
a = nda.NdArray(pa.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]),
|
|
88
|
+
dims={"time": 2, "lat": 3}, attrs={"units": "K"})
|
|
89
|
+
a.values # the flat values as a pyarrow.Array
|
|
90
|
+
|
|
91
|
+
col = nda.NdColumn.from_arrays([a, None])
|
|
92
|
+
arr = col.to_arrow() # pyarrow.StructArray, one row per array
|
|
93
|
+
nda.NdColumn.from_arrow(arr) # back to an NdColumn
|
|
94
|
+
pa.field(col).metadata # the extension name and version
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Record batches and tables
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
lat = nda.NdArray.from_numpy(np.array([-10.0, 0.0, 10.0]), dims=["lat"])
|
|
101
|
+
batch = nda.NdRecordBatch({"sst": a, "lat": lat}, attrs={"title": "demo"})
|
|
102
|
+
|
|
103
|
+
rb = batch.to_arrow() # pyarrow.RecordBatch with one row
|
|
104
|
+
table = pa.table(batch) # pyarrow.Table, through the PyCapsule interface
|
|
105
|
+
|
|
106
|
+
# Many batches with one schema: one row each.
|
|
107
|
+
rows = nda.encode_batches([batch, other_batch])
|
|
108
|
+
batches = nda.NdRecordBatch.from_arrow(rows) # a list, one NdRecordBatch per row
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`NdRecordBatch.from_arrow` accepts a `RecordBatch`, a `Table` or a
|
|
112
|
+
`RecordBatchReader`. It returns one `NdRecordBatch` for each row.
|
|
113
|
+
The global attributes travel in the `__global_attrs__` column.
|
|
114
|
+
|
|
115
|
+
### Files
|
|
116
|
+
|
|
117
|
+
Parquet and Arrow IPC keep the field metadata, so the nd columns survive a
|
|
118
|
+
write and a read:
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
import pyarrow.parquet as pq
|
|
122
|
+
|
|
123
|
+
pq.write_table(pa.Table.from_batches([rows]), "data.parquet")
|
|
124
|
+
batches = nda.NdRecordBatch.from_arrow(pq.read_table("data.parquet"))
|
|
125
|
+
|
|
126
|
+
with pa.OSFile("data.arrow", "wb") as sink, pa.ipc.new_file(sink, rows.schema) as writer:
|
|
127
|
+
writer.write_batch(rows)
|
|
128
|
+
with pa.memory_map("data.arrow") as source:
|
|
129
|
+
batches = nda.NdRecordBatch.from_arrow(pa.ipc.open_file(source).read_all())
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Other libraries
|
|
133
|
+
|
|
134
|
+
`NdArray`, `NdColumn` and `NdRecordBatch` implement the Arrow PyCapsule
|
|
135
|
+
interface (`__arrow_c_array__`, `__arrow_c_stream__`, `__arrow_c_schema__`).
|
|
136
|
+
pyarrow, polars, DuckDB and other Arrow libraries read them without a copy,
|
|
137
|
+
for example `pa.array(col)` or `pa.table(batch)`.
|
|
138
|
+
|
|
139
|
+
## xarray
|
|
140
|
+
|
|
141
|
+
`pip install nd-arrow-array[xarray]` adds xarray and cftime.
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
import xarray as xr
|
|
145
|
+
|
|
146
|
+
ds = xr.Dataset(
|
|
147
|
+
{"air": (("time", "lat"), np.zeros((2, 3)), {"units": "K"})},
|
|
148
|
+
coords={"time": np.array(["2020-01-01", "2020-01-02"], "datetime64[ns]"),
|
|
149
|
+
"lat": [-10.0, 0.0, 10.0]},
|
|
150
|
+
)
|
|
151
|
+
batch = nda.NdRecordBatch.from_xarray(ds) # all variables and coordinates
|
|
152
|
+
batch.to_xarray() # an xarray.Dataset
|
|
153
|
+
|
|
154
|
+
a = nda.NdArray.from_xarray(ds["air"]) # values, dims and attrs only
|
|
155
|
+
a.to_xarray(name="air") # an xarray.DataArray
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
- `NdRecordBatch.from_xarray` accepts a `Dataset` or a `DataArray`.
|
|
159
|
+
Each variable and each coordinate becomes a column.
|
|
160
|
+
- A 1-D column with the name of its dimension becomes an index coordinate.
|
|
161
|
+
- Other coordinates go into the CF `coordinates` attribute of the data
|
|
162
|
+
variables. A coordinate that no data variable uses goes into the
|
|
163
|
+
`coordinates` attribute of the batch.
|
|
164
|
+
- `NdArray.from_xarray` does not keep coordinates.
|
|
165
|
+
- In `to_xarray`, nulls become `NaN` or `NaT`. Integer and bool arrays with
|
|
166
|
+
nulls become `float64`, as in xarray CF decoding.
|
|
167
|
+
- Object arrays must hold `str` or cftime dates. `None` and `NaN` become nulls.
|
|
168
|
+
- cftime dates become `timestamp[us]` in the standard calendar:
|
|
169
|
+
- `standard`, `gregorian`, `proleptic_gregorian` and `julian` keep the
|
|
170
|
+
exact instant;
|
|
171
|
+
- `noleap`, `365_day`, `all_leap`, `366_day` and `360_day` keep the year,
|
|
172
|
+
month, day and time. A date that the standard calendar does not have,
|
|
173
|
+
such as 30 February, raises `ValueError`. Use
|
|
174
|
+
`convert_calendar(..., align_on="date")` in xarray first.
|
|
175
|
+
|
|
176
|
+
## Rules
|
|
177
|
+
|
|
178
|
+
- NumPy dtypes: `bool`, signed and unsigned integers, `float16`, `float32`,
|
|
179
|
+
`float64`, `datetime64` and `timedelta64` (`s`, `ms`, `us`, `ns`) and `str`.
|
|
180
|
+
Other dtypes raise `TypeError`.
|
|
181
|
+
- A `numpy.ma.MaskedArray` mask and `NaT` become nulls. `NaN` stays a value.
|
|
182
|
+
- `to_numpy()` returns a `MaskedArray` when the values have nulls.
|
|
183
|
+
- `attrs` returns a copy. Assign a new dict, or use `set_attr` and `del_attr`.
|
|
184
|
+
- All classes implement the Arrow PyCapsule interface, so pyarrow, polars
|
|
185
|
+
and DuckDB can read them.
|
|
186
|
+
- The package ships type stubs (`py.typed`), so mypy and pyright check calls.
|
|
187
|
+
|
|
188
|
+
## Test
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
pip install pytest mypy numpy pyarrow xarray cftime
|
|
192
|
+
maturin develop && pytest
|
|
193
|
+
python -m mypy.stubtest nd_arrow_array # stubs match the module
|
|
194
|
+
mypy --strict --warn-unused-ignores tests/typing
|
|
195
|
+
```
|
|
196
|
+
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
nd_arrow_array/__init__.py,sha256=NxDl4pFD0kpv0QfUDNGzCQExNH6J5O5LCn3KJwUHUbY,335
|
|
2
|
+
nd_arrow_array/_core.pyd,sha256=PuEpvxiVCZ5im9So5oK-6sSrZA81eWWSDxkNp78m_88,10555904
|
|
3
|
+
nd_arrow_array/_core.pyi,sha256=Oeiu__w2OleaMoxHl0jLauVDm0xft-J0vnWpDa8VyCE,5219
|
|
4
|
+
nd_arrow_array/_xarray.py,sha256=hf0GslIzxj4JHFSCbyiAAWANEbsOuMFPo2WyNcdjopA,8963
|
|
5
|
+
nd_arrow_array/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
nd_arrow_array-0.5.0.dist-info/METADATA,sha256=GEBvkOanQm-FtJXSe5wUHdHo-zMQncZlRfMV0OCKDsU,7569
|
|
7
|
+
nd_arrow_array-0.5.0.dist-info/WHEEL,sha256=xw8F9xBWUOgm5ym2S9tihj_bLJOb723bjhA0kdANo2I,95
|
|
8
|
+
nd_arrow_array-0.5.0.dist-info/sboms/nd-arrow-py.cyclonedx.json,sha256=0n8ENgBmuTv3M_D62YxIQnwJ_1tsiOAkZU5sJiwqO2U,91585
|
|
9
|
+
nd_arrow_array-0.5.0.dist-info/RECORD,,
|