nd-arrow-array 0.5.0__cp39-abi3-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ """N-dimensional Arrow arrays."""
2
+
3
+ from importlib.metadata import version as _version
4
+
5
+ from ._core import NdArray, NdColumn, NdFormatError, NdRecordBatch, encode_batches
6
+
7
+ __all__ = [
8
+ "NdArray",
9
+ "NdColumn",
10
+ "NdFormatError",
11
+ "NdRecordBatch",
12
+ "encode_batches",
13
+ ]
14
+ __version__ = _version("nd-arrow-array")
Binary file
@@ -0,0 +1,148 @@
1
+ from typing import Any, Iterator, Mapping, Optional, Protocol, Sequence, Tuple, Union, final
2
+
3
+ import numpy as np
4
+ import numpy.typing as npt
5
+
6
+ __all__ = [
7
+ "NdArray",
8
+ "NdColumn",
9
+ "NdRecordBatch",
10
+ "_ArrowExport",
11
+ "encode_batches",
12
+ "NdFormatError",
13
+ "_roundtrip_numpy",
14
+ "_roundtrip_attrs",
15
+ ]
16
+
17
+ class _ArrowArrayExportable(Protocol):
18
+ def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
19
+
20
+ class _ArrowScalar(Protocol):
21
+ # A pyarrow.Scalar, without a dependency on pyarrow.
22
+ def as_py(self) -> Any: ...
23
+
24
+ class _ArrowStreamExportable(Protocol):
25
+ def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: ...
26
+
27
+ # Dimensions: {"time": 2, "lat": 3} or [("time", 2), ("lat", 3)].
28
+ _Dims = Union[Mapping[str, int], Sequence[Tuple[str, int]]]
29
+ _AttrValue = Union[
30
+ str,
31
+ bool,
32
+ int,
33
+ float,
34
+ np.generic,
35
+ npt.NDArray[Any],
36
+ Sequence[str],
37
+ Sequence[int],
38
+ Sequence[float],
39
+ _ArrowScalar,
40
+ ]
41
+ _Attrs = Optional[Mapping[str, _AttrValue]]
42
+
43
+ class NdFormatError(ValueError):
44
+ """Data does not follow the nd_arrow.array format."""
45
+
46
+ @final
47
+ class NdArray:
48
+ """One N-dimensional array with named dimensions and attributes."""
49
+
50
+ def __new__(
51
+ cls, values: _ArrowArrayExportable, dims: _Dims, attrs: _Attrs = None
52
+ ) -> NdArray: ...
53
+ @staticmethod
54
+ def from_numpy(
55
+ arr: npt.ArrayLike, dims: Sequence[str], attrs: _Attrs = None
56
+ ) -> NdArray: ...
57
+ @property
58
+ def dims(self) -> Tuple[str, ...]: ...
59
+ @property
60
+ def shape(self) -> Tuple[int, ...]: ...
61
+ @property
62
+ def sizes(self) -> dict[str, int]: ...
63
+ @property
64
+ def ndim(self) -> int: ...
65
+ @property
66
+ def dtype(self) -> str: ...
67
+ @property
68
+ def attrs(self) -> dict[str, Any]: ...
69
+ @attrs.setter
70
+ def attrs(self, value: _Attrs) -> None: ...
71
+ @property
72
+ def values(self) -> Any:
73
+ """The flat values as a pyarrow.Array. Needs pyarrow."""
74
+ def set_attr(self, name: str, value: _AttrValue) -> None: ...
75
+ def del_attr(self, name: str) -> None: ...
76
+ def with_attrs(self, attrs: _Attrs) -> NdArray: ...
77
+ def reshape(self, dims: _Dims) -> NdArray: ...
78
+ def to_numpy(self) -> npt.NDArray[Any]:
79
+ """A numpy.ndarray, or a numpy.ma.MaskedArray when nulls exist."""
80
+ def to_xarray(self, name: Optional[str] = None) -> Any:
81
+ """An xarray.DataArray. Nulls become NaN or NaT. Needs xarray."""
82
+ @staticmethod
83
+ def from_xarray(data_array: Any) -> NdArray:
84
+ """An array from an xarray.DataArray, without coordinates. Needs xarray."""
85
+ def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
86
+ def __eq__(self, other: object, /) -> bool: ...
87
+ def __ne__(self, other: object, /) -> bool: ...
88
+
89
+ @final
90
+ class NdColumn:
91
+ """An Arrow column of the nd_arrow.array extension type."""
92
+
93
+ @staticmethod
94
+ def from_arrays(items: Sequence[Optional[NdArray]]) -> NdColumn: ...
95
+ @staticmethod
96
+ def from_arrow(obj: _ArrowArrayExportable) -> NdColumn: ...
97
+ def to_arrow(self) -> Any:
98
+ """The column as a pyarrow.Array. Needs pyarrow."""
99
+ @property
100
+ def null_count(self) -> int: ...
101
+ def __len__(self) -> int: ...
102
+ def __getitem__(self, index: int, /) -> Optional[NdArray]: ...
103
+ def __iter__(self) -> Iterator[Optional[NdArray]]: ...
104
+ def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
105
+ def __arrow_c_schema__(self) -> Any: ...
106
+
107
+ @final
108
+ class NdRecordBatch:
109
+ """Named nd arrays on one shared grid, like an xarray Dataset."""
110
+
111
+ def __new__(
112
+ cls,
113
+ columns: Mapping[str, NdArray],
114
+ attrs: _Attrs = None,
115
+ target: Optional[_Dims] = None,
116
+ ) -> NdRecordBatch: ...
117
+ @staticmethod
118
+ def from_arrow(
119
+ obj: Union[_ArrowArrayExportable, _ArrowStreamExportable],
120
+ ) -> list[NdRecordBatch]: ...
121
+ @property
122
+ def target(self) -> dict[str, int]: ...
123
+ @property
124
+ def attrs(self) -> dict[str, Any]: ...
125
+ @attrs.setter
126
+ def attrs(self, value: _Attrs) -> None: ...
127
+ def keys(self) -> list[str]: ...
128
+ def __getitem__(self, name: str, /) -> NdArray: ...
129
+ def __len__(self) -> int: ...
130
+ def to_arrow(self) -> Any:
131
+ """The batch as a one-row pyarrow.RecordBatch. Needs pyarrow."""
132
+ def to_xarray(self) -> Any:
133
+ """An xarray.Dataset. Nulls become NaN or NaT. Needs xarray."""
134
+ @staticmethod
135
+ def from_xarray(dataset: Any) -> NdRecordBatch:
136
+ """A batch from an xarray.Dataset or DataArray, with coordinates. Needs xarray."""
137
+ def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: ...
138
+ def __arrow_c_schema__(self) -> Any: ...
139
+
140
+ def encode_batches(batches: Sequence[NdRecordBatch]) -> Any:
141
+ """Several batches as one pyarrow.RecordBatch, one row each. Needs pyarrow."""
142
+
143
+ @final
144
+ class _ArrowExport:
145
+ def __arrow_c_array__(self, requested_schema: Any = None) -> Tuple[Any, Any]: ...
146
+
147
+ def _roundtrip_numpy(arr: npt.ArrayLike) -> npt.NDArray[Any]: ...
148
+ def _roundtrip_attrs(dict: Mapping[str, _AttrValue]) -> dict[str, Any]: ...
@@ -0,0 +1,220 @@
1
+ """Conversion between nd arrays and xarray objects.
2
+
3
+ The methods ``to_xarray`` and ``from_xarray`` of ``NdArray`` and
4
+ ``NdRecordBatch`` call the functions in this module.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from types import ModuleType
10
+ from typing import Any, Hashable, Iterable, Mapping
11
+
12
+ import numpy as np
13
+
14
+ from ._core import NdArray, NdRecordBatch
15
+
16
+ # CF attribute that names the non-index coordinates of a variable.
17
+ COORDINATES = "coordinates"
18
+
19
+
20
+ def _xr(feature: str) -> ModuleType:
21
+ try:
22
+ import xarray
23
+ except ImportError as error:
24
+ raise ImportError(
25
+ f"xarray is required for {feature}: pip install nd-arrow-array[xarray]"
26
+ ) from error
27
+ return xarray
28
+
29
+
30
+ def _name(key: Hashable, kind: str) -> str:
31
+ if not isinstance(key, str):
32
+ raise TypeError(f"{kind} names must be str, got {key!r}")
33
+ return key
34
+
35
+
36
+ def _decode(values: Any) -> np.ndarray[Any, Any]:
37
+ """Nulls become NaN or NaT, as in xarray."""
38
+ if not isinstance(values, np.ma.MaskedArray):
39
+ return np.asarray(values)
40
+ if not values.mask.any():
41
+ return values.data
42
+ kind = values.dtype.kind
43
+ if kind in "fc":
44
+ return values.filled(np.nan)
45
+ if kind in "biu":
46
+ # Like xarray CF decoding: integers with missing values become float64.
47
+ return np.where(values.mask, np.nan, values.data.astype(np.float64))
48
+ if kind in "mM":
49
+ return values.filled(values.dtype.type("NaT", np.datetime_data(values.dtype)[0]))
50
+ out = values.data.astype(object)
51
+ out[values.mask] = np.nan
52
+ return out
53
+
54
+
55
+ def _missing(item: Any) -> bool:
56
+ return item is None or (isinstance(item, float) and item != item)
57
+
58
+
59
+ # Calendars where a cftime date is a real instant. Others are model calendars.
60
+ REAL_CALENDARS = {"standard", "gregorian", "proleptic_gregorian", "julian"}
61
+ MODEL_CALENDARS = {"noleap", "365_day", "all_leap", "366_day", "360_day"}
62
+ US_PER_SECOND = 1_000_000
63
+
64
+
65
+ def _cftime_type() -> type | None:
66
+ try:
67
+ import cftime # type: ignore[import-untyped, unused-ignore]
68
+ except ImportError:
69
+ return None
70
+ return cftime.datetime # type: ignore[no-any-return, unused-ignore]
71
+
72
+
73
+ def _elapsed_us(date: Any, calendar: str) -> int:
74
+ """Microseconds since the Unix epoch of the instant `date`."""
75
+ import cftime # type: ignore[import-untyped, unused-ignore]
76
+
77
+ # The Unix epoch, 1970-01-01 Gregorian, is 1969-12-19 in the Julian calendar.
78
+ ymd = (1969, 12, 19) if calendar == "julian" else (1970, 1, 1)
79
+ epoch = cftime.datetime(*ymd, calendar=calendar, has_year_zero=date.has_year_zero)
80
+ delta = date - epoch
81
+ return int((delta.days * 86_400 + delta.seconds) * US_PER_SECOND + delta.microseconds)
82
+
83
+
84
+ def _fields_us(date: Any, calendar: str) -> int:
85
+ """Microseconds since 1970-01-01 of the same date and time in the standard calendar."""
86
+ month = np.datetime64(date.year - 1970, "Y").astype("M8[M]") + np.timedelta64(date.month - 1, "M")
87
+ first = month.astype("M8[D]")
88
+ days = int(((month + np.timedelta64(1, "M")).astype("M8[D]") - first).astype(np.int64))
89
+ if not 1 <= date.day <= days:
90
+ raise ValueError(
91
+ f"{date} of calendar '{calendar}' does not exist in the standard calendar; "
92
+ "use xarray convert_calendar(..., align_on='date') first"
93
+ )
94
+ day_us = int((first + np.timedelta64(date.day - 1, "D")).astype("M8[us]").astype(np.int64))
95
+ seconds = (date.hour * 60 + date.minute) * 60 + date.second
96
+ return int(day_us + seconds * US_PER_SECOND + date.microsecond)
97
+
98
+
99
+ def _cftime_to_timestamps(flat: np.ndarray[Any, Any], missing: np.ndarray[Any, Any]) -> Any:
100
+ """cftime dates become datetime64[us] in the standard calendar."""
101
+ calendars = {item.calendar for item, gap in zip(flat, missing) if not gap}
102
+ if len(calendars) != 1:
103
+ raise ValueError(f"cftime dates must have one calendar, got {sorted(calendars)}")
104
+ (calendar,) = calendars
105
+ if calendar in REAL_CALENDARS:
106
+ convert = _elapsed_us
107
+ elif calendar in MODEL_CALENDARS:
108
+ convert = _fields_us
109
+ else:
110
+ raise ValueError(f"cftime calendar '{calendar}' is not supported")
111
+ raw = [0 if gap else convert(item, calendar) for item, gap in zip(flat, missing)]
112
+ return np.array(raw, dtype=np.int64).view("M8[us]")
113
+
114
+
115
+ def _encode(values: Any) -> Any:
116
+ """Object arrays of str or cftime dates become typed arrays. None and NaN become nulls."""
117
+ values = np.asarray(values)
118
+ if values.dtype != object:
119
+ return values
120
+ flat = values.ravel()
121
+ missing = np.array([_missing(item) for item in flat], dtype=bool)
122
+ present = [item for item, gap in zip(flat, missing) if not gap]
123
+ cftime_type = _cftime_type()
124
+ if cftime_type is not None and present and all(isinstance(i, cftime_type) for i in present):
125
+ out = _cftime_to_timestamps(flat, missing)
126
+ elif all(isinstance(item, str) for item in present):
127
+ out = np.where(missing, "", flat).astype(str)
128
+ else:
129
+ raise TypeError("object arrays must hold only str or cftime dates, with None and NaN")
130
+ out = out.reshape(values.shape)
131
+ missing = missing.reshape(values.shape)
132
+ return np.ma.MaskedArray(out, missing) if missing.any() else out
133
+
134
+
135
+ def _attrs(attrs: Mapping[Any, Any]) -> dict[str, Any]:
136
+ return {_name(key, "attribute"): value for key, value in attrs.items()}
137
+
138
+
139
+ def _array(dims: Iterable[Hashable], values: Any, attrs: Mapping[Any, Any]) -> NdArray:
140
+ names = [_name(dim, "dimension") for dim in dims]
141
+ return NdArray.from_numpy(_encode(values), dims=names, attrs=_attrs(attrs))
142
+
143
+
144
+ def array_to_xarray(array: NdArray, name: str | None = None) -> Any:
145
+ xr = _xr("NdArray.to_xarray")
146
+ return xr.DataArray(_decode(array.to_numpy()), dims=array.dims, name=name, attrs=array.attrs)
147
+
148
+
149
+ def array_from_xarray(data_array: Any) -> NdArray:
150
+ xr = _xr("NdArray.from_xarray")
151
+ if not isinstance(data_array, xr.DataArray):
152
+ raise TypeError(f"expected an xarray.DataArray, got {type(data_array).__name__}")
153
+ return _array(data_array.dims, data_array.values, data_array.attrs)
154
+
155
+
156
+ def _split(names: Any) -> list[str]:
157
+ if not isinstance(names, str):
158
+ raise TypeError(f"the '{COORDINATES}' attribute must be a str, got {names!r}")
159
+ return names.split()
160
+
161
+
162
+ def batch_to_xarray(batch: NdRecordBatch) -> Any:
163
+ xr = _xr("NdRecordBatch.to_xarray")
164
+ attrs = batch.attrs
165
+ coords = set(_split(attrs.pop(COORDINATES, "")))
166
+ variables = {}
167
+ for name in batch.keys():
168
+ array = batch[name]
169
+ var_attrs = array.attrs
170
+ coords.update(_split(var_attrs.pop(COORDINATES, "")))
171
+ if array.dims == (name,):
172
+ coords.add(name)
173
+ variables[name] = xr.Variable(array.dims, _decode(array.to_numpy()), attrs=var_attrs)
174
+ unknown = coords - variables.keys()
175
+ if unknown:
176
+ raise ValueError(f"'{COORDINATES}' names columns that do not exist: {sorted(unknown)}")
177
+ data_vars = {k: v for k, v in variables.items() if k not in coords}
178
+ coord_vars = {k: v for k, v in variables.items() if k in coords}
179
+ return xr.Dataset(data_vars, coords=coord_vars, attrs=attrs)
180
+
181
+
182
+ def _check_free(attrs: Mapping[str, Any], owner: str) -> None:
183
+ if COORDINATES in attrs:
184
+ raise ValueError(
185
+ f"{owner} has a '{COORDINATES}' attribute; move it to the xarray coordinates"
186
+ )
187
+
188
+
189
+ def batch_from_xarray(dataset: Any) -> NdRecordBatch:
190
+ xr = _xr("NdRecordBatch.from_xarray")
191
+ if isinstance(dataset, xr.DataArray):
192
+ dataset = dataset.to_dataset(name=dataset.name if dataset.name is not None else "data")
193
+ if not isinstance(dataset, xr.Dataset):
194
+ raise TypeError(f"expected an xarray.Dataset, got {type(dataset).__name__}")
195
+
196
+ # Non-index coordinates go to CF 'coordinates' attributes, as in xarray encoding.
197
+ extra = [
198
+ _name(name, "coordinate")
199
+ for name, coord in dataset.coords.items()
200
+ if coord.dims != (name,)
201
+ ]
202
+ unclaimed = set(extra)
203
+ columns = {}
204
+ for key, var in dataset.variables.items():
205
+ name = _name(key, "variable")
206
+ attrs = _attrs(var.attrs)
207
+ _check_free(attrs, f"variable '{name}'")
208
+ if name in dataset.data_vars:
209
+ linked = [c for c in extra if set(dataset[c].dims) <= set(var.dims)]
210
+ if linked:
211
+ attrs[COORDINATES] = " ".join(linked)
212
+ unclaimed.difference_update(linked)
213
+ columns[name] = _array(var.dims, var.values, attrs)
214
+
215
+ attrs = _attrs(dataset.attrs)
216
+ _check_free(attrs, "the dataset")
217
+ if unclaimed:
218
+ attrs[COORDINATES] = " ".join(c for c in extra if c in unclaimed)
219
+ target = {_name(dim, "dimension"): size for dim, size in dataset.sizes.items()}
220
+ return NdRecordBatch(columns, attrs=attrs, target=target)
File without changes
@@ -0,0 +1,196 @@
1
+ Metadata-Version: 2.4
2
+ Name: nd-arrow-array
3
+ Version: 0.5.0
4
+ Classifier: License :: OSI Approved :: Apache Software License
5
+ Classifier: Programming Language :: Python :: 3
6
+ Classifier: Programming Language :: Rust
7
+ Classifier: Topic :: Scientific/Engineering
8
+ Classifier: Typing :: Typed
9
+ Requires-Dist: numpy>=1.21
10
+ Requires-Dist: pyarrow>=14 ; extra == 'arrow'
11
+ Requires-Dist: pytest ; extra == 'test'
12
+ Requires-Dist: pyarrow>=14 ; extra == 'test'
13
+ Requires-Dist: xarray>=2023.1 ; extra == 'test'
14
+ Requires-Dist: cftime>=1.6 ; extra == 'test'
15
+ Requires-Dist: mypy ; extra == 'test'
16
+ Requires-Dist: xarray>=2023.1 ; extra == 'xarray'
17
+ Requires-Dist: cftime>=1.6 ; extra == 'xarray'
18
+ Provides-Extra: arrow
19
+ Provides-Extra: test
20
+ Provides-Extra: xarray
21
+ Summary: N-dimensional Arrow arrays with named dimensions and attributes
22
+ Keywords: arrow,ndarray,xarray,netcdf,tensor
23
+ License: Apache-2.0
24
+ Requires-Python: >=3.9
25
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
26
+ Project-URL: Changelog, https://github.com/robinskil/nd-arrow-array/blob/main/CHANGELOG.md
27
+ Project-URL: Format, https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md
28
+ Project-URL: Repository, https://github.com/robinskil/nd-arrow-array
29
+
30
+ # nd-arrow-array (Python)
31
+
32
+ Python bindings for the `nd_arrow.array` Arrow extension type.
33
+ The PyPI package is `nd-arrow-array`. The import name is `nd_arrow_array`.
34
+ The bindings move N-dimensional arrays between NumPy, pyarrow and the
35
+ format in [FORMAT.md](https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md).
36
+
37
+ ## Install
38
+
39
+ ```bash
40
+ pip install nd-arrow-array # NumPy only
41
+ pip install nd-arrow-array[arrow] # with pyarrow
42
+ pip install nd-arrow-array[xarray] # with xarray and cftime
43
+ ```
44
+
45
+ To build from source, run `pip install maturin` and `maturin develop` in this directory.
46
+
47
+ ## Classes
48
+
49
+ | Class | Content |
50
+ |---|---|
51
+ | `NdArray` | One array: values, named dimensions, attributes. |
52
+ | `NdColumn` | An Arrow column. Each row is one `NdArray` or `None`. |
53
+ | `NdRecordBatch` | Named arrays on one shared grid, with global attributes. |
54
+
55
+ ## Example
56
+
57
+ ```python
58
+ import numpy as np
59
+ import nd_arrow_array as nda
60
+
61
+ a = nda.NdArray.from_numpy(
62
+ np.arange(6.0).reshape(2, 3),
63
+ dims=["time", "lat"],
64
+ attrs={"units": "K", "valid_range": np.array([0, 400], dtype="float32")},
65
+ )
66
+ a.sizes # {'time': 2, 'lat': 3}
67
+ a.set_attr("long_name", "sea surface temperature")
68
+ a.to_numpy()
69
+
70
+ col = nda.NdColumn.from_arrays([a, None])
71
+ arr = col.to_arrow() # pyarrow array
72
+ nda.NdColumn.from_arrow(arr)[0] == a # True
73
+ ```
74
+
75
+ ## pyarrow
76
+
77
+ `pip install nd-arrow-array[arrow]` adds pyarrow. pyarrow sees an nd column as a
78
+ `struct` column. The field metadata `ARROW:extension:name = nd_arrow.array`
79
+ marks it as an nd column. See [FORMAT.md](https://github.com/robinskil/nd-arrow-array/blob/main/FORMAT.md).
80
+
81
+ ### Arrays and columns
82
+
83
+ ```python
84
+ import pyarrow as pa
85
+
86
+ # An array from flat pyarrow values in C order.
87
+ a = nda.NdArray(pa.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]),
88
+ dims={"time": 2, "lat": 3}, attrs={"units": "K"})
89
+ a.values # the flat values as a pyarrow.Array
90
+
91
+ col = nda.NdColumn.from_arrays([a, None])
92
+ arr = col.to_arrow() # pyarrow.StructArray, one row per array
93
+ nda.NdColumn.from_arrow(arr) # back to an NdColumn
94
+ pa.field(col).metadata # the extension name and version
95
+ ```
96
+
97
+ ### Record batches and tables
98
+
99
+ ```python
100
+ lat = nda.NdArray.from_numpy(np.array([-10.0, 0.0, 10.0]), dims=["lat"])
101
+ batch = nda.NdRecordBatch({"sst": a, "lat": lat}, attrs={"title": "demo"})
102
+
103
+ rb = batch.to_arrow() # pyarrow.RecordBatch with one row
104
+ table = pa.table(batch) # pyarrow.Table, through the PyCapsule interface
105
+
106
+ # Many batches with one schema: one row each.
107
+ rows = nda.encode_batches([batch, other_batch])
108
+ batches = nda.NdRecordBatch.from_arrow(rows) # a list, one NdRecordBatch per row
109
+ ```
110
+
111
+ `NdRecordBatch.from_arrow` accepts a `RecordBatch`, a `Table` or a
112
+ `RecordBatchReader`. It returns one `NdRecordBatch` for each row.
113
+ The global attributes travel in the `__global_attrs__` column.
114
+
115
+ ### Files
116
+
117
+ Parquet and Arrow IPC keep the field metadata, so the nd columns survive a
118
+ write and a read:
119
+
120
+ ```python
121
+ import pyarrow.parquet as pq
122
+
123
+ pq.write_table(pa.Table.from_batches([rows]), "data.parquet")
124
+ batches = nda.NdRecordBatch.from_arrow(pq.read_table("data.parquet"))
125
+
126
+ with pa.OSFile("data.arrow", "wb") as sink, pa.ipc.new_file(sink, rows.schema) as writer:
127
+ writer.write_batch(rows)
128
+ with pa.memory_map("data.arrow") as source:
129
+ batches = nda.NdRecordBatch.from_arrow(pa.ipc.open_file(source).read_all())
130
+ ```
131
+
132
+ ### Other libraries
133
+
134
+ `NdArray`, `NdColumn` and `NdRecordBatch` implement the Arrow PyCapsule
135
+ interface (`__arrow_c_array__`, `__arrow_c_stream__`, `__arrow_c_schema__`).
136
+ pyarrow, polars, DuckDB and other Arrow libraries read them without a copy,
137
+ for example `pa.array(col)` or `pa.table(batch)`.
138
+
139
+ ## xarray
140
+
141
+ `pip install nd-arrow-array[xarray]` adds xarray and cftime.
142
+
143
+ ```python
144
+ import xarray as xr
145
+
146
+ ds = xr.Dataset(
147
+ {"air": (("time", "lat"), np.zeros((2, 3)), {"units": "K"})},
148
+ coords={"time": np.array(["2020-01-01", "2020-01-02"], "datetime64[ns]"),
149
+ "lat": [-10.0, 0.0, 10.0]},
150
+ )
151
+ batch = nda.NdRecordBatch.from_xarray(ds) # all variables and coordinates
152
+ batch.to_xarray() # an xarray.Dataset
153
+
154
+ a = nda.NdArray.from_xarray(ds["air"]) # values, dims and attrs only
155
+ a.to_xarray(name="air") # an xarray.DataArray
156
+ ```
157
+
158
+ - `NdRecordBatch.from_xarray` accepts a `Dataset` or a `DataArray`.
159
+ Each variable and each coordinate becomes a column.
160
+ - A 1-D column with the name of its dimension becomes an index coordinate.
161
+ - Other coordinates go into the CF `coordinates` attribute of the data
162
+ variables. A coordinate that no data variable uses goes into the
163
+ `coordinates` attribute of the batch.
164
+ - `NdArray.from_xarray` does not keep coordinates.
165
+ - In `to_xarray`, nulls become `NaN` or `NaT`. Integer and bool arrays with
166
+ nulls become `float64`, as in xarray CF decoding.
167
+ - Object arrays must hold `str` or cftime dates. `None` and `NaN` become nulls.
168
+ - cftime dates become `timestamp[us]` in the standard calendar:
169
+ - `standard`, `gregorian`, `proleptic_gregorian` and `julian` keep the
170
+ exact instant;
171
+ - `noleap`, `365_day`, `all_leap`, `366_day` and `360_day` keep the year,
172
+ month, day and time. A date that the standard calendar does not have,
173
+ such as 30 February, raises `ValueError`. Use
174
+ `convert_calendar(..., align_on="date")` in xarray first.
175
+
176
+ ## Rules
177
+
178
+ - NumPy dtypes: `bool`, signed and unsigned integers, `float16`, `float32`,
179
+ `float64`, `datetime64` and `timedelta64` (`s`, `ms`, `us`, `ns`) and `str`.
180
+ Other dtypes raise `TypeError`.
181
+ - A `numpy.ma.MaskedArray` mask and `NaT` become nulls. `NaN` stays a value.
182
+ - `to_numpy()` returns a `MaskedArray` when the values have nulls.
183
+ - `attrs` returns a copy. Assign a new dict, or use `set_attr` and `del_attr`.
184
+ - All classes implement the Arrow PyCapsule interface, so pyarrow, polars
185
+ and DuckDB can read them.
186
+ - The package ships type stubs (`py.typed`), so mypy and pyright check calls.
187
+
188
+ ## Test
189
+
190
+ ```bash
191
+ pip install pytest mypy numpy pyarrow xarray cftime
192
+ maturin develop && pytest
193
+ python -m mypy.stubtest nd_arrow_array # stubs match the module
194
+ mypy --strict --warn-unused-ignores tests/typing
195
+ ```
196
+
@@ -0,0 +1,9 @@
1
+ nd_arrow_array/__init__.py,sha256=NxDl4pFD0kpv0QfUDNGzCQExNH6J5O5LCn3KJwUHUbY,335
2
+ nd_arrow_array/_core.pyd,sha256=PuEpvxiVCZ5im9So5oK-6sSrZA81eWWSDxkNp78m_88,10555904
3
+ nd_arrow_array/_core.pyi,sha256=Oeiu__w2OleaMoxHl0jLauVDm0xft-J0vnWpDa8VyCE,5219
4
+ nd_arrow_array/_xarray.py,sha256=hf0GslIzxj4JHFSCbyiAAWANEbsOuMFPo2WyNcdjopA,8963
5
+ nd_arrow_array/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ nd_arrow_array-0.5.0.dist-info/METADATA,sha256=GEBvkOanQm-FtJXSe5wUHdHo-zMQncZlRfMV0OCKDsU,7569
7
+ nd_arrow_array-0.5.0.dist-info/WHEEL,sha256=xw8F9xBWUOgm5ym2S9tihj_bLJOb723bjhA0kdANo2I,95
8
+ nd_arrow_array-0.5.0.dist-info/sboms/nd-arrow-py.cyclonedx.json,sha256=0n8ENgBmuTv3M_D62YxIQnwJ_1tsiOAkZU5sJiwqO2U,91585
9
+ nd_arrow_array-0.5.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: maturin (1.16.0)
3
+ Root-Is-Purelib: false
4
+ Tag: cp39-abi3-win_amd64