xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/cftime.py ADDED
@@ -0,0 +1,261 @@
1
+ """Bridge between cftime calendars and Arrow/DataFusion types.
2
+
3
+ cftime (https://unidata.github.io/cftime/) provides datetime objects for
4
+ calendars used in climate science — noleap, 360-day, all-leap, julian, etc.
5
+ Arrow and DataFusion have no native concept of non-Gregorian calendars, so
6
+ this module handles the conversion in two tiers:
7
+
8
+ * **Gregorian-like calendars** (standard, gregorian, proleptic_gregorian,
9
+ noleap/365_day, all_leap/366_day): mapped to ``pa.timestamp('us')`` so
10
+ that string-based SQL filters like ``WHERE time > '1980-01-01'`` work
11
+ naturally. Microsecond resolution avoids the 1678–2262 overflow of
12
+ nanoseconds while preserving sub-second precision.
13
+
14
+ * **Non-Gregorian calendars** (360_day, julian): mapped to ``pa.int64()``
15
+ with ``xarray:units`` and ``xarray:calendar`` metadata on the Arrow field.
16
+ This preserves the original CF-convention encoding losslessly. A
17
+ ``cftime()`` DataFusion UDF (registered automatically by
18
+ ``Context.from_dataset``) provides ergonomic SQL filtering.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from typing import cast
24
+
25
+ import numpy as np
26
+ import pyarrow as pa
27
+ import xarray as xr
28
+
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # Calendar classification
32
+ # ---------------------------------------------------------------------------
33
+
34
+ GREGORIAN_LIKE_CALENDARS: frozenset[str] = frozenset(
35
+ {
36
+ "standard",
37
+ "gregorian",
38
+ "proleptic_gregorian",
39
+ "noleap",
40
+ "365_day",
41
+ "all_leap",
42
+ "366_day",
43
+ }
44
+ )
45
+ """Calendars close enough to proleptic Gregorian for ``pa.timestamp('us')``."""
46
+
47
+ DEFAULT_UNITS: str = "microseconds since 1970-01-01T00:00:00"
48
+ """Default CF-convention units when no encoding is available on the coordinate.
49
+
50
+ Microseconds give sub-second precision and fit int64 for ±292 k years.
51
+ """
52
+
53
+
54
+ def is_gregorian_like(calendar: str) -> bool:
55
+ """Return True if *calendar* is close enough to Gregorian for ``pa.timestamp``."""
56
+ return calendar in GREGORIAN_LIKE_CALENDARS
57
+
58
+
59
+ # ---------------------------------------------------------------------------
60
+ # Detection helpers (avoid materializing Dask/Zarr data where possible)
61
+ # ---------------------------------------------------------------------------
62
+
63
+
64
+ def is_cftime(values: np.ndarray) -> bool:
65
+ """Check if a numpy array contains cftime datetime objects."""
66
+ try:
67
+ import cftime
68
+
69
+ if values.dtype == np.dtype("O") and len(values) > 0:
70
+ sample = values.ravel()[0]
71
+ return isinstance(sample, cftime.datetime)
72
+ except ImportError:
73
+ pass
74
+ return False
75
+
76
+
77
+ def is_cftime_index(ds: xr.Dataset, coord_name: str) -> bool:
78
+ """Check if a coordinate uses a ``CFTimeIndex`` without materializing data."""
79
+ try:
80
+ idx = ds.indexes.get(coord_name)
81
+ if idx is not None:
82
+ from xarray import CFTimeIndex
83
+
84
+ return isinstance(idx, CFTimeIndex)
85
+ except (ImportError, AttributeError):
86
+ pass
87
+ return False
88
+
89
+
90
+ def calendar(ds: xr.Dataset, coord_name: str) -> str | None:
91
+ """Return the calendar name for a cftime coordinate, or ``None``.
92
+
93
+ Checks the xarray index first (no data materialization), then falls
94
+ back to inspecting element 0 of the coordinate values.
95
+ """
96
+ try:
97
+ idx = ds.indexes.get(coord_name)
98
+ if idx is not None:
99
+ from xarray import CFTimeIndex
100
+
101
+ if isinstance(idx, CFTimeIndex):
102
+ return str(idx.calendar) # type: ignore[attr-defined]
103
+ except (ImportError, AttributeError):
104
+ pass
105
+ try:
106
+ values = ds.coords[coord_name].values
107
+ if is_cftime(values):
108
+ return str(values.ravel()[0].calendar)
109
+ except (AttributeError, KeyError):
110
+ pass
111
+ return None
112
+
113
+
114
+ def encoding(ds: xr.Dataset, coord_name: str) -> tuple[str, str]:
115
+ """Return ``(units, calendar)`` for a cftime coordinate.
116
+
117
+ Reads xarray ``.encoding`` metadata (from the originating NetCDF file)
118
+ first, falling back to [DEFAULT_UNITS][xarray_sql.cftime.DEFAULT_UNITS].
119
+ """
120
+ cal = calendar(ds, coord_name) or "standard"
121
+ enc = ds.coords[coord_name].encoding
122
+ units = enc.get("units", DEFAULT_UNITS)
123
+ return units, cal
124
+
125
+
126
+ # ---------------------------------------------------------------------------
127
+ # Numeric conversion
128
+ # ---------------------------------------------------------------------------
129
+
130
+
131
+ def to_microseconds(values) -> np.ndarray:
132
+ """Convert cftime objects to int64 microseconds since Unix epoch.
133
+
134
+ Used for Gregorian-like calendars. Vectorised via ``cftime.date2num``
135
+ (implemented in C).
136
+ """
137
+ import cftime as _cftime
138
+
139
+ us = _cftime.date2num(
140
+ values.ravel(),
141
+ units=DEFAULT_UNITS,
142
+ calendar=values.ravel()[0].calendar,
143
+ )
144
+ return cast(np.ndarray, np.asarray(us, dtype=np.float64).astype(np.int64))
145
+
146
+
147
+ def to_offsets(values, units: str, cal: str) -> np.ndarray:
148
+ """Convert cftime objects to int64 offsets in the given *units*/*calendar*.
149
+
150
+ Used for non-Gregorian calendars where data is stored as ``pa.int64()``.
151
+ """
152
+ import cftime as _cftime
153
+
154
+ raw = _cftime.date2num(values.ravel(), units=units, calendar=cal)
155
+ return cast(np.ndarray, np.asarray(raw, dtype=np.float64).astype(np.int64))
156
+
157
+
158
+ def convert_for_field(values, field: pa.Field) -> np.ndarray:
159
+ """Convert cftime values to the numeric type dictated by *field*.
160
+
161
+ Reads ``xarray:calendar`` and ``xarray:units`` from the field's Arrow
162
+ metadata to choose between the timestamp path and the integer-offset path.
163
+ """
164
+ meta = field.metadata or {}
165
+ cal = meta.get(b"xarray:calendar", b"standard").decode()
166
+ units = meta.get(b"xarray:units", DEFAULT_UNITS.encode()).decode()
167
+ if is_gregorian_like(cal):
168
+ return to_microseconds(values)
169
+ return to_offsets(values, units, cal)
170
+
171
+
172
+ # ---------------------------------------------------------------------------
173
+ # Partition pruning helpers
174
+ # ---------------------------------------------------------------------------
175
+
176
+
177
+ def partition_bounds(
178
+ values,
179
+ ) -> tuple[int, int, str] | None:
180
+ """Return ``(min, max, dtype_tag)`` for a cftime coordinate slice.
181
+
182
+ Gregorian-like calendars return nanosecond bounds tagged
183
+ ``"timestamp_ns"`` (compatible with ``ScalarBound::TimestampNanos``
184
+ in the Rust pruning layer). Non-Gregorian calendars return int64
185
+ offsets tagged ``"int64"``.
186
+
187
+ Returns ``None`` when the nanosecond bound falls outside the int64 range
188
+ (e.g. paleoclimate dates before ~1678), signalling the caller to skip
189
+ pruning for that dimension rather than emit a bound the Rust layer would
190
+ reject.
191
+ """
192
+ cal = values.ravel()[0].calendar
193
+ if is_gregorian_like(cal):
194
+ us = to_microseconds(values)
195
+ lo, hi = int(us.min()) * 1_000, int(us.max()) * 1_000
196
+ int64 = np.iinfo(np.int64)
197
+ if lo < int64.min or hi > int64.max:
198
+ return None
199
+ return lo, hi, "timestamp_ns"
200
+ offsets = to_offsets(values, DEFAULT_UNITS, cal)
201
+ return int(offsets.min()), int(offsets.max()), "int64"
202
+
203
+
204
+ # ---------------------------------------------------------------------------
205
+ # Arrow schema helpers
206
+ # ---------------------------------------------------------------------------
207
+
208
+
209
+ def arrow_field(name: str, units: str, cal: str) -> pa.Field:
210
+ """Build a ``pa.Field`` for a cftime coordinate.
211
+
212
+ Gregorian-like → ``pa.timestamp('us')``; non-Gregorian → ``pa.int64()``.
213
+ Both carry ``xarray:calendar`` and ``xarray:units`` metadata for
214
+ round-trip fidelity.
215
+ """
216
+ meta = {
217
+ b"xarray:calendar": cal.encode(),
218
+ b"xarray:units": units.encode(),
219
+ }
220
+ if is_gregorian_like(cal):
221
+ return pa.field(name, pa.timestamp("us"), metadata=meta)
222
+ return pa.field(name, pa.int64(), metadata=meta)
223
+
224
+
225
+ # ---------------------------------------------------------------------------
226
+ # DataFusion UDF
227
+ # ---------------------------------------------------------------------------
228
+
229
+
230
+ def make_cftime_udf(units: str, calendar: str):
231
+ """Create a DataFusion scalar UDF that converts date strings to int64 offsets.
232
+
233
+ This enables ergonomic SQL filtering on non-Gregorian cftime columns::
234
+
235
+ SELECT * FROM ds360 WHERE time > cftime('0500-01-01')
236
+
237
+ The UDF parses the input string as a cftime datetime in the given
238
+ calendar system and returns the corresponding int64 offset in the
239
+ specified units.
240
+ """
241
+ import cftime as _cftime
242
+ from datafusion import udf
243
+
244
+ def _cftime_scalar(date_strings: pa.Array) -> pa.Array:
245
+ results: list[int | None] = []
246
+ for s in date_strings.to_pylist():
247
+ if s is None:
248
+ results.append(None)
249
+ continue
250
+ dt = _cftime.datetime.strptime(s, "%Y-%m-%d", calendar=calendar)
251
+ val = _cftime.date2num(dt, units=units, calendar=calendar)
252
+ results.append(int(val))
253
+ return pa.array(results, type=pa.int64())
254
+
255
+ return udf(
256
+ _cftime_scalar,
257
+ [pa.utf8()],
258
+ pa.int64(),
259
+ "immutable",
260
+ "cftime",
261
+ )
xarray_sql/core.py ADDED
@@ -0,0 +1,49 @@
1
+ import itertools
2
+ from collections.abc import Iterator
3
+ from typing import Any
4
+
5
+ import numpy as np
6
+ import xarray as xr
7
+
8
+ Row = list[Any]
9
+
10
+
11
+ # deprecated
12
+ def get_columns(ds: xr.Dataset) -> list[str]:
13
+ return list(ds.sizes.keys()) + list(ds.data_vars.keys())
14
+
15
+
16
+ # Deprecated
17
+ def unravel(ds: xr.Dataset) -> Iterator[Row]:
18
+ dim_keys, dim_vals = zip(*ds.sizes.items())
19
+
20
+ for idx in itertools.product(*(range(d) for d in dim_vals)):
21
+ coord_idx = dict(zip(dim_keys, idx))
22
+ data = ds.isel(coord_idx)
23
+ coord_data = [ds.coords[v][coord_idx[v]] for v in dim_keys]
24
+ row = [v.values for v in coord_data + list(data.data_vars.values())]
25
+ yield row
26
+
27
+
28
+ # Deprecated
29
+ def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
30
+ """Unravel with unbounded memory (as a NumPy Array)."""
31
+ dim_keys, dim_vals = zip(*ds.sizes.items())
32
+ columns = get_columns(ds)
33
+
34
+ N = np.prod([d for d in dim_vals])
35
+
36
+ out = np.recarray((N,), dtype=[(c, ds[c].dtype) for c in columns])
37
+
38
+ for name, da in ds.items():
39
+ out[name] = da.values.ravel()
40
+
41
+ prod_vals = (ds.coords[k].values for k in dim_keys)
42
+ coords = np.array(np.meshgrid(*prod_vals), dtype=int).T.reshape(
43
+ -1, len(dim_keys)
44
+ )
45
+
46
+ for i, d in enumerate(dim_keys):
47
+ out[d] = coords[:, i]
48
+
49
+ return out