xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,614 @@
1
+ """Engine-agnostic round-trip: Arrow query results → labeled ``xr.Dataset``.
2
+
3
+ The second seam of xarray-sql. Any engine's result — a DuckDB relation,
4
+ a ``pyarrow.Table``, a ``pyarrow.RecordBatchReader``, or any object
5
+ implementing the Arrow PyCapsule stream protocol — plus the registered
6
+ Dataset as a *template* is enough to rebuild a labeled, metadata-carrying
7
+ Dataset. Nothing here is engine-specific: results arrive as Arrow record
8
+ batches regardless of which engine executed the SQL.
9
+
10
+ Reconstruction of a *re-executable* result — a DuckDB relation, a
11
+ Polars LazyFrame (or eager DataFrame), or a DataFusion DataFrame; see
12
+ [xarray_sql.lazyscan][] — is lazy by default: data variables are read
13
+ on access by re-executing the engine's query narrowed to the selection.
14
+ Passing ``chunks=`` additionally wraps them in a chunk manager, one
15
+ window per chunk. DuckDB relations refuse the chunked path (a
16
+ thread-safety limitation noted on
17
+ [DuckDBHandle][xarray_sql.lazyscan.DuckDBHandle]); pair them with
18
+ ``spill=True`` instead. One-shot Arrow streams have nothing to
19
+ re-execute and are materialized once into a dense in-memory Dataset.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import os
25
+ import tempfile
26
+ import weakref
27
+ from collections.abc import Mapping
28
+ from typing import Any, Literal
29
+
30
+ import numpy as np
31
+ import pandas as pd
32
+ import pyarrow as pa
33
+ import pyarrow.compute as pc
34
+ import pyarrow.parquet as pq
35
+ import xarray as xr
36
+
37
+ from .ds import (
38
+ Sparsity,
39
+ XarrayDataFrame,
40
+ _build_lazy_scan,
41
+ _dataset_from_batches,
42
+ _finish_dataset,
43
+ _read_or_discover,
44
+ _result_dims,
45
+ )
46
+ from .lazyscan import LazyResultHandle, PolarsHandle, resolve_lazy_handle
47
+
48
+
49
+ def _guarded(batches: Any, max_bytes: int | None) -> list[pa.RecordBatch]:
50
+ """Collect a batch iterable, erroring cleanly past ``max_bytes``.
51
+
52
+ A result that would blow past the budget raises with the running
53
+ size instead of exhausting memory, before the (larger) dense
54
+ reconstruction is even attempted.
55
+ """
56
+ if max_bytes is None:
57
+ return list(batches)
58
+ out: list[pa.RecordBatch] = []
59
+ total = 0
60
+ for batch in batches:
61
+ total += batch.nbytes
62
+ if total > max_bytes:
63
+ raise ValueError(
64
+ f"result exceeded max_result_bytes={max_bytes:,} while "
65
+ f"materializing (>= {total:,} bytes after "
66
+ f"{sum(b.num_rows for b in out) + batch.num_rows:,} rows). "
67
+ "Aggregate further, or reconstruct lazily with chunks=."
68
+ )
69
+ out.append(batch)
70
+ return out
71
+
72
+
73
+ def _open_stream(result: Any) -> tuple[pa.Schema, Any] | None:
74
+ """The result's Arrow batches as ``(schema, iterable)``, or ``None``.
75
+
76
+ Probes, in order: ``pyarrow.Table`` / ``pyarrow.RecordBatch``,
77
+ ``pyarrow.RecordBatchReader``, ``__arrow_c_stream__`` (the Arrow
78
+ PyCapsule protocol — DuckDB relations qualify on duckdb >= 1.1), and
79
+ a ``fetch_record_batch()`` method (DuckDB relations on older
80
+ versions).
81
+ """
82
+ if isinstance(result, pa.RecordBatch):
83
+ return result.schema, [result]
84
+ if isinstance(result, pa.Table):
85
+ return result.schema, result.to_batches()
86
+ if isinstance(result, pa.RecordBatchReader):
87
+ return result.schema, result
88
+ if hasattr(result, "__arrow_c_stream__"):
89
+ reader = pa.RecordBatchReader.from_stream(result)
90
+ return reader.schema, reader
91
+ if hasattr(result, "fetch_record_batch"):
92
+ reader = result.fetch_record_batch()
93
+ return reader.schema, reader
94
+ return None
95
+
96
+
97
+ def _result_to_batches(
98
+ result: Any, max_bytes: int | None = None
99
+ ) -> tuple[pa.Schema, list[pa.RecordBatch]]:
100
+ """Normalize an engine result into ``(schema, record batches)``.
101
+
102
+ Accepts everything ``_open_stream`` recognizes, then objects
103
+ with a ``to_arrow_table()`` method (DataFusion DataFrames and the
104
+ [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapper), then re-executable
105
+ results without a stream protocol (a Polars LazyFrame), executed
106
+ once through their lazy handle.
107
+ """
108
+ opened = _open_stream(result)
109
+ if opened is not None:
110
+ schema, batches = opened
111
+ if isinstance(result, (pa.RecordBatch, pa.Table)):
112
+ # Already in memory: nothing left for the budget to bound
113
+ # (the dense-size check still applies downstream).
114
+ return schema, list(batches)
115
+ return schema, _guarded(batches, max_bytes)
116
+ handle = resolve_lazy_handle(result)
117
+ if hasattr(result, "to_arrow_table") and (
118
+ max_bytes is None or handle is None
119
+ ):
120
+ # This branch materializes the whole result in one call before
121
+ # the budget can observe a single batch, so with a budget set a
122
+ # re-executable result streams through its handle below instead;
123
+ # the post-materialization nbytes check is the fallback guard
124
+ # for one-shot results whose only surface is to_arrow_table().
125
+ table = result.to_arrow_table()
126
+ if max_bytes is not None and table.nbytes > max_bytes:
127
+ raise ValueError(
128
+ f"result materialized to {table.nbytes:,} bytes, over "
129
+ f"max_result_bytes={max_bytes:,}. Aggregate further, or "
130
+ "reconstruct lazily with chunks=."
131
+ )
132
+ return table.schema, table.to_batches()
133
+ if handle is not None:
134
+ schema = handle.schema()
135
+ names = list(schema.names)
136
+ if max_bytes is None:
137
+ return schema, handle.fetch({}, names)
138
+ # fetch() may materialize the whole result inside the engine
139
+ # before any batch surfaces (Polars collect()), which would
140
+ # defeat the budget; enforce it on a true batch stream, or
141
+ # refuse up front instead of erroring after the memory is spent.
142
+ stream = getattr(handle, "stream", None)
143
+ if stream is None:
144
+ raise ValueError(
145
+ "max_result_bytes cannot be enforced for "
146
+ f"{type(result).__qualname__}: the result materializes "
147
+ "fully before batches surface. Drop max_result_bytes=, "
148
+ "or reconstruct lazily with chunks=."
149
+ )
150
+ return schema, _guarded(stream(names), max_bytes)
151
+ raise TypeError(
152
+ f"Cannot read an Arrow stream from {type(result).__qualname__}; "
153
+ "expected a pyarrow Table/RecordBatch/RecordBatchReader, an object "
154
+ "implementing __arrow_c_stream__, or an engine result exposing "
155
+ "fetch_record_batch()/to_arrow_table()."
156
+ )
157
+
158
+
159
+ def to_dataset(
160
+ result: Any,
161
+ dims: list[str] | None = None,
162
+ template: xr.Dataset | None = None,
163
+ sparsity: Sparsity = "result",
164
+ fill_value: Any = np.nan,
165
+ chunks: Mapping[str, int] | str | None = None,
166
+ coords: Literal["discover", "template"] = "discover",
167
+ max_result_bytes: int | None = None,
168
+ spill: bool | str | os.PathLike = False,
169
+ ) -> xr.Dataset:
170
+ """Convert an engine's Arrow result into a labeled ``xr.Dataset``.
171
+
172
+ The engine-agnostic counterpart of
173
+ [XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset]: SQL in, array out, for engines
174
+ xarray-sql does not wrap in a session of its own.
175
+
176
+ Example (DuckDB)::
177
+
178
+ con = duckdb.connect()
179
+ xql.register(con, "era5", ds)
180
+ rel = con.sql(
181
+ "SELECT time, lat, lon, AVG(t2m) AS t2m FROM era5 "
182
+ "GROUP BY time, lat, lon"
183
+ )
184
+ out = xql.to_dataset(rel, template=ds)
185
+
186
+ Args:
187
+ result: The engine's query result: a ``pyarrow.Table``,
188
+ ``RecordBatch`` or ``RecordBatchReader``, any object
189
+ implementing ``__arrow_c_stream__`` (DuckDB relations), or an
190
+ object with ``fetch_record_batch()`` / ``to_arrow_table()``.
191
+ The result is consumed once.
192
+ dims: Result columns to use as Dataset dimensions. When ``None``,
193
+ defaults to the ``template``'s dimensions that survive into
194
+ the result columns (so aggregations that drop dims round-trip
195
+ on the remaining ones). Either ``dims`` or ``template`` must
196
+ be given.
197
+ template: The source Dataset registered with the engine. Recovers
198
+ metadata the tabular pivot strips (attrs, encoding, non-dim
199
+ coordinates, dim-coord dtype) and provides the ``dims``
200
+ default.
201
+ sparsity: ``"result"`` (default) keeps only dim values present in
202
+ the result. ``"template"`` reindexes to the template's full
203
+ coord ranges, filling absent cells with ``fill_value``.
204
+ fill_value: Fill for ``sparsity="template"``. Defaults to NaN.
205
+ chunks: ``None`` (default) is lazy without a chunk manager, as
206
+ with ``xr.open_dataset(chunks=None)``: each access
207
+ re-executes the query narrowed to the selection and
208
+ projected to the variable read, so ``.load()`` executes it
209
+ once per variable. Dask is not required, and DuckDB
210
+ relations work, since every access runs on the calling
211
+ thread. Because reads re-execute, the connection must stay
212
+ open while the result is used, later changes to the source
213
+ show through, and a query whose rows vary between runs
214
+ (``LIMIT`` without ``ORDER BY``, random sampling) should be
215
+ read once with ``max_result_bytes=`` instead. One-shot
216
+ streams (Arrow tables, readers, ADBC cursors) have nothing
217
+ to re-execute and are read eagerly. A mapping
218
+ (e.g. ``{"time": 100}``), ``"auto"``, or ``"inherit"``
219
+ selects the lazy/chunked path: data variables are
220
+ reconstructed window by window on access, each window
221
+ re-executing the engine's query narrowed to its coordinate
222
+ range (over a table registered through xarray-sql, that
223
+ filter flows back into chunk pruning at the source).
224
+ Requires a re-executable ``result`` — a Polars
225
+ LazyFrame/DataFrame or a DataFusion DataFrame; DuckDB
226
+ relations refuse the chunked path (add ``spill=True``).
227
+ coords: How the lazy path learns each dimension's coordinate
228
+ values. ``"discover"`` (default) runs one ``DISTINCT`` query
229
+ per dim — correct for any query. ``"template"`` trusts the
230
+ template's coord arrays instead, skipping discovery; only
231
+ valid when the result spans the template's full extent (an
232
+ unfiltered scan), and requires ``template=``.
233
+ max_result_bytes: Optional budget for materializing the result.
234
+ Setting it reads the result eagerly, in memory. Raises a
235
+ clean ``ValueError`` (with the running size) as soon as the
236
+ materializing result exceeds it — both while collecting the
237
+ Arrow stream and before allocating the dense arrays —
238
+ instead of exhausting memory. ``None`` (default) means
239
+ unlimited. Results whose only surface is
240
+ ``to_arrow_table()`` necessarily materialize in full before
241
+ the budget can be checked (the check then runs on the
242
+ materialized size); re-executable results stream instead,
243
+ so the budget fires before full materialization.
244
+ spill: Chunked reconstruction from a one-pass on-disk spill
245
+ instead of per-window re-execution: the result is streamed
246
+ *once* (bounded memory) into a temporary Parquet file, and
247
+ windows re-execute against that file. This serves the two
248
+ results the re-execution path cannot — DuckDB relations and
249
+ one-shot Arrow streams — and trades per-window narrowness
250
+ for a single full pass plus temporary disk. ``True`` spills
251
+ to the system temp dir; a path spills into that directory.
252
+ The file is removed when the returned Dataset is garbage
253
+ collected. Requires Polars; only valid with ``chunks=``.
254
+ Returns:
255
+ An ``xr.Dataset`` with ``dims`` as dimensions and the remaining
256
+ result columns as data variables: lazily indexed by default
257
+ (in memory for one-shot streams or with ``max_result_bytes``), and
258
+ lazily chunked when ``chunks`` is given.
259
+
260
+ Raises:
261
+ ValueError: When neither ``dims`` nor ``template`` resolves the
262
+ dimension columns, a requested dim is missing from the result,
263
+ or ``sparsity="template"`` is used without a template.
264
+ TypeError: When ``result`` exposes no readable Arrow stream, or
265
+ ``chunks`` is requested for a one-shot stream that cannot be
266
+ re-executed.
267
+ """
268
+ if sparsity not in ("result", "template"):
269
+ raise ValueError(
270
+ f"sparsity must be 'result' or 'template', got {sparsity!r}"
271
+ )
272
+ if sparsity == "template" and template is None:
273
+ raise ValueError("sparsity='template' requires template= to be given")
274
+ if coords not in ("discover", "template"):
275
+ raise ValueError(
276
+ f"coords must be 'discover' or 'template', got {coords!r}"
277
+ )
278
+ if coords == "template" and template is None:
279
+ raise ValueError("coords='template' requires template= to be given")
280
+ if spill and chunks is None:
281
+ raise ValueError(
282
+ "spill= only applies to chunked reconstruction; pass chunks=."
283
+ )
284
+
285
+ if chunks is not None:
286
+ if spill:
287
+ return _to_dataset_spilled(
288
+ result,
289
+ dims,
290
+ template,
291
+ sparsity,
292
+ fill_value,
293
+ chunks,
294
+ coords,
295
+ spill,
296
+ )
297
+ return _to_dataset_lazy(
298
+ result, dims, template, sparsity, fill_value, chunks, coords
299
+ )
300
+ if max_result_bytes is None:
301
+ handle = resolve_lazy_handle(result)
302
+ if handle is not None:
303
+ return _to_dataset_lazy(
304
+ result,
305
+ dims,
306
+ template,
307
+ sparsity,
308
+ fill_value,
309
+ None,
310
+ coords,
311
+ _handle=handle,
312
+ )
313
+
314
+ schema, batches = _result_to_batches(result, max_result_bytes)
315
+ field_names = [f.name for f in schema]
316
+ field_types = {f.name: f.type for f in schema}
317
+
318
+ dims = _resolve_dims(dims, template, field_names)
319
+
320
+ if max_result_bytes is not None:
321
+ _check_dense_size(
322
+ batches, dims, field_names, field_types, max_result_bytes
323
+ )
324
+ ds = _dataset_from_batches(batches, dims, field_names, field_types)
325
+ return _finish_dataset(
326
+ ds, dims, template, sparsity, fill_value, None, field_types
327
+ )
328
+
329
+
330
+ def _check_dense_size(
331
+ batches: list[pa.RecordBatch],
332
+ dims: list[str],
333
+ field_names: list[str],
334
+ field_types: dict[str, Any],
335
+ max_bytes: int,
336
+ ) -> None:
337
+ """Error before allocating dense arrays larger than the budget.
338
+
339
+ The dense grid is the coordinate product, which for sparse results
340
+ can dwarf the Arrow input; check it against the same budget before
341
+ a single output array is allocated.
342
+ """
343
+ sizes = []
344
+ for d in dims:
345
+ # Vectorized distinct count: a per-row Python set (to_pylist)
346
+ # costs orders of magnitude more on wide results.
347
+ arrays = [b.column(b.schema.names.index(d)) for b in batches]
348
+ sizes.append(len(pc.unique(pa.chunked_array(arrays))) if arrays else 0)
349
+ cells = int(np.prod(sizes)) if sizes else 0
350
+ total = sum(
351
+ cells * np.dtype(field_types[n].to_pandas_dtype()).itemsize
352
+ for n in field_names
353
+ if n not in dims
354
+ )
355
+ if total > max_bytes:
356
+ raise ValueError(
357
+ f"dense reconstruction needs {total:,} bytes "
358
+ f"({cells:,} grid cells), over max_result_bytes="
359
+ f"{max_bytes:,}. Aggregate further, or reconstruct lazily "
360
+ "with chunks=."
361
+ )
362
+
363
+
364
+ def _resolve_dims(
365
+ dims: list[str] | None,
366
+ template: xr.Dataset | None,
367
+ field_names: list[str],
368
+ ) -> list[str]:
369
+ """Dimension columns, inferred from the template when not given."""
370
+ if dims is None:
371
+ if template is None:
372
+ raise ValueError(
373
+ "dims cannot be inferred without a template; pass "
374
+ "dims=[...] or template=<the registered Dataset>."
375
+ )
376
+ dims = _result_dims(template, field_names)
377
+ if not dims:
378
+ raise ValueError(
379
+ "dims cannot be inferred: no template dimension survives "
380
+ "in the result columns. Pass dims=[...] explicitly."
381
+ )
382
+ missing = [d for d in dims if d not in field_names]
383
+ if missing:
384
+ raise ValueError(
385
+ f"dims {missing} are not columns of the result {field_names}."
386
+ )
387
+ return dims
388
+
389
+
390
+ def _to_dataset_lazy(
391
+ result: Any,
392
+ dims: list[str] | None,
393
+ template: xr.Dataset | None,
394
+ sparsity: Sparsity,
395
+ fill_value: Any,
396
+ chunks: Mapping[str, int] | str | None,
397
+ coords: Literal["discover", "template"],
398
+ _handle: LazyResultHandle | None = None,
399
+ ) -> xr.Dataset:
400
+ """The lazy reconstruction behind ``to_dataset``.
401
+
402
+ ``chunks=None`` returns lazily indexed variables without chunking
403
+ them.
404
+ """
405
+ handle = _handle if _handle is not None else resolve_lazy_handle(result)
406
+ if handle is None:
407
+ raise TypeError(
408
+ "Lazy reconstruction requires a re-executable engine result (a "
409
+ "DuckDB relation, Polars LazyFrame/DataFrame, or DataFusion "
410
+ "DataFrame); got "
411
+ f"{type(result).__qualname__}, which is a one-shot stream. "
412
+ "Pass the engine's lazy handle instead of a materialized "
413
+ "result, add spill=True to reconstruct from a one-pass "
414
+ "on-disk spill, or reconstruct eagerly (the default)."
415
+ )
416
+ schema = handle.schema()
417
+ field_names = [f.name for f in schema]
418
+ field_types = {f.name: f.type for f in schema}
419
+ dims = _resolve_dims(dims, template, field_names)
420
+
421
+ coord_arrays = None
422
+ if coords == "template":
423
+ assert template is not None
424
+ missing = [d for d in dims if d not in template.coords]
425
+ if missing:
426
+ raise ValueError(
427
+ f"coords='template' requires the template to carry coords "
428
+ f"for every dim; missing {missing}."
429
+ )
430
+ coord_arrays = {d: np.asarray(template.coords[d].values) for d in dims}
431
+
432
+ if chunks is None:
433
+ # Lazily indexed and unchunked: every access runs on the calling
434
+ # thread, so even handles that refuse chunking can serve it. A
435
+ # small result is cheaper to read once than to discover lazily.
436
+ batches = None
437
+ if coord_arrays is None:
438
+ batches, coord_arrays = _read_or_discover(handle, field_names, dims)
439
+ if batches is not None:
440
+ ds = _dataset_from_batches(batches, dims, field_names, field_types)
441
+ else:
442
+ ds = _build_lazy_scan(
443
+ handle,
444
+ dims,
445
+ field_names,
446
+ field_types,
447
+ coord_arrays=coord_arrays,
448
+ )
449
+ return _finish_dataset(
450
+ ds, dims, template, sparsity, fill_value, None, field_types
451
+ )
452
+
453
+ resolved = XarrayDataFrame._resolve_chunks(chunks, template, dims)
454
+ if resolved is None:
455
+ # "inherit" with no chunked source dimension to inherit from:
456
+ # eager is the right execution, exactly as on the wrapper path.
457
+ batches = handle.fetch({}, field_names)
458
+ ds = _dataset_from_batches(batches, dims, field_names, field_types)
459
+ return _finish_dataset(
460
+ ds, dims, template, sparsity, fill_value, None, field_types
461
+ )
462
+ if not getattr(handle, "supports_chunked", True):
463
+ raise NotImplementedError(
464
+ "Chunked reconstruction is not supported for "
465
+ f"{type(result).__qualname__}: re-executing a DuckDB "
466
+ "relation from worker threads intermittently deadlocks in "
467
+ "duckdb-python when the query scans a Python-backed table "
468
+ "(see xarray_sql.lazyscan.DuckDBHandle.supports_chunked). "
469
+ "Use chunks=None (lazy and unchunked: each access runs on the "
470
+ "calling thread), add spill=True to reconstruct from a "
471
+ "one-pass on-disk spill, or run the query through "
472
+ "Polars (pl.scan_pyarrow_dataset(xql.arrow_dataset(ds))) "
473
+ "or a DataFusion context."
474
+ )
475
+ ds = _build_lazy_scan(
476
+ handle, dims, field_names, field_types, coord_arrays=coord_arrays
477
+ )
478
+ return _finish_dataset(
479
+ ds, dims, template, sparsity, fill_value, resolved, field_types
480
+ )
481
+
482
+
483
+ def _to_dataset_spilled(
484
+ result: Any,
485
+ dims: list[str] | None,
486
+ template: xr.Dataset | None,
487
+ sparsity: Sparsity,
488
+ fill_value: Any,
489
+ chunks: Mapping[str, int] | str,
490
+ coords: Literal["discover", "template"],
491
+ spill: bool | str | os.PathLike,
492
+ ) -> xr.Dataset:
493
+ """Chunked reconstruction from a one-pass temporary Parquet spill.
494
+
495
+ The result is streamed exactly once with bounded memory — through
496
+ the engine handle where one exists (DuckDB spills on its dedicated
497
+ engine thread; Polars uses its streaming sink), or straight from
498
+ the Arrow stream for one-shot results — and the ordinary lazy
499
+ reconstruction then runs against a Polars scan of the file, whose
500
+ per-window predicates enjoy Parquet row-group pruning. The file is
501
+ removed when the reconstruction handle is garbage collected.
502
+ """
503
+ import polars as pl
504
+
505
+ directory = os.fspath(spill) if not isinstance(spill, bool) else None
506
+ fd, path = tempfile.mkstemp(suffix=".parquet", dir=directory)
507
+ os.close(fd)
508
+ try:
509
+ handle = resolve_lazy_handle(result)
510
+ if handle is not None:
511
+ handle.spill_parquet(path)
512
+ else:
513
+ _stream_to_parquet(result, path, template)
514
+ except BaseException:
515
+ os.unlink(path)
516
+ raise
517
+ spilled = PolarsHandle(pl.scan_parquet(path))
518
+ weakref.finalize(spilled, _unlink_quietly, path)
519
+ return _to_dataset_lazy(
520
+ result,
521
+ dims,
522
+ template,
523
+ sparsity,
524
+ fill_value,
525
+ chunks,
526
+ coords,
527
+ _handle=spilled,
528
+ )
529
+
530
+
531
+ def _unlink_quietly(path: str) -> None:
532
+ try:
533
+ os.unlink(path)
534
+ except OSError:
535
+ pass
536
+
537
+
538
+ _MONTH_DAY_NANO = np.dtype(
539
+ [("months", "<i4"), ("days", "<i4"), ("nanos", "<i8")]
540
+ )
541
+ _NANOS_PER_DAY = 86_400 * 10**9
542
+
543
+
544
+ def _as_durations(array: pa.Array) -> pa.Array:
545
+ """Durations a database returned as intervals or text, as ``duration``.
546
+
547
+ Databases without a duration type return month-day-nano intervals
548
+ (DuckDB, PostgreSQL) or text such as ``'21600s'`` (MySQL, Trino).
549
+ """
550
+ if pa.types.is_interval(array.type):
551
+ parts = np.frombuffer(
552
+ array.buffers()[1],
553
+ dtype=_MONTH_DAY_NANO,
554
+ count=array.offset + len(array),
555
+ )[array.offset :]
556
+ valid = ~np.asarray(array.is_null())
557
+ if parts["months"][valid].any():
558
+ raise ValueError(
559
+ "an interval spans calendar months, which have no fixed "
560
+ "duration"
561
+ )
562
+ nanos = parts["days"].astype(np.int64) * _NANOS_PER_DAY + parts["nanos"]
563
+ return pa.array(nanos, pa.duration("ns"), mask=~valid)
564
+ return pa.array(pd.to_timedelta(array.to_pandas()), pa.duration("ns"))
565
+
566
+
567
+ def _as_timestamps(array: pa.Array) -> pa.Array:
568
+ """Times a database returned as text (SQLite has no time type)."""
569
+ # ISO8601 rather than an inferred format: SQLite text has a fraction
570
+ # of a second only where there is one, so a column mixes both forms.
571
+ times = pd.to_datetime(array.to_pandas(), format="ISO8601")
572
+ return pa.array(times, pa.timestamp("ns"))
573
+
574
+
575
+ def _stream_to_parquet(
576
+ result: Any, path: str, template: xr.Dataset | None = None
577
+ ) -> None:
578
+ """Write a one-shot Arrow result to Parquet, batch by batch.
579
+
580
+ Coordinates the template holds as ``timedelta64`` or ``datetime64``
581
+ but the database returned as intervals or text are written as
582
+ durations and timestamps: Parquet cannot store month-day-nano
583
+ intervals, and the windows the chunked reconstruction builds compare
584
+ times, not text.
585
+ """
586
+ opened = _open_stream(result)
587
+ if opened is None:
588
+ raise TypeError(
589
+ f"cannot spill {type(result).__qualname__}: no readable "
590
+ "Arrow stream."
591
+ )
592
+ schema, batches = opened
593
+ conversions = {}
594
+ for i, field in enumerate(schema):
595
+ if template is None or field.name not in template.coords:
596
+ continue
597
+ kind = template.coords[field.name].dtype.kind
598
+ text = pa.types.is_string(field.type) or pa.types.is_large_string(
599
+ field.type
600
+ )
601
+ if kind == "m" and (text or pa.types.is_interval(field.type)):
602
+ conversions[i] = (_as_durations, pa.duration("ns"))
603
+ elif kind == "M" and text:
604
+ conversions[i] = (_as_timestamps, pa.timestamp("ns"))
605
+ for i, (_, arrow_type) in conversions.items():
606
+ schema = schema.set(i, schema.field(i).with_type(arrow_type))
607
+ with pq.ParquetWriter(path, schema) as writer:
608
+ for batch in batches:
609
+ if conversions:
610
+ columns = list(batch.columns)
611
+ for i, (convert, _) in conversions.items():
612
+ columns[i] = convert(columns[i])
613
+ batch = pa.RecordBatch.from_arrays(columns, schema=schema)
614
+ writer.write_batch(batch)