xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/proj.py ADDED
@@ -0,0 +1,235 @@
1
+ """PROJ-backed CRS transforms for SQL — the optional geo extension.
2
+
3
+ Geospatial SQL dialects expose coordinate reference system (CRS)
4
+ transforms as a scalar function — PostGIS and DuckDB-spatial both call it
5
+ ``ST_Transform`` — because a CRS transform is row-independent: each
6
+ point's new coordinate depends only on its own old coordinate. This
7
+ module brings the same capability to xarray-sql as a vectorized scalar
8
+ UDF over Arrow arrays::
9
+
10
+ SELECT x, y,
11
+ reproject(x, y, 'EPSG:32610', 'EPSG:4326')['x'] AS lon,
12
+ reproject(x, y, 'EPSG:32610', 'EPSG:4326')['y'] AS lat
13
+ FROM grid
14
+
15
+ The CRS pair is part of the *query*, not baked in at registration time,
16
+ so one registered UDF serves any transform — and, because the arguments
17
+ are ordinary SQL expressions, the CRS may even vary per row (e.g. a
18
+ ``CASE`` expression selecting the UTM zone from the longitude).
19
+
20
+ Design notes:
21
+
22
+ * **Both output coordinates come from one call**, returned as an Arrow
23
+ struct ``{x, y}`` (in ``always_xy`` order: easting/longitude first).
24
+ Splitting the transform into two scalar UDFs would run PROJ twice per
25
+ row and, worse, evaluate the two projections concurrently on separate
26
+ expression trees.
27
+ * **All pyproj work runs on a dedicated pool of Python threads.**
28
+ DataFusion's runtime workers are not Python-created threads, and
29
+ pyproj (< 3.8, see pyproj#1541) leaves a dangling ``PJ_CONTEXT``
30
+ behind when their ephemeral Python thread states are torn down, so
31
+ calling pyproj in place segfaults — the UDF hands each batch to the
32
+ pool instead. Pool threads are long-lived, so each caches one
33
+ transformer per CRS pair (transformers must not be shared across
34
+ threads), amortizing the expensive construction — PROJ database
35
+ lookups and candidate-operation selection — across record batches.
36
+ Concurrent partitions still transform in parallel across the pool.
37
+ * Any CRS spelling ``pyproj.CRS`` accepts works: authority codes
38
+ (``EPSG:4326``), WKT, PROJ strings (``+proj=utm +zone=10``), etc.
39
+ An unknown CRS raises ``pyproj.exceptions.CRSError`` and fails the
40
+ query loudly rather than returning wrong coordinates.
41
+ * Non-finite or NULL input coordinates yield NaN output (PROJ itself
42
+ would return ``inf``); NULL CRS arguments yield NaN as well.
43
+
44
+ Requires ``pyproj`` (``pip install xarray-sql[geo]``). When pyproj is
45
+ installed, [xarray_sql.Context][] registers ``reproject()``
46
+ automatically; [register][xarray_sql.proj.register] is the explicit hook for plain
47
+ DataFusion ``SessionContext`` objects or custom UDF names.
48
+ """
49
+
50
+ from __future__ import annotations
51
+
52
+ import os
53
+ import threading
54
+ from concurrent.futures import ThreadPoolExecutor
55
+
56
+ import numpy as np
57
+ import pyarrow as pa
58
+ import pyarrow.compute as pc
59
+ import pyproj
60
+ from datafusion import udf
61
+
62
+ __all__ = ["register"]
63
+
64
+ RETURN_TYPE = pa.struct([("x", pa.float64()), ("y", pa.float64())])
65
+ """Arrow type returned by ``reproject()``: destination coordinates in
66
+ ``always_xy`` order — ``x`` is easting/longitude, ``y`` is
67
+ northing/latitude."""
68
+
69
+
70
+ # ---------------------------------------------------------------------------
71
+ # The PROJ worker pool
72
+ # ---------------------------------------------------------------------------
73
+ #
74
+ # DataFusion evaluates UDFs on its tokio runtime's worker threads, which
75
+ # are not created by Python: a Python thread state is created and
76
+ # destroyed around every UDF call. pyproj keeps its per-thread PJ_CONTEXT
77
+ # in CPython thread-specific storage but does not clear that pointer when
78
+ # the context dies with the ephemeral thread state (fixed by pyproj#1541,
79
+ # unreleased as of 3.7.2), so the next call on the same OS thread
80
+ # dereferences a dangling context and segfaults inside ``proj_create``.
81
+ # Python-owned threads keep their thread state — and thus their contexts —
82
+ # alive for the thread's lifetime, so the UDF never calls pyproj in place:
83
+ # every batch is handed to a small pool of Python-owned worker threads.
84
+ # The pool stays worthwhile on fixed pyproj too: ephemeral thread states
85
+ # would rebuild context and transformer per batch (0.07–12 ms measured)
86
+ # versus ~10 µs for the pool round-trip. pyproj releases the GIL during
87
+ # the transform loop, so concurrent partitions still run in parallel
88
+ # across the pool.
89
+
90
+ _local = threading.local()
91
+ _pool_lock = threading.Lock()
92
+ _pool: ThreadPoolExecutor | None = None
93
+
94
+
95
+ def _proj_pool() -> ThreadPoolExecutor:
96
+ """Return the process-wide pool that runs all pyproj work."""
97
+ global _pool
98
+ if _pool is None:
99
+ with _pool_lock:
100
+ if _pool is None:
101
+ _pool = ThreadPoolExecutor(
102
+ max_workers=os.cpu_count() or 4,
103
+ thread_name_prefix="xarray-sql-proj",
104
+ )
105
+ return _pool
106
+
107
+
108
+ def _transformer(src_crs: str, dst_crs: str) -> pyproj.Transformer:
109
+ """Return a cached ``Transformer`` owned by the calling pool thread.
110
+
111
+ PROJ transformers are not safe to share across threads, so each pool
112
+ thread keeps its own transformer per ``(src, dst)`` pair; the cache
113
+ also amortizes construction (expensive PROJ database lookups) across
114
+ record batches. ``always_xy=True`` fixes the argument order to
115
+ (easting/longitude, northing/latitude) regardless of the CRS's
116
+ declared axis order.
117
+ """
118
+ cache = getattr(_local, "transformers", None)
119
+ if cache is None:
120
+ cache = _local.transformers = {}
121
+ key = (src_crs, dst_crs)
122
+ transformer = cache.get(key)
123
+ if transformer is None:
124
+ transformer = cache[key] = pyproj.Transformer.from_crs(
125
+ src_crs, dst_crs, always_xy=True
126
+ )
127
+ return transformer
128
+
129
+
130
+ def _transform_chunk(
131
+ src_crs: str, dst_crs: str, xs: np.ndarray, ys: np.ndarray
132
+ ) -> tuple[np.ndarray, np.ndarray]:
133
+ """Transform one coordinate chunk; runs on a PROJ pool thread."""
134
+ tx, ty = _transformer(src_crs, dst_crs).transform(xs, ys)
135
+ return tx, ty
136
+
137
+
138
+ # ---------------------------------------------------------------------------
139
+ # The UDF
140
+ # ---------------------------------------------------------------------------
141
+
142
+
143
+ def _reproject(
144
+ x: pa.Array, y: pa.Array, src_crs: pa.Array, dst_crs: pa.Array
145
+ ) -> pa.Array:
146
+ """Vectorized ``reproject`` kernel over one Arrow record batch.
147
+
148
+ DataFusion broadcasts scalar arguments (the usual literal CRS
149
+ strings) to full-length arrays before calling in, so all four
150
+ arguments arrive with one value per row. The common case — one CRS
151
+ pair for the whole batch — never touches the strings row by row:
152
+ uniqueness is established with a vectorized Arrow kernel and the
153
+ batch becomes a single PROJ call. (Materializing the CRS columns
154
+ as Python strings costs two object allocations per row, which at
155
+ billions of rows dwarfs the transform itself.) Only when the CRS
156
+ genuinely varies within the batch are rows grouped by pair and
157
+ transformed per group.
158
+ """
159
+ # Zero-copy read-only views when the batch has no nulls; with nulls,
160
+ # pyarrow must materialize the validity bitmap as NaN (the NULL -> NaN
161
+ # contract). pyproj copies into its own writable buffer either way --
162
+ # PROJ mutates buffers in place -- so this is the minimal-copy path.
163
+ xs = np.asarray(x.to_numpy(zero_copy_only=False), dtype="float64")
164
+ ys = np.asarray(y.to_numpy(zero_copy_only=False), dtype="float64")
165
+
166
+ out_x = np.full(xs.shape, np.nan)
167
+ out_y = np.full(ys.shape, np.nan)
168
+ valid = np.isfinite(xs) & np.isfinite(ys)
169
+ src_unique = pc.unique(src_crs)
170
+ dst_unique = pc.unique(dst_crs)
171
+
172
+ if len(src_unique) == 1 and len(dst_unique) == 1:
173
+ groups = [(src_unique[0].as_py(), dst_unique[0].as_py(), valid)]
174
+ else:
175
+ pairs = list(zip(src_crs.to_pylist(), dst_crs.to_pylist()))
176
+ groups = [
177
+ (
178
+ src,
179
+ dst,
180
+ valid
181
+ & np.fromiter(
182
+ (p == (src, dst) for p in pairs),
183
+ dtype=bool,
184
+ count=len(pairs),
185
+ ),
186
+ )
187
+ for src, dst in set(pairs)
188
+ ]
189
+
190
+ for src, dst, mask in groups:
191
+ if src is None or dst is None or not mask.any():
192
+ continue
193
+ tx, ty = (
194
+ _proj_pool()
195
+ .submit(_transform_chunk, src, dst, xs[mask], ys[mask])
196
+ .result()
197
+ )
198
+ out_x[mask] = tx
199
+ out_y[mask] = ty
200
+
201
+ # PROJ signals out-of-domain points with inf; normalize to NaN so
202
+ # the result round-trips to xarray like any other missing value.
203
+ invalid = ~(np.isfinite(out_x) & np.isfinite(out_y))
204
+ out_x[invalid] = np.nan
205
+ out_y[invalid] = np.nan
206
+
207
+ return pa.StructArray.from_arrays(
208
+ [pa.array(out_x), pa.array(out_y)], names=["x", "y"]
209
+ )
210
+
211
+
212
+ def register(ctx, name: str = "reproject") -> None:
213
+ """Register the ``reproject(x, y, src_crs, dst_crs)`` scalar UDF.
214
+
215
+ Works on any DataFusion ``SessionContext`` (``Context``
216
+ registers it automatically when pyproj is installed). The UDF
217
+ returns a ``{x, y}`` struct of destination coordinates, so a query
218
+ selects components with subscripts::
219
+
220
+ SELECT reproject(x, y, 'EPSG:32610', 'EPSG:4326')['x'] AS lon
221
+ FROM grid
222
+
223
+ Args:
224
+ ctx: The DataFusion session context to register the UDF on.
225
+ name: SQL name for the function (default ``"reproject"``).
226
+ """
227
+ ctx.register_udf(
228
+ udf(
229
+ _reproject,
230
+ [pa.float64(), pa.float64(), pa.utf8(), pa.utf8()],
231
+ RETURN_TYPE,
232
+ "immutable",
233
+ name,
234
+ )
235
+ )
xarray_sql/reader.py ADDED
@@ -0,0 +1,343 @@
1
+ """Lazy Arrow stream reader for xarray Datasets.
2
+
3
+ This module provides XarrayRecordBatchReader, which implements the Arrow
4
+ PyCapsule Interface (__arrow_c_stream__) to enable zero-copy, lazy streaming
5
+ of xarray data to DataFusion and other Arrow consumers.
6
+
7
+ The implementation delegates to PyArrow's RecordBatchReader for the
8
+ actual stream implementation, wrapping xarray block iteration in a generator.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from collections.abc import Callable, Iterator
14
+ from typing import TYPE_CHECKING
15
+
16
+ import numpy as np
17
+ import pyarrow as pa
18
+ import xarray as xr
19
+
20
+ from .df import (
21
+ Block,
22
+ Chunks,
23
+ DEFAULT_BATCH_SIZE,
24
+ _block_len,
25
+ _block_metadata,
26
+ _block_slices_from_resolved,
27
+ _ensure_default_indexes,
28
+ _parse_schema,
29
+ block_slices,
30
+ iter_record_batches,
31
+ resolve_chunks,
32
+ )
33
+
34
+ if TYPE_CHECKING:
35
+ from ._native import LazyArrowStreamTable
36
+
37
+
38
+ class XarrayRecordBatchReader:
39
+ """A lazy Arrow stream reader for xarray Datasets.
40
+
41
+ Implements the Arrow PyCapsule Interface (__arrow_c_stream__) to enable
42
+ zero-copy, lazy streaming of xarray data to DataFusion and other Arrow
43
+ consumers.
44
+
45
+ The key property is that xarray blocks are only converted to Arrow
46
+ RecordBatches when the consumer calls get_next (e.g., during DataFusion's
47
+ collect()), NOT when the reader is created or registered.
48
+
49
+ Attributes:
50
+ schema: The Arrow schema for the stream.
51
+
52
+ Example:
53
+ >>> import xarray as xr
54
+ >>> from xarray_sql import XarrayRecordBatchReader
55
+ >>> ds = xr.tutorial.open_dataset('air_temperature')
56
+ >>> reader = XarrayRecordBatchReader(ds, chunks={'time': 240})
57
+ >>> # At this point, NO data has been read from xarray
58
+ >>> # Data is only read when consumed:
59
+ >>> import pyarrow as pa
60
+ >>> pa_reader = pa.RecordBatchReader.from_stream(reader)
61
+ >>> for batch in pa_reader:
62
+ ... print(batch.num_rows) # Data read here
63
+ """
64
+
65
+ def __init__(
66
+ self,
67
+ ds: xr.Dataset,
68
+ chunks: Chunks = None,
69
+ *,
70
+ batch_size: int = DEFAULT_BATCH_SIZE,
71
+ _iteration_callback: (
72
+ Callable[[Block, list[str] | None], None] | None
73
+ ) = None,
74
+ ):
75
+ """Initialize the lazy reader.
76
+
77
+ Args:
78
+ ds: An xarray Dataset. All data_vars must share the same dimensions.
79
+ chunks: Xarray-like chunks specification. If not provided, uses
80
+ the Dataset's existing chunks, or for an unchunked Dataset
81
+ its storage chunks.
82
+ batch_size: Maximum rows per emitted Arrow RecordBatch. Smaller
83
+ values let DataFusion start processing earlier at the cost of
84
+ more Python→Arrow conversion calls.
85
+ _iteration_callback: Internal callback for testing. Called with
86
+ each block dict just before it's converted to Arrow. This
87
+ allows tests to track when iteration actually occurs.
88
+ """
89
+ self._ds = ds
90
+ self._chunks = chunks
91
+ self._batch_size = batch_size
92
+ self._schema = _parse_schema(ds)
93
+ self._iteration_callback = _iteration_callback
94
+ self._consumed = False
95
+
96
+ # Validate dimensions
97
+ fst = next(iter(ds.values())).dims
98
+ if not all(da.dims == fst for da in ds.values()):
99
+ raise ValueError(
100
+ "All dimensions must be equal. Please filter data_vars in the Dataset."
101
+ )
102
+
103
+ @property
104
+ def schema(self) -> pa.Schema:
105
+ """The Arrow schema for this stream."""
106
+ return self._schema
107
+
108
+ def _generate_batches(self) -> Iterator[pa.RecordBatch]:
109
+ """Generate RecordBatches lazily from xarray blocks.
110
+
111
+ This generator is only consumed when the Arrow stream's get_next
112
+ is called, ensuring true lazy evaluation. Each xarray block is
113
+ emitted as one or more RecordBatches of at most self._batch_size rows.
114
+ """
115
+ for block in block_slices(self._ds, self._chunks):
116
+ # Call the iteration callback if provided (for testing).
117
+ # XarrayRecordBatchReader has no projection concept, so always passes None.
118
+ if self._iteration_callback is not None:
119
+ self._iteration_callback(block, None)
120
+
121
+ yield from iter_record_batches(
122
+ self._ds.isel(block), self._schema, self._batch_size
123
+ )
124
+
125
+ def __arrow_c_stream__(
126
+ self, requested_schema: object | None = None
127
+ ) -> object:
128
+ """Export as Arrow C Stream via PyCapsule.
129
+
130
+ This method is called by Arrow consumers (like DataFusion) to get
131
+ a C-level stream interface. The actual data iteration only begins
132
+ when the consumer calls get_next on the stream.
133
+
134
+ Args:
135
+ requested_schema: Optional schema for type casting. Currently
136
+ passed through to PyArrow's implementation.
137
+
138
+ Returns:
139
+ PyCapsule containing ArrowArrayStream pointer with name
140
+ "arrow_array_stream".
141
+
142
+ Raises:
143
+ RuntimeError: If the stream has already been consumed.
144
+ """
145
+ if self._consumed:
146
+ raise RuntimeError(
147
+ "Stream already consumed. XarrayRecordBatchReader can only "
148
+ "be iterated once. Create a new reader for additional iterations."
149
+ )
150
+ self._consumed = True
151
+
152
+ # Create a PyArrow RecordBatchReader from our generator
153
+ # The generator is NOT consumed here - only when get_next is called
154
+ reader = pa.RecordBatchReader.from_batches(
155
+ self._schema, self._generate_batches()
156
+ )
157
+
158
+ # Delegate to PyArrow's __arrow_c_stream__ implementation
159
+ return reader.__arrow_c_stream__(requested_schema)
160
+
161
+ def __arrow_c_schema__(
162
+ self, requested_schema: object | None = None
163
+ ) -> object:
164
+ """Export the schema as Arrow C Schema via PyCapsule.
165
+
166
+ This allows consumers to inspect the schema without consuming the stream.
167
+
168
+ Args:
169
+ requested_schema: Optional schema for negotiation (unused).
170
+
171
+ Returns:
172
+ PyCapsule containing ArrowSchema pointer.
173
+ """
174
+ return self._schema.__arrow_c_schema__()
175
+
176
+
177
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
178
+ """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
179
+
180
+ Args:
181
+ ds: An Xarray Dataset. All `data_vars` must share the same dimensions.
182
+ chunks: Xarray-like chunks. If not provided, will default to the
183
+ Dataset's chunks, or for an unchunked Dataset its storage chunks.
184
+ The product of the chunk sizes becomes the
185
+ standard length of each dataframe partition.
186
+
187
+ Returns:
188
+ A PyArrow RecordBatchReader, which is a table representation of the input
189
+ Dataset.
190
+ """
191
+ ds = _ensure_default_indexes(ds)
192
+ reader = XarrayRecordBatchReader(ds, chunks=chunks)
193
+ return pa.RecordBatchReader.from_stream(reader)
194
+
195
+
196
+ def read_xarray_table(
197
+ ds: xr.Dataset,
198
+ chunks: Chunks = None,
199
+ *,
200
+ batch_size: int = DEFAULT_BATCH_SIZE,
201
+ coord_arrays: dict[str, np.ndarray] | None = None,
202
+ _iteration_callback: (
203
+ Callable[[Block, list[str] | None], None] | None
204
+ ) = None,
205
+ ) -> "LazyArrowStreamTable":
206
+ """Create a lazy DataFusion table from an xarray Dataset.
207
+
208
+ This is the simplest way to register xarray data with DataFusion.
209
+ Data is only read when queries are executed, not during registration.
210
+ The table can be queried multiple times.
211
+
212
+ Each chunk becomes a separate partition, enabling DataFusion's parallel
213
+ execution across multiple cores.
214
+
215
+ Note:
216
+ SQL queries with WHERE clauses on dimension columns (time, lat, lon, etc.)
217
+ automatically prune partitions that can't contain matching rows — this is
218
+ called *filter pushdown*. For example:
219
+
220
+ # This query will skip loading partitions with time < '2020-02-01'
221
+ result = ctx.sql('SELECT * FROM air WHERE time > "2020-02-01"').collect()
222
+
223
+ Supported operators: `=`, `<`, `>`, `<=`, `>=`, `BETWEEN`, `IN`, `AND`, `OR`.
224
+
225
+ Args:
226
+ ds: An xarray Dataset. All data_vars must share the same dimensions.
227
+ chunks: Xarray-like chunks specification. If not provided, uses
228
+ the Dataset's existing chunks, or for an unchunked Dataset its
229
+ storage chunks.
230
+ batch_size: Maximum rows per Arrow RecordBatch emitted per partition.
231
+ Smaller values let DataFusion start processing earlier; the default
232
+ (65 536) works well for most datasets.
233
+ coord_arrays: Pre-materialised coordinate arrays keyed by dim-name
234
+ string. Hand in to share a single read across multiple tables
235
+ built from the same parent Dataset (e.g. surface + atmosphere
236
+ from ARCO-ERA5); the dim coords are otherwise read once per
237
+ ``read_xarray_table`` call, which is a network round-trip for
238
+ Zarr-backed datasets.
239
+ _iteration_callback: Internal callback for testing. Called with
240
+ each block dict just before it's converted to Arrow.
241
+
242
+ Returns:
243
+ A LazyArrowStreamTable ready for registration with DataFusion.
244
+
245
+ Example:
246
+ >>> from datafusion import SessionContext
247
+ >>> import xarray as xr
248
+ >>> from xarray_sql import read_xarray_table
249
+ >>>
250
+ >>> ds = xr.tutorial.open_dataset('air_temperature')
251
+ >>> table = read_xarray_table(ds, chunks={'time': 240})
252
+ >>>
253
+ >>> ctx = SessionContext()
254
+ >>> ctx.register_table('air', table)
255
+ >>>
256
+ >>> # Data is only read here, during query execution
257
+ >>> # Filters on 'time' will prune partitions automatically!
258
+ >>> result = ctx.sql('SELECT AVG(air) FROM air').collect()
259
+ """
260
+ from ._native import LazyArrowStreamTable
261
+
262
+ ds = _ensure_default_indexes(ds)
263
+ schema = _parse_schema(ds)
264
+
265
+ # Hoist coordinate reads once; avoids N_partitions remote I/O calls for
266
+ # Zarr-backed datasets (e.g. ARCO-ERA5 on GCS). When the caller supplies
267
+ # pre-materialised arrays (e.g. shared across surface + atmosphere
268
+ # tables), reuse them and skip the extra read.
269
+ if coord_arrays is None:
270
+ coord_arrays = {str(dim): ds.coords[dim].values for dim in ds.dims}
271
+
272
+ # Determine which column names are data variables (not dimension coordinates).
273
+ # Used by the factory to skip loading unrequested variables.
274
+ data_var_names = set(ds.data_vars.keys())
275
+
276
+ def make_partition_factory(
277
+ block: Block,
278
+ ) -> Callable[[list[str] | None], pa.RecordBatchReader]:
279
+ def make_stream(
280
+ projection_names: list[str] | None,
281
+ ) -> pa.RecordBatchReader:
282
+ if _iteration_callback is not None:
283
+ _iteration_callback(block, projection_names)
284
+
285
+ if projection_names is not None:
286
+ # Restrict to the data variables mentioned in the projection.
287
+ # Dimension coordinates come along automatically via coords.
288
+ data_vars_needed = [
289
+ c for c in projection_names if c in data_var_names
290
+ ]
291
+ if data_vars_needed:
292
+ ds_block = ds[data_vars_needed].isel(block)
293
+ else:
294
+ # Only dimension coords requested — drop all data vars to avoid
295
+ # loading them unnecessarily (e.g. for queries like SELECT lat, lon).
296
+ ds_block = ds.drop_vars(list(ds.data_vars)).isel(block)
297
+ batch_schema = pa.schema(
298
+ [schema.field(name) for name in projection_names]
299
+ )
300
+ else:
301
+ ds_block = ds.isel(block)
302
+ batch_schema = schema
303
+
304
+ return pa.RecordBatchReader.from_batches(
305
+ batch_schema,
306
+ iter_record_batches(ds_block, batch_schema, batch_size),
307
+ )
308
+
309
+ return make_stream
310
+
311
+ # Separate dims whose chunk bounds vary across partitions from those
312
+ # whose bounds are constant (one chunk spanning the whole axis). For the
313
+ # latter we compute min/max once instead of re-scanning the full coord
314
+ # array on every partition — dominant cost when registering hundreds of
315
+ # thousands of single-time-step partitions on a 4-D dataset like ERA5.
316
+ resolved = resolve_chunks(ds, chunks)
317
+ varying_dims = [d for d, tup in resolved.items() if len(tup) > 1]
318
+ static_dims = [d for d in ds.dims if d not in varying_dims]
319
+ static_block: Block = {d: slice(None) for d in static_dims}
320
+ static_ranges = _block_metadata(
321
+ coord_arrays, static_block, dims=static_dims
322
+ )
323
+
324
+ def partition_pairs():
325
+ """Lazily yield (factory, metadata, num_rows) for each partition.
326
+
327
+ Consuming this generator one item at a time means Python never holds
328
+ all N block dicts, metadata dicts, and factory closures simultaneously.
329
+ Peak Python memory during registration is O(1) per partition instead
330
+ of O(N_partitions).
331
+ """
332
+ for block in _block_slices_from_resolved(ds, resolved):
333
+ dynamic = _block_metadata(coord_arrays, block, dims=varying_dims)
334
+ yield (
335
+ make_partition_factory(block),
336
+ {**static_ranges, **dynamic},
337
+ # Exact row count for this partition (product of the chunk's
338
+ # per-dimension sizes), so the scan can report exact
339
+ # Statistics::num_rows to the optimizer.
340
+ _block_len(block),
341
+ )
342
+
343
+ return LazyArrowStreamTable(partition_pairs(), schema)