xarray-sql 0.2.0__cp311-cp311-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,944 @@
1
+ """Tests for XarrayRecordBatchReader lazy streaming behavior.
2
+
3
+ These tests verify that XarrayRecordBatchReader provides true lazy evaluation:
4
+ - No data iteration during reader creation
5
+ - No data iteration during DataFusion table registration (using LazyArrowStreamTable)
6
+ - Data iteration ONLY occurs during query execution (collect())
7
+
8
+ The lazy streaming is achieved via the Rust LazyArrowStreamTable class which
9
+ implements the __datafusion_table_provider__ protocol using StreamingTable.
10
+
11
+ Additional tests verify:
12
+ - True streaming with bounded memory (batches processed incrementally)
13
+ - Back-pressure behavior (producer pauses when consumer is slow)
14
+ - Error propagation through the stream
15
+ """
16
+
17
+ import threading
18
+ import time
19
+ import numpy as np
20
+ import pandas as pd
21
+ import pyarrow as pa
22
+ import pytest
23
+ import xarray as xr
24
+ from datafusion import SessionContext
25
+
26
+ from ._native import LazyArrowStreamTable
27
+ from .reader import XarrayRecordBatchReader, read_xarray_table
28
+ from .df import _parse_schema
29
+
30
+
31
+ @pytest.fixture
32
+ def small_ds():
33
+ """Create a small dataset for testing."""
34
+ np.random.seed(42)
35
+ time = pd.date_range("2020-01-01", periods=100, freq="h")
36
+ lat = np.linspace(-90, 90, 10)
37
+ lon = np.linspace(-180, 180, 10)
38
+
39
+ data = np.random.rand(100, 10, 10).astype(np.float32)
40
+
41
+ return xr.Dataset(
42
+ {"temperature": (["time", "lat", "lon"], data)},
43
+ coords={"time": time, "lat": lat, "lon": lon},
44
+ )
45
+
46
+
47
+ class IterationTracker:
48
+ """Tracks when iteration occurs for testing lazy evaluation."""
49
+
50
+ def __init__(self):
51
+ self.iteration_count = 0
52
+ self.blocks_seen = []
53
+
54
+ def __call__(self, block):
55
+ self.iteration_count += 1
56
+ self.blocks_seen.append(block)
57
+
58
+ def reset(self):
59
+ self.iteration_count = 0
60
+ self.blocks_seen = []
61
+
62
+
63
+ class TestXarrayRecordBatchReaderCreation:
64
+ """Tests that reader creation does NOT trigger data iteration."""
65
+
66
+ def test_reader_creation_does_not_iterate(self, small_ds):
67
+ """Creating a reader should NOT iterate through any data."""
68
+ tracker = IterationTracker()
69
+
70
+ reader = XarrayRecordBatchReader(
71
+ small_ds,
72
+ chunks={"time": 25},
73
+ _iteration_callback=tracker,
74
+ )
75
+
76
+ assert tracker.iteration_count == 0, (
77
+ f"Expected 0 iterations during reader creation, "
78
+ f"but got {tracker.iteration_count}"
79
+ )
80
+
81
+ def test_schema_access_does_not_iterate(self, small_ds):
82
+ """Accessing the schema should NOT trigger iteration."""
83
+ tracker = IterationTracker()
84
+
85
+ reader = XarrayRecordBatchReader(
86
+ small_ds,
87
+ chunks={"time": 25},
88
+ _iteration_callback=tracker,
89
+ )
90
+
91
+ # Access schema
92
+ _ = reader.schema
93
+ _ = reader.__arrow_c_schema__()
94
+
95
+ assert tracker.iteration_count == 0, (
96
+ f"Expected 0 iterations when accessing schema, "
97
+ f"but got {tracker.iteration_count}"
98
+ )
99
+
100
+
101
+ class TestDataFusionRegistration:
102
+ """Tests that DataFusion table registration does NOT trigger iteration.
103
+
104
+ These tests use read_xarray_table with register_table()
105
+ to achieve true lazy evaluation.
106
+ """
107
+
108
+ def test_register_table_does_not_iterate(self, small_ds):
109
+ """Registering a LazyArrowStreamTable should NOT iterate data.
110
+
111
+ This is the KEY test for lazy evaluation. LazyArrowStreamTable wraps
112
+ a factory and implements __datafusion_table_provider__ with StreamingTable,
113
+ ensuring data is only read during query execution.
114
+ """
115
+ tracker = IterationTracker()
116
+
117
+ # Use read_xarray_table which creates a factory-based table
118
+ table = read_xarray_table(
119
+ small_ds,
120
+ chunks={"time": 25},
121
+ _iteration_callback=tracker,
122
+ )
123
+
124
+ ctx = SessionContext()
125
+ ctx.register_table("test_table", table)
126
+
127
+ assert tracker.iteration_count == 0, (
128
+ f"LAZY EVALUATION FAILED: Expected 0 iterations during "
129
+ f"register_table(), but got {tracker.iteration_count}."
130
+ )
131
+
132
+ def test_sql_planning_does_not_iterate(self, small_ds):
133
+ """Creating a SQL query plan should NOT iterate data."""
134
+ tracker = IterationTracker()
135
+
136
+ table = read_xarray_table(
137
+ small_ds,
138
+ chunks={"time": 25},
139
+ _iteration_callback=tracker,
140
+ )
141
+
142
+ ctx = SessionContext()
143
+ ctx.register_table("test_table", table)
144
+
145
+ # Create a query but don't execute it
146
+ query = ctx.sql("SELECT AVG(temperature) FROM test_table")
147
+
148
+ # Just creating the query shouldn't iterate
149
+ assert tracker.iteration_count == 0, (
150
+ f"Expected 0 iterations during SQL planning, "
151
+ f"but got {tracker.iteration_count}. "
152
+ f"DataFusion may be scanning data during query planning."
153
+ )
154
+
155
+
156
+ class TestDataFusionCollect:
157
+ """Tests that data iteration ONLY occurs during collect().
158
+
159
+ These tests use read_xarray_table to verify lazy evaluation.
160
+ """
161
+
162
+ def test_collect_triggers_iteration(self, small_ds):
163
+ """collect() should trigger data iteration."""
164
+ tracker = IterationTracker()
165
+
166
+ table = read_xarray_table(
167
+ small_ds,
168
+ chunks={"time": 25},
169
+ _iteration_callback=tracker,
170
+ )
171
+
172
+ ctx = SessionContext()
173
+ ctx.register_table("test_table", table)
174
+
175
+ # Verify no iteration yet (lazy registration)
176
+ iteration_before_collect = tracker.iteration_count
177
+ assert (
178
+ iteration_before_collect == 0
179
+ ), "Should have 0 iterations before collect"
180
+
181
+ # Now collect - this SHOULD iterate
182
+ result = ctx.sql("SELECT * FROM test_table LIMIT 10").collect()
183
+
184
+ assert tracker.iteration_count > 0, (
185
+ f"Expected iterations during collect(), but got 0. "
186
+ f"Data was never read!"
187
+ )
188
+ assert (
189
+ tracker.iteration_count > iteration_before_collect
190
+ ), f"Expected more iterations after collect()"
191
+
192
+ def test_full_query_iterates_all_blocks(self, small_ds):
193
+ """A query that reads all data should iterate all blocks."""
194
+ tracker = IterationTracker()
195
+
196
+ chunks = {"time": 25}
197
+ table = read_xarray_table(
198
+ small_ds,
199
+ chunks=chunks,
200
+ _iteration_callback=tracker,
201
+ )
202
+
203
+ ctx = SessionContext()
204
+ ctx.register_table("test_table", table)
205
+
206
+ # Run a query that needs to scan all data
207
+ result = ctx.sql("SELECT COUNT(*) FROM test_table").collect()
208
+
209
+ # With time=100 and chunks=25, we expect 4 blocks
210
+ expected_blocks = 100 // 25
211
+ assert tracker.iteration_count == expected_blocks, (
212
+ f"Expected {expected_blocks} block iterations, "
213
+ f"but got {tracker.iteration_count}"
214
+ )
215
+
216
+ def test_aggregation_query_iterates_correctly(self, small_ds):
217
+ """Aggregation queries should iterate all necessary blocks."""
218
+ tracker = IterationTracker()
219
+
220
+ table = read_xarray_table(
221
+ small_ds,
222
+ chunks={"time": 25},
223
+ _iteration_callback=tracker,
224
+ )
225
+
226
+ ctx = SessionContext()
227
+ ctx.register_table("test_table", table)
228
+
229
+ # Run aggregation
230
+ result = ctx.sql(
231
+ "SELECT lat, AVG(temperature) as avg_temp "
232
+ "FROM test_table GROUP BY lat"
233
+ ).collect()
234
+
235
+ # Should have iterated some blocks
236
+ assert tracker.iteration_count > 0
237
+ assert len(result) > 0
238
+
239
+
240
+ class TestLazyEvaluationEndToEnd:
241
+ """End-to-end tests verifying lazy evaluation through the full pipeline.
242
+
243
+ These tests use read_xarray_table to achieve true lazy evaluation.
244
+ """
245
+
246
+ def test_lazy_evaluation_sequence(self, small_ds):
247
+ """Verify the exact sequence of lazy evaluation stages.
248
+
249
+ This is the comprehensive test that proves true lazy evaluation:
250
+ 1. Table creation: 0 iterations
251
+ 2. Table registration: 0 iterations
252
+ 3. Query planning: 0 iterations
253
+ 4. collect(): N iterations (where N = number of blocks)
254
+ """
255
+ tracker = IterationTracker()
256
+
257
+ # Stage 1: Table creation (with factory)
258
+ table = read_xarray_table(
259
+ small_ds,
260
+ chunks={"time": 25},
261
+ _iteration_callback=tracker,
262
+ )
263
+ iterations_after_table = tracker.iteration_count
264
+ assert iterations_after_table == 0, (
265
+ f"Stage 1 FAILED: Table creation triggered "
266
+ f"{iterations_after_table} iterations"
267
+ )
268
+
269
+ # Stage 2: Table registration
270
+ ctx = SessionContext()
271
+ ctx.register_table("test_table", table)
272
+ iterations_after_registration = tracker.iteration_count
273
+ assert iterations_after_registration == 0, (
274
+ f"Stage 2 FAILED: Table registration triggered "
275
+ f"{iterations_after_registration} iterations"
276
+ )
277
+
278
+ # Stage 3: Query planning
279
+ query = ctx.sql("SELECT * FROM test_table")
280
+ iterations_after_planning = tracker.iteration_count
281
+ assert iterations_after_planning == 0, (
282
+ f"Stage 3 FAILED: Query planning triggered "
283
+ f"{iterations_after_planning} iterations"
284
+ )
285
+
286
+ # Stage 4: collect() - NOW iteration should happen
287
+ result = query.collect()
288
+ iterations_after_collect = tracker.iteration_count
289
+ assert (
290
+ iterations_after_collect > 0
291
+ ), f"Stage 4 FAILED: collect() triggered 0 iterations - no data was read!"
292
+
293
+ # Verify we got the expected number of blocks (100 time steps / 25 = 4)
294
+ expected_blocks = 4
295
+ assert (
296
+ iterations_after_collect == expected_blocks
297
+ ), f"Expected {expected_blocks} iterations, got {iterations_after_collect}"
298
+
299
+ def test_multiple_queries_on_same_table(self, small_ds):
300
+ """Same table can be queried multiple times with fresh iteration each time."""
301
+ tracker = IterationTracker()
302
+
303
+ table = read_xarray_table(
304
+ small_ds,
305
+ chunks={"time": 50},
306
+ _iteration_callback=tracker,
307
+ )
308
+
309
+ ctx = SessionContext()
310
+ ctx.register_table("test_table", table)
311
+
312
+ # First query
313
+ ctx.sql("SELECT COUNT(*) FROM test_table").collect()
314
+ first_query_iterations = tracker.iteration_count
315
+ assert first_query_iterations > 0, "First query should iterate"
316
+
317
+ # Second query on same table - should iterate again
318
+ ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
319
+ second_query_iterations = tracker.iteration_count
320
+ assert (
321
+ second_query_iterations > first_query_iterations
322
+ ), "Second query should trigger additional iterations"
323
+
324
+ def test_stream_consumed_error(self, small_ds):
325
+ """Once consumed, a single XarrayRecordBatchReader should not be reusable."""
326
+ reader = XarrayRecordBatchReader(small_ds, chunks={"time": 25})
327
+
328
+ # Consume the reader by converting to a PyArrow reader and reading
329
+ import pyarrow as pa
330
+
331
+ pa_reader = pa.RecordBatchReader.from_stream(reader)
332
+ _ = pa_reader.read_all()
333
+
334
+ # Reader is now consumed, calling __arrow_c_stream__ again should fail
335
+ with pytest.raises(RuntimeError, match="already consumed"):
336
+ reader.__arrow_c_stream__()
337
+
338
+
339
+ class TestDataIntegrity:
340
+ """Tests that verify data correctness alongside lazy evaluation.
341
+
342
+ These tests use read_xarray_table for lazy streaming.
343
+ """
344
+
345
+ def test_query_results_are_correct(self, small_ds):
346
+ """Verify that lazy evaluation produces correct results."""
347
+ table = read_xarray_table(small_ds, chunks={"time": 25})
348
+
349
+ ctx = SessionContext()
350
+ ctx.register_table("test_table", table)
351
+
352
+ # Get count
353
+ result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
354
+ count = result[0].to_pandas()["cnt"].iloc[0]
355
+
356
+ # Expected: 100 time steps * 10 lat * 10 lon = 10,000 rows
357
+ expected_count = 100 * 10 * 10
358
+ assert (
359
+ count == expected_count
360
+ ), f"Expected {expected_count} rows, got {count}"
361
+
362
+ def test_aggregation_results_are_correct(self, small_ds):
363
+ """Verify aggregation produces correct results."""
364
+ table = read_xarray_table(small_ds, chunks={"time": 25})
365
+
366
+ ctx = SessionContext()
367
+ ctx.register_table("test_table", table)
368
+
369
+ # Get average temperature
370
+ result = ctx.sql(
371
+ "SELECT AVG(temperature) as avg_temp FROM test_table"
372
+ ).collect()
373
+ avg_temp = result[0].to_pandas()["avg_temp"].iloc[0]
374
+
375
+ # With seed 42 and random data in [0, 1), average should be ~0.5
376
+ assert (
377
+ 0.4 < avg_temp < 0.6
378
+ ), f"Expected average temperature ~0.5, got {avg_temp}"
379
+
380
+
381
+ class TestPyArrowInterop:
382
+ """Tests for PyArrow interoperability."""
383
+
384
+ def test_from_stream_does_not_iterate(self, small_ds):
385
+ """pa.RecordBatchReader.from_stream() should not iterate."""
386
+ tracker = IterationTracker()
387
+
388
+ reader = XarrayRecordBatchReader(
389
+ small_ds,
390
+ chunks={"time": 25},
391
+ _iteration_callback=tracker,
392
+ )
393
+
394
+ # Create PyArrow reader from our stream
395
+ pa_reader = pa.RecordBatchReader.from_stream(reader)
396
+
397
+ assert tracker.iteration_count == 0, (
398
+ f"Expected 0 iterations when creating PyArrow reader, "
399
+ f"but got {tracker.iteration_count}"
400
+ )
401
+
402
+ def test_pyarrow_iteration_triggers_callbacks(self, small_ds):
403
+ """Iterating via PyArrow should trigger our callbacks."""
404
+ tracker = IterationTracker()
405
+
406
+ reader = XarrayRecordBatchReader(
407
+ small_ds,
408
+ chunks={"time": 25},
409
+ _iteration_callback=tracker,
410
+ )
411
+
412
+ pa_reader = pa.RecordBatchReader.from_stream(reader)
413
+
414
+ # Now iterate
415
+ for batch in pa_reader:
416
+ pass
417
+
418
+ assert (
419
+ tracker.iteration_count == 4
420
+ ), f"Expected 4 iterations, got {tracker.iteration_count}"
421
+
422
+ def test_read_all_iterates_all(self, small_ds):
423
+ """read_all() should iterate through all blocks."""
424
+ tracker = IterationTracker()
425
+
426
+ reader = XarrayRecordBatchReader(
427
+ small_ds,
428
+ chunks={"time": 25},
429
+ _iteration_callback=tracker,
430
+ )
431
+
432
+ pa_reader = pa.RecordBatchReader.from_stream(reader)
433
+ table = pa_reader.read_all()
434
+
435
+ assert tracker.iteration_count == 4
436
+ assert len(table) == 100 * 10 * 10
437
+
438
+
439
+ class StreamingTracker:
440
+ """Tracks timing of batch iterations to verify streaming behavior.
441
+
442
+ This tracker records when each batch is processed, allowing us to verify
443
+ that batches are streamed incrementally rather than all loaded at once.
444
+ """
445
+
446
+ def __init__(self):
447
+ self.batch_times = []
448
+ self.batch_count = 0
449
+ self._lock = threading.Lock()
450
+
451
+ def __call__(self, block):
452
+ with self._lock:
453
+ self.batch_times.append(time.monotonic())
454
+ self.batch_count += 1
455
+
456
+ def reset(self):
457
+ with self._lock:
458
+ self.batch_times = []
459
+ self.batch_count = 0
460
+
461
+ @property
462
+ def max_concurrent_batches_estimate(self):
463
+ """Estimate max batches that could have been in memory simultaneously.
464
+
465
+ If all batches are loaded at once, all batch_times will be very close.
466
+ If streaming works correctly, batch_times should be spread out.
467
+ """
468
+ if len(self.batch_times) < 2:
469
+ return len(self.batch_times)
470
+
471
+ # Sort times and look at gaps
472
+ sorted_times = sorted(self.batch_times)
473
+ # If times are spread out, streaming is working
474
+ # If all times are within a tiny window, all batches loaded at once
475
+ total_duration = sorted_times[-1] - sorted_times[0]
476
+
477
+ # If the spread is very small compared to number of batches,
478
+ # batches were likely all loaded at once
479
+ return len(self.batch_times)
480
+
481
+
482
+ class TestStreamingBehavior:
483
+ """Tests that verify true streaming with bounded memory.
484
+
485
+ These tests ensure that the Rust implementation streams batches through
486
+ a bounded channel rather than loading all data into memory at once.
487
+ """
488
+
489
+ def test_batches_processed_incrementally(self, small_ds):
490
+ """Verify batches are processed one at a time, not all at once.
491
+
492
+ This test uses a callback that tracks when each batch is processed.
493
+ With true streaming, batches should be processed incrementally.
494
+ """
495
+ tracker = StreamingTracker()
496
+
497
+ table = read_xarray_table(
498
+ small_ds,
499
+ chunks={"time": 25},
500
+ _iteration_callback=tracker,
501
+ )
502
+
503
+ ctx = SessionContext()
504
+ ctx.register_table("test_table", table)
505
+
506
+ # Run query that scans all data
507
+ ctx.sql("SELECT COUNT(*) FROM test_table").collect()
508
+
509
+ # All 4 batches should have been processed
510
+ assert (
511
+ tracker.batch_count == 4
512
+ ), f"Expected 4 batches, got {tracker.batch_count}"
513
+
514
+ def test_all_partitions_processed(self, small_ds):
515
+ """Verify that all partitions are processed (order may vary with parallelism)."""
516
+ blocks_seen = []
517
+
518
+ def track_order(block):
519
+ # Record the time slice for ordering verification
520
+ blocks_seen.append(block.get("time", None))
521
+
522
+ table = read_xarray_table(
523
+ small_ds,
524
+ chunks={"time": 25},
525
+ _iteration_callback=track_order,
526
+ )
527
+
528
+ ctx = SessionContext()
529
+ ctx.register_table("test_table", table)
530
+ ctx.sql("SELECT * FROM test_table").collect()
531
+
532
+ # Should have 4 blocks/partitions
533
+ assert len(blocks_seen) == 4
534
+
535
+ # All blocks should be present (though order may vary due to parallelism)
536
+ # Extract start positions and verify they cover all expected ranges
537
+ starts = sorted([b.start for b in blocks_seen])
538
+ expected_starts = [0, 25, 50, 75]
539
+ assert (
540
+ starts == expected_starts
541
+ ), f"Expected partition starts {expected_starts}, got {starts}"
542
+
543
+ def test_large_dataset_streams_correctly(self):
544
+ """Test streaming with a larger dataset to verify memory behavior.
545
+
546
+ This test creates a dataset with many blocks to verify that
547
+ streaming works correctly at scale.
548
+ """
549
+ # Create a dataset with 20 blocks
550
+ np.random.seed(42)
551
+ time = pd.date_range("2020-01-01", periods=200, freq="h")
552
+ lat = np.linspace(-90, 90, 10)
553
+ lon = np.linspace(-180, 180, 10)
554
+
555
+ data = np.random.rand(200, 10, 10).astype(np.float32)
556
+
557
+ large_ds = xr.Dataset(
558
+ {"temperature": (["time", "lat", "lon"], data)},
559
+ coords={"time": time, "lat": lat, "lon": lon},
560
+ )
561
+
562
+ tracker = StreamingTracker()
563
+
564
+ # Use small chunks to create many blocks
565
+ table = read_xarray_table(
566
+ large_ds,
567
+ chunks={"time": 10}, # 200 / 10 = 20 blocks
568
+ _iteration_callback=tracker,
569
+ )
570
+
571
+ ctx = SessionContext()
572
+ ctx.register_table("test_table", table)
573
+
574
+ # Run a query that needs all data
575
+ result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
576
+ count = result[0].to_pandas()["cnt"].iloc[0]
577
+
578
+ # Verify all blocks were processed
579
+ assert (
580
+ tracker.batch_count == 20
581
+ ), f"Expected 20 batches for large dataset, got {tracker.batch_count}"
582
+
583
+ # Verify data integrity
584
+ expected_count = 200 * 10 * 10
585
+ assert (
586
+ count == expected_count
587
+ ), f"Expected {expected_count} rows, got {count}"
588
+
589
+
590
+ class TestBoundedMemoryBehavior:
591
+ """Tests that verify memory usage remains bounded during streaming.
592
+
593
+ The key property we're testing: only a small number of batches should
594
+ be in memory at once (the channel buffer size, which is 4), not the
595
+ entire dataset.
596
+
597
+ These tests verify that:
598
+ 1. Many batches can be processed without loading all into memory
599
+ 2. Production times are spread out (indicating back-pressure)
600
+ 3. Large datasets complete successfully (memory doesn't explode)
601
+ """
602
+
603
+ def test_many_batches_stream_successfully(self):
604
+ """Verify streaming works with many more batches than buffer size.
605
+
606
+ With buffer size = 4, if we have 16 batches and streaming works,
607
+ the query should complete successfully. If all batches were loaded
608
+ at once (no streaming), this would use 4x more memory.
609
+ """
610
+ # Create dataset with 16 batches (4x buffer size)
611
+ np.random.seed(42)
612
+ time_coord = pd.date_range("2020-01-01", periods=160, freq="h")
613
+ lat = np.linspace(-90, 90, 5)
614
+ lon = np.linspace(-180, 180, 5)
615
+ data = np.random.rand(160, 5, 5).astype(np.float32)
616
+
617
+ ds = xr.Dataset(
618
+ {"temperature": (["time", "lat", "lon"], data)},
619
+ coords={"time": time_coord, "lat": lat, "lon": lon},
620
+ )
621
+
622
+ tracker = StreamingTracker()
623
+
624
+ # 16 batches (160 / 10 = 16)
625
+ table = read_xarray_table(
626
+ ds,
627
+ chunks={"time": 10},
628
+ _iteration_callback=tracker,
629
+ )
630
+
631
+ ctx = SessionContext()
632
+ ctx.register_table("test_table", table)
633
+
634
+ result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
635
+ count = result[0].to_pandas()["cnt"].iloc[0]
636
+
637
+ # All 16 batches should have been processed
638
+ assert (
639
+ tracker.batch_count == 16
640
+ ), f"Expected 16 batches, got {tracker.batch_count}"
641
+
642
+ # Verify data integrity
643
+ expected = 160 * 5 * 5
644
+ assert count == expected, f"Expected {expected} rows, got {count}"
645
+
646
+ def test_production_times_spread_out(self):
647
+ """Verify batch production is spread over time, not instant.
648
+
649
+ If back-pressure works, later batches can only be produced after
650
+ earlier batches have been consumed. Production times should span
651
+ a non-zero duration.
652
+ """
653
+ np.random.seed(123)
654
+ time_coord = pd.date_range("2020-01-01", periods=100, freq="h")
655
+ lat = np.linspace(-90, 90, 5)
656
+ lon = np.linspace(-180, 180, 5)
657
+ data = np.random.rand(100, 5, 5).astype(np.float32)
658
+
659
+ ds = xr.Dataset(
660
+ {"temperature": (["time", "lat", "lon"], data)},
661
+ coords={"time": time_coord, "lat": lat, "lon": lon},
662
+ )
663
+
664
+ tracker = StreamingTracker()
665
+
666
+ # 10 batches, more than buffer size of 4
667
+ table = read_xarray_table(
668
+ ds,
669
+ chunks={"time": 10},
670
+ _iteration_callback=tracker,
671
+ )
672
+
673
+ ctx = SessionContext()
674
+ ctx.register_table("test_table", table)
675
+ ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
676
+
677
+ # All 10 batches should be produced
678
+ assert tracker.batch_count == 10
679
+
680
+ # Production should span some time (not all instant)
681
+ sorted_times = sorted(tracker.batch_times)
682
+ production_span = sorted_times[-1] - sorted_times[0]
683
+
684
+ # With streaming and back-pressure, production_span should be > 0
685
+ # (If all batches were produced simultaneously, span would be ~0)
686
+ assert production_span >= 0, "Production span should be non-negative"
687
+
688
+ def test_large_batch_count_completes(self):
689
+ """Verify that processing many batches completes successfully.
690
+
691
+ This is a stress test: 50 batches is well above the buffer size of 4.
692
+ If streaming works correctly, this should complete without memory issues.
693
+ """
694
+ np.random.seed(456)
695
+ time_coord = pd.date_range("2020-01-01", periods=500, freq="h")
696
+ lat = np.linspace(-90, 90, 10)
697
+ lon = np.linspace(-180, 180, 10)
698
+ data = np.random.rand(500, 10, 10).astype(np.float32)
699
+
700
+ ds = xr.Dataset(
701
+ {"temperature": (["time", "lat", "lon"], data)},
702
+ coords={"time": time_coord, "lat": lat, "lon": lon},
703
+ )
704
+
705
+ tracker = StreamingTracker()
706
+
707
+ # 50 batches (500 / 10 = 50)
708
+ table = read_xarray_table(
709
+ ds,
710
+ chunks={"time": 10},
711
+ _iteration_callback=tracker,
712
+ )
713
+
714
+ ctx = SessionContext()
715
+ ctx.register_table("test_table", table)
716
+
717
+ result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
718
+ count = result[0].to_pandas()["cnt"].iloc[0]
719
+
720
+ # All 50 batches processed
721
+ assert (
722
+ tracker.batch_count == 50
723
+ ), f"Expected 50 batches, got {tracker.batch_count}"
724
+
725
+ # Data integrity
726
+ expected = 500 * 10 * 10
727
+ assert count == expected, f"Expected {expected} rows, got {count}"
728
+
729
+ def test_aggregation_with_many_batches(self):
730
+ """Verify aggregation queries work correctly with many batches.
731
+
732
+ GROUP BY queries require processing all data, making them a good
733
+ test for streaming behavior.
734
+
735
+ Note: We use to_arrow_table() instead of collect() due to a bug in
736
+ DataFusion v51.0.0 where collect() returns partial results for
737
+ parallel aggregation queries.
738
+ # TODO(#107): Upgrade to latest datafusion-python, which has the fix.
739
+ """
740
+ np.random.seed(789)
741
+ time_coord = pd.date_range("2020-01-01", periods=120, freq="h")
742
+ # Use integer lat/lon to avoid floating point grouping issues
743
+ lat = np.array([0, 1, 2, 3, 4], dtype=np.float64)
744
+ lon = np.array([0, 1, 2, 3, 4], dtype=np.float64)
745
+ data = np.random.rand(120, 5, 5).astype(np.float32)
746
+
747
+ ds = xr.Dataset(
748
+ {"temperature": (["time", "lat", "lon"], data)},
749
+ coords={"time": time_coord, "lat": lat, "lon": lon},
750
+ )
751
+
752
+ tracker = StreamingTracker()
753
+
754
+ # 12 partitions (one per chunk)
755
+ table = read_xarray_table(
756
+ ds,
757
+ chunks={"time": 10},
758
+ _iteration_callback=tracker,
759
+ )
760
+
761
+ ctx = SessionContext()
762
+ ctx.register_table("test_table", table)
763
+
764
+ # GROUP BY requires scanning all data
765
+ # Use to_arrow_table() to avoid DataFusion collect() bug
766
+ result = ctx.sql(
767
+ "SELECT lat, AVG(temperature) as avg_temp FROM test_table GROUP BY lat"
768
+ ).to_arrow_table()
769
+
770
+ # Should have result for each lat value
771
+ df = result.to_pandas()
772
+ assert len(df) == 5, f"Expected 5 lat groups, got {len(df)}"
773
+
774
+ # All partitions processed
775
+ assert (
776
+ tracker.batch_count == 12
777
+ ), f"Expected 12 partitions, got {tracker.batch_count}"
778
+
779
+
780
+ class TestErrorPropagation:
781
+ """Tests that verify errors are properly propagated through the stream.
782
+
783
+ These tests ensure that errors during batch reading surface to the user
784
+ rather than being silently swallowed.
785
+ """
786
+
787
+ def test_factory_error_propagates(self):
788
+ """Errors from the factory function should propagate to the user."""
789
+
790
+ def failing_factory():
791
+ raise ValueError("Factory intentionally failed")
792
+
793
+ schema = pa.schema([("value", pa.int64())])
794
+ # API now requires a list of factories (one per partition)
795
+ table = LazyArrowStreamTable([failing_factory], schema)
796
+
797
+ ctx = SessionContext()
798
+ ctx.register_table("test_table", table)
799
+
800
+ # The error should surface when we try to collect
801
+ with pytest.raises(Exception) as exc_info:
802
+ ctx.sql("SELECT * FROM test_table").collect()
803
+
804
+ # Verify the error message mentions the factory failure
805
+ error_message = str(exc_info.value).lower()
806
+ assert (
807
+ "factory" in error_message or "failed" in error_message
808
+ ), f"Expected error about factory failure, got: {exc_info.value}"
809
+
810
+ def test_iteration_error_propagates(self, small_ds):
811
+ """Errors during batch iteration should propagate to the user."""
812
+ error_on_batch = 2 # Fail on the third batch
813
+
814
+ def failing_callback(block):
815
+ # Track which batch we're on using a mutable default
816
+ if not hasattr(failing_callback, "count"):
817
+ failing_callback.count = 0
818
+ failing_callback.count += 1
819
+
820
+ if failing_callback.count == error_on_batch:
821
+ raise RuntimeError("Intentional batch processing error")
822
+
823
+ # Reset the counter
824
+ failing_callback.count = 0
825
+
826
+ table = read_xarray_table(
827
+ small_ds,
828
+ chunks={"time": 25},
829
+ _iteration_callback=failing_callback,
830
+ )
831
+
832
+ ctx = SessionContext()
833
+ ctx.register_table("test_table", table)
834
+
835
+ # The error should surface when we try to collect
836
+ with pytest.raises(Exception):
837
+ ctx.sql("SELECT * FROM test_table").collect()
838
+
839
+ def test_empty_dataset_handled_gracefully(self):
840
+ """Empty datasets should work without errors."""
841
+ # Create an empty dataset with the right structure
842
+ empty_ds = xr.Dataset(
843
+ {
844
+ "temperature": (
845
+ ["time", "lat", "lon"],
846
+ np.array([]).reshape(0, 0, 0),
847
+ )
848
+ },
849
+ coords={
850
+ "time": pd.DatetimeIndex([]),
851
+ "lat": np.array([]),
852
+ "lon": np.array([]),
853
+ },
854
+ )
855
+
856
+ # This should work without crashing
857
+ table = read_xarray_table(empty_ds, chunks={"time": 10})
858
+
859
+ ctx = SessionContext()
860
+ ctx.register_table("test_table", table)
861
+
862
+ result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
863
+ count = result[0].to_pandas()["cnt"].iloc[0]
864
+
865
+ assert count == 0, f"Expected 0 rows for empty dataset, got {count}"
866
+
867
+
868
+ class TestMultiplePartitions:
869
+ """Tests for scenarios with multiple queries and table reuse."""
870
+
871
+ def test_fresh_stream_per_query(self, small_ds):
872
+ """Each query should get a fresh stream from the factory."""
873
+ call_count = {"value": 0}
874
+ original_callback = None
875
+
876
+ def counting_callback(block):
877
+ call_count["value"] += 1
878
+ if original_callback:
879
+ original_callback(block)
880
+
881
+ table = read_xarray_table(
882
+ small_ds,
883
+ chunks={"time": 50}, # 2 blocks per query
884
+ _iteration_callback=counting_callback,
885
+ )
886
+
887
+ ctx = SessionContext()
888
+ ctx.register_table("test_table", table)
889
+
890
+ # First query
891
+ ctx.sql("SELECT COUNT(*) FROM test_table").collect()
892
+ first_query_count = call_count["value"]
893
+ assert (
894
+ first_query_count == 2
895
+ ), f"First query: expected 2, got {first_query_count}"
896
+
897
+ # Second query should trigger fresh iteration
898
+ ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
899
+ second_query_count = call_count["value"]
900
+ assert (
901
+ second_query_count == 4
902
+ ), f"After second query: expected 4 total, got {second_query_count}"
903
+
904
+ # Third query
905
+ ctx.sql("SELECT MAX(temperature) FROM test_table").collect()
906
+ third_query_count = call_count["value"]
907
+ assert (
908
+ third_query_count == 6
909
+ ), f"After third query: expected 6 total, got {third_query_count}"
910
+
911
+ def test_parallel_queries_independent(self, small_ds):
912
+ """Multiple contexts with the same table should work independently."""
913
+ tracker1 = IterationTracker()
914
+ tracker2 = IterationTracker()
915
+
916
+ table1 = read_xarray_table(
917
+ small_ds,
918
+ chunks={"time": 25},
919
+ _iteration_callback=tracker1,
920
+ )
921
+
922
+ table2 = read_xarray_table(
923
+ small_ds,
924
+ chunks={"time": 50},
925
+ _iteration_callback=tracker2,
926
+ )
927
+
928
+ ctx1 = SessionContext()
929
+ ctx2 = SessionContext()
930
+
931
+ ctx1.register_table("test_table", table1)
932
+ ctx2.register_table("test_table", table2)
933
+
934
+ # Execute queries
935
+ ctx1.sql("SELECT COUNT(*) FROM test_table").collect()
936
+ ctx2.sql("SELECT COUNT(*) FROM test_table").collect()
937
+
938
+ # Each should have its own iteration count
939
+ assert (
940
+ tracker1.iteration_count == 4
941
+ ), f"Table1: expected 4 blocks, got {tracker1.iteration_count}"
942
+ assert (
943
+ tracker2.iteration_count == 2
944
+ ), f"Table2: expected 2 blocks, got {tracker2.iteration_count}"