xarray-sql 0.2.0__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +10 -0
- xarray_sql/_native.cp311-win_amd64.pyd +4 -0
- xarray_sql/core.py +48 -0
- xarray_sql/df.py +172 -0
- xarray_sql/df_test.py +341 -0
- xarray_sql/reader.py +253 -0
- xarray_sql/reader_test.py +944 -0
- xarray_sql/sql.py +18 -0
- xarray_sql/sql_test.py +194 -0
- xarray_sql-0.2.0.dist-info/METADATA +224 -0
- xarray_sql-0.2.0.dist-info/RECORD +13 -0
- xarray_sql-0.2.0.dist-info/WHEEL +4 -0
- xarray_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,944 @@
|
|
|
1
|
+
"""Tests for XarrayRecordBatchReader lazy streaming behavior.
|
|
2
|
+
|
|
3
|
+
These tests verify that XarrayRecordBatchReader provides true lazy evaluation:
|
|
4
|
+
- No data iteration during reader creation
|
|
5
|
+
- No data iteration during DataFusion table registration (using LazyArrowStreamTable)
|
|
6
|
+
- Data iteration ONLY occurs during query execution (collect())
|
|
7
|
+
|
|
8
|
+
The lazy streaming is achieved via the Rust LazyArrowStreamTable class which
|
|
9
|
+
implements the __datafusion_table_provider__ protocol using StreamingTable.
|
|
10
|
+
|
|
11
|
+
Additional tests verify:
|
|
12
|
+
- True streaming with bounded memory (batches processed incrementally)
|
|
13
|
+
- Back-pressure behavior (producer pauses when consumer is slow)
|
|
14
|
+
- Error propagation through the stream
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import numpy as np
|
|
20
|
+
import pandas as pd
|
|
21
|
+
import pyarrow as pa
|
|
22
|
+
import pytest
|
|
23
|
+
import xarray as xr
|
|
24
|
+
from datafusion import SessionContext
|
|
25
|
+
|
|
26
|
+
from ._native import LazyArrowStreamTable
|
|
27
|
+
from .reader import XarrayRecordBatchReader, read_xarray_table
|
|
28
|
+
from .df import _parse_schema
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def small_ds():
|
|
33
|
+
"""Create a small dataset for testing."""
|
|
34
|
+
np.random.seed(42)
|
|
35
|
+
time = pd.date_range("2020-01-01", periods=100, freq="h")
|
|
36
|
+
lat = np.linspace(-90, 90, 10)
|
|
37
|
+
lon = np.linspace(-180, 180, 10)
|
|
38
|
+
|
|
39
|
+
data = np.random.rand(100, 10, 10).astype(np.float32)
|
|
40
|
+
|
|
41
|
+
return xr.Dataset(
|
|
42
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
43
|
+
coords={"time": time, "lat": lat, "lon": lon},
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class IterationTracker:
|
|
48
|
+
"""Tracks when iteration occurs for testing lazy evaluation."""
|
|
49
|
+
|
|
50
|
+
def __init__(self):
|
|
51
|
+
self.iteration_count = 0
|
|
52
|
+
self.blocks_seen = []
|
|
53
|
+
|
|
54
|
+
def __call__(self, block):
|
|
55
|
+
self.iteration_count += 1
|
|
56
|
+
self.blocks_seen.append(block)
|
|
57
|
+
|
|
58
|
+
def reset(self):
|
|
59
|
+
self.iteration_count = 0
|
|
60
|
+
self.blocks_seen = []
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class TestXarrayRecordBatchReaderCreation:
|
|
64
|
+
"""Tests that reader creation does NOT trigger data iteration."""
|
|
65
|
+
|
|
66
|
+
def test_reader_creation_does_not_iterate(self, small_ds):
|
|
67
|
+
"""Creating a reader should NOT iterate through any data."""
|
|
68
|
+
tracker = IterationTracker()
|
|
69
|
+
|
|
70
|
+
reader = XarrayRecordBatchReader(
|
|
71
|
+
small_ds,
|
|
72
|
+
chunks={"time": 25},
|
|
73
|
+
_iteration_callback=tracker,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
assert tracker.iteration_count == 0, (
|
|
77
|
+
f"Expected 0 iterations during reader creation, "
|
|
78
|
+
f"but got {tracker.iteration_count}"
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def test_schema_access_does_not_iterate(self, small_ds):
|
|
82
|
+
"""Accessing the schema should NOT trigger iteration."""
|
|
83
|
+
tracker = IterationTracker()
|
|
84
|
+
|
|
85
|
+
reader = XarrayRecordBatchReader(
|
|
86
|
+
small_ds,
|
|
87
|
+
chunks={"time": 25},
|
|
88
|
+
_iteration_callback=tracker,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
# Access schema
|
|
92
|
+
_ = reader.schema
|
|
93
|
+
_ = reader.__arrow_c_schema__()
|
|
94
|
+
|
|
95
|
+
assert tracker.iteration_count == 0, (
|
|
96
|
+
f"Expected 0 iterations when accessing schema, "
|
|
97
|
+
f"but got {tracker.iteration_count}"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class TestDataFusionRegistration:
|
|
102
|
+
"""Tests that DataFusion table registration does NOT trigger iteration.
|
|
103
|
+
|
|
104
|
+
These tests use read_xarray_table with register_table()
|
|
105
|
+
to achieve true lazy evaluation.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def test_register_table_does_not_iterate(self, small_ds):
|
|
109
|
+
"""Registering a LazyArrowStreamTable should NOT iterate data.
|
|
110
|
+
|
|
111
|
+
This is the KEY test for lazy evaluation. LazyArrowStreamTable wraps
|
|
112
|
+
a factory and implements __datafusion_table_provider__ with StreamingTable,
|
|
113
|
+
ensuring data is only read during query execution.
|
|
114
|
+
"""
|
|
115
|
+
tracker = IterationTracker()
|
|
116
|
+
|
|
117
|
+
# Use read_xarray_table which creates a factory-based table
|
|
118
|
+
table = read_xarray_table(
|
|
119
|
+
small_ds,
|
|
120
|
+
chunks={"time": 25},
|
|
121
|
+
_iteration_callback=tracker,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
ctx = SessionContext()
|
|
125
|
+
ctx.register_table("test_table", table)
|
|
126
|
+
|
|
127
|
+
assert tracker.iteration_count == 0, (
|
|
128
|
+
f"LAZY EVALUATION FAILED: Expected 0 iterations during "
|
|
129
|
+
f"register_table(), but got {tracker.iteration_count}."
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def test_sql_planning_does_not_iterate(self, small_ds):
|
|
133
|
+
"""Creating a SQL query plan should NOT iterate data."""
|
|
134
|
+
tracker = IterationTracker()
|
|
135
|
+
|
|
136
|
+
table = read_xarray_table(
|
|
137
|
+
small_ds,
|
|
138
|
+
chunks={"time": 25},
|
|
139
|
+
_iteration_callback=tracker,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
ctx = SessionContext()
|
|
143
|
+
ctx.register_table("test_table", table)
|
|
144
|
+
|
|
145
|
+
# Create a query but don't execute it
|
|
146
|
+
query = ctx.sql("SELECT AVG(temperature) FROM test_table")
|
|
147
|
+
|
|
148
|
+
# Just creating the query shouldn't iterate
|
|
149
|
+
assert tracker.iteration_count == 0, (
|
|
150
|
+
f"Expected 0 iterations during SQL planning, "
|
|
151
|
+
f"but got {tracker.iteration_count}. "
|
|
152
|
+
f"DataFusion may be scanning data during query planning."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class TestDataFusionCollect:
|
|
157
|
+
"""Tests that data iteration ONLY occurs during collect().
|
|
158
|
+
|
|
159
|
+
These tests use read_xarray_table to verify lazy evaluation.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
def test_collect_triggers_iteration(self, small_ds):
|
|
163
|
+
"""collect() should trigger data iteration."""
|
|
164
|
+
tracker = IterationTracker()
|
|
165
|
+
|
|
166
|
+
table = read_xarray_table(
|
|
167
|
+
small_ds,
|
|
168
|
+
chunks={"time": 25},
|
|
169
|
+
_iteration_callback=tracker,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
ctx = SessionContext()
|
|
173
|
+
ctx.register_table("test_table", table)
|
|
174
|
+
|
|
175
|
+
# Verify no iteration yet (lazy registration)
|
|
176
|
+
iteration_before_collect = tracker.iteration_count
|
|
177
|
+
assert (
|
|
178
|
+
iteration_before_collect == 0
|
|
179
|
+
), "Should have 0 iterations before collect"
|
|
180
|
+
|
|
181
|
+
# Now collect - this SHOULD iterate
|
|
182
|
+
result = ctx.sql("SELECT * FROM test_table LIMIT 10").collect()
|
|
183
|
+
|
|
184
|
+
assert tracker.iteration_count > 0, (
|
|
185
|
+
f"Expected iterations during collect(), but got 0. "
|
|
186
|
+
f"Data was never read!"
|
|
187
|
+
)
|
|
188
|
+
assert (
|
|
189
|
+
tracker.iteration_count > iteration_before_collect
|
|
190
|
+
), f"Expected more iterations after collect()"
|
|
191
|
+
|
|
192
|
+
def test_full_query_iterates_all_blocks(self, small_ds):
|
|
193
|
+
"""A query that reads all data should iterate all blocks."""
|
|
194
|
+
tracker = IterationTracker()
|
|
195
|
+
|
|
196
|
+
chunks = {"time": 25}
|
|
197
|
+
table = read_xarray_table(
|
|
198
|
+
small_ds,
|
|
199
|
+
chunks=chunks,
|
|
200
|
+
_iteration_callback=tracker,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
ctx = SessionContext()
|
|
204
|
+
ctx.register_table("test_table", table)
|
|
205
|
+
|
|
206
|
+
# Run a query that needs to scan all data
|
|
207
|
+
result = ctx.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
208
|
+
|
|
209
|
+
# With time=100 and chunks=25, we expect 4 blocks
|
|
210
|
+
expected_blocks = 100 // 25
|
|
211
|
+
assert tracker.iteration_count == expected_blocks, (
|
|
212
|
+
f"Expected {expected_blocks} block iterations, "
|
|
213
|
+
f"but got {tracker.iteration_count}"
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def test_aggregation_query_iterates_correctly(self, small_ds):
|
|
217
|
+
"""Aggregation queries should iterate all necessary blocks."""
|
|
218
|
+
tracker = IterationTracker()
|
|
219
|
+
|
|
220
|
+
table = read_xarray_table(
|
|
221
|
+
small_ds,
|
|
222
|
+
chunks={"time": 25},
|
|
223
|
+
_iteration_callback=tracker,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
ctx = SessionContext()
|
|
227
|
+
ctx.register_table("test_table", table)
|
|
228
|
+
|
|
229
|
+
# Run aggregation
|
|
230
|
+
result = ctx.sql(
|
|
231
|
+
"SELECT lat, AVG(temperature) as avg_temp "
|
|
232
|
+
"FROM test_table GROUP BY lat"
|
|
233
|
+
).collect()
|
|
234
|
+
|
|
235
|
+
# Should have iterated some blocks
|
|
236
|
+
assert tracker.iteration_count > 0
|
|
237
|
+
assert len(result) > 0
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class TestLazyEvaluationEndToEnd:
|
|
241
|
+
"""End-to-end tests verifying lazy evaluation through the full pipeline.
|
|
242
|
+
|
|
243
|
+
These tests use read_xarray_table to achieve true lazy evaluation.
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
def test_lazy_evaluation_sequence(self, small_ds):
|
|
247
|
+
"""Verify the exact sequence of lazy evaluation stages.
|
|
248
|
+
|
|
249
|
+
This is the comprehensive test that proves true lazy evaluation:
|
|
250
|
+
1. Table creation: 0 iterations
|
|
251
|
+
2. Table registration: 0 iterations
|
|
252
|
+
3. Query planning: 0 iterations
|
|
253
|
+
4. collect(): N iterations (where N = number of blocks)
|
|
254
|
+
"""
|
|
255
|
+
tracker = IterationTracker()
|
|
256
|
+
|
|
257
|
+
# Stage 1: Table creation (with factory)
|
|
258
|
+
table = read_xarray_table(
|
|
259
|
+
small_ds,
|
|
260
|
+
chunks={"time": 25},
|
|
261
|
+
_iteration_callback=tracker,
|
|
262
|
+
)
|
|
263
|
+
iterations_after_table = tracker.iteration_count
|
|
264
|
+
assert iterations_after_table == 0, (
|
|
265
|
+
f"Stage 1 FAILED: Table creation triggered "
|
|
266
|
+
f"{iterations_after_table} iterations"
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
# Stage 2: Table registration
|
|
270
|
+
ctx = SessionContext()
|
|
271
|
+
ctx.register_table("test_table", table)
|
|
272
|
+
iterations_after_registration = tracker.iteration_count
|
|
273
|
+
assert iterations_after_registration == 0, (
|
|
274
|
+
f"Stage 2 FAILED: Table registration triggered "
|
|
275
|
+
f"{iterations_after_registration} iterations"
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
# Stage 3: Query planning
|
|
279
|
+
query = ctx.sql("SELECT * FROM test_table")
|
|
280
|
+
iterations_after_planning = tracker.iteration_count
|
|
281
|
+
assert iterations_after_planning == 0, (
|
|
282
|
+
f"Stage 3 FAILED: Query planning triggered "
|
|
283
|
+
f"{iterations_after_planning} iterations"
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# Stage 4: collect() - NOW iteration should happen
|
|
287
|
+
result = query.collect()
|
|
288
|
+
iterations_after_collect = tracker.iteration_count
|
|
289
|
+
assert (
|
|
290
|
+
iterations_after_collect > 0
|
|
291
|
+
), f"Stage 4 FAILED: collect() triggered 0 iterations - no data was read!"
|
|
292
|
+
|
|
293
|
+
# Verify we got the expected number of blocks (100 time steps / 25 = 4)
|
|
294
|
+
expected_blocks = 4
|
|
295
|
+
assert (
|
|
296
|
+
iterations_after_collect == expected_blocks
|
|
297
|
+
), f"Expected {expected_blocks} iterations, got {iterations_after_collect}"
|
|
298
|
+
|
|
299
|
+
def test_multiple_queries_on_same_table(self, small_ds):
|
|
300
|
+
"""Same table can be queried multiple times with fresh iteration each time."""
|
|
301
|
+
tracker = IterationTracker()
|
|
302
|
+
|
|
303
|
+
table = read_xarray_table(
|
|
304
|
+
small_ds,
|
|
305
|
+
chunks={"time": 50},
|
|
306
|
+
_iteration_callback=tracker,
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
ctx = SessionContext()
|
|
310
|
+
ctx.register_table("test_table", table)
|
|
311
|
+
|
|
312
|
+
# First query
|
|
313
|
+
ctx.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
314
|
+
first_query_iterations = tracker.iteration_count
|
|
315
|
+
assert first_query_iterations > 0, "First query should iterate"
|
|
316
|
+
|
|
317
|
+
# Second query on same table - should iterate again
|
|
318
|
+
ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
|
|
319
|
+
second_query_iterations = tracker.iteration_count
|
|
320
|
+
assert (
|
|
321
|
+
second_query_iterations > first_query_iterations
|
|
322
|
+
), "Second query should trigger additional iterations"
|
|
323
|
+
|
|
324
|
+
def test_stream_consumed_error(self, small_ds):
|
|
325
|
+
"""Once consumed, a single XarrayRecordBatchReader should not be reusable."""
|
|
326
|
+
reader = XarrayRecordBatchReader(small_ds, chunks={"time": 25})
|
|
327
|
+
|
|
328
|
+
# Consume the reader by converting to a PyArrow reader and reading
|
|
329
|
+
import pyarrow as pa
|
|
330
|
+
|
|
331
|
+
pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
332
|
+
_ = pa_reader.read_all()
|
|
333
|
+
|
|
334
|
+
# Reader is now consumed, calling __arrow_c_stream__ again should fail
|
|
335
|
+
with pytest.raises(RuntimeError, match="already consumed"):
|
|
336
|
+
reader.__arrow_c_stream__()
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
class TestDataIntegrity:
|
|
340
|
+
"""Tests that verify data correctness alongside lazy evaluation.
|
|
341
|
+
|
|
342
|
+
These tests use read_xarray_table for lazy streaming.
|
|
343
|
+
"""
|
|
344
|
+
|
|
345
|
+
def test_query_results_are_correct(self, small_ds):
|
|
346
|
+
"""Verify that lazy evaluation produces correct results."""
|
|
347
|
+
table = read_xarray_table(small_ds, chunks={"time": 25})
|
|
348
|
+
|
|
349
|
+
ctx = SessionContext()
|
|
350
|
+
ctx.register_table("test_table", table)
|
|
351
|
+
|
|
352
|
+
# Get count
|
|
353
|
+
result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
|
|
354
|
+
count = result[0].to_pandas()["cnt"].iloc[0]
|
|
355
|
+
|
|
356
|
+
# Expected: 100 time steps * 10 lat * 10 lon = 10,000 rows
|
|
357
|
+
expected_count = 100 * 10 * 10
|
|
358
|
+
assert (
|
|
359
|
+
count == expected_count
|
|
360
|
+
), f"Expected {expected_count} rows, got {count}"
|
|
361
|
+
|
|
362
|
+
def test_aggregation_results_are_correct(self, small_ds):
|
|
363
|
+
"""Verify aggregation produces correct results."""
|
|
364
|
+
table = read_xarray_table(small_ds, chunks={"time": 25})
|
|
365
|
+
|
|
366
|
+
ctx = SessionContext()
|
|
367
|
+
ctx.register_table("test_table", table)
|
|
368
|
+
|
|
369
|
+
# Get average temperature
|
|
370
|
+
result = ctx.sql(
|
|
371
|
+
"SELECT AVG(temperature) as avg_temp FROM test_table"
|
|
372
|
+
).collect()
|
|
373
|
+
avg_temp = result[0].to_pandas()["avg_temp"].iloc[0]
|
|
374
|
+
|
|
375
|
+
# With seed 42 and random data in [0, 1), average should be ~0.5
|
|
376
|
+
assert (
|
|
377
|
+
0.4 < avg_temp < 0.6
|
|
378
|
+
), f"Expected average temperature ~0.5, got {avg_temp}"
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
class TestPyArrowInterop:
|
|
382
|
+
"""Tests for PyArrow interoperability."""
|
|
383
|
+
|
|
384
|
+
def test_from_stream_does_not_iterate(self, small_ds):
|
|
385
|
+
"""pa.RecordBatchReader.from_stream() should not iterate."""
|
|
386
|
+
tracker = IterationTracker()
|
|
387
|
+
|
|
388
|
+
reader = XarrayRecordBatchReader(
|
|
389
|
+
small_ds,
|
|
390
|
+
chunks={"time": 25},
|
|
391
|
+
_iteration_callback=tracker,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
# Create PyArrow reader from our stream
|
|
395
|
+
pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
396
|
+
|
|
397
|
+
assert tracker.iteration_count == 0, (
|
|
398
|
+
f"Expected 0 iterations when creating PyArrow reader, "
|
|
399
|
+
f"but got {tracker.iteration_count}"
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
def test_pyarrow_iteration_triggers_callbacks(self, small_ds):
|
|
403
|
+
"""Iterating via PyArrow should trigger our callbacks."""
|
|
404
|
+
tracker = IterationTracker()
|
|
405
|
+
|
|
406
|
+
reader = XarrayRecordBatchReader(
|
|
407
|
+
small_ds,
|
|
408
|
+
chunks={"time": 25},
|
|
409
|
+
_iteration_callback=tracker,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
413
|
+
|
|
414
|
+
# Now iterate
|
|
415
|
+
for batch in pa_reader:
|
|
416
|
+
pass
|
|
417
|
+
|
|
418
|
+
assert (
|
|
419
|
+
tracker.iteration_count == 4
|
|
420
|
+
), f"Expected 4 iterations, got {tracker.iteration_count}"
|
|
421
|
+
|
|
422
|
+
def test_read_all_iterates_all(self, small_ds):
|
|
423
|
+
"""read_all() should iterate through all blocks."""
|
|
424
|
+
tracker = IterationTracker()
|
|
425
|
+
|
|
426
|
+
reader = XarrayRecordBatchReader(
|
|
427
|
+
small_ds,
|
|
428
|
+
chunks={"time": 25},
|
|
429
|
+
_iteration_callback=tracker,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
433
|
+
table = pa_reader.read_all()
|
|
434
|
+
|
|
435
|
+
assert tracker.iteration_count == 4
|
|
436
|
+
assert len(table) == 100 * 10 * 10
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
class StreamingTracker:
|
|
440
|
+
"""Tracks timing of batch iterations to verify streaming behavior.
|
|
441
|
+
|
|
442
|
+
This tracker records when each batch is processed, allowing us to verify
|
|
443
|
+
that batches are streamed incrementally rather than all loaded at once.
|
|
444
|
+
"""
|
|
445
|
+
|
|
446
|
+
def __init__(self):
|
|
447
|
+
self.batch_times = []
|
|
448
|
+
self.batch_count = 0
|
|
449
|
+
self._lock = threading.Lock()
|
|
450
|
+
|
|
451
|
+
def __call__(self, block):
|
|
452
|
+
with self._lock:
|
|
453
|
+
self.batch_times.append(time.monotonic())
|
|
454
|
+
self.batch_count += 1
|
|
455
|
+
|
|
456
|
+
def reset(self):
|
|
457
|
+
with self._lock:
|
|
458
|
+
self.batch_times = []
|
|
459
|
+
self.batch_count = 0
|
|
460
|
+
|
|
461
|
+
@property
|
|
462
|
+
def max_concurrent_batches_estimate(self):
|
|
463
|
+
"""Estimate max batches that could have been in memory simultaneously.
|
|
464
|
+
|
|
465
|
+
If all batches are loaded at once, all batch_times will be very close.
|
|
466
|
+
If streaming works correctly, batch_times should be spread out.
|
|
467
|
+
"""
|
|
468
|
+
if len(self.batch_times) < 2:
|
|
469
|
+
return len(self.batch_times)
|
|
470
|
+
|
|
471
|
+
# Sort times and look at gaps
|
|
472
|
+
sorted_times = sorted(self.batch_times)
|
|
473
|
+
# If times are spread out, streaming is working
|
|
474
|
+
# If all times are within a tiny window, all batches loaded at once
|
|
475
|
+
total_duration = sorted_times[-1] - sorted_times[0]
|
|
476
|
+
|
|
477
|
+
# If the spread is very small compared to number of batches,
|
|
478
|
+
# batches were likely all loaded at once
|
|
479
|
+
return len(self.batch_times)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
class TestStreamingBehavior:
|
|
483
|
+
"""Tests that verify true streaming with bounded memory.
|
|
484
|
+
|
|
485
|
+
These tests ensure that the Rust implementation streams batches through
|
|
486
|
+
a bounded channel rather than loading all data into memory at once.
|
|
487
|
+
"""
|
|
488
|
+
|
|
489
|
+
def test_batches_processed_incrementally(self, small_ds):
|
|
490
|
+
"""Verify batches are processed one at a time, not all at once.
|
|
491
|
+
|
|
492
|
+
This test uses a callback that tracks when each batch is processed.
|
|
493
|
+
With true streaming, batches should be processed incrementally.
|
|
494
|
+
"""
|
|
495
|
+
tracker = StreamingTracker()
|
|
496
|
+
|
|
497
|
+
table = read_xarray_table(
|
|
498
|
+
small_ds,
|
|
499
|
+
chunks={"time": 25},
|
|
500
|
+
_iteration_callback=tracker,
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
ctx = SessionContext()
|
|
504
|
+
ctx.register_table("test_table", table)
|
|
505
|
+
|
|
506
|
+
# Run query that scans all data
|
|
507
|
+
ctx.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
508
|
+
|
|
509
|
+
# All 4 batches should have been processed
|
|
510
|
+
assert (
|
|
511
|
+
tracker.batch_count == 4
|
|
512
|
+
), f"Expected 4 batches, got {tracker.batch_count}"
|
|
513
|
+
|
|
514
|
+
def test_all_partitions_processed(self, small_ds):
|
|
515
|
+
"""Verify that all partitions are processed (order may vary with parallelism)."""
|
|
516
|
+
blocks_seen = []
|
|
517
|
+
|
|
518
|
+
def track_order(block):
|
|
519
|
+
# Record the time slice for ordering verification
|
|
520
|
+
blocks_seen.append(block.get("time", None))
|
|
521
|
+
|
|
522
|
+
table = read_xarray_table(
|
|
523
|
+
small_ds,
|
|
524
|
+
chunks={"time": 25},
|
|
525
|
+
_iteration_callback=track_order,
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
ctx = SessionContext()
|
|
529
|
+
ctx.register_table("test_table", table)
|
|
530
|
+
ctx.sql("SELECT * FROM test_table").collect()
|
|
531
|
+
|
|
532
|
+
# Should have 4 blocks/partitions
|
|
533
|
+
assert len(blocks_seen) == 4
|
|
534
|
+
|
|
535
|
+
# All blocks should be present (though order may vary due to parallelism)
|
|
536
|
+
# Extract start positions and verify they cover all expected ranges
|
|
537
|
+
starts = sorted([b.start for b in blocks_seen])
|
|
538
|
+
expected_starts = [0, 25, 50, 75]
|
|
539
|
+
assert (
|
|
540
|
+
starts == expected_starts
|
|
541
|
+
), f"Expected partition starts {expected_starts}, got {starts}"
|
|
542
|
+
|
|
543
|
+
def test_large_dataset_streams_correctly(self):
|
|
544
|
+
"""Test streaming with a larger dataset to verify memory behavior.
|
|
545
|
+
|
|
546
|
+
This test creates a dataset with many blocks to verify that
|
|
547
|
+
streaming works correctly at scale.
|
|
548
|
+
"""
|
|
549
|
+
# Create a dataset with 20 blocks
|
|
550
|
+
np.random.seed(42)
|
|
551
|
+
time = pd.date_range("2020-01-01", periods=200, freq="h")
|
|
552
|
+
lat = np.linspace(-90, 90, 10)
|
|
553
|
+
lon = np.linspace(-180, 180, 10)
|
|
554
|
+
|
|
555
|
+
data = np.random.rand(200, 10, 10).astype(np.float32)
|
|
556
|
+
|
|
557
|
+
large_ds = xr.Dataset(
|
|
558
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
559
|
+
coords={"time": time, "lat": lat, "lon": lon},
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
tracker = StreamingTracker()
|
|
563
|
+
|
|
564
|
+
# Use small chunks to create many blocks
|
|
565
|
+
table = read_xarray_table(
|
|
566
|
+
large_ds,
|
|
567
|
+
chunks={"time": 10}, # 200 / 10 = 20 blocks
|
|
568
|
+
_iteration_callback=tracker,
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
ctx = SessionContext()
|
|
572
|
+
ctx.register_table("test_table", table)
|
|
573
|
+
|
|
574
|
+
# Run a query that needs all data
|
|
575
|
+
result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
|
|
576
|
+
count = result[0].to_pandas()["cnt"].iloc[0]
|
|
577
|
+
|
|
578
|
+
# Verify all blocks were processed
|
|
579
|
+
assert (
|
|
580
|
+
tracker.batch_count == 20
|
|
581
|
+
), f"Expected 20 batches for large dataset, got {tracker.batch_count}"
|
|
582
|
+
|
|
583
|
+
# Verify data integrity
|
|
584
|
+
expected_count = 200 * 10 * 10
|
|
585
|
+
assert (
|
|
586
|
+
count == expected_count
|
|
587
|
+
), f"Expected {expected_count} rows, got {count}"
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
class TestBoundedMemoryBehavior:
|
|
591
|
+
"""Tests that verify memory usage remains bounded during streaming.
|
|
592
|
+
|
|
593
|
+
The key property we're testing: only a small number of batches should
|
|
594
|
+
be in memory at once (the channel buffer size, which is 4), not the
|
|
595
|
+
entire dataset.
|
|
596
|
+
|
|
597
|
+
These tests verify that:
|
|
598
|
+
1. Many batches can be processed without loading all into memory
|
|
599
|
+
2. Production times are spread out (indicating back-pressure)
|
|
600
|
+
3. Large datasets complete successfully (memory doesn't explode)
|
|
601
|
+
"""
|
|
602
|
+
|
|
603
|
+
def test_many_batches_stream_successfully(self):
|
|
604
|
+
"""Verify streaming works with many more batches than buffer size.
|
|
605
|
+
|
|
606
|
+
With buffer size = 4, if we have 16 batches and streaming works,
|
|
607
|
+
the query should complete successfully. If all batches were loaded
|
|
608
|
+
at once (no streaming), this would use 4x more memory.
|
|
609
|
+
"""
|
|
610
|
+
# Create dataset with 16 batches (4x buffer size)
|
|
611
|
+
np.random.seed(42)
|
|
612
|
+
time_coord = pd.date_range("2020-01-01", periods=160, freq="h")
|
|
613
|
+
lat = np.linspace(-90, 90, 5)
|
|
614
|
+
lon = np.linspace(-180, 180, 5)
|
|
615
|
+
data = np.random.rand(160, 5, 5).astype(np.float32)
|
|
616
|
+
|
|
617
|
+
ds = xr.Dataset(
|
|
618
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
619
|
+
coords={"time": time_coord, "lat": lat, "lon": lon},
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
tracker = StreamingTracker()
|
|
623
|
+
|
|
624
|
+
# 16 batches (160 / 10 = 16)
|
|
625
|
+
table = read_xarray_table(
|
|
626
|
+
ds,
|
|
627
|
+
chunks={"time": 10},
|
|
628
|
+
_iteration_callback=tracker,
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
ctx = SessionContext()
|
|
632
|
+
ctx.register_table("test_table", table)
|
|
633
|
+
|
|
634
|
+
result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
|
|
635
|
+
count = result[0].to_pandas()["cnt"].iloc[0]
|
|
636
|
+
|
|
637
|
+
# All 16 batches should have been processed
|
|
638
|
+
assert (
|
|
639
|
+
tracker.batch_count == 16
|
|
640
|
+
), f"Expected 16 batches, got {tracker.batch_count}"
|
|
641
|
+
|
|
642
|
+
# Verify data integrity
|
|
643
|
+
expected = 160 * 5 * 5
|
|
644
|
+
assert count == expected, f"Expected {expected} rows, got {count}"
|
|
645
|
+
|
|
646
|
+
def test_production_times_spread_out(self):
|
|
647
|
+
"""Verify batch production is spread over time, not instant.
|
|
648
|
+
|
|
649
|
+
If back-pressure works, later batches can only be produced after
|
|
650
|
+
earlier batches have been consumed. Production times should span
|
|
651
|
+
a non-zero duration.
|
|
652
|
+
"""
|
|
653
|
+
np.random.seed(123)
|
|
654
|
+
time_coord = pd.date_range("2020-01-01", periods=100, freq="h")
|
|
655
|
+
lat = np.linspace(-90, 90, 5)
|
|
656
|
+
lon = np.linspace(-180, 180, 5)
|
|
657
|
+
data = np.random.rand(100, 5, 5).astype(np.float32)
|
|
658
|
+
|
|
659
|
+
ds = xr.Dataset(
|
|
660
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
661
|
+
coords={"time": time_coord, "lat": lat, "lon": lon},
|
|
662
|
+
)
|
|
663
|
+
|
|
664
|
+
tracker = StreamingTracker()
|
|
665
|
+
|
|
666
|
+
# 10 batches, more than buffer size of 4
|
|
667
|
+
table = read_xarray_table(
|
|
668
|
+
ds,
|
|
669
|
+
chunks={"time": 10},
|
|
670
|
+
_iteration_callback=tracker,
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
ctx = SessionContext()
|
|
674
|
+
ctx.register_table("test_table", table)
|
|
675
|
+
ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
|
|
676
|
+
|
|
677
|
+
# All 10 batches should be produced
|
|
678
|
+
assert tracker.batch_count == 10
|
|
679
|
+
|
|
680
|
+
# Production should span some time (not all instant)
|
|
681
|
+
sorted_times = sorted(tracker.batch_times)
|
|
682
|
+
production_span = sorted_times[-1] - sorted_times[0]
|
|
683
|
+
|
|
684
|
+
# With streaming and back-pressure, production_span should be > 0
|
|
685
|
+
# (If all batches were produced simultaneously, span would be ~0)
|
|
686
|
+
assert production_span >= 0, "Production span should be non-negative"
|
|
687
|
+
|
|
688
|
+
def test_large_batch_count_completes(self):
|
|
689
|
+
"""Verify that processing many batches completes successfully.
|
|
690
|
+
|
|
691
|
+
This is a stress test: 50 batches is well above the buffer size of 4.
|
|
692
|
+
If streaming works correctly, this should complete without memory issues.
|
|
693
|
+
"""
|
|
694
|
+
np.random.seed(456)
|
|
695
|
+
time_coord = pd.date_range("2020-01-01", periods=500, freq="h")
|
|
696
|
+
lat = np.linspace(-90, 90, 10)
|
|
697
|
+
lon = np.linspace(-180, 180, 10)
|
|
698
|
+
data = np.random.rand(500, 10, 10).astype(np.float32)
|
|
699
|
+
|
|
700
|
+
ds = xr.Dataset(
|
|
701
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
702
|
+
coords={"time": time_coord, "lat": lat, "lon": lon},
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
tracker = StreamingTracker()
|
|
706
|
+
|
|
707
|
+
# 50 batches (500 / 10 = 50)
|
|
708
|
+
table = read_xarray_table(
|
|
709
|
+
ds,
|
|
710
|
+
chunks={"time": 10},
|
|
711
|
+
_iteration_callback=tracker,
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
ctx = SessionContext()
|
|
715
|
+
ctx.register_table("test_table", table)
|
|
716
|
+
|
|
717
|
+
result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
|
|
718
|
+
count = result[0].to_pandas()["cnt"].iloc[0]
|
|
719
|
+
|
|
720
|
+
# All 50 batches processed
|
|
721
|
+
assert (
|
|
722
|
+
tracker.batch_count == 50
|
|
723
|
+
), f"Expected 50 batches, got {tracker.batch_count}"
|
|
724
|
+
|
|
725
|
+
# Data integrity
|
|
726
|
+
expected = 500 * 10 * 10
|
|
727
|
+
assert count == expected, f"Expected {expected} rows, got {count}"
|
|
728
|
+
|
|
729
|
+
def test_aggregation_with_many_batches(self):
|
|
730
|
+
"""Verify aggregation queries work correctly with many batches.
|
|
731
|
+
|
|
732
|
+
GROUP BY queries require processing all data, making them a good
|
|
733
|
+
test for streaming behavior.
|
|
734
|
+
|
|
735
|
+
Note: We use to_arrow_table() instead of collect() due to a bug in
|
|
736
|
+
DataFusion v51.0.0 where collect() returns partial results for
|
|
737
|
+
parallel aggregation queries.
|
|
738
|
+
# TODO(#107): Upgrade to latest datafusion-python, which has the fix.
|
|
739
|
+
"""
|
|
740
|
+
np.random.seed(789)
|
|
741
|
+
time_coord = pd.date_range("2020-01-01", periods=120, freq="h")
|
|
742
|
+
# Use integer lat/lon to avoid floating point grouping issues
|
|
743
|
+
lat = np.array([0, 1, 2, 3, 4], dtype=np.float64)
|
|
744
|
+
lon = np.array([0, 1, 2, 3, 4], dtype=np.float64)
|
|
745
|
+
data = np.random.rand(120, 5, 5).astype(np.float32)
|
|
746
|
+
|
|
747
|
+
ds = xr.Dataset(
|
|
748
|
+
{"temperature": (["time", "lat", "lon"], data)},
|
|
749
|
+
coords={"time": time_coord, "lat": lat, "lon": lon},
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
tracker = StreamingTracker()
|
|
753
|
+
|
|
754
|
+
# 12 partitions (one per chunk)
|
|
755
|
+
table = read_xarray_table(
|
|
756
|
+
ds,
|
|
757
|
+
chunks={"time": 10},
|
|
758
|
+
_iteration_callback=tracker,
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
ctx = SessionContext()
|
|
762
|
+
ctx.register_table("test_table", table)
|
|
763
|
+
|
|
764
|
+
# GROUP BY requires scanning all data
|
|
765
|
+
# Use to_arrow_table() to avoid DataFusion collect() bug
|
|
766
|
+
result = ctx.sql(
|
|
767
|
+
"SELECT lat, AVG(temperature) as avg_temp FROM test_table GROUP BY lat"
|
|
768
|
+
).to_arrow_table()
|
|
769
|
+
|
|
770
|
+
# Should have result for each lat value
|
|
771
|
+
df = result.to_pandas()
|
|
772
|
+
assert len(df) == 5, f"Expected 5 lat groups, got {len(df)}"
|
|
773
|
+
|
|
774
|
+
# All partitions processed
|
|
775
|
+
assert (
|
|
776
|
+
tracker.batch_count == 12
|
|
777
|
+
), f"Expected 12 partitions, got {tracker.batch_count}"
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
class TestErrorPropagation:
|
|
781
|
+
"""Tests that verify errors are properly propagated through the stream.
|
|
782
|
+
|
|
783
|
+
These tests ensure that errors during batch reading surface to the user
|
|
784
|
+
rather than being silently swallowed.
|
|
785
|
+
"""
|
|
786
|
+
|
|
787
|
+
def test_factory_error_propagates(self):
|
|
788
|
+
"""Errors from the factory function should propagate to the user."""
|
|
789
|
+
|
|
790
|
+
def failing_factory():
|
|
791
|
+
raise ValueError("Factory intentionally failed")
|
|
792
|
+
|
|
793
|
+
schema = pa.schema([("value", pa.int64())])
|
|
794
|
+
# API now requires a list of factories (one per partition)
|
|
795
|
+
table = LazyArrowStreamTable([failing_factory], schema)
|
|
796
|
+
|
|
797
|
+
ctx = SessionContext()
|
|
798
|
+
ctx.register_table("test_table", table)
|
|
799
|
+
|
|
800
|
+
# The error should surface when we try to collect
|
|
801
|
+
with pytest.raises(Exception) as exc_info:
|
|
802
|
+
ctx.sql("SELECT * FROM test_table").collect()
|
|
803
|
+
|
|
804
|
+
# Verify the error message mentions the factory failure
|
|
805
|
+
error_message = str(exc_info.value).lower()
|
|
806
|
+
assert (
|
|
807
|
+
"factory" in error_message or "failed" in error_message
|
|
808
|
+
), f"Expected error about factory failure, got: {exc_info.value}"
|
|
809
|
+
|
|
810
|
+
def test_iteration_error_propagates(self, small_ds):
|
|
811
|
+
"""Errors during batch iteration should propagate to the user."""
|
|
812
|
+
error_on_batch = 2 # Fail on the third batch
|
|
813
|
+
|
|
814
|
+
def failing_callback(block):
|
|
815
|
+
# Track which batch we're on using a mutable default
|
|
816
|
+
if not hasattr(failing_callback, "count"):
|
|
817
|
+
failing_callback.count = 0
|
|
818
|
+
failing_callback.count += 1
|
|
819
|
+
|
|
820
|
+
if failing_callback.count == error_on_batch:
|
|
821
|
+
raise RuntimeError("Intentional batch processing error")
|
|
822
|
+
|
|
823
|
+
# Reset the counter
|
|
824
|
+
failing_callback.count = 0
|
|
825
|
+
|
|
826
|
+
table = read_xarray_table(
|
|
827
|
+
small_ds,
|
|
828
|
+
chunks={"time": 25},
|
|
829
|
+
_iteration_callback=failing_callback,
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
ctx = SessionContext()
|
|
833
|
+
ctx.register_table("test_table", table)
|
|
834
|
+
|
|
835
|
+
# The error should surface when we try to collect
|
|
836
|
+
with pytest.raises(Exception):
|
|
837
|
+
ctx.sql("SELECT * FROM test_table").collect()
|
|
838
|
+
|
|
839
|
+
def test_empty_dataset_handled_gracefully(self):
|
|
840
|
+
"""Empty datasets should work without errors."""
|
|
841
|
+
# Create an empty dataset with the right structure
|
|
842
|
+
empty_ds = xr.Dataset(
|
|
843
|
+
{
|
|
844
|
+
"temperature": (
|
|
845
|
+
["time", "lat", "lon"],
|
|
846
|
+
np.array([]).reshape(0, 0, 0),
|
|
847
|
+
)
|
|
848
|
+
},
|
|
849
|
+
coords={
|
|
850
|
+
"time": pd.DatetimeIndex([]),
|
|
851
|
+
"lat": np.array([]),
|
|
852
|
+
"lon": np.array([]),
|
|
853
|
+
},
|
|
854
|
+
)
|
|
855
|
+
|
|
856
|
+
# This should work without crashing
|
|
857
|
+
table = read_xarray_table(empty_ds, chunks={"time": 10})
|
|
858
|
+
|
|
859
|
+
ctx = SessionContext()
|
|
860
|
+
ctx.register_table("test_table", table)
|
|
861
|
+
|
|
862
|
+
result = ctx.sql("SELECT COUNT(*) as cnt FROM test_table").collect()
|
|
863
|
+
count = result[0].to_pandas()["cnt"].iloc[0]
|
|
864
|
+
|
|
865
|
+
assert count == 0, f"Expected 0 rows for empty dataset, got {count}"
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
class TestMultiplePartitions:
|
|
869
|
+
"""Tests for scenarios with multiple queries and table reuse."""
|
|
870
|
+
|
|
871
|
+
def test_fresh_stream_per_query(self, small_ds):
|
|
872
|
+
"""Each query should get a fresh stream from the factory."""
|
|
873
|
+
call_count = {"value": 0}
|
|
874
|
+
original_callback = None
|
|
875
|
+
|
|
876
|
+
def counting_callback(block):
|
|
877
|
+
call_count["value"] += 1
|
|
878
|
+
if original_callback:
|
|
879
|
+
original_callback(block)
|
|
880
|
+
|
|
881
|
+
table = read_xarray_table(
|
|
882
|
+
small_ds,
|
|
883
|
+
chunks={"time": 50}, # 2 blocks per query
|
|
884
|
+
_iteration_callback=counting_callback,
|
|
885
|
+
)
|
|
886
|
+
|
|
887
|
+
ctx = SessionContext()
|
|
888
|
+
ctx.register_table("test_table", table)
|
|
889
|
+
|
|
890
|
+
# First query
|
|
891
|
+
ctx.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
892
|
+
first_query_count = call_count["value"]
|
|
893
|
+
assert (
|
|
894
|
+
first_query_count == 2
|
|
895
|
+
), f"First query: expected 2, got {first_query_count}"
|
|
896
|
+
|
|
897
|
+
# Second query should trigger fresh iteration
|
|
898
|
+
ctx.sql("SELECT AVG(temperature) FROM test_table").collect()
|
|
899
|
+
second_query_count = call_count["value"]
|
|
900
|
+
assert (
|
|
901
|
+
second_query_count == 4
|
|
902
|
+
), f"After second query: expected 4 total, got {second_query_count}"
|
|
903
|
+
|
|
904
|
+
# Third query
|
|
905
|
+
ctx.sql("SELECT MAX(temperature) FROM test_table").collect()
|
|
906
|
+
third_query_count = call_count["value"]
|
|
907
|
+
assert (
|
|
908
|
+
third_query_count == 6
|
|
909
|
+
), f"After third query: expected 6 total, got {third_query_count}"
|
|
910
|
+
|
|
911
|
+
def test_parallel_queries_independent(self, small_ds):
|
|
912
|
+
"""Multiple contexts with the same table should work independently."""
|
|
913
|
+
tracker1 = IterationTracker()
|
|
914
|
+
tracker2 = IterationTracker()
|
|
915
|
+
|
|
916
|
+
table1 = read_xarray_table(
|
|
917
|
+
small_ds,
|
|
918
|
+
chunks={"time": 25},
|
|
919
|
+
_iteration_callback=tracker1,
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
table2 = read_xarray_table(
|
|
923
|
+
small_ds,
|
|
924
|
+
chunks={"time": 50},
|
|
925
|
+
_iteration_callback=tracker2,
|
|
926
|
+
)
|
|
927
|
+
|
|
928
|
+
ctx1 = SessionContext()
|
|
929
|
+
ctx2 = SessionContext()
|
|
930
|
+
|
|
931
|
+
ctx1.register_table("test_table", table1)
|
|
932
|
+
ctx2.register_table("test_table", table2)
|
|
933
|
+
|
|
934
|
+
# Execute queries
|
|
935
|
+
ctx1.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
936
|
+
ctx2.sql("SELECT COUNT(*) FROM test_table").collect()
|
|
937
|
+
|
|
938
|
+
# Each should have its own iteration count
|
|
939
|
+
assert (
|
|
940
|
+
tracker1.iteration_count == 4
|
|
941
|
+
), f"Table1: expected 4 blocks, got {tracker1.iteration_count}"
|
|
942
|
+
assert (
|
|
943
|
+
tracker2.iteration_count == 2
|
|
944
|
+
), f"Table2: expected 2 blocks, got {tracker2.iteration_count}"
|