vastdb 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

vastdb/tests/test_nested.py

@@ -1,11 +1,15 @@
+import functools
 import itertools
+import operator
 
 import pyarrow as pa
+import pyarrow.compute as pc
+import pytest
 
 from .util import prepare_data
 
 
-def test_nested(session, clean_bucket_name):
+def test_nested_select(session, clean_bucket_name):
     columns = pa.schema([
         ('l', pa.list_(pa.int8())),
         ('m', pa.map_(pa.utf8(), pa.float64())),
@@ -18,11 +22,81 @@ def test_nested(session, clean_bucket_name):
     ])
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
-        actual = pa.Table.from_batches(t.select())
+        actual = t.select().read_all()
         assert actual == expected
 
         names = [f.name for f in columns]
         for n in range(len(names) + 1):
             for cols in itertools.permutations(names, n):
-                actual = pa.Table.from_batches(t.select(columns=cols))
+                actual = t.select(columns=cols).read_all()
                 assert actual == expected.select(cols)
+
+
+def test_nested_filter(session, clean_bucket_name):
+    columns = pa.schema([
+        ('x', pa.int64()),
+        ('l', pa.list_(pa.int8())),
+        ('y', pa.int64()),
+        ('m', pa.map_(pa.utf8(), pa.float64())),
+        ('z', pa.int64()),
+        ('s', pa.struct([('x', pa.int16()), ('y', pa.int32())])),
+        ('w', pa.int64()),
+    ])
+    expected = pa.table(schema=columns, data=[
+        [1, 2, 3, None],
+        [[1], [], [2, 3], None],
+        [1, 2, None, 3],
+        [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
+        [1, None, 2, 3],
+        [{'x': 1, 'y': None}, None, {'x': 2, 'y': 3}, {'x': None, 'y': 4}],
+        [None, 1, 2, 3],
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
+        actual = t.select().read_all()
+        assert actual == expected
+
+        names = list('xyzw')
+        for n in range(1, len(names) + 1):
+            for cols in itertools.permutations(names, n):
+                ibis_predicate = functools.reduce(
+                    operator.and_,
+                    (t[col] > 2 for col in cols))
+                actual = t.select(predicate=ibis_predicate).read_all()
+
+                arrow_predicate = functools.reduce(
+                    operator.and_,
+                    (pc.field(col) > 2 for col in cols))
+                assert actual == expected.filter(arrow_predicate)
+
+
+def test_nested_unsupported_filter(session, clean_bucket_name):
+    columns = pa.schema([
+        ('x', pa.int64()),
+        ('l', pa.list_(pa.int8())),
+        ('y', pa.int64()),
+        ('m', pa.map_(pa.utf8(), pa.float64())),
+        ('z', pa.int64()),
+        ('s', pa.struct([('x', pa.int16()), ('y', pa.int32())])),
+        ('w', pa.int64()),
+    ])
+    expected = pa.table(schema=columns, data=[
+        [1, 2, 3, None],
+        [[1], [], [2, 3], None],
+        [1, 2, None, 3],
+        [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
+        [1, None, 2, 3],
+        [{'x': 1, 'y': None}, None, {'x': 2, 'y': 3}, {'x': None, 'y': 4}],
+        [None, 1, 2, 3],
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
+
+        with pytest.raises(NotImplementedError):
+            list(t.select(predicate=(t['l'].isnull())))
+
+        with pytest.raises(NotImplementedError):
+            list(t.select(predicate=(t['m'].isnull())))
+
+        with pytest.raises(NotImplementedError):
+            list(t.select(predicate=(t['s'].isnull())))

vastdb/tests/test_projections.py

@@ -1,7 +1,10 @@
 import logging
+import time
 
 import pyarrow as pa
 
+from vastdb.table import QueryConfig
+
 log = logging.getLogger(__name__)
 
 
@@ -41,3 +44,78 @@ def test_basic_projections(session, clean_bucket_name):
         projs = t.projections()
         assert len(projs) == 1
         assert projs[0].name == 'p_new'
+
+
+def test_query_data_with_projection(session, clean_bucket_name):
+    columns = pa.schema([
+        ('a', pa.int64()),
+        ('b', pa.int64()),
+        ('s', pa.utf8()),
+    ])
+    # need to be large enough in order to consider as projection
+
+    GROUP_SIZE = 128 * 1024
+    expected = pa.table(schema=columns, data=[
+        [i for i in range(GROUP_SIZE)],
+        [i for i in reversed(range(GROUP_SIZE))],
+        [f's{i}' for i in range(GROUP_SIZE)],
+    ])
+
+    expected_projection_p1 = pa.table(schema=columns, data=[
+        [i for i in reversed(range(GROUP_SIZE - 5, GROUP_SIZE))],
+        [i for i in range(5)],
+        [f's{i}' for i in reversed(range(GROUP_SIZE - 5, GROUP_SIZE))],
+    ])
+
+    expected_projection_p2 = pa.table(schema=columns, data=[
+        [i for i in range(GROUP_SIZE - 5, GROUP_SIZE)],
+        [i for i in reversed(range(5))],
+        [f's{i}' for i in range(GROUP_SIZE - 5, GROUP_SIZE)],
+    ])
+
+    schema_name = "schema"
+    table_name = "table"
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema(schema_name)
+        t = s.create_table(table_name, expected.schema)
+
+        sorted_columns = ['b']
+        unsorted_columns = ['a', 's']
+        t.create_projection('p1', sorted_columns, unsorted_columns)
+
+        sorted_columns = ['a']
+        unsorted_columns = ['b', 's']
+        t.create_projection('p2', sorted_columns, unsorted_columns)
+
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).schema(schema_name)
+        t = s.table(table_name)
+        t.insert(expected)
+        actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's']))
+        assert actual == expected
+
+    time.sleep(3)
+
+    with session.transaction() as tx:
+        config = QueryConfig()
+        # in nfs mock server num row groups per row block is 1 so need to change this in the config
+        config.num_row_groups_per_sub_split = 1
+
+        s = tx.bucket(clean_bucket_name).schema(schema_name)
+        t = s.table(table_name)
+        projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))
+        # no projection supply - need to be with p1 projeciton
+        assert expected_projection_p1 == projection_actual
+
+        config.semi_sorted_projection_name = 'p1'
+        projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))
+        # expecting results of projection p1 since we asked it specificaly
+        assert expected_projection_p1 == projection_actual
+
+        config.semi_sorted_projection_name = 'p2'
+        projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))
+        # expecting results of projection p2 since we asked it specificaly
+        assert expected_projection_p2 == projection_actual
+
+        t.drop()
+        s.drop()

vastdb/tests/test_schemas.py

@@ -61,3 +61,52 @@ def test_list_snapshots(session, clean_bucket_name):
     with session.transaction() as tx:
         b = tx.bucket(clean_bucket_name)
         b.snapshots()  # VAST Catalog may create some snapshots
+
+
+def test_nested_schemas(session, clean_bucket_name):
+    with session.transaction() as tx:
+        b = tx.bucket(clean_bucket_name)
+        s1 = b.create_schema('s1')
+        s1_s2 = s1.create_schema('s2')
+        s1_s3 = s1.create_schema('s3')
+        s1_s3_s4 = s1_s3.create_schema('s4')
+        s5 = b.create_schema('s5')
+
+        assert b.schema('s1') == s1
+        assert s1.schema('s2') == s1_s2
+        assert s1.schema('s3') == s1_s3
+        assert s1_s3.schema('s4') == s1_s3_s4
+        assert b.schema('s5') == s5
+
+        assert b.schemas() == [s1, s5]
+        assert s1.schemas() == [s1_s2, s1_s3]
+        assert s1_s2.schemas() == []
+        assert s1_s3.schemas() == [s1_s3_s4]
+        assert s1_s3_s4.schemas() == []
+        assert s5.schemas() == []
+
+        s1_s3_s4.drop()
+        assert s1_s3.schemas() == []
+        s1_s3.drop()
+        assert s1.schemas() == [s1_s2]
+        s1_s2.drop()
+        assert s1.schemas() == []
+
+        assert b.schemas() == [s1, s5]
+        s1.drop()
+        assert b.schemas() == [s5]
+        s5.drop()
+        assert b.schemas() == []
+
+
+def test_schema_pagination(session, clean_bucket_name):
+    with session.transaction() as tx:
+        b = tx.bucket(clean_bucket_name)
+        names = [f's{i}' for i in range(10)]
+        schemas = [b.create_schema(name) for name in names]
+        assert b.schemas(batch_size=3) == schemas
+
+        s0 = b.schema('s0')
+        names = [f'q{i}' for i in range(10)]
+        subschemas = [s0.create_schema(name) for name in names]
+        assert s0.schemas(batch_size=3) == subschemas

vastdb/tests/test_tables.py

@@ -3,10 +3,10 @@ import decimal
 import logging
 import random
 import threading
-import time
 from contextlib import closing
 from tempfile import NamedTemporaryFile
 
+import ibis
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
@@ -32,25 +32,25 @@ def test_tables(session, clean_bucket_name):
         ['a', 'bb', 'ccc'],
     ])
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's']))
+        actual = t.select(columns=['a', 'b', 's']).read_all()
         assert actual == expected
 
-        actual = pa.Table.from_batches(t.select())
+        actual = t.select().read_all()
         assert actual == expected
 
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b']))
+        actual = t.select(columns=['a', 'b']).read_all()
         assert actual == expected.select(['a', 'b'])
 
-        actual = pa.Table.from_batches(t.select(columns=['b', 's', 'a']))
+        actual = t.select(columns=['b', 's', 'a']).read_all()
         assert actual == expected.select(['b', 's', 'a'])
 
-        actual = pa.Table.from_batches(t.select(columns=['s']))
+        actual = t.select(columns=['s']).read_all()
         assert actual == expected.select(['s'])
 
-        actual = pa.Table.from_batches(t.select(columns=[]))
+        actual = t.select(columns=[]).read_all()
         assert actual == expected.select([])
 
-        actual = pa.Table.from_batches(t.select(columns=['s'], internal_row_id=True))
+        actual = t.select(columns=['s'], internal_row_id=True).read_all()
         log.debug("actual=%s", actual)
         assert actual.to_pydict() == {
             's': ['a', 'bb', 'ccc'],
@@ -61,9 +61,9 @@ def test_tables(session, clean_bucket_name):
         rb = pa.record_batch(schema=columns_to_delete, data=[[0]])  # delete rows 0,1
         t.delete(rb)
 
-        selected_rows = pa.Table.from_batches(t.select(columns=['b'], predicate=(t['a'] == 222), internal_row_id=True))
+        selected_rows = t.select(columns=['b'], predicate=(t['a'] == 222), internal_row_id=True).read_all()
         t.delete(selected_rows)
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's']))
+        actual = t.select(columns=['a', 'b', 's']).read_all()
         assert actual.to_pydict() == {
             'a': [333],
             'b': [2.5],
@@ -77,7 +77,7 @@ def test_insert_wide_row(session, clean_bucket_name):
     expected = pa.table(schema=columns, data=data)
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
-        actual = pa.Table.from_batches(t.select())
+        actual = t.select().read_all()
         assert actual == expected
 
 
@@ -124,33 +124,33 @@ def test_update_table(session, clean_bucket_name):
         ])
 
         t.update(rb)
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b']))
+        actual = t.select(columns=['a', 'b']).read_all()
         assert actual.to_pydict() == {
             'a': [1110, 222, 3330],
             'b': [0.5, 1.5, 2.5]
         }
 
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b'], predicate=(t['a'] < 1000), internal_row_id=True))
+        actual = t.select(columns=['a', 'b'], predicate=(t['a'] < 1000), internal_row_id=True).read_all()
         column_index = actual.column_names.index('a')
         column_field = actual.field(column_index)
         new_data = pc.add(actual.column('a'), 2000)
         update_table = actual.set_column(column_index, column_field, new_data)
 
         t.update(update_table, columns=['a'])
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b']))
+        actual = t.select(columns=['a', 'b']).read_all()
         assert actual.to_pydict() == {
             'a': [1110, 2222, 3330],
             'b': [0.5, 1.5, 2.5]
         }
 
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b'], predicate=(t['a'] != 2222), internal_row_id=True))
+        actual = t.select(columns=['a', 'b'], predicate=(t['a'] != 2222), internal_row_id=True).read_all()
         column_index = actual.column_names.index('a')
         column_field = actual.field(column_index)
         new_data = pc.divide(actual.column('a'), 10)
         update_table = actual.set_column(column_index, column_field, new_data)
 
         t.update(update_table.to_batches()[0], columns=['a'])
-        actual = pa.Table.from_batches(t.select(columns=['a', 'b']))
+        actual = t.select(columns=['a', 'b']).read_all()
         assert actual.to_pydict() == {
             'a': [111, 2222, 333],
             'b': [0.5, 1.5, 2.5]
@@ -170,7 +170,7 @@ def test_select_with_multisplits(session, clean_bucket_name):
     config.rows_per_split = 1000
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
-        actual = pa.Table.from_batches(t.select(columns=['a'], config=config))
+        actual = t.select(columns=['a'], config=config).read_all()
         assert actual == expected
 
 
@@ -215,46 +215,47 @@ def test_types(session, clean_bucket_name):
         [dt.datetime(2024, 4, 10, 12, 34, 56, 789789), dt.datetime(2025, 4, 10, 12, 34, 56, 789789), dt.datetime(2026, 4, 10, 12, 34, 56, 789789)],
     ])
 
-    with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
         def select(predicate):
-            return pa.Table.from_batches(t.select(predicate=predicate))
+            return table.select(predicate=predicate).read_all()
 
         assert select(None) == expected
-        assert select(t['tb'] == False) == expected.filter(pc.field('tb') == False)  # noqa: E712
-        assert select(t['a1'] == 2) == expected.filter(pc.field('a1') == 2)
-        assert select(t['a2'] == 2000) == expected.filter(pc.field('a2') == 2000)
-        assert select(t['a4'] == 222111122) == expected.filter(pc.field('a4') == 222111122)
-        assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
-        assert select(t['s'] == "v") == expected.filter(pc.field('s') == "v")
-        assert select(t['d'] == 231.15) == expected.filter(pc.field('d') == 231.15)
-        assert select(t['bin'] == b"\x01\x02") == expected.filter(pc.field('bin') == b"\x01\x02")
+        for t in [table, ibis._]:
+            assert select(t['tb'] == False) == expected.filter(pc.field('tb') == False)  # noqa: E712
+            assert select(t['a1'] == 2) == expected.filter(pc.field('a1') == 2)
+            assert select(t['a2'] == 2000) == expected.filter(pc.field('a2') == 2000)
+            assert select(t['a4'] == 222111122) == expected.filter(pc.field('a4') == 222111122)
+            assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
+            assert select(t['s'] == "v") == expected.filter(pc.field('s') == "v")
+            assert select(t['d'] == 231.15) == expected.filter(pc.field('d') == 231.15)
+            assert select(t['bin'] == b"\x01\x02") == expected.filter(pc.field('bin') == b"\x01\x02")
 
-        date_literal = dt.date(2024, 4, 10)
-        assert select(t['date'] == date_literal) == expected.filter(pc.field('date') == date_literal)
+            date_literal = dt.date(2024, 4, 10)
+            assert select(t['date'] == date_literal) == expected.filter(pc.field('date') == date_literal)
 
-        time_literal = dt.time(12, 34, 56)
-        assert select(t['t0'] == time_literal) == expected.filter(pc.field('t0') == time_literal)
+            time_literal = dt.time(12, 34, 56)
+            assert select(t['t0'] == time_literal) == expected.filter(pc.field('t0') == time_literal)
 
-        time_literal = dt.time(12, 34, 56, 789000)
-        assert select(t['t3'] == time_literal) == expected.filter(pc.field('t3') == time_literal)
+            time_literal = dt.time(12, 34, 56, 789000)
+            assert select(t['t3'] == time_literal) == expected.filter(pc.field('t3') == time_literal)
 
-        time_literal = dt.time(12, 34, 56, 789789)
-        assert select(t['t6'] == time_literal) == expected.filter(pc.field('t6') == time_literal)
+            time_literal = dt.time(12, 34, 56, 789789)
+            assert select(t['t6'] == time_literal) == expected.filter(pc.field('t6') == time_literal)
 
-        time_literal = dt.time(12, 34, 56, 789789)
-        assert select(t['t9'] == time_literal) == expected.filter(pc.field('t9') == time_literal)
+            time_literal = dt.time(12, 34, 56, 789789)
+            assert select(t['t9'] == time_literal) == expected.filter(pc.field('t9') == time_literal)
 
-        ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56)
-        assert select(t['ts0'] == ts_literal) == expected.filter(pc.field('ts0') == ts_literal)
+            ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56)
+            assert select(t['ts0'] == ts_literal) == expected.filter(pc.field('ts0') == ts_literal)
 
-        ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789000)
-        assert select(t['ts3'] == ts_literal) == expected.filter(pc.field('ts3') == ts_literal)
+            ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789000)
+            assert select(t['ts3'] == ts_literal) == expected.filter(pc.field('ts3') == ts_literal)
 
-        ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789789)
-        assert select(t['ts6'] == ts_literal) == expected.filter(pc.field('ts6') == ts_literal)
+            ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789789)
+            assert select(t['ts6'] == ts_literal) == expected.filter(pc.field('ts6') == ts_literal)
 
-        ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789789)
-        assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
+            ts_literal = dt.datetime(2024, 4, 10, 12, 34, 56, 789789)
+            assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
 def test_filters(session, clean_bucket_name):
@@ -270,62 +271,70 @@ def test_filters(session, clean_bucket_name):
         ['a', 'bb', 'ccc', None, 'xyz'],
     ])
 
-    with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
         def select(predicate):
-            return pa.Table.from_batches(t.select(predicate=predicate), t.arrow_schema)
+            return table.select(predicate=predicate).read_all()
 
         assert select(None) == expected
         assert select(True) == expected
         assert select(False) == pa.Table.from_batches([], schema=columns)
 
-        assert select(t['a'].between(222, 444)) == expected.filter((pc.field('a') >= 222) & (pc.field('a') <= 444))
-        assert select((t['a'].between(222, 444)) & (t['b'] > 2.5)) == expected.filter((pc.field('a') >= 222) & (pc.field('a') <= 444) & (pc.field('b') > 2.5))
+        for t in [table, ibis._]:
+
+            select(t['a'].isin(list(range(100))))
+            select(t['a'].isin(list(range(1000))))
+            select(t['a'].isin(list(range(10000))))
+            with pytest.raises(errors.TooLargeRequest):
+                select(t['a'].isin(list(range(100000))))
 
-        assert select(t['a'] > 222) == expected.filter(pc.field('a') > 222)
-        assert select(t['a'] < 222) == expected.filter(pc.field('a') < 222)
-        assert select(t['a'] == 222) == expected.filter(pc.field('a') == 222)
-        assert select(t['a'] != 222) == expected.filter(pc.field('a') != 222)
-        assert select(t['a'] <= 222) == expected.filter(pc.field('a') <= 222)
-        assert select(t['a'] >= 222) == expected.filter(pc.field('a') >= 222)
+            assert select(t['a'].between(222, 444)) == expected.filter((pc.field('a') >= 222) & (pc.field('a') <= 444))
+            assert select((t['a'].between(222, 444)) & (t['b'] > 2.5)) == expected.filter((pc.field('a') >= 222) & (pc.field('a') <= 444) & (pc.field('b') > 2.5))
 
-        assert select(t['b'] > 1.5) == expected.filter(pc.field('b') > 1.5)
-        assert select(t['b'] < 1.5) == expected.filter(pc.field('b') < 1.5)
-        assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
-        assert select(t['b'] != 1.5) == expected.filter(pc.field('b') != 1.5)
-        assert select(t['b'] <= 1.5) == expected.filter(pc.field('b') <= 1.5)
-        assert select(t['b'] >= 1.5) == expected.filter(pc.field('b') >= 1.5)
+            assert select(t['a'] > 222) == expected.filter(pc.field('a') > 222)
+            assert select(t['a'] < 222) == expected.filter(pc.field('a') < 222)
+            assert select(t['a'] == 222) == expected.filter(pc.field('a') == 222)
+            assert select(t['a'] != 222) == expected.filter(pc.field('a') != 222)
+            assert select(t['a'] <= 222) == expected.filter(pc.field('a') <= 222)
+            assert select(t['a'] >= 222) == expected.filter(pc.field('a') >= 222)
 
-        assert select(t['s'] > 'bb') == expected.filter(pc.field('s') > 'bb')
-        assert select(t['s'] < 'bb') == expected.filter(pc.field('s') < 'bb')
-        assert select(t['s'] == 'bb') == expected.filter(pc.field('s') == 'bb')
-        assert select(t['s'] != 'bb') == expected.filter(pc.field('s') != 'bb')
-        assert select(t['s'] <= 'bb') == expected.filter(pc.field('s') <= 'bb')
-        assert select(t['s'] >= 'bb') == expected.filter(pc.field('s') >= 'bb')
+            assert select(t['b'] > 1.5) == expected.filter(pc.field('b') > 1.5)
+            assert select(t['b'] < 1.5) == expected.filter(pc.field('b') < 1.5)
+            assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
+            assert select(t['b'] != 1.5) == expected.filter(pc.field('b') != 1.5)
+            assert select(t['b'] <= 1.5) == expected.filter(pc.field('b') <= 1.5)
+            assert select(t['b'] >= 1.5) == expected.filter(pc.field('b') >= 1.5)
 
-        assert select((t['a'] > 111) & (t['b'] > 0) & (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) & (pc.field('b') > 0) & (pc.field('s') < 'ccc'))
-        assert select((t['a'] > 111) & (t['b'] < 2.5)) == expected.filter((pc.field('a') > 111) & (pc.field('b') < 2.5))
-        assert select((t['a'] > 111) & (t['a'] < 333)) == expected.filter((pc.field('a') > 111) & (pc.field('a') < 333))
+            assert select(t['s'] > 'bb') == expected.filter(pc.field('s') > 'bb')
+            assert select(t['s'] < 'bb') == expected.filter(pc.field('s') < 'bb')
+            assert select(t['s'] == 'bb') == expected.filter(pc.field('s') == 'bb')
+            assert select(t['s'] != 'bb') == expected.filter(pc.field('s') != 'bb')
+            assert select(t['s'] <= 'bb') == expected.filter(pc.field('s') <= 'bb')
+            assert select(t['s'] >= 'bb') == expected.filter(pc.field('s') >= 'bb')
 
-        assert select((t['a'] > 111) | (t['a'] < 333)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333))
-        assert select(((t['a'] > 111) | (t['a'] < 333)) & (t['b'] < 2.5)) == expected.filter(((pc.field('a') > 111) | (pc.field('a') < 333)) & (pc.field('b') < 2.5))
-        with pytest.raises(NotImplementedError):
-            assert select((t['a'] > 111) | (t['b'] > 0) | (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) | (pc.field('b') > 0) | (pc.field('s') < 'ccc'))
-        assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
+            assert select((t['a'] > 111) & (t['b'] > 0) & (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) & (pc.field('b') > 0) & (pc.field('s') < 'ccc'))
+            assert select((t['a'] > 111) & (t['b'] < 2.5)) == expected.filter((pc.field('a') > 111) & (pc.field('b') < 2.5))
+            assert select((t['a'] > 111) & (t['a'] < 333)) == expected.filter((pc.field('a') > 111) & (pc.field('a') < 333))
 
-        assert select(t['s'].isnull()) == expected.filter(pc.field('s').is_null())
-        assert select((t['s'].isnull()) | (t['s'] == 'bb'))  == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
-        assert select((t['s'].isnull()) & (t['b'] == 3.5))  == expected.filter((pc.field('s').is_null()) & (pc.field('b') == 3.5))
+            assert select((t['a'] > 111) | (t['a'] < 333)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333))
+            assert select(((t['a'] > 111) | (t['a'] < 333)) & (t['b'] < 2.5)) == expected.filter(((pc.field('a') > 111) | (pc.field('a') < 333)) & (pc.field('b') < 2.5))
+            with pytest.raises(NotImplementedError):
+                assert select((t['a'] > 111) | (t['b'] > 0) | (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) | (pc.field('b') > 0) | (pc.field('s') < 'ccc'))
+            assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
 
-        assert select(~t['s'].isnull()) == expected.filter(~pc.field('s').is_null())
-        assert select(t['s'].contains('b')) == expected.filter(pc.field('s') == 'bb')
-        assert select(t['s'].contains('y')) == expected.filter(pc.field('s') == 'xyz')
+            assert select(t['s'].isnull()) == expected.filter(pc.field('s').is_null())
+            assert select((t['s'].isnull()) | (t['s'] == 'bb'))  == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
+            assert select((t['s'].isnull()) & (t['b'] == 3.5))  == expected.filter((pc.field('s').is_null()) & (pc.field('b') == 3.5))
 
-        assert select(t['a'].isin([555])) == expected.filter(pc.field('a').isin([555]))
-        assert select(t['a'].isin([111, 222, 999])) == expected.filter(pc.field('a').isin([111, 222, 999]))
-        assert select((t['a'] == 111) | t['a'].isin([333, 444]) | (t['a'] > 600)) == expected.filter((pc.field('a') == 111) | pc.field('a').isin([333, 444]) | (pc.field('a') > 600))
+            assert select(~t['s'].isnull()) == expected.filter(~pc.field('s').is_null())
+            assert select(t['s'].contains('b')) == expected.filter(pc.field('s') == 'bb')
+            assert select(t['s'].contains('y')) == expected.filter(pc.field('s') == 'xyz')
 
-        with pytest.raises(NotImplementedError):
-            select(t['a'].isin([]))
+            assert select(t['a'].isin([555])) == expected.filter(pc.field('a').isin([555]))
+            assert select(t['a'].isin([111, 222, 999])) == expected.filter(pc.field('a').isin([111, 222, 999]))
+            assert select((t['a'] == 111) | t['a'].isin([333, 444]) | (t['a'] > 600)) == expected.filter((pc.field('a') == 111) | pc.field('a').isin([333, 444]) | (pc.field('a') > 600))
+
+            with pytest.raises(NotImplementedError):
+                select(t['a'].isin([]))
 
 
 def test_parquet_export(session, clean_bucket_name):
@@ -348,7 +357,7 @@ def test_parquet_export(session, clean_bucket_name):
         expected = pa.Table.from_batches([rb])
         rb = t.insert(rb)
         assert rb.to_pylist() == [0, 1]
-        actual = pa.Table.from_batches(t.select())
+        actual = t.select().read_all()
         assert actual == expected
 
         table_batches = t.select()
@@ -664,18 +673,37 @@ def test_select_stop(session, clean_bucket_name):
     assert active_threads() == 0
 
 
-def test_big_catalog_select(session, clean_bucket_name):
+def test_catalog_select(session, clean_bucket_name):
     with session.transaction() as tx:
         bc = tx.catalog()
-        actual = pa.Table.from_batches(bc.select(['name']))
-        assert actual
-        log.info("actual=%s", actual)
+        assert bc.columns()
+        rows = bc.select(['name']).read_all()
+        assert len(rows) > 0, rows
+
 
+class NotReady(Exception):
+    pass
 
+
+@pytest.mark.flaky(retries=30, delay=1, only_on=[NotReady])
 def test_audit_log_select(session, clean_bucket_name):
     with session.transaction() as tx:
         a = tx.audit_log()
-        a.columns()
-        time.sleep(1)
-        actual = pa.Table.from_batches(a.select(), a.arrow_schema)
-        log.info("actual=%s", actual)
+        assert a.columns()
+        rows = a.select().read_all()
+        if len(rows) == 0:
+            raise NotReady
+
+
+@pytest.mark.flaky(retries=30, delay=1, only_on=[NotReady])
+def test_catalog_snapshots_select(session, clean_bucket_name):
+    with session.transaction() as tx:
+        snaps = tx.catalog_snapshots()
+        if not snaps:
+            raise NotReady
+        latest = snaps[-1]
+        t = tx.catalog(latest)
+        assert t.columns()
+        rows = t.select().read_all()
+        if not rows:
+            raise NotReady