vastdb 1.1.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

vastdb/_internal.py

@@ -34,7 +34,7 @@ from ibis.expr.operations.logical import (
     Or,
 )
 from ibis.expr.operations.relations import Field
-from ibis.expr.operations.strings import StringContains
+from ibis.expr.operations.strings import StartsWith, StringContains
 from ibis.expr.operations.structs import StructField
 
 import vast_flatbuf.org.apache.arrow.computeir.flatbuf.BinaryLiteral as fb_binary_lit
@@ -103,7 +103,7 @@ from vast_flatbuf.tabular.ListProjectionsResponse import (
 from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list_schemas
 from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
 
-from . import errors
+from . import errors, util
 from .config import BackoffConfig
 
 UINT64_MAX = 18446744073709551615
@@ -168,6 +168,7 @@ class Predicate:
             IsNull: self.build_is_null,
             Not: self.build_is_not_null,
             StringContains: self.build_match_substring,
+            StartsWith: self.build_starts_with,
             Between: self.build_between,
         }
 
@@ -207,6 +208,14 @@ class Predicate:
                     elif builder_func == self.build_between:
                         column, lower, upper = inner_op.args
                         literals = (None,)
+                    elif builder_func == self.build_starts_with:
+                        column, prefix = inner_op.args
+                        literals = (None,)
+                        if prefix.value:
+                            lower_bytes, upper_bytes = util.prefix_to_range(prefix.value)
+                        else:
+                            # `col.starts_with('')` is equivalent to `col IS NOT NULL`
+                            builder_func = self.build_is_not_null
                     else:
                         column, arg = inner_op.args
                         if isinstance(arg, tuple):
@@ -249,6 +258,9 @@ class Predicate:
                         if builder_func == self.build_between:
                             args_offsets.append(self.build_literal(field=node.field, value=lower.value))
                             args_offsets.append(self.build_literal(field=node.field, value=upper.value))
+                        if builder_func == self.build_starts_with:
+                            args_offsets.append(self.build_literal(field=node.field, value=lower_bytes))
+                            args_offsets.append(self.build_literal(field=node.field, value=upper_bytes))
 
                         inner_offsets.append(builder_func(*args_offsets))
 
@@ -550,6 +562,13 @@ class Predicate:
         ]
         return self.build_and(offsets)
 
+    def build_starts_with(self, column: int, lower: int, upper: int):
+        offsets = [
+            self.build_greater_equal(column, lower),
+            self.build_less(column, upper),
+        ]
+        return self.build_and(offsets)
+
 
 class FieldNodesState:
     def __init__(self) -> None:

vastdb/schema.py

@@ -103,9 +103,7 @@ class Schema:
         log.debug("Found table: %s", t[0])
         return t[0]
 
-    def tables(self, table_name=None) -> List["Table"]:
-        """List all tables under this schema."""
-        tables = []
+    def _iter_tables(self, table_name=None):
         next_key = 0
         name_prefix = table_name if table_name else ""
         exact_match = bool(table_name)
@@ -116,11 +114,20 @@ class Schema:
                     exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match)
             if not curr_tables:
                 break
-            tables.extend(curr_tables)
+            yield from curr_tables
             if not is_truncated:
                 break
 
-        return [_parse_table_info(table, self) for table in tables]
+    def tables(self, table_name=None) -> List["Table"]:
+        """List all tables under this schema."""
+        return [
+            _parse_table_info(table_info, self)
+            for table_info in self._iter_tables(table_name=table_name)
+        ]
+
+    def tablenames(self) -> List[str]:
+        """List all table names under this schema."""
+        return [table_info.name for table_info in self._iter_tables()]
 
     def drop(self) -> None:
         """Delete this schema."""

vastdb/tests/test_tables.py

@@ -127,6 +127,21 @@ def test_exists(session, clean_bucket_name):
         assert s.tables() == [t]
 
 
+def test_list_tables(session, clean_bucket_name):
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema('s1')
+        assert s.tables() == []
+        assert s.tablenames() == []
+
+        tables = [
+            s.create_table(f't{i}', pa.schema([(f'x{i}', pa.int64())]))
+            for i in range(10)
+        ]
+        assert tables == s.tables()
+        tablenames = [t.name for t in tables]
+        assert s.tablenames() == tablenames
+
+
 def test_update_table(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int64()),
@@ -822,3 +837,48 @@ def test_catalog_snapshots_select(session, clean_bucket_name):
         rows = t.select().read_all()
         if not rows:
             raise NotReady
+
+
+def test_starts_with(session, clean_bucket_name):
+    columns = pa.schema([
+        ('s', pa.utf8()),
+        ('i', pa.int16()),
+    ])
+
+    expected = pa.table(schema=columns, data=[
+        ['a', 'ab', 'abc', None, 'abd', 'α', '', 'b'],
+        [0, 1, 2, 3, 4, 5, 6, 7],
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
+        def select(prefix):
+            res = table.select(predicate=table['s'].startswith(prefix)).read_all()
+            return res.to_pydict()
+
+        assert select('')['s'] == ['a', 'ab', 'abc', 'abd', 'α', '', 'b']
+        assert select('a')['s'] == ['a', 'ab', 'abc', 'abd']
+        assert select('b')['s'] == ['b']
+        assert select('ab')['s'] == ['ab', 'abc', 'abd']
+        assert select('abc')['s'] == ['abc']
+        assert select('α')['s'] == ['α']
+
+        res = table.select(predicate=(table['s'].startswith('ab') | (table['s'].isnull()))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', None, 'abd']
+
+        res = table.select(predicate=(table['s'].startswith('ab') | (table['s'] == 'b'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', 'abd', 'b']
+
+        res = table.select(predicate=((table['s'] == 'b') | table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', 'abd', 'b']
+
+        res = table.select(predicate=(table['s'].startswith('ab') & (table['s'] != 'abc'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abd']
+
+        res = table.select(predicate=((table['s'] != 'abc') & table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abd']
+
+        res = table.select(predicate=((table['i'] > 3) & table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict() == {'i': [4], 's': ['abd']}
+
+        res = table.select(predicate=(table['s'].startswith('ab')) & (table['i'] > 3)).read_all()
+        assert res.to_pydict() == {'i': [4], 's': ['abd']}

vastdb/tests/test_util.py

@@ -43,3 +43,16 @@ def _parse(bufs):
     for buf in bufs:
         with pa.ipc.open_stream(buf) as reader:
             yield from reader
+
+
+def test_prefix():
+    assert util.prefix_to_range('a') == (b'a', b'b')
+    assert util.prefix_to_range('abc') == (b'abc', b'abd')
+    assert util.prefix_to_range('abc\x00') == (b'abc\x00', b'abc\x01')
+    assert util.prefix_to_range('abc\x7f') == (b'abc\x7f', b'abc\x80')
+    assert util.prefix_to_range('/a/b/c') == (b'/a/b/c', b'/a/b/d')
+    assert util.prefix_to_range('/123α') == (b'/123\xce\xb1', b'/123\xce\xb2')
+    assert util.prefix_to_range('/123αA') == (b'/123\xce\xb1A', b'/123\xce\xb1B')
+    assert util.prefix_to_range('\U0010ffff') == (b'\xf4\x8f\xbf\xbf', b'\xf4\x8f\xbf\xc0')  # max unicode codepoint
+    with pytest.raises(AssertionError):
+        util.prefix_to_range('')

vastdb/util.py

@@ -157,3 +157,13 @@ def sort_record_batch_if_needed(record_batch, sort_column):
         return record_batch.sort_by(sort_column)
     else:
         return record_batch
+
+
+def prefix_to_range(prefix: str):
+    """Compute (L, U) such that `s.starts_with(prefix)` is equivalent to `L <= s.encode() < H`."""
+    assert prefix, "Empty prefix is not convertible to range predicate"
+    lower = prefix.encode()
+    upper = bytearray(lower)
+    # https://en.wikipedia.org/wiki/UTF-8#Encoding guarantees that the last byte is not 0xFF
+    upper[-1] = upper[-1] + 1
+    return (lower, bytes(upper))

{vastdb-1.1.2.dist-info → vastdb-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.1.2
+Version: 1.3.0
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.1.2.dist-info → vastdb-1.3.0.dist-info}/RECORD

@@ -149,17 +149,17 @@ vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=J1JjKiFkKC95BHowfh9kJfQFTjRce-QMsc6zF_FfxC0,432
-vastdb/_internal.py,sha256=KRFNlB25Ckj4nowVUneC5tBSYXpnIv-MDdw4Vn6KYnc,91255
+vastdb/_internal.py,sha256=N_vCPwyPxyvRtN8Gvhjc1_tNmvUsfn-rdz_27-h-gaY,92312
 vastdb/bucket.py,sha256=5KuKhPjZOevznZqWHDVVocejvAy7dcwobPuV6BJCfPc,2544
 vastdb/config.py,sha256=1tMYtzKXerGcIUjH4tIGEvZNWvO4fviCEdcNCnELJZo,2269
 vastdb/conftest.py,sha256=X2kVveySPQYZlVBXUMoo7Oea5IsvmJzjdqq3fpH2kVw,3469
 vastdb/errors.py,sha256=2XR1ko7J5nkfiHSAgwuVAADw0SsyqxOwSeFaGgKZEXM,4186
 vastdb/features.py,sha256=DxV746LSkORwVSD6MP2hdXRfnyoLkJwtOwGmp1dnquo,1322
-vastdb/schema.py,sha256=X7IRrogXH7Z0kes-DsDh1bRqIhvjH6owlFigGBXy7XQ,5913
+vastdb/schema.py,sha256=IaZDJsx0ms_dJVXeyCcSD8Dt3TNJkqR3739XOnDBM_E,6177
 vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
 vastdb/table.py,sha256=C6Zz0zolRRRbf5EQBvDRAofl3kGvfz4PjZwGeQongTI,31106
 vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
-vastdb/util.py,sha256=eunfTuqbCrqQEFZEO9T15N-Bu8Fqpw7Zlqp2TAGfYaY,5870
+vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/test_perf.py,sha256=gZIqfHva6lNFpD-9bHAe7M8COBjUyrPkHu3E7F8J2L0,1072
 vastdb/bench/test_sample.py,sha256=LgF4syzij09sH3Noiv1EyCAJ9pvrUE5bxR4RJTVEYag,7881
@@ -196,14 +196,14 @@ vastdb/tests/test_nested.py,sha256=LPU6uV3Ri23dBzAEMFQqRPbqapV5LfmiHSHkhILPIY0,6
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
 vastdb/tests/test_sanity.py,sha256=oiV2gb05aPyG5RMNUQZlyjNlg3T7Fig1_8OJzpAgcsk,3038
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=URfd0_rh5sleCFGbNsJBdIGdXOvIiiLGyhKTNtswU98,32578
-vastdb/tests/test_util.py,sha256=Ok_sAEBJsRGF5Voa_v5eu3eAd52GWu8jMjjQbadwW-s,1260
+vastdb/tests/test_tables.py,sha256=17-t9VkEJRIW43Yf-lwEI7jHn8teOJvv-eZgANcvTkM,35023
+vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
 vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
 vastdb/vast_tests/test_scale.py,sha256=EpjCJmVAQrNBxVnHGJ-KHCoxevhqOcyqYFPMIIY9s60,2714
-vastdb-1.1.2.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.1.2.dist-info/METADATA,sha256=n2QV0OrTzVR9pQKUmVhsH3c5dIS29Vl5C1h0AyDEJDg,1340
-vastdb-1.1.2.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vastdb-1.1.2.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-1.1.2.dist-info/RECORD,,
+vastdb-1.3.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.3.0.dist-info/METADATA,sha256=Idrpl3by0yNRJPAYlVgcqZEiLPSsA3vvte2KneNonEA,1340
+vastdb-1.3.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.3.0.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-1.3.0.dist-info/RECORD,,