vastdb 1.2.0__py3-none-any.whl → 1.3.1__py3-none-any.whl

vastdb/_internal.py

@@ -34,7 +34,7 @@ from ibis.expr.operations.logical import (
     Or,
 )
 from ibis.expr.operations.relations import Field
-from ibis.expr.operations.strings import StringContains
+from ibis.expr.operations.strings import StartsWith, StringContains
 from ibis.expr.operations.structs import StructField
 
 import vast_flatbuf.org.apache.arrow.computeir.flatbuf.BinaryLiteral as fb_binary_lit
@@ -103,7 +103,7 @@ from vast_flatbuf.tabular.ListProjectionsResponse import (
 from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list_schemas
 from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
 
-from . import errors
+from . import errors, util
 from .config import BackoffConfig
 
 UINT64_MAX = 18446744073709551615
@@ -168,6 +168,7 @@ class Predicate:
             IsNull: self.build_is_null,
             Not: self.build_is_not_null,
             StringContains: self.build_match_substring,
+            StartsWith: self.build_starts_with,
             Between: self.build_between,
         }
 
@@ -207,6 +208,14 @@ class Predicate:
                     elif builder_func == self.build_between:
                         column, lower, upper = inner_op.args
                         literals = (None,)
+                    elif builder_func == self.build_starts_with:
+                        column, prefix = inner_op.args
+                        literals = (None,)
+                        if prefix.value:
+                            lower_bytes, upper_bytes = util.prefix_to_range(prefix.value)
+                        else:
+                            # `col.starts_with('')` is equivalent to `col IS NOT NULL`
+                            builder_func = self.build_is_not_null
                     else:
                         column, arg = inner_op.args
                         if isinstance(arg, tuple):
@@ -249,6 +258,9 @@ class Predicate:
                         if builder_func == self.build_between:
                             args_offsets.append(self.build_literal(field=node.field, value=lower.value))
                             args_offsets.append(self.build_literal(field=node.field, value=upper.value))
+                        if builder_func == self.build_starts_with:
+                            args_offsets.append(self.build_literal(field=node.field, value=lower_bytes))
+                            args_offsets.append(self.build_literal(field=node.field, value=upper_bytes))
 
                         inner_offsets.append(builder_func(*args_offsets))
 
@@ -550,6 +562,13 @@ class Predicate:
         ]
         return self.build_and(offsets)
 
+    def build_starts_with(self, column: int, lower: int, upper: int):
+        offsets = [
+            self.build_greater_equal(column, lower),
+            self.build_less(column, upper),
+        ]
+        return self.build_and(offsets)
+
 
 class FieldNodesState:
     def __init__(self) -> None:
@@ -847,7 +866,7 @@ class VastdbApi:
             res = self._session.request(method=method, url=url, timeout=self.timeout, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # low-level connection issue, it is safe to retry only read-only requests
-            may_retry = (method == "GET")
+            may_retry = (method in {"HEAD", "GET"})
             raise errors.ConnectionError(cause=err, may_retry=may_retry) from err
 
         if not skip_status_check:

vastdb/tests/test_tables.py

@@ -837,3 +837,48 @@ def test_catalog_snapshots_select(session, clean_bucket_name):
         rows = t.select().read_all()
         if not rows:
             raise NotReady
+
+
+def test_starts_with(session, clean_bucket_name):
+    columns = pa.schema([
+        ('s', pa.utf8()),
+        ('i', pa.int16()),
+    ])
+
+    expected = pa.table(schema=columns, data=[
+        ['a', 'ab', 'abc', None, 'abd', 'α', '', 'b'],
+        [0, 1, 2, 3, 4, 5, 6, 7],
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
+        def select(prefix):
+            res = table.select(predicate=table['s'].startswith(prefix)).read_all()
+            return res.to_pydict()
+
+        assert select('')['s'] == ['a', 'ab', 'abc', 'abd', 'α', '', 'b']
+        assert select('a')['s'] == ['a', 'ab', 'abc', 'abd']
+        assert select('b')['s'] == ['b']
+        assert select('ab')['s'] == ['ab', 'abc', 'abd']
+        assert select('abc')['s'] == ['abc']
+        assert select('α')['s'] == ['α']
+
+        res = table.select(predicate=(table['s'].startswith('ab') | (table['s'].isnull()))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', None, 'abd']
+
+        res = table.select(predicate=(table['s'].startswith('ab') | (table['s'] == 'b'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', 'abd', 'b']
+
+        res = table.select(predicate=((table['s'] == 'b') | table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abc', 'abd', 'b']
+
+        res = table.select(predicate=(table['s'].startswith('ab') & (table['s'] != 'abc'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abd']
+
+        res = table.select(predicate=((table['s'] != 'abc') & table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict()['s'] == ['ab', 'abd']
+
+        res = table.select(predicate=((table['i'] > 3) & table['s'].startswith('ab'))).read_all()
+        assert res.to_pydict() == {'i': [4], 's': ['abd']}
+
+        res = table.select(predicate=(table['s'].startswith('ab')) & (table['i'] > 3)).read_all()
+        assert res.to_pydict() == {'i': [4], 's': ['abd']}

vastdb/tests/test_util.py

@@ -43,3 +43,16 @@ def _parse(bufs):
     for buf in bufs:
         with pa.ipc.open_stream(buf) as reader:
             yield from reader
+
+
+def test_prefix():
+    assert util.prefix_to_range('a') == (b'a', b'b')
+    assert util.prefix_to_range('abc') == (b'abc', b'abd')
+    assert util.prefix_to_range('abc\x00') == (b'abc\x00', b'abc\x01')
+    assert util.prefix_to_range('abc\x7f') == (b'abc\x7f', b'abc\x80')
+    assert util.prefix_to_range('/a/b/c') == (b'/a/b/c', b'/a/b/d')
+    assert util.prefix_to_range('/123α') == (b'/123\xce\xb1', b'/123\xce\xb2')
+    assert util.prefix_to_range('/123αA') == (b'/123\xce\xb1A', b'/123\xce\xb1B')
+    assert util.prefix_to_range('\U0010ffff') == (b'\xf4\x8f\xbf\xbf', b'\xf4\x8f\xbf\xc0')  # max unicode codepoint
+    with pytest.raises(AssertionError):
+        util.prefix_to_range('')

vastdb/util.py

@@ -157,3 +157,13 @@ def sort_record_batch_if_needed(record_batch, sort_column):
         return record_batch.sort_by(sort_column)
     else:
         return record_batch
+
+
+def prefix_to_range(prefix: str):
+    """Compute (L, U) such that `s.starts_with(prefix)` is equivalent to `L <= s.encode() < H`."""
+    assert prefix, "Empty prefix is not convertible to range predicate"
+    lower = prefix.encode()
+    upper = bytearray(lower)
+    # https://en.wikipedia.org/wiki/UTF-8#Encoding guarantees that the last byte is not 0xFF
+    upper[-1] = upper[-1] + 1
+    return (lower, bytes(upper))

vastdb/vast_tests/test_scale.py

@@ -66,3 +66,26 @@ def test_table_stats(session, test_bucket_name, schema_name, table_name):
     logger.info("inserted to table")
     assert new_stat.size_in_bytes != initial_stat.size_in_bytes
     assert new_stat.num_rows - NUM_TIMES_TO_INSERT == initial_stat.num_rows
+
+
+def test_ndu_while_querying(session, test_bucket_name, schema_name, table_name):
+    """
+    Executing queries while a NDU takes place.
+    """
+    # TODO: Before merging run mypy and print query result
+
+    config = QueryConfig(num_splits=1, num_sub_splits=1)
+
+    logger.info(f'{test_bucket_name=}, {schema_name=}, {table_name=}')
+
+    for query in range(300):
+        with session.transaction() as tx:
+            t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+            s = time.time()
+            if query == 0:
+                res = t.select(config=config).read_all()
+                logger.info(f'{res=}')
+            else:
+                assert res == t.select(config=config).read_all()
+            e = time.time()
+            logger.info(f'{query=} took {e - s}')

{vastdb-1.2.0.dist-info → vastdb-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.2.0
+Version: 1.3.1
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.2.0.dist-info → vastdb-1.3.1.dist-info}/RECORD

@@ -149,7 +149,7 @@ vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=J1JjKiFkKC95BHowfh9kJfQFTjRce-QMsc6zF_FfxC0,432
-vastdb/_internal.py,sha256=KRFNlB25Ckj4nowVUneC5tBSYXpnIv-MDdw4Vn6KYnc,91255
+vastdb/_internal.py,sha256=LmiiNmUqJFfNGUoe362sHwU8M3aKbN1VWYIvobrgeK8,92322
 vastdb/bucket.py,sha256=5KuKhPjZOevznZqWHDVVocejvAy7dcwobPuV6BJCfPc,2544
 vastdb/config.py,sha256=1tMYtzKXerGcIUjH4tIGEvZNWvO4fviCEdcNCnELJZo,2269
 vastdb/conftest.py,sha256=X2kVveySPQYZlVBXUMoo7Oea5IsvmJzjdqq3fpH2kVw,3469
@@ -159,7 +159,7 @@ vastdb/schema.py,sha256=IaZDJsx0ms_dJVXeyCcSD8Dt3TNJkqR3739XOnDBM_E,6177
 vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
 vastdb/table.py,sha256=C6Zz0zolRRRbf5EQBvDRAofl3kGvfz4PjZwGeQongTI,31106
 vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
-vastdb/util.py,sha256=eunfTuqbCrqQEFZEO9T15N-Bu8Fqpw7Zlqp2TAGfYaY,5870
+vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/test_perf.py,sha256=gZIqfHva6lNFpD-9bHAe7M8COBjUyrPkHu3E7F8J2L0,1072
 vastdb/bench/test_sample.py,sha256=LgF4syzij09sH3Noiv1EyCAJ9pvrUE5bxR4RJTVEYag,7881
@@ -196,14 +196,14 @@ vastdb/tests/test_nested.py,sha256=LPU6uV3Ri23dBzAEMFQqRPbqapV5LfmiHSHkhILPIY0,6
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
 vastdb/tests/test_sanity.py,sha256=oiV2gb05aPyG5RMNUQZlyjNlg3T7Fig1_8OJzpAgcsk,3038
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=H5BK3Zm0Ocug8kZ2cJeGTC7o8YgZgBDSqkOgwW01hAo,33056
-vastdb/tests/test_util.py,sha256=Ok_sAEBJsRGF5Voa_v5eu3eAd52GWu8jMjjQbadwW-s,1260
+vastdb/tests/test_tables.py,sha256=17-t9VkEJRIW43Yf-lwEI7jHn8teOJvv-eZgANcvTkM,35023
+vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
 vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
-vastdb/vast_tests/test_scale.py,sha256=EpjCJmVAQrNBxVnHGJ-KHCoxevhqOcyqYFPMIIY9s60,2714
-vastdb-1.2.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.2.0.dist-info/METADATA,sha256=O4U-OWABQccLaEF7kOdwqoLO55YG3FkZ3SG5IQs6PS4,1340
-vastdb-1.2.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vastdb-1.2.0.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-1.2.0.dist-info/RECORD,,
+vastdb/vast_tests/test_scale.py,sha256=yPF5sL2X7fiP_QooV3OnZAzsW38ANfkHFXq8B1_Uh-s,3515
+vastdb-1.3.1.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.3.1.dist-info/METADATA,sha256=oyVkT5bnbEKiuHPfiYuQKtu0C1N7yfeQTSOnhRWpF8c,1340
+vastdb-1.3.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.3.1.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-1.3.1.dist-info/RECORD,,