vastdb 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

vast_flatbuf/tabular/CreateViewRequest.py

@@ -0,0 +1,104 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: tabular
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+np = import_numpy()
+
+class CreateViewRequest(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = CreateViewRequest()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsCreateViewRequest(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+    # CreateViewRequest
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # CreateViewRequest
+    def ViewMetadataArrowBuffer(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
+        return 0
+
+    # CreateViewRequest
+    def ViewMetadataArrowBufferAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
+    # CreateViewRequest
+    def ViewMetadataArrowBufferLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # CreateViewRequest
+    def ViewMetadataArrowBufferIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        return o == 0
+
+    # CreateViewRequest
+    def ViewDataArrowSchema(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
+        return 0
+
+    # CreateViewRequest
+    def ViewDataArrowSchemaAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
+    # CreateViewRequest
+    def ViewDataArrowSchemaLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # CreateViewRequest
+    def ViewDataArrowSchemaIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        return o == 0
+
+def Start(builder): builder.StartObject(2)
+def CreateViewRequestStart(builder):
+    """This method is deprecated. Please switch to Start."""
+    return Start(builder)
+def AddViewMetadataArrowBuffer(builder, viewMetadataArrowBuffer): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(viewMetadataArrowBuffer), 0)
+def CreateViewRequestAddViewMetadataArrowBuffer(builder, viewMetadataArrowBuffer):
+    """This method is deprecated. Please switch to AddViewMetadataArrowBuffer."""
+    return AddViewMetadataArrowBuffer(builder, viewMetadataArrowBuffer)
+def StartViewMetadataArrowBufferVector(builder, numElems): return builder.StartVector(1, numElems, 1)
+def CreateViewRequestStartViewMetadataArrowBufferVector(builder, numElems):
+    """This method is deprecated. Please switch to Start."""
+    return StartViewMetadataArrowBufferVector(builder, numElems)
+def AddViewDataArrowSchema(builder, viewDataArrowSchema): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(viewDataArrowSchema), 0)
+def CreateViewRequestAddViewDataArrowSchema(builder, viewDataArrowSchema):
+    """This method is deprecated. Please switch to AddViewDataArrowSchema."""
+    return AddViewDataArrowSchema(builder, viewDataArrowSchema)
+def StartViewDataArrowSchemaVector(builder, numElems): return builder.StartVector(1, numElems, 1)
+def CreateViewRequestStartViewDataArrowSchemaVector(builder, numElems):
+    """This method is deprecated. Please switch to Start."""
+    return StartViewDataArrowSchemaVector(builder, numElems)
+def End(builder): return builder.EndObject()
+def CreateViewRequestEnd(builder):
+    """This method is deprecated. Please switch to End."""
+    return End(builder)

vast_flatbuf/tabular/ListViewsResponse.py

@@ -0,0 +1,89 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: tabular
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+np = import_numpy()
+
+class ListViewsResponse(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = ListViewsResponse()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsListViewsResponse(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+    # ListViewsResponse
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # ListViewsResponse
+    def BucketName(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ListViewsResponse
+    def SchemaName(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ListViewsResponse
+    def Views(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Vector(o)
+            x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
+            x = self._tab.Indirect(x)
+            from vast_flatbuf.tabular.ObjectDetails import ObjectDetails
+            obj = ObjectDetails()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # ListViewsResponse
+    def ViewsLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # ListViewsResponse
+    def ViewsIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        return o == 0
+
+def Start(builder): builder.StartObject(3)
+def ListViewsResponseStart(builder):
+    """This method is deprecated. Please switch to Start."""
+    return Start(builder)
+def AddBucketName(builder, bucketName): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(bucketName), 0)
+def ListViewsResponseAddBucketName(builder, bucketName):
+    """This method is deprecated. Please switch to AddBucketName."""
+    return AddBucketName(builder, bucketName)
+def AddSchemaName(builder, schemaName): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(schemaName), 0)
+def ListViewsResponseAddSchemaName(builder, schemaName):
+    """This method is deprecated. Please switch to AddSchemaName."""
+    return AddSchemaName(builder, schemaName)
+def AddViews(builder, views): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(views), 0)
+def ListViewsResponseAddViews(builder, views):
+    """This method is deprecated. Please switch to AddViews."""
+    return AddViews(builder, views)
+def StartViewsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def ListViewsResponseStartViewsVector(builder, numElems):
+    """This method is deprecated. Please switch to Start."""
+    return StartViewsVector(builder, numElems)
+def End(builder): return builder.EndObject()
+def ListViewsResponseEnd(builder):
+    """This method is deprecated. Please switch to End."""
+    return End(builder)

vast_flatbuf/tabular/ObjectDetails.py

@@ -60,7 +60,21 @@ class ObjectDetails(object):
             return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos)
         return 0
 
-def Start(builder): builder.StartObject(5)
+    # ObjectDetails
+    def NumPartitions(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos)
+        return 0
+
+    # ObjectDetails
+    def SortingKeyEnabled(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
+        if o != 0:
+            return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos))
+        return False
+
+def Start(builder): builder.StartObject(7)
 def ObjectDetailsStart(builder):
     """This method is deprecated. Please switch to Start."""
     return Start(builder)
@@ -84,6 +98,14 @@ def AddSizeInBytes(builder, sizeInBytes): builder.PrependUint64Slot(4, sizeInByt
 def ObjectDetailsAddSizeInBytes(builder, sizeInBytes):
     """This method is deprecated. Please switch to AddSizeInBytes."""
     return AddSizeInBytes(builder, sizeInBytes)
+def AddNumPartitions(builder, numPartitions): builder.PrependUint64Slot(5, numPartitions, 0)
+def ObjectDetailsAddNumPartitions(builder, numPartitions):
+    """This method is deprecated. Please switch to AddNumPartitions."""
+    return AddNumPartitions(builder, numPartitions)
+def AddSortingKeyEnabled(builder, sortingKeyEnabled): builder.PrependBoolSlot(6, sortingKeyEnabled, 0)
+def ObjectDetailsAddSortingKeyEnabled(builder, sortingKeyEnabled):
+    """This method is deprecated. Please switch to AddSortingKeyEnabled."""
+    return AddSortingKeyEnabled(builder, sortingKeyEnabled)
 def End(builder): return builder.EndObject()
 def ObjectDetailsEnd(builder):
     """This method is deprecated. Please switch to End."""

vastdb/_internal.py

@@ -735,17 +735,57 @@ def _iter_nested_arrays(column: pa.Array) -> Iterator[pa.Array]:
         yield from _iter_nested_arrays(column.values)  # Note: Map is serialized in VAST as a List<Struct<K, V>>
 
 
-TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes')
+class ValidateInList:
+    def __init__(self, *args):
+        self.candidates = [x.strip() for x in args]
 
+    def __call__(self, x):
+        x = x.strip()
+        if x not in self.candidates:
+            raise Exception(f'{x} is not in {self.candidates}')
+        return x
+
+    def __getitem__(self, x):
+        return self
+
+
+_int_coding = (lambda x: str(int(x)), lambda x: int(x))
+_prop_coding = {
+        "message.timestamp.type": ValidateInList('CreateTime', 'LogAppendTime'),
+        "retention.ms": _int_coding,
+        "message.timestamp.after.max.ms": _int_coding,
+        "message.timestamp.before.max.ms": _int_coding
+        }
+
+
+def _encode_table_props(**kwargs):
+    if all([v is None for v in kwargs.values()]):
+        return None
+    else:
+        pairs = [(k.replace("_", ".").strip(), v) for k, v in kwargs.items() if v is not None]
+        return "$".join([f"{k}={_prop_coding[k][0](v)}" for k, v in pairs])
+
+
+def _decode_table_props(s):
+    if s.strip() == '':
+        return {}
+    triplets = [(x.strip(), x.strip().replace(".", "_"), y.strip()) for x, y in [z.split('=') for z in s.strip().split("$")]]
+    return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
-def _parse_table_info(obj):
 
+TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions')
+
+
+def _parse_table_info(obj):
     name = obj.Name().decode()
     properties = obj.Properties().decode()
     handle = obj.Handle().decode()
     num_rows = obj.NumRows()
     used_bytes = obj.SizeInBytes()
-    return TableInfo(name, properties, handle, num_rows, used_bytes)
+    num_partitions = obj.NumPartitions()
+    if num_partitions != 0:
+        properties = _decode_table_props(properties)
+    return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions)
 
 
 # Results that returns from tablestats
@@ -774,7 +814,13 @@ def _backoff_giveup(exc: Exception) -> bool:
     return True  # give up in case of other exceptions
 
 
+class UnsupportedServer(NotImplementedError):
+    """Raised when the response comes back from non-VAST DB server."""
+    pass
+
+
 class VastdbApi:
+    VAST_SERVER_PREFIX = 'vast'
     # we expect the vast version to be <major>.<minor>.<patch>.<protocol>
     VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
 
@@ -829,6 +875,9 @@ class VastdbApi:
         if server_header is None:
             _logger.error("Response doesn't contain 'Server' header")
         else:
+            if not server_header.startswith(self.VAST_SERVER_PREFIX):
+                raise UnsupportedServer(f'{self.url} is not a VAST DB server endpoint ("{server_header}")')
+
             if m := self.VAST_VERSION_REGEX.match(server_header):
                 self.vast_version: Tuple[int, ...] = tuple(int(v) for v in m.group(1).split("."))
                 return
@@ -1049,8 +1098,9 @@ class VastdbApi:
 
         return snapshots, is_truncated, marker
 
-    def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[],
-                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False):
+    def create_table(self, bucket, schema, name, arrow_schema=None, txid=0, client_tags=[], expected_retvals=[],
+                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False,
+                     message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
         """
         Create a table, use the following request
         POST /bucket/schema/table?table HTTP/1.1
@@ -1069,6 +1119,9 @@ class VastdbApi:
         """
         headers = self._fill_common_headers(txid=txid, client_tags=client_tags)
 
+        if arrow_schema is None:
+            arrow_schema = pa.schema([])
+
         serialized_schema = arrow_schema.serialize()
         headers['Content-Length'] = str(len(serialized_schema))
         if use_external_row_ids_allocation:
@@ -1078,6 +1131,11 @@ class VastdbApi:
         if create_imports_table:
             url_params['sub-table'] = IMPORTED_OBJECTS_TABLE_NAME
 
+        if topic_partitions > 0:
+            table_props = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+            if table_props is not None:
+                url_params['table-props'] = table_props
+
         self._request(
             method="POST",
             url=self._url(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
@@ -1106,7 +1164,8 @@ class VastdbApi:
         return TableStatsResult(num_rows, size_in_bytes, is_external_rowid_alloc, tuple(endpoints))
 
     def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
-                    new_name="", expected_retvals=[]):
+                    new_name="", expected_retvals=[],
+                    message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
         """
         PUT /mybucket/myschema/mytable?table HTTP/1.1
         Content-Length: ContentLength
@@ -1118,6 +1177,11 @@ class VastdbApi:
         """
         builder = flatbuffers.Builder(1024)
 
+        if message_timestamp_type is not None or retention_ms is not None or message_timestamp_after_max_ms is not None or message_timestamp_before_max_ms is not None:
+            table_properties = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+            if table_properties is None:
+                table_properties = ""
+
         properties = builder.CreateString(table_properties)
         tabular_alter_table.Start(builder)
         if len(table_properties):

vastdb/schema.py

@@ -77,7 +77,11 @@ class Schema:
         return result
 
     def create_table(self, table_name: str, columns: pa.Schema, fail_if_exists=True, use_external_row_ids_allocation=False) -> "Table":
-        """Create a new table under this schema."""
+        """Create a new table under this schema.
+
+        A virtual `vastdb_rowid` column (of `int64` type) can be created to access and filter by internal VAST row IDs.
+        See https://support.vastdata.com/s/article/UUID-48d0a8cf-5786-5ef3-3fa3-9c64e63a0967 for more details.
+        """
         if current := self.table(table_name, fail_if_missing=False):
             if fail_if_exists:
                 raise errors.TableExists(self.bucket.name, self.name, table_name)
@@ -142,5 +146,4 @@ class Schema:
 
 
 def _parse_table_info(table_info, schema: "schema.Schema"):
-    stats = table.TableStats(num_rows=table_info.num_rows, size_in_bytes=table_info.size_in_bytes)
-    return table.Table(name=table_info.name, schema=schema, handle=int(table_info.handle), stats=stats, _imports_table=False)
+    return table.Table(name=table_info.name, schema=schema, handle=int(table_info.handle), _imports_table=False)

vastdb/table.py

@@ -112,7 +112,6 @@ class Table:
     name: str
     schema: "schema.Schema"
     handle: int
-    stats: TableStats
     arrow_schema: pa.Schema = field(init=False, compare=False, repr=False)
     _ibis_table: ibis.Schema = field(init=False, compare=False, repr=False)
     _imports_table: bool
@@ -139,6 +138,11 @@ class Table:
         """Return bucket."""
         return self.schema.bucket
 
+    @property
+    def stats(self):
+        """Fetch table's statistics from server."""
+        return self.get_stats()
+
     def columns(self) -> pa.Schema:
         """Return columns' metadata."""
         fields = []
@@ -576,7 +580,7 @@ class Table:
     def imports_table(self) -> Optional["Table"]:
         """Get the imports table of this table."""
         self.tx._rpc.features.check_imports_table()
-        return Table(name=self.name, schema=self.schema, handle=int(self.handle), stats=self.stats, _imports_table=True)
+        return Table(name=self.name, schema=self.schema, handle=int(self.handle), _imports_table=True)
 
     def __getitem__(self, col_name):
         """Allow constructing ibis-like column expressions from this table.

vastdb/tests/test_sanity.py

@@ -7,6 +7,7 @@ from itertools import cycle
 import pytest
 
 import vastdb.errors
+from vastdb._internal import UnsupportedServer
 
 log = logging.getLogger(__name__)
 
@@ -32,13 +33,14 @@ def test_bad_endpoint(session):
 def test_version_extraction():
     # A list of version and expected version parsed by API
     TEST_CASES = [
-            (None, None),                                   # vast server without version in header
-            ("5", None),                                    # major
-            ("5.2", None),                                  # major.minor
-            ("5.2.0", None),                                # major.minor.patch
-            ("5.2.0.10", (5, 2, 0, 10)),                    # major.minor.patch.protocol
-            ("5.2.0.10 some other things", None),           # suffix
-            ("5.2.0.10.20", None),                          # extra version
+            ("nginx", UnsupportedServer),                               # non-vast server
+            ("vast", NotImplementedError),                              # vast server without version in header
+            ("vast 5", NotImplementedError),                            # major
+            ("vast 5.2", NotImplementedError),                          # major.minor
+            ("vast 5.2.0", NotImplementedError),                        # major.minor.patch
+            ("vast 5.2.0.10", (5, 2, 0, 10)),                           # major.minor.patch.protocol
+            ("vast 5.2.0.10 some other things", NotImplementedError),   # suffix
+            ("vast 5.2.0.10.20", NotImplementedError),                  # extra version
     ]
 
     # Mock OPTIONS handle that cycles through the test cases response
@@ -53,8 +55,7 @@ def test_version_extraction():
             self.end_headers()
 
         def version_string(self):
-            version = next(self.versions_iterator)[0]
-            return f"vast {version}" if version else "vast"
+            return next(self.versions_iterator)[0]
 
         def log_message(self, format, *args):
             log.debug(format, *args)
@@ -74,7 +75,10 @@ def test_version_extraction():
 
     try:
         for _, expected in TEST_CASES:
-            with (pytest.raises(NotImplementedError) if expected is None else contextlib.nullcontext()):
+            manager = contextlib.nullcontext()
+            if isinstance(expected, type) and issubclass(expected, NotImplementedError):
+                manager = pytest.raises(expected)
+            with manager:
                 s = vastdb.connect(endpoint=f"http://localhost:{httpd.server_port}", access="abc", secret="abc")
                 assert s.api.vast_version == expected
     finally:

vastdb/tests/test_tables.py

@@ -452,8 +452,8 @@ def test_filters(session, clean_bucket_name):
             assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
 
             assert select(t['s'].isnull()) == expected.filter(pc.field('s').is_null())
-            assert select((t['s'].isnull()) | (t['s'] == 'bb'))  == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
-            assert select((t['s'].isnull()) & (t['b'] == 3.5))  == expected.filter((pc.field('s').is_null()) & (pc.field('b') == 3.5))
+            assert select((t['s'].isnull()) | (t['s'] == 'bb')) == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
+            assert select((t['s'].isnull()) & (t['b'] == 3.5)) == expected.filter((pc.field('s').is_null()) & (pc.field('b') == 3.5))
 
             assert select(~t['s'].isnull()) == expected.filter(~pc.field('s').is_null())
             assert select(t['s'].contains('b')) == expected.filter(pc.field('s') == 'bb')
@@ -882,3 +882,84 @@ def test_starts_with(session, clean_bucket_name):
 
         res = table.select(predicate=(table['s'].startswith('ab')) & (table['i'] > 3)).read_all()
         assert res.to_pydict() == {'i': [4], 's': ['abd']}
+
+
+def test_external_row_id(session, clean_bucket_name):
+    columns = [
+        ('vastdb_rowid', pa.int64()),
+        ('x', pa.float32()),
+        ('y', pa.utf8()),
+    ]
+
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema('s')
+
+        t = s.create_table('t1', pa.schema(columns))
+        assert not t.stats.is_external_rowid_alloc
+        t.insert(pa.record_batch(schema=pa.schema(columns), data=[[0], [1.5], ['ABC']]))
+        assert t.stats.is_external_rowid_alloc
+
+        t = s.create_table('t2', pa.schema(columns))
+        assert not t.stats.is_external_rowid_alloc
+        t.insert(pa.record_batch(schema=pa.schema(columns[1:]), data=[[1.5], ['ABC']]))
+        assert not t.stats.is_external_rowid_alloc
+
+
+def test_multiple_contains_clauses(session, clean_bucket_name):
+    columns = pa.schema([
+        ('theint', pa.int32()),
+        ('thestring', pa.string()),
+        ('theotherstring', pa.string()),
+    ])
+
+    expected = pa.table(schema=columns, data=[
+        [111, 222, 333, 444, 555],
+        ['abc', 'efg', 'hij', 'klm', 'nop'],
+        ['abcd', 'bcde', 'cdef', 'defg', 'efgh'],
+    ])
+    with (prepare_data(session, clean_bucket_name, 's', 't', expected) as t):
+        failed_preds = [
+            lambda t: (t["thestring"].contains("b") | t["theotherstring"].contains("a")),
+            lambda t: (~t["thestring"].contains("a")),
+        ]
+
+        assert t.select(predicate=t["thestring"].contains("b")).read_all() == expected.filter(pc.match_substring(expected["thestring"], "b"))
+        assert (t.select(predicate=(t["thestring"].contains("b")) & (t["theotherstring"].startswith("a"))).read_all() ==
+                expected.filter(pc.and_(
+                    pc.match_substring(expected["thestring"], "b"),
+                    pc.starts_with(expected["thestring"], "a")
+                    )))
+        assert (t.select(predicate=(t["thestring"].contains("b") & (t["thestring"].contains("y")))).read_all() ==
+        t.select(predicate=t["thestring"].contains("y") & (t["thestring"].contains("b"))).read_all())
+
+        assert (t.select(predicate=(t["thestring"].contains("o") & (t["theint"] > 500))).read_all() ==
+        pc.filter(expected, pc.and_(
+            pc.match_substring(expected["thestring"], "o"),
+            pc.greater(expected["theint"], 500)
+        )))
+        assert (t.select(predicate=((t["thestring"].contains("bc")) | (t["thestring"].contains("kl")) |
+                                   (t["thestring"] == "hi") | (t["thestring"].startswith("e")))).read_all() ==
+                expected.filter(
+                    pc.or_(pc.or_(pc.match_substring(expected["thestring"], "bc"),
+                                  pc.match_substring(expected["thestring"], "kl")),
+                            pc.or_(pc.equal(expected["thestring"], "hi"),
+                                   pc.starts_with(expected["thestring"], "e"))
+                )))
+        assert (t.select(predicate=((t["thestring"].contains("abc")) | (t["thestring"].contains("xyz")) |
+                                    (t["thestring"].startswith("z")) | (t["thestring"].isnull()))).read_all() ==
+                pc.filter(expected,
+                    pc.or_(pc.or_(pc.match_substring(expected["thestring"], "abc"),
+                                  pc.match_substring(expected["thestring"], "xyz")),
+                           pc.or_(pc.starts_with(expected["thestring"], "z"),
+                                  pc.is_null(expected["thestring"]))
+                    )))
+        assert (t.select(predicate=((t["thestring"].contains("k")) & (t["theotherstring"].contains("h")) &
+                                    (t["theint"] > 500))).read_all() ==
+                         pc.filter(expected, pc.and_(
+                             pc.and_(pc.match_substring(expected["thestring"], "k"),
+                             pc.match_substring(expected["theotherstring"], "h")),
+                             pc.greater(expected["theint"], 500)
+                         )))
+        for pred in failed_preds:
+            with pytest.raises(NotImplementedError):
+                t.select(predicate=pred(t)).read_all()

vastdb/vast_tests/test_scale.py

@@ -72,20 +72,31 @@ def test_ndu_while_querying(session, test_bucket_name, schema_name, table_name):
     """
     Executing queries while a NDU takes place.
     """
-    # TODO: Before merging run mypy and print query result
-
+    amount_of_queries_in_parallel = 5
     config = QueryConfig(num_splits=1, num_sub_splits=1)
-
     logger.info(f'{test_bucket_name=}, {schema_name=}, {table_name=}')
 
-    for query in range(300):
+    seed = random.randint(0, 100)
+    logger.info(f'{seed=}')
+    r = random.Random(seed)
+
+    def _execute_single_query(expected=None):
+        time.sleep(r.random())
         with session.transaction() as tx:
             t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
             s = time.time()
-            if query == 0:
-                res = t.select(config=config).read_all()
-                logger.info(f'{res=}')
-            else:
-                assert res == t.select(config=config).read_all()
+            res = t.select(config=config).read_all()
             e = time.time()
-            logger.info(f'{query=} took {e - s}')
+            if expected:
+                assert expected == res
+            logger.info(f'query took {e - s}')
+            return res
+
+    initial_res = _execute_single_query()
+    for i in range(300):
+        with ThreadPoolExecutor() as executor:
+            futures = [executor.submit(_execute_single_query, initial_res) for _ in range(amount_of_queries_in_parallel)]
+            for future in futures:
+                future.result()
+
+        logger.info(f"finished running {amount_of_queries_in_parallel} queries, loop {i}")

{vastdb-1.3.1.dist-info → vastdb-1.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.1
+Version: 1.3.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.3.1.dist-info → vastdb-1.3.3.dist-info}/RECORD

@@ -138,26 +138,28 @@ vast_flatbuf/tabular/Column.py,sha256=2iXTlN8pynb_Sf7n15k8n2uXP_8eoz2RqKrUbTBrIy
 vast_flatbuf/tabular/ColumnType.py,sha256=_4-jMG08VR2zdn1ZH7F4aahYPxWsBSm7adUoVf-HFSU,151
 vast_flatbuf/tabular/CreateProjectionRequest.py,sha256=POlK1DrYMAldNJscLIRL3j4jAT0Sv_fRzfvBXwZAAMw,2516
 vast_flatbuf/tabular/CreateSchemaRequest.py,sha256=MrOfWaFu0Q1-mxLlGV8YMPajZ5kASyvowVSrKU-NPx8,1626
+vast_flatbuf/tabular/CreateViewRequest.py,sha256=zagEVFrcWZLDY2exRHxr_uf-Yirchp7uYLk5b6Zvah8,4497
 vast_flatbuf/tabular/GetProjectionTableStatsResponse.py,sha256=Bp-ln-0lcZEiUvp3vWYmnCP6t2UsZ5J-lezgkUUWhzo,3474
 vast_flatbuf/tabular/GetTableStatsResponse.py,sha256=_UsKj6-VAvyDZ8Eku9fegQlRKV-T_0Dsb7qjulYoZus,4655
 vast_flatbuf/tabular/ImportDataRequest.py,sha256=f1chKp5d5NUxfNjI8YI1o4MYInF8UDhIhpWkT3vG4Do,2450
 vast_flatbuf/tabular/ListProjectionsResponse.py,sha256=secqrBsJY3ydbA28j09rmxzBqj-c1JNqaP7JMuib7nE,4240
 vast_flatbuf/tabular/ListSchemasResponse.py,sha256=V8tbwcWAC96eNwuoqDNqCSb02BnMdq60TpyISuWTVMk,3036
 vast_flatbuf/tabular/ListTablesResponse.py,sha256=V7jZAS8ryKY8s6o_QyjWzgan-rsGm17zjKEmi7K6qTM,3550
-vast_flatbuf/tabular/ObjectDetails.py,sha256=qW0WtbkCYYE_L-Kw6VNRDCLYaRm5lKvTbLNkfD4zV4A,3589
+vast_flatbuf/tabular/ListViewsResponse.py,sha256=tn7mfW9afjWxMyBs3tsBcS_JA7uTAVNkFyaM7tctyns,3521
+vast_flatbuf/tabular/ObjectDetails.py,sha256=yS16tYAu8BCYe7h_B4w7C9BDRVVxFMFg8_mkCyU7nOs,4697
 vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI0,4450
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=J1JjKiFkKC95BHowfh9kJfQFTjRce-QMsc6zF_FfxC0,432
-vastdb/_internal.py,sha256=LmiiNmUqJFfNGUoe362sHwU8M3aKbN1VWYIvobrgeK8,92322
+vastdb/_internal.py,sha256=nLa2G6FwjPiirjXo4HBc3Cs6P2XH8ub1yBsy1Zxu0R8,95257
 vastdb/bucket.py,sha256=5KuKhPjZOevznZqWHDVVocejvAy7dcwobPuV6BJCfPc,2544
 vastdb/config.py,sha256=1tMYtzKXerGcIUjH4tIGEvZNWvO4fviCEdcNCnELJZo,2269
 vastdb/conftest.py,sha256=X2kVveySPQYZlVBXUMoo7Oea5IsvmJzjdqq3fpH2kVw,3469
 vastdb/errors.py,sha256=2XR1ko7J5nkfiHSAgwuVAADw0SsyqxOwSeFaGgKZEXM,4186
 vastdb/features.py,sha256=DxV746LSkORwVSD6MP2hdXRfnyoLkJwtOwGmp1dnquo,1322
-vastdb/schema.py,sha256=IaZDJsx0ms_dJVXeyCcSD8Dt3TNJkqR3739XOnDBM_E,6177
+vastdb/schema.py,sha256=N9GLEoSFPrbpreJbCwcWGCtknoNQkavqra8UQvCzy3E,6306
 vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
-vastdb/table.py,sha256=C6Zz0zolRRRbf5EQBvDRAofl3kGvfz4PjZwGeQongTI,31106
+vastdb/table.py,sha256=PHtirFNL8j9PlXAgJp_rcLU37FgnWj0dPaFPFYWm28o,31186
 vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
 vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -194,16 +196,16 @@ vastdb/tests/test_duckdb.py,sha256=STw_1PwTQR8Naz6s0p6lQTV1ZTKKhe3LPBUbhqzTCu0,1
 vastdb/tests/test_imports.py,sha256=xKub3-bisFjH0BsZM8COfiUWuMrtoOoQKprF6VQT9RI,5669
 vastdb/tests/test_nested.py,sha256=LPU6uV3Ri23dBzAEMFQqRPbqapV5LfmiHSHkhILPIY0,6332
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
-vastdb/tests/test_sanity.py,sha256=oiV2gb05aPyG5RMNUQZlyjNlg3T7Fig1_8OJzpAgcsk,3038
+vastdb/tests/test_sanity.py,sha256=bv1ypGDzvOgmMvGbucDYiLQu8krQLlE6NB3M__q87x8,3303
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=17-t9VkEJRIW43Yf-lwEI7jHn8teOJvv-eZgANcvTkM,35023
+vastdb/tests/test_tables.py,sha256=711oTcdw431zs5LAyDxvTaJqF86xYwVCo5iZxSpH58o,39184
 vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
 vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
-vastdb/vast_tests/test_scale.py,sha256=yPF5sL2X7fiP_QooV3OnZAzsW38ANfkHFXq8B1_Uh-s,3515
-vastdb-1.3.1.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.3.1.dist-info/METADATA,sha256=oyVkT5bnbEKiuHPfiYuQKtu0C1N7yfeQTSOnhRWpF8c,1340
-vastdb-1.3.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vastdb-1.3.1.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-1.3.1.dist-info/RECORD,,
+vastdb/vast_tests/test_scale.py,sha256=5jGwOdZH6Tv5tPdZYPWoqcxOceI2jA5i2D1zNKZHER4,3958
+vastdb-1.3.3.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.3.3.dist-info/METADATA,sha256=xqBDm4Puvz348pzEDM7b_IIs-nZDSha7Sgg0RHHh94w,1340
+vastdb-1.3.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.3.3.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-1.3.3.dist-info/RECORD,,