vastdb 1.3.10__py3-none-any.whl → 1.4.0__py3-none-any.whl

vastdb/_internal.py

@@ -69,6 +69,7 @@ import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.Date as fb_date
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.Decimal as fb_decimal
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.Field as fb_field
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.FixedSizeBinary as fb_fixed_size_binary
+import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.FixedSizeList as fb_fixed_size_list
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.FloatingPoint as fb_floating_point
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.Int as fb_int
 import vastdb.vast_flatbuf.org.apache.arrow.flatbuf.List as fb_list
@@ -497,7 +498,13 @@ class Predicate:
             fb_bool.Start(self.builder)
             field_type = fb_bool.End(self.builder)
 
-            value = True if value == 'true' else False  # not cover all cases
+            # Handle both boolean values and string representations
+            if isinstance(value, bool):
+                value = value
+            elif isinstance(value, str):
+                value = value.lower() == 'true'
+            else:
+                value = bool(value)
         elif isinstance(field.type, pa.Decimal128Type):
             literal_type = fb_decimal_lit
             literal_impl = LiteralImpl.DecimalLiteral
@@ -608,7 +615,7 @@ class FieldNode:
         self.debug = debug
         if isinstance(self.type, pa.StructType):
             self.children = [FieldNode(field, index_iter, parent=self) for field in self.type]
-        elif isinstance(self.type, pa.ListType):
+        elif pa.types.is_list(self.type) or pa.types.is_fixed_size_list(self.type):
             self.children = [FieldNode(self.type.value_field, index_iter, parent=self)]
         elif isinstance(self.type, pa.MapType):
             # Map is represented as List<Struct<K, V>> in Arrow
@@ -752,7 +759,7 @@ def _iter_nested_arrays(column: pa.Array) -> Iterator[pa.Array]:
         if not column.type.num_fields == 1:  # Note: VAST serializes only a single struct field at a time
             raise ValueError(f'column.type.num_fields: {column.type.num_fields} not eq to 1')
         yield from _iter_nested_arrays(column.field(0))
-    elif isinstance(column.type, pa.ListType):
+    elif pa.types.is_list(column.type) or pa.types.is_fixed_size_list(column.type):
         yield from _iter_nested_arrays(column.values)  # Note: Map is serialized in VAST as a List<Struct<K, V>>
 
 
@@ -853,10 +860,11 @@ class VastdbApi:
     VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
 
     def __init__(self, endpoint, access_key, secret_key,
-            *,
-            ssl_verify=True,
-            timeout=None,
-            backoff_config: Optional[BackoffConfig] = None):
+                 *,
+                 ssl_verify=True,
+                 timeout=None,
+                 backoff_config: Optional[BackoffConfig] = None,
+                 version_check=True):
 
         from . import version  # import lazily here (to avoid circular dependencies)
         self.client_sdk_version = f"VAST Database Python SDK {version()} - 2024 (c)"
@@ -896,29 +904,30 @@ class VastdbApi:
                                             aws_region='',
                                             aws_service='s3')
 
-        # probe the cluster for its version
-        res = self._request(method="GET", url=self._url(command="transaction"), skip_status_check=True)  # used only for the response headers
-        _logger.debug("headers=%s code=%s content=%s", res.headers, res.status_code, res.content)
-        server_header = res.headers.get("Server")
-        if server_header is None:
-            _logger.error("Response doesn't contain 'Server' header")
-        else:
-            if not server_header.startswith(self.VAST_SERVER_PREFIX):
-                raise UnsupportedServer(f'{self.url} is not a VAST DB server endpoint ("{server_header}")')
-
-            if m := self.VAST_VERSION_REGEX.match(server_header):
-                self.vast_version: Tuple[int, ...] = tuple(int(v) for v in m.group(1).split("."))
-                return
+        if version_check:
+            # probe the cluster for its version
+            res = self._request(method="GET", url=self._url(command="transaction"), skip_status_check=True)  # used only for the response headers
+            _logger.debug("headers=%s code=%s content=%s", res.headers, res.status_code, res.content)
+            server_header = res.headers.get("Server")
+            if server_header is None:
+                _logger.error("Response doesn't contain 'Server' header")
             else:
-                _logger.error("'Server' header '%s' doesn't match the expected pattern", server_header)
+                if not server_header.startswith(self.VAST_SERVER_PREFIX):
+                    raise UnsupportedServer(f'{self.url} is not a VAST DB server endpoint ("{server_header}")')
 
-        msg = (
-            f'Please use `vastdb` <= 0.0.5.x with current VAST cluster version ("{server_header or "N/A"}"). '
-            'To use the latest SDK, please upgrade your cluster to the latest service pack. '
-            'Please contact customer.support@vastdata.com for more details.'
-        )
-        _logger.critical(msg)
-        raise NotImplementedError(msg)
+                if m := self.VAST_VERSION_REGEX.match(server_header):
+                    self.vast_version: Tuple[int, ...] = tuple(int(v) for v in m.group(1).split("."))
+                    return
+                else:
+                    _logger.error("'Server' header '%s' doesn't match the expected pattern", server_header)
+
+            msg = (
+                f'Please use `vastdb` <= 0.0.5.x with current VAST cluster version ("{server_header or "N/A"}"). '
+                'To use the latest SDK, please upgrade your cluster to the latest service pack. '
+                'Please contact customer.support@vastdata.com for more details.'
+            )
+            _logger.critical(msg)
+            raise NotImplementedError(msg)
 
     def __enter__(self):
         """Allow using this session as a context manager."""
@@ -935,7 +944,8 @@ class VastdbApi:
             secret_key=self.secret_key,
             ssl_verify=self._session.verify,
             timeout=self.timeout,
-            backoff_config=self.backoff_config)
+            backoff_config=self.backoff_config,
+            version_check=False)
 
     def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
         _logger.debug("Sending request: %s %s %s timeout=%s", method, url, kwargs, self.timeout)
@@ -1349,12 +1359,12 @@ class VastdbApi:
         lists = list_tables.GetRootAs(res.content)
         tables_length = lists.TablesLength()
         count = int(res_headers['tabular-list-count']) if 'tabular-list-count' in res_headers else tables_length
-        return lists, is_truncated, count
+        return lists, next_key, is_truncated, count
 
     def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
                               exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
         tables = []
-        lists, is_truncated, count = self._list_tables_raw(bucket, schema, txid=txid, client_tags=client_tags, max_keys=max_keys,
+        lists, next_key, is_truncated, count = self._list_tables_raw(bucket, schema, txid=txid, client_tags=client_tags, max_keys=max_keys,
                                  next_key=next_key, name_prefix=name_prefix, exact_match=exact_match, expected_retvals=expected_retvals,
                                  include_list_stats=include_list_stats, count_only=count_only)
         bucket_name = lists.BucketName().decode()
@@ -1368,7 +1378,7 @@ class VastdbApi:
         return bucket_name, schema_name, tables, next_key, is_truncated, count
 
     def raw_sorting_score(self, bucket, schema, txid, name):
-        lists, _, _ = self._list_tables_raw(bucket, schema, txid=txid, exact_match=True, name_prefix=name, include_list_stats=True)
+        lists, _, _, _ = self._list_tables_raw(bucket, schema, txid=txid, exact_match=True, name_prefix=name, include_list_stats=True)
         bucket_name = lists.BucketName().decode()
         if not bucket.startswith(bucket_name):  # ignore snapshot name
             raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
@@ -2267,11 +2277,17 @@ def get_field_type(builder: flatbuffers.Builder, field: pa.Field):
         fb_struct.Start(builder)
         field_type = fb_struct.End(builder)
 
-    elif isinstance(field.type, pa.ListType):
+    elif pa.types.is_list(field.type):
         field_type_type = Type.List
         fb_list.Start(builder)
         field_type = fb_list.End(builder)
 
+    elif pa.types.is_fixed_size_list(field.type):
+        field_type_type = Type.FixedSizeList
+        fb_fixed_size_list.Start(builder)
+        fb_fixed_size_list.AddListSize(builder, field.type.list_size)
+        field_type = fb_fixed_size_list.End(builder)
+
     elif isinstance(field.type, pa.MapType):
         field_type_type = Type.Map
         fb_map.Start(builder)
@@ -2293,7 +2309,7 @@ def build_field(builder: flatbuffers.Builder, f: pa.Field, name: str):
     children = None
     if isinstance(f.type, pa.StructType):
         children = [build_field(builder, child, child.name) for child in list(f.type)]
-    if isinstance(f.type, pa.ListType):
+    if pa.types.is_list(f.type) or pa.types.is_fixed_size_list(f.type):
         children = [build_field(builder, f.type.value_field, "item")]
     if isinstance(f.type, pa.MapType):
         children = [

vastdb/conftest.py

@@ -6,6 +6,7 @@ import boto3
 import pytest
 
 import vastdb
+import vastdb.errors
 
 
 def pytest_addoption(parser):
@@ -65,8 +66,14 @@ def clean_bucket_name(request, test_bucket_name, session):
         b = tx.bucket(test_bucket_name)
         for top_schema in b.schemas():
             for s in iter_schemas(top_schema):
-                for t in s.tables():
-                    t.drop()
+                for t_name in s.tablenames():
+                    try:
+                        t = s.table(t_name)
+                        t.drop()
+                    except vastdb.errors.NotSupportedSchema:
+                        # Use internal API to drop the table in case unsupported schema prevents creating a table
+                        # object.
+                        tx._rpc.api.drop_table(b.name, s.name, t_name, txid=tx.txid)
                 s.drop()
     return test_bucket_name
 

vastdb/errors.py

@@ -2,7 +2,9 @@ import logging
 import xml.etree.ElementTree
 from dataclasses import dataclass
 from enum import Enum
+from typing import Optional
 
+import pyarrow as pa
 import requests
 
 
@@ -89,6 +91,9 @@ class ImportFilesError(Exception):
     message: str
     error_dict: dict
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 class InvalidArgument(Exception):
     pass
@@ -122,18 +127,27 @@ class NotSupported(Exception):
 class MissingBucket(Missing):
     bucket: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class MissingSnapshot(Missing):
     bucket: str
     snapshot: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class MissingSchema(Missing):
     bucket: str
     schema: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class MissingTable(Missing):
@@ -141,6 +155,9 @@ class MissingTable(Missing):
     schema: str
     table: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class MissingProjection(Missing):
@@ -149,6 +166,9 @@ class MissingProjection(Missing):
     table: str
     projection: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 class Exists(Exception):
     pass
@@ -159,6 +179,9 @@ class SchemaExists(Exists):
     bucket: str
     schema: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class TableExists(Exists):
@@ -166,6 +189,9 @@ class TableExists(Exists):
     schema: str
     table: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class NotSupportedCommand(NotSupported):
@@ -173,18 +199,37 @@ class NotSupportedCommand(NotSupported):
     schema: str
     table: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class NotSupportedVersion(NotSupported):
     err_msg: str
     version: str
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
+
+@dataclass
+class NotSupportedSchema(NotSupported):
+    message: Optional[str] = None
+    schema: Optional[pa.Schema] = None
+    cause: Optional[Exception] = None
+
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 @dataclass
 class ConnectionError(Exception):
     cause: Exception
     may_retry: bool
 
+    def __post_init__(self):
+        self.args = [vars(self)]
+
 
 def handle_unavailable(**kwargs):
     if kwargs['code'] == 'SlowDown':
@@ -192,7 +237,7 @@ def handle_unavailable(**kwargs):
     raise ServiceUnavailable(**kwargs)
 
 
-ERROR_TYPES_MAP = {
+HTTP_ERROR_TYPES_MAP = {
     HttpStatus.BAD_REQUEST: BadRequest,
     HttpStatus.FOBIDDEN: Forbidden,
     HttpStatus.NOT_FOUND: NotFound,
@@ -205,6 +250,10 @@ ERROR_TYPES_MAP = {
     HttpStatus.INSUFFICIENT_CAPACITY: InsufficientCapacity,
 }
 
+SPECIFIC_ERROR_TYPES_MAP = {
+    'TabularUnsupportedColumnType': NotSupportedSchema,
+}
+
 
 def from_response(res: requests.Response):
     if res.status_code == HttpStatus.SUCCESS.value:
@@ -234,5 +283,10 @@ def from_response(res: requests.Response):
     )
     log.warning("RPC failed: %s", kwargs)
     status = HttpStatus(res.status_code)
-    error_type = ERROR_TYPES_MAP.get(status, UnexpectedError)
-    return error_type(**kwargs)  # type: ignore
+    http_error_type = HTTP_ERROR_TYPES_MAP.get(status, UnexpectedError)
+    http_error = http_error_type(**kwargs)  # type: ignore
+    # Wrap specific error types if applicable
+    if code_str in SPECIFIC_ERROR_TYPES_MAP:
+        error_type = SPECIFIC_ERROR_TYPES_MAP[code_str]
+        return error_type(message=message_str, cause=http_error)
+    return http_error

vastdb/schema.py

@@ -91,6 +91,7 @@ class Schema:
         if use_external_row_ids_allocation:
             self.tx._rpc.features.check_external_row_ids_allocation()
 
+        table.Table.validate_ibis_support_schema(columns)
         self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
                                       use_external_row_ids_allocation=use_external_row_ids_allocation,
                                       sorting_key=sorting_key)
@@ -109,14 +110,14 @@ class Schema:
         log.debug("Found table: %s", t[0])
         return t[0]
 
-    def _iter_tables(self, table_name=None):
+    def _iter_tables(self, table_name=None, page_size=1000):
         next_key = 0
         name_prefix = table_name if table_name else ""
         exact_match = bool(table_name)
         while True:
             _bucket_name, _schema_name, curr_tables, next_key, is_truncated, _ = \
                 self.tx._rpc.api.list_tables(
-                    bucket=self.bucket.name, schema=self.name, next_key=next_key, txid=self.tx.txid,
+                    bucket=self.bucket.name, schema=self.name, next_key=next_key, max_keys=page_size, txid=self.tx.txid,
                     exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match)
             if not curr_tables:
                 break
@@ -124,19 +125,19 @@ class Schema:
             if not is_truncated:
                 break
 
-    def tables(self, table_name: str = "") -> List["Table"]:
+    def tables(self, table_name: str = "", page_size=1000) -> List["Table"]:
         """List all tables under this schema if `table_name` is empty.
 
         Otherwise, list only the specific table (if exists).
         """
         return [
             _parse_table_info(table_info, self)
-            for table_info in self._iter_tables(table_name=table_name)
+            for table_info in self._iter_tables(table_name=table_name, page_size=page_size)
         ]
 
-    def tablenames(self) -> List[str]:
+    def tablenames(self, page_size=1000) -> List[str]:
         """List all table names under this schema."""
-        return [table_info.name for table_info in self._iter_tables()]
+        return [table_info.name for table_info in self._iter_tables(page_size=page_size)]
 
     def drop(self) -> None:
         """Delete this schema."""

vastdb/table.py

@@ -126,11 +126,35 @@ class Table:
     _imports_table: bool
     sorted_table: bool
 
+    @staticmethod
+    def validate_ibis_support_schema(arrow_schema: pa.Schema):
+        """Validate that the provided Arrow schema is compatible with Ibis.
+
+        Raises NotSupportedSchema if the schema contains unsupported fields.
+        """
+        unsupported_fields = []
+        first_exception = None
+        for f in arrow_schema:
+            try:
+                ibis.Schema.from_pyarrow(pa.schema([f]))
+            except Exception as e:
+                if first_exception is None:
+                    first_exception = e
+                unsupported_fields.append(f)
+
+        if unsupported_fields:
+            raise errors.NotSupportedSchema(
+                message=f"Ibis does not support the schema {unsupported_fields=}",
+                schema=arrow_schema,
+                cause=first_exception
+            )
+
     def __post_init__(self):
         """Also, load columns' metadata."""
         self.arrow_schema = self.columns()
 
         self._table_path = f'{self.schema.bucket.name}/{self.schema.name}/{self.name}'
+        self.validate_ibis_support_schema(self.arrow_schema)
         self._ibis_table = ibis.table(ibis.Schema.from_pyarrow(self.arrow_schema), self._table_path)
 
     @property
@@ -350,12 +374,8 @@ class Table:
         if limit_rows:
             config.limit_rows_per_sub_split = limit_rows
 
-        stats = None
-        # Retrieve snapshots only if needed
         if config.data_endpoints is None:
-            stats = self.get_stats()
-            log.debug("stats: %s", stats)
-            endpoints = stats.endpoints
+            endpoints = tuple([self.tx._rpc.api.url])
         else:
             endpoints = tuple(config.data_endpoints)
         log.debug("endpoints: %s", endpoints)
@@ -385,8 +405,7 @@ class Table:
                 num_rows = self._get_row_estimate(columns, predicate, query_schema)
                 log.debug(f'sorted estimate: {num_rows}')
             if num_rows == 0:
-                if stats is None:
-                    stats = self.get_stats()
+                stats = self.get_stats()
                 num_rows = stats.num_rows
 
             config.num_splits = max(1, num_rows // config.rows_per_split)
@@ -465,7 +484,7 @@ class Table:
 
             total_num_rows = limit_rows if limit_rows else sys.maxsize
             with concurrent.futures.ThreadPoolExecutor(max_workers=len(endpoints), thread_name_prefix=threads_prefix) as tp:  # TODO: concurrency == enpoints is just a heuristic
-                futures = [tp.submit(single_endpoint_worker, endpoint) for endpoint in endpoints]
+                futures = [tp.submit(single_endpoint_worker, endpoint) for endpoint in endpoints[:config.num_splits]]
                 tasks_running = len(futures)
                 try:
                     while tasks_running > 0:
@@ -513,16 +532,24 @@ class Table:
             columns_name_chunk = columns_names[start:end]
             columns_chunks = columns[start:end]
             arrays_chunks = arrays[start:end]
-            columns_chunks.append(INTERNAL_ROW_ID_FIELD)
+            columns_chunks.append(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD)
             arrays_chunks.append(row_ids.to_pylist())
             column_record_batch = pa.RecordBatch.from_arrays(arrays_chunks, schema=pa.schema(columns_chunks))
             self.update(rows=column_record_batch, columns=columns_name_chunk)
         return row_ids
 
-    def insert(self, rows: Union[pa.RecordBatch, pa.Table]):
+    def insert(self, rows: Union[pa.RecordBatch, pa.Table], by_columns: bool = False):
         """Insert a RecordBatch into this table."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
+        if 0 == rows.num_rows:
+            log.debug("Ignoring empty insert into %s", self.name)
+            return pa.chunked_array([], type=(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD).type)
+
+        if by_columns:
+            self.tx._rpc.features.check_return_row_ids()
+            return self.insert_in_column_batches(rows)
+
         try:
             row_ids = []
             serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
@@ -535,7 +562,7 @@ class Table:
                 self.tx._rpc.features.check_return_row_ids()
             except errors.NotSupportedVersion:
                 return  # type: ignore
-            return pa.chunked_array(row_ids)
+            return pa.chunked_array(row_ids, type=(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD).type)
         except errors.TooWideRow:
             self.tx._rpc.features.check_return_row_ids()
             return self.insert_in_column_batches(rows)
@@ -619,6 +646,7 @@ class Table:
         """Add a new column."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
+        self.validate_ibis_support_schema(new_column)
         self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
         log.info("Added column(s): %s", new_column)
         self.arrow_schema = self.columns()

vastdb/tests/test_fixed_list.py

@@ -0,0 +1,294 @@
+import datetime
+import decimal
+import itertools
+import random
+from typing import Any, Union, cast
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+import pytest
+
+import vastdb.errors
+
+from .util import prepare_data
+
+supported_fixed_list_element_types = [
+    pa.uint8(),
+    pa.uint16(),
+    pa.uint32(),
+    pa.uint64(),
+    pa.int8(),
+    pa.int16(),
+    pa.int32(),
+    pa.int64(),
+    pa.float32(),
+    pa.float64(),
+    pa.decimal128(10),
+    pa.date32(),
+    pa.timestamp("s"),
+    pa.time32("ms"),
+    pa.time64("us"),
+]
+
+# All the supported element types are supported as non-nullable.
+supported_fixed_list_element_fields = [
+    pa.field(name="item", type=element_type, nullable=False)
+    for element_type in supported_fixed_list_element_types
+]
+
+unsupported_fixed_list_element_types = [
+    pa.string(),
+    pa.list_(pa.int64()),
+    pa.list_(pa.int64(), 1),
+    pa.map_(pa.utf8(), pa.float64()),
+    pa.struct([("x", pa.int16())]),
+    pa.bool_(),
+    pa.binary(),
+]
+
+unsupported_fixed_list_element_fields = [  # Nullable types are not supported.
+                                            pa.field(name="item", type=element_type, nullable=True)
+                                            for element_type in itertools.chain(
+        supported_fixed_list_element_types, unsupported_fixed_list_element_types
+    )
+                                        ] + [  # Not nullable unsupported type are unsupported.
+                                            pa.field(name="item", type=element_type, nullable=False)
+                                            for element_type in unsupported_fixed_list_element_types
+                                        ]
+
+unsupported_fixed_list_types = (
+        [
+            pa.list_(element_field, 1)
+            for element_field in unsupported_fixed_list_element_fields
+        ] +
+        # Fixed list with amount of elements exceeding the supported limit.
+        [pa.list_(
+            pa.field("item", pa.int64(), nullable=False), np.iinfo(np.int32).max
+        )]
+)
+
+invalid_fixed_list_types = [
+    # Fixed list 0 elements.
+    pa.list_(pa.field("item", pa.int64(), nullable=False), 0),
+]
+
+
+def test_vectors(session, clean_bucket_name):
+    """
+    Test table with efficient vector type - pa.FixedSizeListArray[not nullable numeric].
+    """
+    dimension = 100
+    element_type = pa.float32()
+    num_rows = 50
+
+    columns = pa.schema(
+        [("id", pa.int64()), ("vec", pa.list_(pa.field(name="item", type=element_type, nullable=False), dimension),)]
+    )
+    ids = range(num_rows)
+    expected = pa.table(
+        schema=columns,
+        data=[
+            ids,
+            [[i] * dimension for i in ids],
+        ],
+    )
+
+    with prepare_data(session, clean_bucket_name, "s", "t", expected) as t:
+        assert t.arrow_schema == columns
+
+        # Full scan.
+        actual = t.select().read_all()
+        assert actual == expected
+
+        # Select by id.
+        select_id = random.randint(0, num_rows)
+        actual = t.select(predicate=(t["id"] == select_id)).read_all()
+        assert actual.to_pydict()["vec"] == [[select_id] * dimension]
+        assert actual == expected.filter(pc.field("id") == select_id)
+
+
+def convert_scalar_type_pyarrow_to_numpy(arrow_type: pa.DataType):
+    return pa.array([], type=arrow_type).to_numpy().dtype.type
+
+
+def generate_random_pyarrow_value(
+        element: Union[pa.DataType, pa.Field], nulls_prob: float = 0.2
+) -> Any:
+    """
+    Generates a random value compatible with the provided PyArrow type.
+
+    Args:
+        element: The pyarrow field/type to generate values for.
+        nulls_prob: Probability of creating nulls.
+    """
+    assert 0 <= nulls_prob <= 1
+
+    nullable = True
+
+    # Convert Field to DataType.
+    if isinstance(element, pa.DataType):
+        pa_type = element
+    elif isinstance(element, pa.Field):
+        pa_type = element.type
+        nullable = element.nullable
+    else:
+        raise TypeError(
+            f"Expected pyarrow.DataType or pyarrow.Field, got {type(element)}"
+        )
+
+    if nullable and random.random() < nulls_prob:
+        return None
+
+    if pa.types.is_boolean(pa_type):
+        return random.choice([True, False])
+    if pa.types.is_integer(pa_type):
+        np_type = convert_scalar_type_pyarrow_to_numpy(pa_type)
+        iinfo = np.iinfo(np_type)
+        return np.random.randint(iinfo.min, iinfo.max, dtype=np_type)
+    if pa.types.is_floating(pa_type):
+        np_type = convert_scalar_type_pyarrow_to_numpy(pa_type)
+        finfo = np.finfo(np_type)
+        return np_type(random.uniform(float(finfo.min), float(finfo.max)))
+    if pa.types.is_string(pa_type) or pa.types.is_large_string(pa_type):
+        return "".join(
+            random.choices("abcdefghijklmnopqrstuvwxyz ", k=random.randint(5, 20))
+        )
+    if pa.types.is_binary(pa_type) or pa.types.is_large_binary(pa_type):
+        return random.randbytes(random.randint(5, 20))
+    if pa.types.is_timestamp(pa_type):
+        # Generate a random timestamp within a range (e.g., last 10 years)
+        start_datetime = datetime.datetime(2015, 1, 1, tzinfo=datetime.timezone.utc)
+        end_datetime = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc)
+        random_seconds = random.uniform(
+            0, (end_datetime - start_datetime).total_seconds()
+        )
+        return start_datetime + datetime.timedelta(seconds=random_seconds)
+    if pa.types.is_date(pa_type):
+        start_date = datetime.date(2000, 1, 1)
+        end_date = datetime.date(2025, 1, 1)
+        random_days = random.randint(0, (end_date - start_date).days)
+        return start_date + datetime.timedelta(days=random_days)
+    if pa.types.is_time(pa_type):
+        return datetime.time(
+            random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)
+        )
+    if pa.types.is_decimal(pa_type):
+        pa_type = cast(pa.Decimal128Type, pa_type)
+        decimal_value = decimal.Decimal(
+            round(random.uniform(-1000.0, 1000.0), pa_type.precision)
+        )
+        quantize_template = decimal.Decimal("1e-%d" % pa_type.scale)
+        return decimal_value.quantize(quantize_template)
+    if pa.types.is_null(pa_type):  # Explicit NullType
+        return None
+    if pa.types.is_list(pa_type) or pa.types.is_fixed_size_list(pa_type):
+        # For ListType, recursively generate elements for the value_type
+        pa_type = (
+            cast(pa.FixedSizeListType, pa_type)
+            if pa.types.is_fixed_size_list(pa_type)
+            else cast(pa.ListType, pa_type)
+        )
+        list_size = (
+            pa_type.list_size
+            if pa.types.is_fixed_size_list(pa_type)
+            else random.randint(0, 5)
+        )
+        list_elements = [
+            generate_random_pyarrow_value(pa_type.value_field, nulls_prob)
+            for _ in range(list_size)
+        ]
+        return list_elements
+    if pa.types.is_struct(pa_type):
+        struct_dict = {}
+        for field in cast(pa.StructType, pa_type):
+            # Recursively generate value for each field in the struct
+            struct_dict[field.name] = generate_random_pyarrow_value(field, nulls_prob)
+        return struct_dict
+    if pa.types.is_map(pa_type):
+        num_entries = random.randint(0, 3)  # Random number of map entries
+        pa_type = cast(pa.MapType, pa_type)
+        return {
+            generate_random_pyarrow_value(pa_type.key_field, nulls_prob): generate_random_pyarrow_value(
+                pa_type.item_field, nulls_prob)
+            for _ in range(num_entries)
+        }
+
+    raise NotImplementedError(
+        f"Generation for PyArrow type {pa_type} not implemented yet."
+    )
+
+
+@pytest.mark.parametrize("element_field", supported_fixed_list_element_fields)
+def test_fixed_list_type_values(session, clean_bucket_name, element_field):
+    list_size = random.randint(1, 1000)
+    num_rows = random.randint(1, 100)
+
+    vec_type = pa.list_(element_field, list_size)
+    schema = pa.schema(
+        {"id": pa.int64(), "vec": vec_type, "random_int": pa.int64()})
+    expected = pa.table(
+        schema=schema,
+        data=[list(range(num_rows))] + [[generate_random_pyarrow_value(schema.field(col_name)) for _ in range(num_rows)]
+                                        for col_name in
+                                        schema.names[1:]],
+    )
+
+    with prepare_data(session, clean_bucket_name, "s", "t", expected) as table:
+        assert table.arrow_schema == schema
+        actual = table.select().read_all()
+        assert actual == expected
+
+
+@pytest.mark.parametrize("list_type", unsupported_fixed_list_types)
+def test_unsupported_fixed_list_types(session, clean_bucket_name, list_type):
+    schema = pa.schema({"fixed_list": list_type})
+    empty_table = pa.table(schema=schema, data=[[]])
+
+    with pytest.raises((vastdb.errors.BadRequest, vastdb.errors.NotSupported), match=r'TabularUnsupportedColumnType'):
+        with prepare_data(session, clean_bucket_name, "s", "t", empty_table):
+            pass
+
+
+@pytest.mark.parametrize("list_type", invalid_fixed_list_types)
+def test_invalid_fixed_list_types(session, clean_bucket_name, list_type):
+    schema = pa.schema({"fixed_list": list_type})
+    empty_table = pa.table(schema=schema, data=[[]])
+
+    with pytest.raises(vastdb.errors.BadRequest, match=r'TabularInvalidColumnTypeParam'):
+        with prepare_data(session, clean_bucket_name, "s", "t", empty_table):
+            pass
+
+
+def test_invalid_values_fixed_list(session, clean_bucket_name):
+    dimension = 10
+    element_type = pa.float32()
+
+    col_name = "vec"
+    schema = pa.schema([(col_name, pa.list_(pa.field(name="item", type=element_type, nullable=False), dimension))])
+    empty_table = pa.table(schema=schema, data=[[]])
+
+    with prepare_data(session, clean_bucket_name, "s", "t", empty_table) as table:
+        invalid_fields = [
+            pa.field(col_name, pa.list_(pa.field(name="item", type=element_type, nullable=False), dimension - 1)),
+            pa.field(col_name, pa.list_(pa.field(name="item", type=element_type, nullable=False), dimension + 1)),
+            pa.field(col_name, pa.list_(pa.field(name="item", type=element_type, nullable=True), dimension)),
+            schema.field(0).with_nullable(False),
+        ]
+        for field in invalid_fields:
+            # Everything that could be null should be in order to be invalid regarding the values and not just the type.
+            rb = pa.record_batch(
+                schema=pa.schema([field]),
+                data=[[[1] * field.type.list_size]]
+            )
+            with pytest.raises((vastdb.errors.BadRequest, vastdb.errors.NotFound, vastdb.errors.NotSupported),
+                               match=r'(TabularInvalidColumnTypeParam)|(TabularUnsupportedColumnType)|(TabularMismatchColumnType)'):
+                table.insert(rb)
+
+        # Amount of elements in fixed list is not equal to the list size is enforced by Arrow.
+        with pytest.raises(pa.ArrowInvalid):
+            # Insert with empty list.
+            pa.record_batch(
+                schema=schema,
+                data=[[[generate_random_pyarrow_value(element_type, 0) for _ in range(dimension + 1)]]],
+            )

vastdb/tests/test_imports.py

@@ -222,6 +222,45 @@ def test_zip_imports(zip_import_session, clean_bucket_name, s3):
     # Step 3: Import files into the table
     attempt_import(zip_import_session, clean_bucket_name, 's1', 't1', files, key_names=['id', 'symbol'])
 
+    # Step 4: Construct expected rows
+    expected_rows = []
+    for i in range(num_rows):
+        row = {
+            'vastdb_rowid': 10 + i,  # Initial vastdb_rowid values (10-19)
+            'id': i,  # ID values (0-9)
+            'symbol': chr(ord('a') + i),  # Symbol values ('a' to 'j')
+            'feature0': 0 * 10 + i,  # Values from file 1 (0-9)
+            'feature1': 1 * 10 + i,  # Values from file 2 (10-19)
+            'feature2': 2 * 10 + i,  # Values from file 3 (20-29)
+            'feature3': 3 * 10 + i,  # Values from file 4 (30-39)
+            'feature4': 4 * 10 + i,  # Values from file 5 (40-49)
+        }
+        expected_rows.append(row)
+
+    # Step 5: Query the actual data from the table
+    with zip_import_session.transaction() as tx:
+        t = tx.bucket(clean_bucket_name).schema('s1').table('t1')
+        arrow_table = t.select().read_all()
+        actual_data = arrow_table.to_pydict()
+
+    # Step 6: Compare expected and actual data
+    num_actual_rows = len(next(iter(actual_data.values()), []))
+    assert num_actual_rows == len(expected_rows), f"Expected {len(expected_rows)} rows but got {num_actual_rows}"
+
+    # Convert expected_rows to a comparable format (pydict format)
+    expected_data = {k: [] for k in expected_rows[0].keys()}
+    for row in expected_rows:
+        for k, v in row.items():
+            expected_data[k].append(v)
+
+    # Check that all expected columns exist in actual data
+    for col in expected_data:
+        assert col in actual_data, f"Expected column {col} not found in actual data"
+
+    # Compare column values
+    for col in expected_data:
+        assert actual_data[col] == expected_data[col], f"Values in column {col} don't match expected values"
+
 
 def test_zip_imports_scale(zip_import_session, clean_bucket_name, s3):
     """Verify that many key names, and large amounts of data of different kind work as expected."""

vastdb/tests/test_nested.py

@@ -12,11 +12,15 @@ from .util import prepare_data
 def test_nested_select(session, clean_bucket_name):
     columns = pa.schema([
         ('l', pa.list_(pa.int8())),
+        ('fl', pa.list_(pa.field(name='item', type=pa.int64(), nullable=False), 2)),
+        ('lfl', pa.list_(pa.list_(pa.field(name='item', type=pa.int64(), nullable=False), 2))),
         ('m', pa.map_(pa.utf8(), pa.float64())),
         ('s', pa.struct([('x', pa.int16()), ('y', pa.int32())])),
     ])
     expected = pa.table(schema=columns, data=[
         [[1], [], [2, 3], None],
+        [[1, 2], None, [3, 4], None],
+        [[[1, 2], [3, 4], [4, 5]], None, [[5, 6], [7, 8]], [None, None]],
         [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
         [{'x': 1, 'y': None}, None, {'x': 2, 'y': 3}, {'x': None, 'y': 4}],
     ])
@@ -36,6 +40,7 @@ def test_nested_filter(session, clean_bucket_name):
     columns = pa.schema([
         ('x', pa.int64()),
         ('l', pa.list_(pa.int8())),
+        ('fl', pa.list_(pa.field(name='item', type=pa.int64(), nullable=False), 2)),
         ('y', pa.int64()),
         ('m', pa.map_(pa.utf8(), pa.float64())),
         ('z', pa.int64()),
@@ -45,6 +50,7 @@ def test_nested_filter(session, clean_bucket_name):
     expected = pa.table(schema=columns, data=[
         [1, 2, 3, None],
         [[1], [], [2, 3], None],
+        [[1, 2], None, [3, 4], None],
         [1, 2, None, 3],
         [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
         [1, None, 2, 3],
@@ -72,22 +78,16 @@ def test_nested_filter(session, clean_bucket_name):
 
 def test_nested_unsupported_filter(session, clean_bucket_name):
     columns = pa.schema([
-        ('x', pa.int64()),
         ('l', pa.list_(pa.int8())),
-        ('y', pa.int64()),
+        ('fl', pa.list_(pa.field(name='item', type=pa.int64(), nullable=False), 2)),
         ('m', pa.map_(pa.utf8(), pa.float64())),
-        ('z', pa.int64()),
         ('s', pa.struct([('x', pa.int16()), ('y', pa.int32())])),
-        ('w', pa.int64()),
     ])
     expected = pa.table(schema=columns, data=[
-        [1, 2, 3, None],
         [[1], [], [2, 3], None],
-        [1, 2, None, 3],
+        [[1, 2], None, [3, 4], None],
         [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
-        [1, None, 2, 3],
         [{'x': 1, 'y': None}, None, {'x': 2, 'y': 3}, {'x': None, 'y': 4}],
-        [None, 1, 2, 3],
     ])
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
@@ -95,6 +95,9 @@ def test_nested_unsupported_filter(session, clean_bucket_name):
         with pytest.raises(NotImplementedError):
             list(t.select(predicate=(t['l'].isnull())))
 
+        with pytest.raises(NotImplementedError):
+            list(t.select(predicate=(t['fl'].isnull())))
+
         with pytest.raises(NotImplementedError):
             list(t.select(predicate=(t['m'].isnull())))
 
@@ -106,6 +109,7 @@ def test_nested_subfields_predicate_pushdown(session, clean_bucket_name):
     columns = pa.schema([
         ('x', pa.int64()),
         ('l', pa.list_(pa.int8())),
+        ('fl', pa.list_(pa.field(name='item', type=pa.int64(), nullable=False), 2)),
         ('y', pa.int64()),
         ('m', pa.map_(pa.utf8(), pa.float64())),
         ('z', pa.int64()),
@@ -122,6 +126,7 @@ def test_nested_subfields_predicate_pushdown(session, clean_bucket_name):
     expected = pa.table(schema=columns, data=[
         [1, 2, 3, None],
         [[1], [], [2, 3], None],
+        [[1, 2], None, [3, 4], None],
         [1, 2, None, 3],
         [None, {'a': 2.5}, {'b': 0.25, 'c': 0.025}, {}],
         [1, None, 2, 3],

vastdb/tests/test_tables.py

@@ -15,10 +15,11 @@ import pytest
 from requests.exceptions import HTTPError
 
 from vastdb.errors import BadRequest
+from vastdb.session import Session
 
 from .. import errors
 from ..table import INTERNAL_ROW_ID, QueryConfig
-from .util import prepare_data
+from .util import assert_row_ids_ascending_on_first_insertion_to_table, prepare_data
 
 log = logging.getLogger(__name__)
 
@@ -112,13 +113,18 @@ def test_insert_empty(session, clean_bucket_name):
     data = [[None] * 5, [None] * 5]
     all_nulls = pa.table(schema=columns, data=data)
     no_columns = all_nulls.select([])
+    no_rows = pa.table(schema=columns, data=[[] for _ in columns])
 
     with session.transaction() as tx:
         t = tx.bucket(clean_bucket_name).create_schema('s').create_table('t', columns)
         t.insert(all_nulls)
+
         with pytest.raises(errors.NotImplemented):
             t.insert(no_columns)
 
+        row_ids = t.insert(no_rows).to_pylist()
+        assert row_ids == []
+
 
 def test_exists(session, clean_bucket_name):
     with session.transaction() as tx:
@@ -140,7 +146,8 @@ def test_exists(session, clean_bucket_name):
         assert s.tables() == [t]
 
 
-def test_list_tables(session, clean_bucket_name):
+@pytest.mark.parametrize("num_tables,page_size", [(10, 3)])
+def test_list_tables(session, clean_bucket_name, num_tables, page_size):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s1')
         assert s.tables() == []
@@ -148,12 +155,14 @@ def test_list_tables(session, clean_bucket_name):
 
         tables = [
             s.create_table(f't{i}', pa.schema([(f'x{i}', pa.int64())]))
-            for i in range(10)
+            for i in range(num_tables)
         ]
         assert tables == s.tables()
         tablenames = [t.name for t in tables]
         assert s.tablenames() == tablenames
 
+        assert s.tablenames(page_size=page_size) == tablenames
+
 
 def test_update_table(session, clean_bucket_name):
     columns = pa.schema([
@@ -388,6 +397,7 @@ def test_types(session, clean_bucket_name):
         assert select(None) == expected
         for t in [table, ibis._]:
             assert select(t['tb'] == False) == expected.filter(pc.field('tb') == False)  # noqa: E712
+            assert select(t['tb'] == True) == expected.filter(pc.field('tb') == True)  # noqa: E712
             assert select(t['a1'] == 2) == expected.filter(pc.field('a1') == 2)
             assert select(t['a2'] == 2000) == expected.filter(pc.field('a2') == 2000)
             assert select(t['a4'] == 222111122) == expected.filter(pc.field('a4') == 222111122)
@@ -424,6 +434,58 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
+@pytest.mark.parametrize("arrow_type,internal_support", [
+    # Types not supported by Vast.
+    (pa.null(), False),
+    (pa.dictionary(pa.int64(), pa.int64()), False),
+    (pa.dense_union([pa.field('1', pa.int32()), pa.field('2', pa.int64())]), False),
+    # Arrow.FixedSizeBinaryType is not supported by Ibis, but Vast supports it internally.
+    (pa.binary(1), True)
+])
+def test_unsupported_types(session, clean_bucket_name, arrow_type, internal_support):
+    """ Test that unsupported types cannot be used in table creation or modification."""
+    unsupported_field = pa.field('u', arrow_type)
+    schema_name = 's'
+    table_name = 't'
+
+    # Create the schema
+    with session.transaction() as tx:
+        tx.bucket(clean_bucket_name).create_schema(schema_name)
+
+    # Creation of a table with unsupported types should fail
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).schema(schema_name)
+        with pytest.raises(errors.NotSupportedSchema):
+            s.create_table(table_name, pa.schema([unsupported_field]))
+
+    with session.transaction() as tx:
+        tx.bucket(clean_bucket_name).schema(schema_name).create_table(table_name,
+                                                                      pa.schema([pa.field('a', pa.int32())]))
+
+    # Adding unsupported types to an existing table should fail
+    with session.transaction() as tx:
+        t = tx.bucket(clean_bucket_name).schema(schema_name).table(table_name)
+        with pytest.raises(errors.NotSupportedSchema):
+            t.add_column(pa.schema([unsupported_field]))
+
+    if internal_support:
+        # Using internal API to add unsupported types
+        with session.transaction() as tx:
+            tx._rpc.api.add_columns(clean_bucket_name, schema_name, table_name, pa.schema([unsupported_field]),
+                                    txid=tx.txid)
+
+        # Attempt to open a table with unsupported types should fail
+        with session.transaction() as tx:
+            s = tx.bucket(clean_bucket_name).schema(schema_name)
+            with pytest.raises(errors.NotSupportedSchema):
+                s.table(table_name)
+
+        # Even though the table is with unsupported types, it should still be listed
+        with session.transaction() as tx:
+            s = tx.bucket(clean_bucket_name).schema(schema_name)
+            assert [table_name] == s.tablenames()
+
+
 def test_unsigned_filters(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.uint8()),
@@ -1089,7 +1151,7 @@ def test_elysium_tx(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert len(sorted_columns) == 0
         t.add_sorting_key(sorting)
@@ -1125,7 +1187,7 @@ def test_elysium_double_enable(elysium_session, clean_bucket_name):
             t.add_sorting_key(sorting)
 
 
-def test_elysium_update_table_tx(elysium_session, clean_bucket_name):
+def test_elysium_update_table_tx(elysium_session: Session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int64()),
         ('b', pa.float32()),
@@ -1144,7 +1206,7 @@ def test_elysium_update_table_tx(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert sorted_columns[0].name == 's'
         assert sorted_columns[1].name == 'b'
@@ -1217,7 +1279,7 @@ def test_elysium_splits(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert sorted_columns[0].name == 'a'
 

vastdb/tests/util.py

@@ -3,17 +3,27 @@ from contextlib import contextmanager
 
 import pyarrow as pa
 
+from vastdb.session import Session
+
 log = logging.getLogger(__name__)
 
 
+def assert_row_ids_ascending_on_first_insertion_to_table(row_ids, expected_num_rows, sorted_table):
+    adjusted_row_ids = [
+        int(row_id) & 0xFFFFFFFFFFFFFF for row_id in row_ids
+        ] if sorted_table else row_ids
+
+    assert adjusted_row_ids == list(range(expected_num_rows))
+
+
 @contextmanager
-def prepare_data(session, clean_bucket_name, schema_name, table_name, arrow_table, sorting_key=[]):
+def prepare_data(session: Session, clean_bucket_name, schema_name, table_name, arrow_table, sorting_key=[]):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema(schema_name)
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting_key)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         yield t
         t.drop()
         s.drop()

{vastdb-1.3.10.dist-info → vastdb-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.10
+Version: 1.4.0
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.3.10.dist-info → vastdb-1.4.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 vastdb/__init__.py,sha256=uf-AXdzsD4nPxFP7WxkcAXGG0whv8BHLrrXCJtsPGaQ,436
-vastdb/_internal.py,sha256=tGNU-9wOtRoK7OXFmX1-uEgQRjpKQXPA0H4rZy86-JM,107257
+vastdb/_internal.py,sha256=YoZGgliQfNYNKuKG8M9ziZpu4R1pmp3PLdLcyOgo9Hc,108129
 vastdb/bucket.py,sha256=aomUbrfK5Oa6FdGPVsoBXgRW39IzYnmsorF8642r990,2549
 vastdb/config.py,sha256=OehnsWrjzv0-SUouEXmkrKBugiWyhXOn4XiSLV3s9yk,2342
-vastdb/conftest.py,sha256=X2kVveySPQYZlVBXUMoo7Oea5IsvmJzjdqq3fpH2kVw,3469
-vastdb/errors.py,sha256=B_FNFONDE8apoTRL8wkMNjUJWAjXu36mO0HI4cGSBgY,4328
+vastdb/conftest.py,sha256=Cl98Hg4kkLmx83F5dFMbVb-sTnn0zHxruE5B1hYXbMk,3866
+vastdb/errors.py,sha256=NiKdwbfVsWJIixP2Tf3JgiBoEt8rRaZ0VeCyD9mXnoM,5645
 vastdb/features.py,sha256=ivYbvhiGA858B00vhs_CNzlVV9QDUe53yW6V3J5EoxM,1874
-vastdb/schema.py,sha256=UR1WzQvfAdnpDaNsEaGZLYGC65Blri5MYOWinCcl8Hc,6552
+vastdb/schema.py,sha256=5BZ0f3b_c-fGRKAaBBL6B3avHel5EDwwxte7t17WeTw,6718
 vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
-vastdb/table.py,sha256=NGImmz_KltU80B0u-CYDgEdGOMHSppf7mmVs72WD8wM,35937
+vastdb/table.py,sha256=rM-xJgpSuvowUH3oilVVuW-7JsPDmMJ4rbHnru1EA0Y,37268
 vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
 vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,14 +41,15 @@ vastdb/bench/perf_bench/query/query_vastdb.py,sha256=SZYem_EmsaynEftAa_VFobjSJZD
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/tests/metrics.py,sha256=ZCSeBYFSPMG3yI0JrAHs2CrY6wFjx_5GwRTYHVAwLKA,1026
 vastdb/tests/test_duckdb.py,sha256=STw_1PwTQR8Naz6s0p6lQTV1ZTKKhe3LPBUbhqzTCu0,1880
-vastdb/tests/test_imports.py,sha256=R-ExC6IYaf4REGQw0v7iVAz7TPY9vd8S3S892vy86R0,20011
-vastdb/tests/test_nested.py,sha256=LPU6uV3Ri23dBzAEMFQqRPbqapV5LfmiHSHkhILPIY0,6332
+vastdb/tests/test_fixed_list.py,sha256=qwtFNvw5fdMkOsYAcFfqPN3JOnJn31XGYtFWVe0vuOQ,11187
+vastdb/tests/test_imports.py,sha256=1Xi5s0qWxuoVunW5iMQGzofTNOXxXP8eOARs9HWOiGE,21734
+vastdb/tests/test_nested.py,sha256=c7q9a3MsyDymqAtShPC4cMHlzjCr18kbu_Db3u_c4IQ,6893
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
 vastdb/tests/test_sanity.py,sha256=bv1ypGDzvOgmMvGbucDYiLQu8krQLlE6NB3M__q87x8,3303
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=wBPUewfJVEJNyDHwO49qld3lMVjVjUiAzP7ngX07fFA,48478
+vastdb/tests/test_tables.py,sha256=2m7Ao97mLwvCgzGShxTHMxFbt3vnNfb6InNe96DTJMA,51502
 vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
-vastdb/tests/util.py,sha256=YsCBCcx7n1QOH-IPDpCsl6KEaUQQJRZwGPeayijHNb4,1307
+vastdb/tests/util.py,sha256=j8WSYOsScz7uvJljIcI9ZQWz3cA3WPykurwymnDd3s0,1690
 vastdb/vast_flatbuf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_flatbuf/org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_flatbuf/org/apache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -209,8 +210,8 @@ vastdb/vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
 vastdb/vast_tests/test_scale.py,sha256=5jGwOdZH6Tv5tPdZYPWoqcxOceI2jA5i2D1zNKZHER4,3958
-vastdb-1.3.10.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.3.10.dist-info/METADATA,sha256=BFeEhZ0mgwoCyAKM_EkijrPcI5RWTME4tDtdq-fcWwc,1341
-vastdb-1.3.10.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vastdb-1.3.10.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
-vastdb-1.3.10.dist-info/RECORD,,
+vastdb-1.4.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.4.0.dist-info/METADATA,sha256=7bga0VnmmT_RYdlB7abV_l-SEDegSuLMIRf7rB8UGpQ,1340
+vastdb-1.4.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.4.0.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
+vastdb-1.4.0.dist-info/RECORD,,