vastdb 2.0.5__tar.gz → 2.0.8__tar.gz

{vastdb-2.0.5 → vastdb-2.0.8}/CHANGELOG.md

@@ -4,6 +4,35 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [2.0.8] (2026-01-21)
+[2.0.8]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.7...v2.0.8
+
+### Changed
+- populate _metadata._vector_index from list_tables response.
+
+## [2.0.7] (2026-01-19)
+[2.0.7]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.5...v2.0.6
+
+### Added
+- Vector Index table support (ORION-276190):
+  - `sorting_key_enabled` field in `TableInfo` is now exclusive to Elysium tables (no longer `true` for vector index tables)
+  - Vector index tables are now classified as `TableType.Regular` instead of `TableType.Elysium`
+  - Requires VAST Cluster with ORION-276190 support
+- Vector Index metadata fields in `TableInfo` (conditioned on `include_vector_index_metadata`):
+  - `vector_index_enabled`: boolean flag indicating vector index presence
+  - `vector_index_column_name`: name of the column with vector index
+  - `vector_index_distance_metric`: distance metric used (L2, COSINE, etc.)
+  - `vector_index_sql_function_name`: SQL function name for distance calculation (e.g., "array_distance")
+
+## [2.0.6] (2026-01-19)
+[2.0.6]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.5...v2.0.6
+
+### Added
+- Float16 (half-precision floating point) type support
+  - Schema serialization for float16 columns with precision.half
+  - Predicate literal handling for float16 values
+  - Fixes 'unsupported column type halffloat' error when creating tables with float16 columns
+
 ## [2.0.5] (2025-01-08)
 [2.0.4]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.4...v2.0.5
 

{vastdb-2.0.5 → vastdb-2.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.5
+Version: 2.0.8
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
@@ -20,6 +20,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aws-requests-auth
 Requires-Dist: ibis-framework~=10.1
+Requires-Dist: sqlglot<28.7,>=23.4
 Requires-Dist: pyarrow~=18.0
 Requires-Dist: pyarrow-hotfix==0.7
 Requires-Dist: flatbuffers

{vastdb-2.0.5 → vastdb-2.0.8}/requirements.txt

@@ -1,5 +1,6 @@
 aws-requests-auth
 ibis-framework~=10.1
+sqlglot>=23.4,<28.7
 pyarrow~=18.0
 pyarrow-hotfix==0.7
 flatbuffers

{vastdb-2.0.5 → vastdb-2.0.8}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.10.0',
     description='VAST Data SDK',
-    version='2.0.5' + suffix,
+    version='2.0.8' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-2.0.5 → vastdb-2.0.8}/vastdb/_internal.py

@@ -47,6 +47,7 @@ import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.DecimalLiteral as
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Expression as fb_expression
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.FieldIndex as fb_field_index
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.FieldRef as fb_field_ref
+import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float16Literal as fb_float16_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float32Literal as fb_float32_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float64Literal as fb_float64_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Int8Literal as fb_int8_lit
@@ -411,15 +412,19 @@ class Predicate:
 
         if pa.types.is_floating(pa_type):
             impl_type, impl_class = None, None
-            value = float(value)
 
-            if pa.types.is_float32(pa_type):
-                impl_type, impl_class = LiteralImpl.Float32Literal, fb_float32_lit
-            elif pa.types.is_float64(pa_type):
-                impl_type, impl_class = LiteralImpl.Float64Literal, fb_float64_lit
+            if pa.types.is_float16(pa_type):
+                import numpy as np
+                value = np.float16(value).view(np.uint16).item()
+                impl_type, impl_class = LiteralImpl.Float16Literal, fb_float16_lit
             else:
-                # Float16 is not supported by Vast.
-                raise ValueError(f'unsupported floating point predicate type: {pa_type}, value={value}')
+                value = float(value)
+                if pa.types.is_float32(pa_type):
+                    impl_type, impl_class = LiteralImpl.Float32Literal, fb_float32_lit
+                elif pa.types.is_float64(pa_type):
+                    impl_type, impl_class = LiteralImpl.Float64Literal, fb_float64_lit
+                else:
+                    raise ValueError(f'unsupported floating point predicate type: {pa_type}, value={value}')
 
             impl_class.Start(self.builder)
             impl_class.AddValue(self.builder, value)
@@ -783,7 +788,7 @@ def _decode_table_props(s):
     return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
 
-TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled sorting_score write_amplification acummulative_row_insertion_count sorting_done')
+TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled sorting_score write_amplification acummulative_row_insertion_count sorting_done vector_index_enabled vector_index_column_name vector_index_distance_metric vector_index_sql_function_name')
 
 
 def _parse_table_info(obj, parse_properties):
@@ -801,8 +806,15 @@ def _parse_table_info(obj, parse_properties):
 
     sorting_score = sorting_score_raw & ((1 << 63) - 1)
     sorting_done = bool(sorting_score_raw >> 63)
+
+    vector_index_enabled = obj.VectorIndexEnabled() if hasattr(obj, 'VectorIndexEnabled') else False
+    vector_index_column_name = obj.VectorIndexColumnName().decode() if hasattr(obj, 'VectorIndexColumnName') and obj.VectorIndexColumnName() else ""
+    vector_index_distance_metric = obj.VectorIndexDistanceMetric().decode() if hasattr(obj, 'VectorIndexDistanceMetric') and obj.VectorIndexDistanceMetric() else ""
+    vector_index_sql_function_name = obj.VectorIndexSqlFunctionName().decode() if hasattr(obj, 'VectorIndexSqlFunctionName') and obj.VectorIndexSqlFunctionName() else ""
+
     return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions, sorting_key_enabled,
-                     sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done)
+                     sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done,
+                     vector_index_enabled, vector_index_column_name, vector_index_distance_metric, vector_index_sql_function_name)
 
 
 @dataclass
@@ -1384,20 +1396,24 @@ class VastdbApi:
                                           parse_properties=_decode_table_props, max_keys=max_keys,
                                           next_key=next_key, name_prefix=name_prefix, exact_match=exact_match,
                                           expected_retvals=expected_retvals,
-                                          include_list_stats=include_list_stats, count_only=count_only)
+                                          include_list_stats=include_list_stats, count_only=count_only,
+                                          include_vector_index_metadata=False)
 
     def list_tables(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                    include_vector_index_metadata=False):
         def parse_properties(x):
             return x
         return self._list_tables_internal(bucket=bucket, schema=schema, txid=txid, client_tags=client_tags,
                                           parse_properties=parse_properties, max_keys=max_keys, next_key=next_key,
                                           name_prefix=name_prefix, exact_match=exact_match,
                                           expected_retvals=expected_retvals,
-                                          include_list_stats=include_list_stats, count_only=count_only)
+                                          include_list_stats=include_list_stats, count_only=count_only,
+                                          include_vector_index_metadata=include_vector_index_metadata)
 
     def _list_tables_raw(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                         exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                         exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                         include_vector_index_metadata=False):
         """
         GET /mybucket/schema_path?table HTTP/1.1
         tabular-txid: TransactionId
@@ -1416,6 +1432,7 @@ class VastdbApi:
 
         headers['tabular-list-count-only'] = str(count_only)
         headers['tabular-include-list-stats'] = str(include_list_stats)
+        headers['tabular-include-vector-index-meta-data'] = str(include_vector_index_metadata).lower()
 
         res = self._request(
             method="GET",
@@ -1431,11 +1448,13 @@ class VastdbApi:
         return lists, next_key, is_truncated, count
 
     def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                              include_vector_index_metadata=False):
         tables = []
         lists, next_key, is_truncated, count = self._list_tables_raw(bucket, schema, txid=txid, client_tags=client_tags, max_keys=max_keys,
                                  next_key=next_key, name_prefix=name_prefix, exact_match=exact_match, expected_retvals=expected_retvals,
-                                 include_list_stats=include_list_stats, count_only=count_only)
+                                 include_list_stats=include_list_stats, count_only=count_only,
+                                 include_vector_index_metadata=include_vector_index_metadata)
         bucket_name = lists.BucketName().decode()
         schema_name = lists.SchemaName().decode()
         if not bucket.startswith(bucket_name):  # ignore snapshot name
@@ -2283,6 +2302,12 @@ def get_field_type(builder: flatbuffers.Builder, field: pa.Field):
         fb_int.AddIsSigned(builder, False)
         field_type = fb_int.End(builder)
 
+    elif field.type.equals(pa.float16()):
+        field_type_type = Type.FloatingPoint
+        fb_floating_point.Start(builder)
+        fb_floating_point.AddPrecision(builder, 0)  # half
+        field_type = fb_floating_point.End(builder)
+
     elif field.type.equals(pa.float32()):
         field_type_type = Type.FloatingPoint
         fb_floating_point.Start(builder)

{vastdb-2.0.5 → vastdb-2.0.8}/vastdb/schema.py

@@ -14,7 +14,7 @@ from vastdb.table_metadata import TableMetadata, TableRef, TableType
 
 from . import bucket, errors, schema, table
 from ._ibis_support import validate_ibis_support_schema
-from ._internal import VectorIndexSpec
+from ._internal import VectorIndex, VectorIndexSpec
 
 if TYPE_CHECKING:
     from .table import Table
@@ -130,7 +130,7 @@ class Schema:
         log.debug("Found table: %s", t[0])
         return t[0]
 
-    def _iter_tables(self, table_name=None, page_size=1000):
+    def _iter_tables(self, table_name=None, page_size=1000, include_vector_index_metadata=False):
         next_key = 0
         name_prefix = table_name if table_name else ""
         exact_match = bool(table_name)
@@ -138,7 +138,8 @@ class Schema:
             _bucket_name, _schema_name, curr_tables, next_key, is_truncated, _ = \
                 self.tx._rpc.api.list_tables(
                     bucket=self.bucket.name, schema=self.name, next_key=next_key, max_keys=page_size, txid=self.tx.active_txid,
-                    exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match)
+                    exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match,
+                    include_vector_index_metadata=include_vector_index_metadata)
             if not curr_tables:
                 break
             yield from curr_tables
@@ -177,7 +178,17 @@ def _parse_table_info(table_info, schema: "schema.Schema"):
                    table=table_info.name)
 
     table_type = TableType.Elysium if table_info.sorting_key_enabled else TableType.Regular
-    table_metadata = TableMetadata(ref, table_type=table_type)
+
+    # populate vector_index from list_tables if vector index is enabled
+    vector_index = None
+    if table_info.vector_index_enabled:
+        vector_index = VectorIndex(
+            column=table_info.vector_index_column_name,
+            distance_metric=table_info.vector_index_distance_metric,
+            sql_distance_function=table_info.vector_index_sql_function_name
+        )
+
+    table_metadata = TableMetadata(ref, table_type=table_type, vector_index=vector_index)
 
     return table.Table(handle=int(table_info.handle),
                        metadata=table_metadata,

{vastdb-2.0.5 → vastdb-2.0.8}/vastdb/table.py

@@ -226,7 +226,7 @@ class TableInTransaction(ITable):
 
     @property
     def _internal_rowid_field(self) -> pa.Field:
-        return INTERNAL_ROW_ID_SORTED_FIELD if self._is_sorted_table else INTERNAL_ROW_ID_FIELD
+        return INTERNAL_ROW_ID_SORTED_FIELD if self._uses_global_row_ids else INTERNAL_ROW_ID_FIELD
 
     def sorted_columns(self) -> list[pa.Field]:
         """Return sorted columns' metadata."""
@@ -818,6 +818,16 @@ class TableInTransaction(ITable):
     def _is_sorted_table(self) -> bool:
         return self._metadata.table_type is TableType.Elysium
 
+    @property
+    def _uses_global_row_ids(self) -> bool:
+        """Check if table uses global row IDs (decimal128: ehandle + row_id).
+
+        Both Elysium and Vector Index tables use global row IDs.
+        """
+        # _vector_index is set from list_tables or synced from stats.vector_index by _parse_stats_vector_index()
+        has_vector_index = self._metadata._vector_index is not None
+        return self._is_sorted_table or has_vector_index
+
     def vector_search(
         self,
         vec: list[float],

{vastdb-2.0.5 → vastdb-2.0.8}/vastdb/table_metadata.py

@@ -173,14 +173,14 @@ class TableMetadata:
         self._parse_stats_vector_index()
 
     def _parse_stats_vector_index(self):
-        vector_index_is_set = self._vector_index is not None
+        if self._vector_index is not None and self._stats.vector_index != self._vector_index:
+            is_empty_placeholder = not self._vector_index.column and not self._vector_index.distance_metric
+            if not is_empty_placeholder:
+                raise ValueError(
+                    f"Table has index {self._stats.vector_index}, but was initialized as {self._vector_index}"
+                    )
 
-        if vector_index_is_set and self._stats.vector_index != self._vector_index:
-            raise ValueError(
-                f"Table has index {self._stats.vector_index}, but was initialized as {self._vector_index}"
-                )
-        else:
-            self._vector_index = self._stats.vector_index
+        self._vector_index = self._stats.vector_index
 
     def _set_sorted_table(self, tx: "Transaction"):
         self._table_type = TableType.Elysium

{vastdb-2.0.5 → vastdb-2.0.8}/vastdb/tests/test_tables.py

@@ -9,6 +9,7 @@ from contextlib import closing
 from tempfile import NamedTemporaryFile
 
 import ibis
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -390,6 +391,7 @@ def test_types(session, clean_bucket_name):
         ('a2', pa.int16()),
         ('a4', pa.int64()),
         ('b', pa.float32()),
+        ('f16', pa.float16()),
         ('s', pa.string()),
         ('d', pa.decimal128(7, 3)),
         ('bin', pa.binary()),
@@ -410,6 +412,7 @@ def test_types(session, clean_bucket_name):
         [1999, 2000, 2001],
         [11122221, 222111122, 333333],
         [0.5, 1.5, 2.5],
+        [np.float16(0.5), np.float16(1.5), np.float16(2.5)],
         ["a", "v", "s"],
         [decimal.Decimal('110.52'), decimal.Decimal('231.15'), decimal.Decimal('3332.44')],
         [b"\x01\x02", b"\x01\x05", b"\x01\x07"],
@@ -436,6 +439,13 @@ def test_types(session, clean_bucket_name):
             assert select(t['a2'] == 2000) == expected.filter(pc.field('a2') == 2000)
             assert select(t['a4'] == 222111122) == expected.filter(pc.field('a4') == 222111122)
             assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
+
+            # Test float16 predicate (PyArrow compute doesn't support float16, so validate manually)
+            f16_literal = np.float16(1.5)
+            result = select(t['f16'] == f16_literal)
+            assert len(result) == 1, f"Expected 1 row for f16==1.5, got {len(result)}"
+            assert np.float16(result.column('f16')[0].as_py()) == f16_literal
+
             assert select(t['s'] == "v") == expected.filter(pc.field('s') == "v")
             assert select(t['d'] == 231.15) == expected.filter(pc.field('d') == 231.15)
             assert select(t['bin'] == b"\x01\x02") == expected.filter(pc.field('bin') == b"\x01\x02")
@@ -468,6 +478,59 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
+@pytest.mark.parametrize("element_type,test_name", [
+    (pa.float32(), "float32"),
+    (pa.float16(), "float16"),
+])
+def test_vector_types(session, clean_bucket_name, element_type, test_name):
+    """Test vector (fixed-size list) columns with different element types."""
+    vector_dim = 3
+    vec_type = pa.list_(pa.field('', element_type, False), vector_dim)
+
+    columns = pa.schema([
+        ('id', pa.int32()),
+        ('vector', vec_type),
+    ])
+
+    # Create test data based on element type
+    if element_type == pa.float16():
+        test_vectors = [
+            [np.float16(1.0), np.float16(2.0), np.float16(3.0)],
+            [np.float16(4.5), np.float16(5.5), np.float16(6.5)],
+            [np.float16(-1.0), np.float16(0.0), np.float16(1.0)],
+        ]
+    else:  # float32
+        test_vectors = [
+            [1.0, 2.0, 3.0],
+            [4.5, 5.5, 6.5],
+            [-1.0, 0.0, 1.0],
+        ]
+
+    expected = pa.table(schema=columns, data=[
+        [0, 1, 2],
+        test_vectors,
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
+        # Read back and verify
+        actual = table.select().read_all()
+        assert actual.schema == columns
+        assert len(actual) == 3
+
+        # Verify vector data
+        actual_vectors = actual.column('vector').to_pylist()
+        for i, (actual_vec, expected_vec) in enumerate(zip(actual_vectors, test_vectors)):
+            assert len(actual_vec) == vector_dim, f"Wrong vector dimension at row {i}"
+            for j, (act, exp) in enumerate(zip(actual_vec, expected_vec)):
+                if element_type == pa.float16():
+                    assert np.float16(act) == np.float16(exp), \
+                        f"Mismatch at row {i}, element {j}: {act} != {exp}"
+                else:
+                    assert act == exp, f"Mismatch at row {i}, element {j}: {act} != {exp}"
+
+        log.info(f"Vector type test ({test_name}) passed successfully")
+
+
 @pytest.mark.parametrize("arrow_type,internal_support", [
     # Types not supported by Vast.
     (pa.null(), False),