vastdb 2.0.5__tar.gz → 2.0.7__tar.gz

{vastdb-2.0.5 → vastdb-2.0.7}/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [2.0.7] (2026-01-19)
+[2.0.7]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.5...v2.0.6
+
+### Added
+- Vector Index table support (ORION-276190):
+  - `sorting_key_enabled` field in `TableInfo` is now exclusive to Elysium tables (no longer `true` for vector index tables)
+  - Vector index tables are now classified as `TableType.Regular` instead of `TableType.Elysium`
+  - Requires VAST Cluster with ORION-276190 support
+- Vector Index metadata fields in `TableInfo` (conditioned on `include_vector_index_metadata`):
+  - `vector_index_enabled`: boolean flag indicating vector index presence
+  - `vector_index_column_name`: name of the column with vector index
+  - `vector_index_distance_metric`: distance metric used (L2, COSINE, etc.)
+  - `vector_index_sql_function_name`: SQL function name for distance calculation (e.g., "array_distance")
+
+## [2.0.6] (2026-01-19)
+[2.0.6]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.5...v2.0.6
+
+### Added
+- Float16 (half-precision floating point) type support
+  - Schema serialization for float16 columns with precision.half
+  - Predicate literal handling for float16 values
+  - Fixes 'unsupported column type halffloat' error when creating tables with float16 columns
+
 ## [2.0.5] (2025-01-08)
 [2.0.4]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.4...v2.0.5
 

{vastdb-2.0.5 → vastdb-2.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.5
+Version: 2.0.7
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-2.0.5 → vastdb-2.0.7}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.10.0',
     description='VAST Data SDK',
-    version='2.0.5' + suffix,
+    version='2.0.7' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb/_internal.py

@@ -47,6 +47,7 @@ import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.DecimalLiteral as
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Expression as fb_expression
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.FieldIndex as fb_field_index
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.FieldRef as fb_field_ref
+import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float16Literal as fb_float16_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float32Literal as fb_float32_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Float64Literal as fb_float64_lit
 import vastdb.vast_flatbuf.org.apache.arrow.computeir.flatbuf.Int8Literal as fb_int8_lit
@@ -411,15 +412,19 @@ class Predicate:
 
         if pa.types.is_floating(pa_type):
             impl_type, impl_class = None, None
-            value = float(value)
 
-            if pa.types.is_float32(pa_type):
-                impl_type, impl_class = LiteralImpl.Float32Literal, fb_float32_lit
-            elif pa.types.is_float64(pa_type):
-                impl_type, impl_class = LiteralImpl.Float64Literal, fb_float64_lit
+            if pa.types.is_float16(pa_type):
+                import numpy as np
+                value = np.float16(value).view(np.uint16).item()
+                impl_type, impl_class = LiteralImpl.Float16Literal, fb_float16_lit
             else:
-                # Float16 is not supported by Vast.
-                raise ValueError(f'unsupported floating point predicate type: {pa_type}, value={value}')
+                value = float(value)
+                if pa.types.is_float32(pa_type):
+                    impl_type, impl_class = LiteralImpl.Float32Literal, fb_float32_lit
+                elif pa.types.is_float64(pa_type):
+                    impl_type, impl_class = LiteralImpl.Float64Literal, fb_float64_lit
+                else:
+                    raise ValueError(f'unsupported floating point predicate type: {pa_type}, value={value}')
 
             impl_class.Start(self.builder)
             impl_class.AddValue(self.builder, value)
@@ -783,7 +788,7 @@ def _decode_table_props(s):
     return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
 
-TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled sorting_score write_amplification acummulative_row_insertion_count sorting_done')
+TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled sorting_score write_amplification acummulative_row_insertion_count sorting_done vector_index_enabled vector_index_column_name vector_index_distance_metric vector_index_sql_function_name')
 
 
 def _parse_table_info(obj, parse_properties):
@@ -801,8 +806,15 @@ def _parse_table_info(obj, parse_properties):
 
     sorting_score = sorting_score_raw & ((1 << 63) - 1)
     sorting_done = bool(sorting_score_raw >> 63)
+
+    vector_index_enabled = obj.VectorIndexEnabled() if hasattr(obj, 'VectorIndexEnabled') else False
+    vector_index_column_name = obj.VectorIndexColumnName().decode() if hasattr(obj, 'VectorIndexColumnName') and obj.VectorIndexColumnName() else None
+    vector_index_distance_metric = obj.VectorIndexDistanceMetric().decode() if hasattr(obj, 'VectorIndexDistanceMetric') and obj.VectorIndexDistanceMetric() else None
+    vector_index_sql_function_name = obj.VectorIndexSqlFunctionName().decode() if hasattr(obj, 'VectorIndexSqlFunctionName') and obj.VectorIndexSqlFunctionName() else None
+
     return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions, sorting_key_enabled,
-                     sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done)
+                     sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done,
+                     vector_index_enabled, vector_index_column_name, vector_index_distance_metric, vector_index_sql_function_name)
 
 
 @dataclass
@@ -1384,20 +1396,24 @@ class VastdbApi:
                                           parse_properties=_decode_table_props, max_keys=max_keys,
                                           next_key=next_key, name_prefix=name_prefix, exact_match=exact_match,
                                           expected_retvals=expected_retvals,
-                                          include_list_stats=include_list_stats, count_only=count_only)
+                                          include_list_stats=include_list_stats, count_only=count_only,
+                                          include_vector_index_metadata=False)
 
     def list_tables(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                    include_vector_index_metadata=False):
         def parse_properties(x):
             return x
         return self._list_tables_internal(bucket=bucket, schema=schema, txid=txid, client_tags=client_tags,
                                           parse_properties=parse_properties, max_keys=max_keys, next_key=next_key,
                                           name_prefix=name_prefix, exact_match=exact_match,
                                           expected_retvals=expected_retvals,
-                                          include_list_stats=include_list_stats, count_only=count_only)
+                                          include_list_stats=include_list_stats, count_only=count_only,
+                                          include_vector_index_metadata=include_vector_index_metadata)
 
     def _list_tables_raw(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                         exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                         exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                         include_vector_index_metadata=False):
         """
         GET /mybucket/schema_path?table HTTP/1.1
         tabular-txid: TransactionId
@@ -1416,6 +1432,7 @@ class VastdbApi:
 
         headers['tabular-list-count-only'] = str(count_only)
         headers['tabular-include-list-stats'] = str(include_list_stats)
+        headers['tabular-include-vector-index-meta-data'] = str(include_vector_index_metadata).lower()
 
         res = self._request(
             method="GET",
@@ -1431,11 +1448,13 @@ class VastdbApi:
         return lists, next_key, is_truncated, count
 
     def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False,
+                              include_vector_index_metadata=False):
         tables = []
         lists, next_key, is_truncated, count = self._list_tables_raw(bucket, schema, txid=txid, client_tags=client_tags, max_keys=max_keys,
                                  next_key=next_key, name_prefix=name_prefix, exact_match=exact_match, expected_retvals=expected_retvals,
-                                 include_list_stats=include_list_stats, count_only=count_only)
+                                 include_list_stats=include_list_stats, count_only=count_only,
+                                 include_vector_index_metadata=include_vector_index_metadata)
         bucket_name = lists.BucketName().decode()
         schema_name = lists.SchemaName().decode()
         if not bucket.startswith(bucket_name):  # ignore snapshot name
@@ -2283,6 +2302,12 @@ def get_field_type(builder: flatbuffers.Builder, field: pa.Field):
         fb_int.AddIsSigned(builder, False)
         field_type = fb_int.End(builder)
 
+    elif field.type.equals(pa.float16()):
+        field_type_type = Type.FloatingPoint
+        fb_floating_point.Start(builder)
+        fb_floating_point.AddPrecision(builder, 0)  # half
+        field_type = fb_floating_point.End(builder)
+
     elif field.type.equals(pa.float32()):
         field_type_type = Type.FloatingPoint
         fb_floating_point.Start(builder)

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb/schema.py

@@ -130,7 +130,7 @@ class Schema:
         log.debug("Found table: %s", t[0])
         return t[0]
 
-    def _iter_tables(self, table_name=None, page_size=1000):
+    def _iter_tables(self, table_name=None, page_size=1000, include_vector_index_metadata=False):
         next_key = 0
         name_prefix = table_name if table_name else ""
         exact_match = bool(table_name)
@@ -138,7 +138,8 @@ class Schema:
             _bucket_name, _schema_name, curr_tables, next_key, is_truncated, _ = \
                 self.tx._rpc.api.list_tables(
                     bucket=self.bucket.name, schema=self.name, next_key=next_key, max_keys=page_size, txid=self.tx.active_txid,
-                    exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match)
+                    exact_match=exact_match, name_prefix=name_prefix, include_list_stats=exact_match,
+                    include_vector_index_metadata=include_vector_index_metadata)
             if not curr_tables:
                 break
             yield from curr_tables

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb/table.py

@@ -226,7 +226,7 @@ class TableInTransaction(ITable):
 
     @property
     def _internal_rowid_field(self) -> pa.Field:
-        return INTERNAL_ROW_ID_SORTED_FIELD if self._is_sorted_table else INTERNAL_ROW_ID_FIELD
+        return INTERNAL_ROW_ID_SORTED_FIELD if self._uses_global_row_ids else INTERNAL_ROW_ID_FIELD
 
     def sorted_columns(self) -> list[pa.Field]:
         """Return sorted columns' metadata."""
@@ -818,6 +818,17 @@ class TableInTransaction(ITable):
     def _is_sorted_table(self) -> bool:
         return self._metadata.table_type is TableType.Elysium
 
+    @property
+    def _uses_global_row_ids(self) -> bool:
+        """Check if table uses global row IDs (decimal128: ehandle + row_id).
+
+        Both Elysium and Vector Index tables use global row IDs.
+        """
+        # check if table has vector index from loaded stats
+        has_vector_index = (self._metadata.stats is not None and
+                           self._metadata.stats.vector_index is not None)
+        return self._is_sorted_table or has_vector_index
+
     def vector_search(
         self,
         vec: list[float],

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb/vast_flatbuf/tabular/ObjectDetails.py

@@ -95,7 +95,42 @@ class ObjectDetails(object):
             return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos)
         return 0
 
-def Start(builder): builder.StartObject(10)
+    # ObjectDetails
+    def PartitioningKeyEnabled(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(24))
+        if o != 0:
+            return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos))
+        return False
+
+    # ObjectDetails
+    def VectorIndexEnabled(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(26))
+        if o != 0:
+            return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos))
+        return False
+
+    # ObjectDetails
+    def VectorIndexColumnName(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(28))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ObjectDetails
+    def VectorIndexDistanceMetric(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ObjectDetails
+    def VectorIndexSqlFunctionName(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(32))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+def Start(builder): builder.StartObject(15)
 def ObjectDetailsStart(builder):
     """This method is deprecated. Please switch to Start."""
     return Start(builder)
@@ -139,6 +174,26 @@ def AddAcummulativeRowInseritionCount(builder, acummulativeRowInseritionCount):
 def ObjectDetailsAddAcummulativeRowInseritionCount(builder, acummulativeRowInseritionCount):
     """This method is deprecated. Please switch to AddAcummulativeRowInseritionCount."""
     return AddAcummulativeRowInseritionCount(builder, acummulativeRowInseritionCount)
+def AddPartitioningKeyEnabled(builder, partitioningKeyEnabled): builder.PrependBoolSlot(10, partitioningKeyEnabled, 0)
+def ObjectDetailsAddPartitioningKeyEnabled(builder, partitioningKeyEnabled):
+    """This method is deprecated. Please switch to AddPartitioningKeyEnabled."""
+    return AddPartitioningKeyEnabled(builder, partitioningKeyEnabled)
+def AddVectorIndexEnabled(builder, vectorIndexEnabled): builder.PrependBoolSlot(11, vectorIndexEnabled, 0)
+def ObjectDetailsAddVectorIndexEnabled(builder, vectorIndexEnabled):
+    """This method is deprecated. Please switch to AddVectorIndexEnabled."""
+    return AddVectorIndexEnabled(builder, vectorIndexEnabled)
+def AddVectorIndexColumnName(builder, vectorIndexColumnName): builder.PrependUOffsetTRelativeSlot(12, flatbuffers.number_types.UOffsetTFlags.py_type(vectorIndexColumnName), 0)
+def ObjectDetailsAddVectorIndexColumnName(builder, vectorIndexColumnName):
+    """This method is deprecated. Please switch to AddVectorIndexColumnName."""
+    return AddVectorIndexColumnName(builder, vectorIndexColumnName)
+def AddVectorIndexDistanceMetric(builder, vectorIndexDistanceMetric): builder.PrependUOffsetTRelativeSlot(13, flatbuffers.number_types.UOffsetTFlags.py_type(vectorIndexDistanceMetric), 0)
+def ObjectDetailsAddVectorIndexDistanceMetric(builder, vectorIndexDistanceMetric):
+    """This method is deprecated. Please switch to AddVectorIndexDistanceMetric."""
+    return AddVectorIndexDistanceMetric(builder, vectorIndexDistanceMetric)
+def AddVectorIndexSqlFunctionName(builder, vectorIndexSqlFunctionName): builder.PrependUOffsetTRelativeSlot(14, flatbuffers.number_types.UOffsetTFlags.py_type(vectorIndexSqlFunctionName), 0)
+def ObjectDetailsAddVectorIndexSqlFunctionName(builder, vectorIndexSqlFunctionName):
+    """This method is deprecated. Please switch to AddVectorIndexSqlFunctionName."""
+    return AddVectorIndexSqlFunctionName(builder, vectorIndexSqlFunctionName)
 def End(builder): return builder.EndObject()
 def ObjectDetailsEnd(builder):
     """This method is deprecated. Please switch to End."""

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb/vast_tests/test_scale.py

@@ -4,6 +4,7 @@ import time
 from concurrent.futures import ThreadPoolExecutor
 
 import pyarrow as pa
+import pytest
 
 from vastdb.table import QueryConfig
 
@@ -34,6 +35,7 @@ def test_concurrent_query(session, test_bucket_name, schema_name, table_name):
     logger.info(f"finished running {amount_of_queries_in_parallel} queries")
 
 
+@pytest.mark.skip(reason="see https://vastdata.atlassian.net/browse/ORION-319356")
 def test_table_stats(session, test_bucket_name, schema_name, table_name):
     """
     Testing stats integrity while altering table

{vastdb-2.0.5 → vastdb-2.0.7}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.5
+Version: 2.0.7
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA