vastdb 2.0.7__py3-none-any.whl → 2.0.8__py3-none-any.whl

vastdb/_internal.py

@@ -808,9 +808,9 @@ def _parse_table_info(obj, parse_properties):
     sorting_done = bool(sorting_score_raw >> 63)
 
     vector_index_enabled = obj.VectorIndexEnabled() if hasattr(obj, 'VectorIndexEnabled') else False
-    vector_index_column_name = obj.VectorIndexColumnName().decode() if hasattr(obj, 'VectorIndexColumnName') and obj.VectorIndexColumnName() else None
-    vector_index_distance_metric = obj.VectorIndexDistanceMetric().decode() if hasattr(obj, 'VectorIndexDistanceMetric') and obj.VectorIndexDistanceMetric() else None
-    vector_index_sql_function_name = obj.VectorIndexSqlFunctionName().decode() if hasattr(obj, 'VectorIndexSqlFunctionName') and obj.VectorIndexSqlFunctionName() else None
+    vector_index_column_name = obj.VectorIndexColumnName().decode() if hasattr(obj, 'VectorIndexColumnName') and obj.VectorIndexColumnName() else ""
+    vector_index_distance_metric = obj.VectorIndexDistanceMetric().decode() if hasattr(obj, 'VectorIndexDistanceMetric') and obj.VectorIndexDistanceMetric() else ""
+    vector_index_sql_function_name = obj.VectorIndexSqlFunctionName().decode() if hasattr(obj, 'VectorIndexSqlFunctionName') and obj.VectorIndexSqlFunctionName() else ""
 
     return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions, sorting_key_enabled,
                      sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done,

vastdb/schema.py

@@ -14,7 +14,7 @@ from vastdb.table_metadata import TableMetadata, TableRef, TableType
 
 from . import bucket, errors, schema, table
 from ._ibis_support import validate_ibis_support_schema
-from ._internal import VectorIndexSpec
+from ._internal import VectorIndex, VectorIndexSpec
 
 if TYPE_CHECKING:
     from .table import Table
@@ -178,7 +178,17 @@ def _parse_table_info(table_info, schema: "schema.Schema"):
                    table=table_info.name)
 
     table_type = TableType.Elysium if table_info.sorting_key_enabled else TableType.Regular
-    table_metadata = TableMetadata(ref, table_type=table_type)
+
+    # populate vector_index from list_tables if vector index is enabled
+    vector_index = None
+    if table_info.vector_index_enabled:
+        vector_index = VectorIndex(
+            column=table_info.vector_index_column_name,
+            distance_metric=table_info.vector_index_distance_metric,
+            sql_distance_function=table_info.vector_index_sql_function_name
+        )
+
+    table_metadata = TableMetadata(ref, table_type=table_type, vector_index=vector_index)
 
     return table.Table(handle=int(table_info.handle),
                        metadata=table_metadata,

vastdb/table.py

@@ -824,9 +824,8 @@ class TableInTransaction(ITable):
 
         Both Elysium and Vector Index tables use global row IDs.
         """
-        # check if table has vector index from loaded stats
-        has_vector_index = (self._metadata.stats is not None and
-                           self._metadata.stats.vector_index is not None)
+        # _vector_index is set from list_tables or synced from stats.vector_index by _parse_stats_vector_index()
+        has_vector_index = self._metadata._vector_index is not None
         return self._is_sorted_table or has_vector_index
 
     def vector_search(

vastdb/table_metadata.py

@@ -173,14 +173,14 @@ class TableMetadata:
         self._parse_stats_vector_index()
 
     def _parse_stats_vector_index(self):
-        vector_index_is_set = self._vector_index is not None
+        if self._vector_index is not None and self._stats.vector_index != self._vector_index:
+            is_empty_placeholder = not self._vector_index.column and not self._vector_index.distance_metric
+            if not is_empty_placeholder:
+                raise ValueError(
+                    f"Table has index {self._stats.vector_index}, but was initialized as {self._vector_index}"
+                    )
 
-        if vector_index_is_set and self._stats.vector_index != self._vector_index:
-            raise ValueError(
-                f"Table has index {self._stats.vector_index}, but was initialized as {self._vector_index}"
-                )
-        else:
-            self._vector_index = self._stats.vector_index
+        self._vector_index = self._stats.vector_index
 
     def _set_sorted_table(self, tx: "Transaction"):
         self._table_type = TableType.Elysium

vastdb/tests/test_tables.py

@@ -9,6 +9,7 @@ from contextlib import closing
 from tempfile import NamedTemporaryFile
 
 import ibis
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -390,6 +391,7 @@ def test_types(session, clean_bucket_name):
         ('a2', pa.int16()),
         ('a4', pa.int64()),
         ('b', pa.float32()),
+        ('f16', pa.float16()),
         ('s', pa.string()),
         ('d', pa.decimal128(7, 3)),
         ('bin', pa.binary()),
@@ -410,6 +412,7 @@ def test_types(session, clean_bucket_name):
         [1999, 2000, 2001],
         [11122221, 222111122, 333333],
         [0.5, 1.5, 2.5],
+        [np.float16(0.5), np.float16(1.5), np.float16(2.5)],
         ["a", "v", "s"],
         [decimal.Decimal('110.52'), decimal.Decimal('231.15'), decimal.Decimal('3332.44')],
         [b"\x01\x02", b"\x01\x05", b"\x01\x07"],
@@ -436,6 +439,13 @@ def test_types(session, clean_bucket_name):
             assert select(t['a2'] == 2000) == expected.filter(pc.field('a2') == 2000)
             assert select(t['a4'] == 222111122) == expected.filter(pc.field('a4') == 222111122)
             assert select(t['b'] == 1.5) == expected.filter(pc.field('b') == 1.5)
+
+            # Test float16 predicate (PyArrow compute doesn't support float16, so validate manually)
+            f16_literal = np.float16(1.5)
+            result = select(t['f16'] == f16_literal)
+            assert len(result) == 1, f"Expected 1 row for f16==1.5, got {len(result)}"
+            assert np.float16(result.column('f16')[0].as_py()) == f16_literal
+
             assert select(t['s'] == "v") == expected.filter(pc.field('s') == "v")
             assert select(t['d'] == 231.15) == expected.filter(pc.field('d') == 231.15)
             assert select(t['bin'] == b"\x01\x02") == expected.filter(pc.field('bin') == b"\x01\x02")
@@ -468,6 +478,59 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
+@pytest.mark.parametrize("element_type,test_name", [
+    (pa.float32(), "float32"),
+    (pa.float16(), "float16"),
+])
+def test_vector_types(session, clean_bucket_name, element_type, test_name):
+    """Test vector (fixed-size list) columns with different element types."""
+    vector_dim = 3
+    vec_type = pa.list_(pa.field('', element_type, False), vector_dim)
+
+    columns = pa.schema([
+        ('id', pa.int32()),
+        ('vector', vec_type),
+    ])
+
+    # Create test data based on element type
+    if element_type == pa.float16():
+        test_vectors = [
+            [np.float16(1.0), np.float16(2.0), np.float16(3.0)],
+            [np.float16(4.5), np.float16(5.5), np.float16(6.5)],
+            [np.float16(-1.0), np.float16(0.0), np.float16(1.0)],
+        ]
+    else:  # float32
+        test_vectors = [
+            [1.0, 2.0, 3.0],
+            [4.5, 5.5, 6.5],
+            [-1.0, 0.0, 1.0],
+        ]
+
+    expected = pa.table(schema=columns, data=[
+        [0, 1, 2],
+        test_vectors,
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as table:
+        # Read back and verify
+        actual = table.select().read_all()
+        assert actual.schema == columns
+        assert len(actual) == 3
+
+        # Verify vector data
+        actual_vectors = actual.column('vector').to_pylist()
+        for i, (actual_vec, expected_vec) in enumerate(zip(actual_vectors, test_vectors)):
+            assert len(actual_vec) == vector_dim, f"Wrong vector dimension at row {i}"
+            for j, (act, exp) in enumerate(zip(actual_vec, expected_vec)):
+                if element_type == pa.float16():
+                    assert np.float16(act) == np.float16(exp), \
+                        f"Mismatch at row {i}, element {j}: {act} != {exp}"
+                else:
+                    assert act == exp, f"Mismatch at row {i}, element {j}: {act} != {exp}"
+
+        log.info(f"Vector type test ({test_name}) passed successfully")
+
+
 @pytest.mark.parametrize("arrow_type,internal_support", [
     # Types not supported by Vast.
     (pa.null(), False),

vastdb/tests/test_vector_index.py

@@ -160,3 +160,165 @@ def test_vector_index_metadata_from_stats(session: Session, clean_bucket_name: s
         assert table._metadata._vector_index.distance_metric == "l2sq"
 
         log.info("✓ Vector index metadata correctly retrieved from stats")
+
+
+@pytest.fixture
+def vector_index_test_tables(session: Session, clean_bucket_name: str):
+    """
+    Fixture that creates a schema with 4 test tables:
+    - regular_table: no vector index
+    - vector_table_l2: l2sq distance metric
+    - vector_table_ip: ip distance metric
+    - vector_table_cosine: cosine distance metric
+
+    Returns a tuple of (schema, table_names_dict, expected_metrics) where:
+    - table_names_dict contains the names of all created tables
+    - expected_metrics is a list of (metric_key, column, distance_metric, sql_function) tuples
+    """
+    schema_name = "vector_list_schema"
+    vector_dimension = 5
+
+    with session.transaction() as tx:
+        bucket = tx.bucket(clean_bucket_name)
+        schema = bucket.create_schema(schema_name)
+
+        # Create test tables
+        vec_type = pa.list_(pa.field('', pa.float32(), False), vector_dimension)
+        arrow_schema = pa.schema([
+            ('id', pa.int64()),
+            ('embedding', vec_type)
+        ])
+
+        table_names = {}
+        expected_metrics = []
+
+        # Regular table without vector index
+        table_name = "regular_table"
+        schema.create_table(table_name, arrow_schema)
+        table_names["regular"] = table_name
+        log.info(f"Created regular table: {table_name}")
+
+        # Vector index table with l2sq metric
+        table_name = "vector_table_l2"
+        schema.create_table(table_name, arrow_schema, vector_index=VectorIndexSpec("embedding", "l2sq"))
+        table_names["l2"] = table_name
+        expected_metrics.append(("l2", "embedding", "l2sq", "array_distance"))
+        log.info(f"Created vector index table (l2sq): {table_name}")
+
+        # Vector index table with ip metric
+        table_name = "vector_table_ip"
+        schema.create_table(table_name, arrow_schema, vector_index=VectorIndexSpec("embedding", "ip"))
+        table_names["ip"] = table_name
+        expected_metrics.append(("ip", "embedding", "ip", "array_inner_product"))
+        log.info(f"Created vector index table (ip): {table_name}")
+
+        # Vector index table with cosine metric
+        table_name = "vector_table_cosine"
+        schema.create_table(table_name, arrow_schema, vector_index=VectorIndexSpec("embedding", "cosine"))
+        table_names["cosine"] = table_name
+        expected_metrics.append(("cosine", "embedding", "cosine", "array_cosine_distance"))
+        log.info(f"Created vector index table (cosine): {table_name}")
+
+        yield schema, table_names, expected_metrics
+
+
+def _check_vector_index_metadata(actual_column, actual_metric, actual_sql_func,
+                                 expected_column, expected_metric, expected_sql_func,
+                                 expect_full_metadata: bool,
+                                 table_name: str = "table"):
+    """Pure checker helper to validate vector index metadata values.
+
+    Args:
+        actual_column: Actual column name from result
+        actual_metric: Actual distance metric from result
+        actual_sql_func: Actual SQL function name from result
+        expected_column: Expected column name (when full metadata is present)
+        expected_metric: Expected distance metric (when full metadata is present)
+        expected_sql_func: Expected SQL function name (when full metadata is present)
+        expect_full_metadata: Whether full metadata should be present
+        table_name: Name of table for error messages
+    """
+    # Determine actual expected values based on metadata presence
+    if expect_full_metadata:
+        expected_col = expected_column
+        expected_met = expected_metric
+        expected_sql = expected_sql_func
+    else:
+        # When metadata is not loaded, both Table objects and TableInfo use empty strings
+        expected_col = ""
+        expected_met = ""
+        expected_sql = ""
+
+    assert actual_column == expected_col, \
+        f"Expected column='{expected_col}', got '{actual_column}' for {table_name}"
+    assert actual_metric == expected_met, \
+        f"Expected metric='{expected_met}', got '{actual_metric}' for {table_name}"
+    assert actual_sql_func == expected_sql, \
+        f"Expected sql_func='{expected_sql}', got '{actual_sql_func}' for {table_name}"
+
+
+@pytest.mark.parametrize("test_case,use_public_api,include_metadata,expect_full_metadata", [
+    # Test 1: Public API schema.tables() - returns Table objects with empty metadata
+    ("schema.tables() public API", True, None, False),
+    # Test 2: Internal API with full metadata enabled
+    ("_iter_tables(include_vector_index_metadata=True)", False, True, True),
+    # Test 3: Internal API with metadata disabled - returns flag only
+    ("_iter_tables(include_vector_index_metadata=False)", False, False, False),
+])
+def test_list_tables_vector_index_metadata(session: Session, vector_index_test_tables,
+                                          test_case: str, use_public_api: bool,
+                                          include_metadata: Optional[bool], expect_full_metadata: bool):
+    """
+    Test that list_tables APIs return correct vector index metadata based on parameters.
+
+    Tests three scenarios:
+    1. schema.tables() - public API returns Table objects with vector_index placeholder (empty strings)
+    2. _iter_tables(include_vector_index_metadata=True) - returns full metadata
+    3. _iter_tables(include_vector_index_metadata=False) - returns flag only, no expensive metadata
+    """
+    schema, table_names, expected_metrics = vector_index_test_tables
+
+    log.info(f"Testing: {test_case}")
+
+    # Call the appropriate API and extract vector index info into uniform structure
+    if use_public_api:
+        # Public API returns Table objects with _vector_index
+        results = schema.tables()
+        results_by_name = {t.name: (t, t._metadata._vector_index) for t in results}
+    else:
+        # Internal API returns table info objects with individual vector_index fields
+        # Create a simple object to match _vector_index interface
+        from types import SimpleNamespace
+        results = list(schema._iter_tables(include_vector_index_metadata=include_metadata))
+        results_by_name = {}
+        for t in results:
+            # Create object with same attributes as VectorIndex for uniform access
+            vi = SimpleNamespace(
+                column=t.vector_index_column_name,
+                distance_metric=t.vector_index_distance_metric,
+                sql_distance_function=t.vector_index_sql_function_name,
+            ) if t.vector_index_enabled else None
+            results_by_name[t.name] = (t, vi)
+
+    assert len(results) == 4, f"Expected 4 tables, got {len(results)}"
+
+    # Validate regular table (no vector index)
+    _, regular_vi = results_by_name[table_names["regular"]]
+    assert regular_vi is None, \
+        "Expected vector_index=None for regular table"
+
+    # Validate vector index tables
+    for metric_key, expected_column, expected_metric, expected_sql_func in expected_metrics:
+        _, vi = results_by_name[table_names[metric_key]]
+
+        assert vi is not None, \
+            f"Expected vector_index present for {metric_key}"
+
+        _check_vector_index_metadata(
+            vi.column, vi.distance_metric, vi.sql_distance_function,
+            expected_column, expected_metric, expected_sql_func,
+            expect_full_metadata,
+            table_name=metric_key
+        )
+
+    log.info(f"{test_case} validated successfully")

{vastdb-2.0.7.dist-info → vastdb-2.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.7
+Version: 2.0.8
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
@@ -20,6 +20,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aws-requests-auth
 Requires-Dist: ibis-framework~=10.1
+Requires-Dist: sqlglot<28.7,>=23.4
 Requires-Dist: pyarrow~=18.0
 Requires-Dist: pyarrow-hotfix==0.7
 Requires-Dist: flatbuffers

{vastdb-2.0.7.dist-info → vastdb-2.0.8.dist-info}/RECORD

@@ -1,17 +1,17 @@
 vastdb/__init__.py,sha256=uf-AXdzsD4nPxFP7WxkcAXGG0whv8BHLrrXCJtsPGaQ,436
 vastdb/_adbc.py,sha256=aUjn0hC92U5Jh2jQYeMFzTHYnVP086dVHlMoIoxhvh0,5985
 vastdb/_ibis_support.py,sha256=sJieOMvDWpsciPKh1mJzS56jxLtCRVlvK41hW84vexM,866
-vastdb/_internal.py,sha256=jpgfZFDz12Yo1Lwj6JbhFLWo8PmLCKyppqpc32IBvgM,114719
+vastdb/_internal.py,sha256=-xN2v4nVn8CERbKT2Ub_xDqqy337YjK_d2Dllb8yhEY,114713
 vastdb/_table_interface.py,sha256=dRcXzC7j1SSpBQ4jCOPF_Zgg66LSt0tBeILM8W_A71c,4130
 vastdb/bucket.py,sha256=ulkTI_7xw5FYVzcrTFC7G2ijmTTVSmvJZUdgzycGHR0,2588
 vastdb/config.py,sha256=OehnsWrjzv0-SUouEXmkrKBugiWyhXOn4XiSLV3s9yk,2342
 vastdb/conftest.py,sha256=vDNqhYMsF630_ueXofnT2zhuSEpjcc4hxcXAjPqlxDI,5525
 vastdb/errors.py,sha256=GVmiwyrfHrag8GL55ivcF5Kwc1Ro4wvvaJAcuIWvmO0,5764
 vastdb/features.py,sha256=ivYbvhiGA858B00vhs_CNzlVV9QDUe53yW6V3J5EoxM,1874
-vastdb/schema.py,sha256=84h2volDYlaHMhnm0Cg4-sQrdr2rXoQfa_TKjylAUyY,7900
+vastdb/schema.py,sha256=jYEjddYnWxp0-3UsN3B7Kqthagrfy4NtULRwreaMM2c,8325
 vastdb/session.py,sha256=oUiTx8x_5XOs6EnNvFjon9dSVTk89XhOZ-SfLR8sITQ,2529
-vastdb/table.py,sha256=JyHZ6U1fTg9G4XAAQHC5cWlxb2b6hnYRWcA0kM_03RY,46716
-vastdb/table_metadata.py,sha256=PSwx1A25AmcCs4ig-Sx_xyXr7xqyGaNIk2LMnIvrwmM,7358
+vastdb/table.py,sha256=1fdzD2eNYBTh9PjGrGamHZ9pjtF5aZULzxhyXSD4z04,46698
+vastdb/table_metadata.py,sha256=HWs3s0LP2gx9EemMb5N7OYnd3zdAletX_DubhvtIOEs,7451
 vastdb/transaction.py,sha256=JaTvUw86HZ-qFiqjR1fmHndF5bSgMtVWj549MtGsZ5Q,4734
 vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -53,9 +53,9 @@ vastdb/tests/test_projections.py,sha256=v43WrIdz_Ru4j3VeZ7medRPafzuw1eKMZfbAE56x
 vastdb/tests/test_sanity.py,sha256=bv1ypGDzvOgmMvGbucDYiLQu8krQLlE6NB3M__q87x8,3303
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
 vastdb/tests/test_table_in_tx.py,sha256=-OFGaZMZQc78HOmR23xHIsH7d0VBneFG3tGdDlogceM,9105
-vastdb/tests/test_tables.py,sha256=yU5fBeHentNe22Rc1e73mj33M0nl63Y_acOy6VTzF-4,54496
+vastdb/tests/test_tables.py,sha256=HZBulgWhLh74MSSkOzOnWYiG4cQOi8T7-N8E3IMnIPo,56927
 vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
-vastdb/tests/test_vector_index.py,sha256=qgppSQG77PUgmdXzfprX1mxUkkqq7tbxCA3MRaYFFgQ,6460
+vastdb/tests/test_vector_index.py,sha256=3JEyL2JqTWOACxJ0nihmOfLxX-ICVisCZMK1EfOOu-w,13871
 vastdb/tests/test_vector_search.py,sha256=50mnBFuw0PAApMLkYwDUMV5VxQorswzD_f_XdpaEXTo,7103
 vastdb/tests/util.py,sha256=cv8Qcvy92vpBbiiuqXr7pAfbh2K8ziBVZ_j1EQ4vT6o,5994
 vastdb/vast_flatbuf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -219,8 +219,8 @@ vastdb/vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
 vastdb/vast_tests/test_scale.py,sha256=RZ4K4DQB1N4XOBIfNqp3-aIDeXHIqXN115gQpjSyIXA,4055
-vastdb-2.0.7.dist-info/licenses/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-2.0.7.dist-info/METADATA,sha256=P1r6sOebX-XL4NPzidEDDprrQHp5cIaLJ6iyro5fKBU,1721
-vastdb-2.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-vastdb-2.0.7.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
-vastdb-2.0.7.dist-info/RECORD,,
+vastdb-2.0.8.dist-info/licenses/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-2.0.8.dist-info/METADATA,sha256=YCzUbEjArA0qr6cY1xAuZ-I6BXcShcrrqY6VzMscqIs,1756
+vastdb-2.0.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+vastdb-2.0.8.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
+vastdb-2.0.8.dist-info/RECORD,,

{vastdb-2.0.7.dist-info → vastdb-2.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any