vastdb 2.0.0__tar.gz → 2.0.2__tar.gz

{vastdb-2.0.0 → vastdb-2.0.2}/CHANGELOG.md

@@ -4,7 +4,20 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [2.0.2] (2025-10-22)
+[2.0.2]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.1...v2.0.2
+
+### Fixed
+- Get correct rows count and size in bytes when asking for a table projections
+
+## [2.0.1] (2025-10-22)
+[2.0.1]: https://github.com/vast-data/vastdb_sdk/compare/v2.0.0...v2.0.1
+
+### Fixed
+- Performance improvement in query data flow - sending only required columns instead of entire schema
+
 ## [2.0.0] (2025-09-17)
+[2.0.0]: https://github.com/vast-data/vastdb_sdk/compare/v1.4.0...v2.0.0
 
 ### BREAKING CHANGE
 - Removed properties `Table.bucket` and `Table.schema`
@@ -31,7 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Support Tables with FixedSizeListArray of Numerics
 
-## Fixed
+### Fixed
 - Table insert with no rows
 - Boolean predicates
 - List tables with pagination

{vastdb-2.0.0 → vastdb-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.0
+Version: 2.0.2
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-2.0.0 → vastdb-2.0.2}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.10.0',
     description='VAST Data SDK',
-    version='2.0.0' + suffix,
+    version='2.0.2' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-2.0.0 → vastdb-2.0.2}/vastdb/_internal.py

@@ -17,6 +17,7 @@ import requests
 import urllib3
 import xmltodict
 from aws_requests_auth.aws_auth import AWSRequestsAuth
+from ibis.expr.operations import Field, Node
 from ibis.expr.operations.generic import (
     IsNull,
     Literal,
@@ -34,7 +35,6 @@ from ibis.expr.operations.logical import (
     NotEquals,
     Or,
 )
-from ibis.expr.operations.relations import Field
 from ibis.expr.operations.strings import StartsWith, StringContains
 from ibis.expr.operations.structs import StructField
 
@@ -2350,9 +2350,36 @@ def get_response_schema(schema: 'pa.Schema' = pa.schema([]), field_names: Option
     return pa.schema([schema.field(name) for name in field_names])
 
 
-def build_query_data_request(schema: 'pa.Schema' = pa.schema([]),
+def _column_names_in_node_tree(expr: ibis.expr.types.Expr) -> set[str]:
+    def walk_op(op: Node):
+        if isinstance(op, Field):
+            names.add(op.name)
+        elif isinstance(op, (list, tuple)):
+            for item in op:
+                walk_op(item)
+        else:
+            for arg in getattr(op, "args", ()):
+                walk_op(arg)
+
+    names: set[str] = set()
+    walk_op(expr.op())
+    return names
+
+
+def build_query_data_request(schema: pa.Schema,
                              predicate: ibis.expr.types.BooleanColumn = None,
-                             field_names: Optional[List[str]] = None) -> QueryDataRequest:
+                             field_names: Optional[list[str]] = None) -> QueryDataRequest:
+    if field_names is None:
+        queried_columns = [f.name for f in schema]
+    else:
+        # apparently there are some tests that send a tuple despite the signature asking for a list
+        queried_columns = list(field_names)
+
+    column_names_in_predicate = set() if predicate is None else _column_names_in_node_tree(predicate)
+    column_names_required_by_predicate = column_names_in_predicate - set(queried_columns)
+    queried_columns.extend(column_names_required_by_predicate)
+    schema = pa.schema((schema.field(name) for name in queried_columns))
+
     builder = flatbuffers.Builder(1024)
 
     source_name = builder.CreateString('')  # required
@@ -2374,18 +2401,17 @@ def build_query_data_request(schema: 'pa.Schema' = pa.schema([]),
     parser = QueryDataParser(schema)
     leaves_map = {node.field.name: [leaf.index for leaf in node._iter_leaves()] for node in parser.nodes}
 
-    response_schema = get_response_schema(schema, field_names)
-    field_names = [field.name for field in response_schema]
-
     projection_fields = []
-    for field_name in field_names:
+    for field in schema:
         # TODO: only root-level projection pushdown is supported (i.e. no support for SELECT s.x FROM t)
-        positions = leaves_map[field_name]
+        positions = leaves_map[field.name]
+
         for leaf_position in positions:
             fb_field_index.Start(builder)
             fb_field_index.AddPosition(builder, leaf_position)
             offset = fb_field_index.End(builder)
             projection_fields.append(offset)
+
     fb_source.StartProjectionVector(builder, len(projection_fields))
     for offset in reversed(projection_fields):
         builder.PrependUOffsetTRelative(offset)
@@ -2405,4 +2431,4 @@ def build_query_data_request(schema: 'pa.Schema' = pa.schema([]),
 
     builder.Finish(relation)
 
-    return QueryDataRequest(serialized=builder.Output(), response_schema=response_schema, response_parser=QueryDataParser(response_schema))
+    return QueryDataRequest(serialized=builder.Output(), response_schema=schema, response_parser=QueryDataParser(schema))

{vastdb-2.0.0 → vastdb-2.0.2}/vastdb/table.py

@@ -231,11 +231,11 @@ class TableInTransaction(ITable):
             raise errors.NotSupportedCommand(
                 self.ref.bucket, self.ref.schema, self.ref.table)
 
-    def projection(self, name: str) -> "Projection":
+    def projection(self, name: str, include_stats: bool = True) -> "Projection":
         """Get a specific semi-sorted projection of this table."""
         self._assert_not_imports_table()
 
-        projs = tuple(self.projections(projection_name=name))
+        projs = tuple(self.projections(projection_name=name, include_stats=include_stats))
         if not projs:
             raise errors.MissingProjection(
                 self.ref.bucket, self.ref.schema, self.ref.table, name)
@@ -245,7 +245,7 @@ class TableInTransaction(ITable):
         log.debug("Found projection: %s", projs[0])
         return projs[0]
 
-    def projections(self, projection_name: str = "") -> Iterable["Projection"]:
+    def projections(self, projection_name: str = "", include_stats: bool = True) -> Iterable["Projection"]:
         """List all semi-sorted projections of this table if `projection_name` is empty.
 
         Otherwise, list only the specific projection (if exists).
@@ -260,7 +260,7 @@ class TableInTransaction(ITable):
             _bucket_name, _schema_name, _table_name, curr_projections, next_key, is_truncated, _ = \
                 self._tx._rpc.api.list_projections(
                     bucket=self.ref.bucket, schema=self.ref.schema, table=self.ref.table, next_key=next_key, txid=self._tx.active_txid,
-                    exact_match=exact_match, name_prefix=name_prefix)
+                    exact_match=exact_match, name_prefix=name_prefix, include_list_stats=include_stats)
             if not curr_projections:
                 break
             projections.extend(curr_projections)

{vastdb-2.0.0 → vastdb-2.0.2}/vastdb/tests/test_projections.py

@@ -8,6 +8,9 @@ from vastdb.table import QueryConfig
 log = logging.getLogger(__name__)
 
 
+DELAY_TO_LET_SERVER_UPDATE: int = 3
+
+
 def test_basic_projections(session, clean_bucket_name):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s1')
@@ -94,7 +97,7 @@ def test_query_data_with_projection(session, clean_bucket_name):
         actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's']))
         assert actual == expected
 
-    time.sleep(3)
+    time.sleep(DELAY_TO_LET_SERVER_UPDATE)
 
     with session.transaction() as tx:
         config = QueryConfig()
@@ -123,3 +126,48 @@ def test_query_data_with_projection(session, clean_bucket_name):
 
         t.drop()
         s.drop()
+
+
+def test_projection_stats(session, clean_bucket_name):
+    columns = pa.schema([
+        ('a', pa.int64()),
+        ('b', pa.int64()),
+    ])
+
+    # min size to be considered as a projection
+    GROUP_SIZE = 64 * 1024
+    expected = pa.table(schema=columns, data=[
+        [i for i in range(GROUP_SIZE)],
+        [i for i in reversed(range(GROUP_SIZE))],
+    ])
+
+    schema_name = "schema"
+    table_name = "table"
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema(schema_name)
+        t = s.create_table(table_name, expected.schema)
+
+        sorted_columns = ['b']
+        unsorted_columns = ['a']
+        t.create_projection('p1', sorted_columns, unsorted_columns)
+
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).schema(schema_name)
+        t = s.table(table_name)
+        t.insert(expected)
+        actual = pa.Table.from_batches(t.select(columns=['a', 'b']))
+        assert actual == expected
+
+    time.sleep(DELAY_TO_LET_SERVER_UPDATE)
+
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).schema(schema_name)
+        t = s.table(table_name)
+        projections = t.projections()
+        assert len(projections) == 1
+        stats = projections[0].stats
+        assert stats.num_rows == GROUP_SIZE
+        assert stats.size_in_bytes > 0
+
+        t.drop()
+        s.drop()

{vastdb-2.0.0 → vastdb-2.0.2}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vastdb
-Version: 2.0.0
+Version: 2.0.2
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA