vastdb 1.3.8__tar.gz → 1.3.10__tar.gz

{vastdb-1.3.8 → vastdb-1.3.10}/CHANGELOG.md

@@ -4,6 +4,30 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.3.10] (2025-05-11)
+[1.3.10]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.9...v1.3.10
+
+### Added
+  - Support limit on query data
+  - Support for timestamp with timezone
+
+## Fixed
+  - Don't modify the user-passed config
+  - Name the feature logger
+
+
+## [1.3.9] (2025-04-22)
+[1.3.9]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.8...v1.3.9
+
+### Fixed
+  - Elysium-check validation
+  - Fix logging level
+
+### Added
+  - Support elysium new stats
+  - Exposing `sorting_done` flag for Elysium tables (for other tables, `sorting_done()` will always return false)
+
+
 ## [1.3.8] (2025-03-31)
 [1.3.8]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.7...v1.3.8
 

{vastdb-1.3.8 → vastdb-1.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.8
+Version: 1.3.10
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.3.8 → vastdb-1.3.10}/README.md

@@ -2,14 +2,18 @@
 
 # VAST DB Python SDK
 
-## Introduction
+A Python SDK for seamless interaction with [VAST Database](https://vastdata.com/database) and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database). Enables powerful data operations including:
 
-`vastdb` is a Python-based SDK designed for interacting with a [VAST Database](https://vastdata.com/database) and the [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database), enabling schema and table management, efficient ingest, query, and modification of columnar data. 
-
-For more details about the VAST Database, see [this whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
+- Schema and table management
+- Efficient data ingest and querying
+- Advanced filtering with predicate pushdown
+- Direct integration with PyArrow and DuckDB
+- File system querying through VAST Catalog
 
 [![vastdb](docs/vastdb.png)](https://vastdata.com/database)
 
+For technical details about VAST Database architecture, see the [whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
+
 ## Getting Started
 
 ### Requirements
@@ -217,4 +221,4 @@ See these blog posts for more examples:
 - https://vastdata.com/blog/the-vast-catalog-in-action-part-1
 - https://vastdata.com/blog/the-vast-catalog-in-action-part-2
 
-See also the [full Vast DB Python SDK documentation](https://vastdb-sdk.readthedocs.io/en/v1.1.0/)
+See also the [full Vast DB Python SDK documentation](https://vastdb-sdk.readthedocs.io/en/latest/)

{vastdb-1.3.8 → vastdb-1.3.10}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='1.3.8' + suffix,
+    version='1.3.10' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-1.3.8 → vastdb-1.3.10}/vastdb/__init__.py

@@ -13,5 +13,5 @@ def connect(*args, **kwargs):  # noqa: D103
 
 def version():
     """Return VAST DB SDK version."""
-    import importlib
-    return importlib.metadata.distribution(__package__).version
+    from importlib import metadata
+    return metadata.distribution(__package__).version

{vastdb-1.3.8 → vastdb-1.3.10}/vastdb/_internal.py

@@ -794,7 +794,7 @@ def _decode_table_props(s):
     return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
 
-TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled')
+TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions sorting_key_enabled sorting_score write_amplification acummulative_row_insertion_count sorting_done')
 
 
 def _parse_table_info(obj, parse_properties):
@@ -806,13 +806,20 @@ def _parse_table_info(obj, parse_properties):
     num_partitions = obj.NumPartitions()
     properties = parse_properties(properties)
     sorting_key_enabled = obj.SortingKeyEnabled()
-    return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions, sorting_key_enabled)
+    sorting_score_raw = obj.SortingScore()
+    write_amplification = obj.WriteAmplification()
+    acummulative_row_insertion_count = obj.AcummulativeRowInseritionCount()
+
+    sorting_score = sorting_score_raw & ((1 << 63) - 1)
+    sorting_done = bool(sorting_score_raw >> 63)
+    return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions, sorting_key_enabled,
+                     sorting_score, write_amplification, acummulative_row_insertion_count, sorting_done)
 
 
 # Results that returns from tablestats
 
 
-TableStatsResult = namedtuple("TableStatsResult", ["num_rows", "size_in_bytes", "is_external_rowid_alloc", "endpoints"])
+TableStatsResult = namedtuple("TableStatsResult", 'num_rows size_in_bytes is_external_rowid_alloc sorting_key_enabled sorting_score write_amplification acummulative_row_inserition_count sorting_done endpoints')
 
 
 _RETRIABLE_EXCEPTIONS = (
@@ -1213,8 +1220,16 @@ class VastdbApi:
         num_rows = stats.NumRows()
         size_in_bytes = stats.SizeInBytes()
         is_external_rowid_alloc = stats.IsExternalRowidAlloc()
+        sorting_key_enabled = stats.SortingKeyEnabled()
+        sorting_score_raw = stats.SortingScore()
+        write_amplification = stats.WriteAmplification()
+        acummulative_row_inserition_count = stats.AcummulativeRowInseritionCount()
+
+        sorting_score = sorting_score_raw & ((1 << 63) - 1)
+        sorting_done = bool(sorting_score_raw >> 63)
+
         endpoints = [self.url]  # we cannot replace the host by a VIP address in HTTPS-based URLs
-        return TableStatsResult(num_rows, size_in_bytes, is_external_rowid_alloc, tuple(endpoints))
+        return TableStatsResult(num_rows, size_in_bytes, is_external_rowid_alloc, sorting_key_enabled, sorting_score, write_amplification, acummulative_row_inserition_count, sorting_done, tuple(endpoints))
 
     def alter_topic(self, bucket, name,
                     new_name="", expected_retvals=[],
@@ -1302,8 +1317,8 @@ class VastdbApi:
                                           expected_retvals=expected_retvals,
                                           include_list_stats=include_list_stats, count_only=count_only)
 
-    def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
-                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+    def _list_tables_raw(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
+                         exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
         """
         GET /mybucket/schema_path?table HTTP/1.1
         tabular-txid: TransactionId
@@ -1323,7 +1338,6 @@ class VastdbApi:
         headers['tabular-list-count-only'] = str(count_only)
         headers['tabular-include-list-stats'] = str(include_list_stats)
 
-        tables = []
         res = self._request(
             method="GET",
             url=self._url(bucket=bucket, schema=schema, command="table"),
@@ -1333,17 +1347,36 @@ class VastdbApi:
         next_key = int(res_headers['tabular-next-key'])
         is_truncated = res_headers['tabular-is-truncated'] == 'true'
         lists = list_tables.GetRootAs(res.content)
+        tables_length = lists.TablesLength()
+        count = int(res_headers['tabular-list-count']) if 'tabular-list-count' in res_headers else tables_length
+        return lists, is_truncated, count
+
+    def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
+                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+        tables = []
+        lists, is_truncated, count = self._list_tables_raw(bucket, schema, txid=txid, client_tags=client_tags, max_keys=max_keys,
+                                 next_key=next_key, name_prefix=name_prefix, exact_match=exact_match, expected_retvals=expected_retvals,
+                                 include_list_stats=include_list_stats, count_only=count_only)
         bucket_name = lists.BucketName().decode()
         schema_name = lists.SchemaName().decode()
         if not bucket.startswith(bucket_name):  # ignore snapshot name
             raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
         tables_length = lists.TablesLength()
-        count = int(res_headers['tabular-list-count']) if 'tabular-list-count' in res_headers else tables_length
         for i in range(tables_length):
             tables.append(_parse_table_info(lists.Tables(i), parse_properties))
 
         return bucket_name, schema_name, tables, next_key, is_truncated, count
 
+    def raw_sorting_score(self, bucket, schema, txid, name):
+        lists, _, _ = self._list_tables_raw(bucket, schema, txid=txid, exact_match=True, name_prefix=name, include_list_stats=True)
+        bucket_name = lists.BucketName().decode()
+        if not bucket.startswith(bucket_name):  # ignore snapshot name
+            raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
+        tables_length = lists.TablesLength()
+        if tables_length != 1:
+            raise ValueError(f'table: {name} received {tables_length} response')
+        return lists.Tables(0).SortingScore()
+
     def add_columns(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[]):
         """
         Add a column to table, use the following request

vastdb-1.3.10/vastdb/bench/test_perf.py

@@ -0,0 +1,87 @@
+import datetime as dt
+import logging
+import time
+
+import pytest
+
+from vastdb import util
+from vastdb.table import ImportConfig, QueryConfig
+from vastdb.tests.util import compare_pyarrow_tables
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.benchmark
+def test_bench(session, test_bucket_name, parquets_path, crater_path):
+    files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
+    stats = None
+
+    with session.transaction() as tx:
+        b = tx.bucket(test_bucket_name)
+        s = b.create_schema('s1')
+        util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
+        t2 = util.create_table_from_files(s, 't2', files, config=ImportConfig(import_concurrency=8))
+        # Enabling Elysium with 4 sorting keys - ts, sid, ask_open, ask_close
+        t2.add_sorting_key([2, 0, 3, 4])
+        stats = t2.get_stats()
+        log.info("Added sorting keys")
+
+    assert stats
+    # Waiting up to 2 hours for sorting to complete.
+    start_time = time.time()
+    while not stats.sorting_done:
+        if time.time() - start_time > 7200:
+            raise TimeoutError("Sorting did not complete after waiting for 2 hours.")
+        time.sleep(30)
+        with session.transaction() as tx:
+            table = tx.bucket(test_bucket_name).schema('s1').table('t2')
+            stats = table.get_stats()
+    log.info("Sorting completed")
+
+    queries = [
+        {'query_str': "select sid from {t} where sid = 10033007".format, 'columns': ['sid'],
+         'predicate': lambda t: t['sid'] == 10033007},
+        {'query_str': "select last_trade_price from {t} where ts between "
+                      "TIMESTAMP'2018-01-04 20:30:00' AND TIMESTAMP'2018-01-05 20:30:00'".format,
+         'columns': ['last_trade_price'], 'predicate': lambda t: (t['ts'].between(
+            dt.datetime(2018, 1, 4, 20, 30, 00, 00), dt.datetime(2018, 1, 5, 20, 30, 00, 00)))},
+        {'query_str': "select ts,ask_close,ask_open from {t} where bid_qty = 684000 and ask_close > 1".format,
+         'columns': ['ts', 'ask_close', 'ask_open'],
+         'predicate': lambda t: ((t['bid_qty'] == 684000) & (t['ask_close'] > 1))},
+        {'query_str': "select ts,ticker from {t} where "
+                      "ask_open between 4374 and 4375 OR ask_open between 380 and 381".format,
+         'columns': ['ts', 'ticker'],
+         'predicate': lambda t: ((t['ask_open'].between(4374, 4375)) | (t['ask_open'].between(380, 381)))},
+        {
+         'query_str': "select trade_close, trade_high, trade_low, trade_open from {t} where ticker in ('BANR', 'KELYB')".format,
+         'columns': ['trade_close', 'trade_high', 'trade_low', 'trade_open'],
+         'predicate': lambda t: (t['ticker'].isin(['BANR', 'KELYB']))}
+    ]
+
+    log.info("Starting to run queries")
+    with session.transaction() as tx:
+        schema = tx.bucket(test_bucket_name).schema('s1')
+        t1 = schema.table("t1")
+        t2 = schema.table("t2")
+
+        config = QueryConfig(num_splits=8, num_sub_splits=4)
+
+        for q in queries:
+            normal_table_res, els_table_res = None, None
+            for table in [t1, t2]:
+                log.info("Starting query: %s", q['query_str'](t=table.name))
+                s = time.time()
+                res = table.select(columns=q['columns'], predicate=q['predicate'](table), config=config).read_all()
+                e = time.time()
+                if table == t1:
+                    normal_table_res = res
+                else:
+                    els_table_res = res
+                log.info("Query %s returned in %s seconds.", q['query_str'](t=table.name), e - s)
+                if crater_path:
+                    with open(f'{crater_path}/bench_results', 'a') as f:
+                        f.write(f"Query '{q['query_str'](t=table)}' returned in {e - s} seconds")
+
+            assert normal_table_res, f"missing result for {t1} table"
+            assert els_table_res, f"missing result for {t2} table"
+            assert compare_pyarrow_tables(normal_table_res, els_table_res)

{vastdb-1.3.8 → vastdb-1.3.10}/vastdb/features.py

@@ -4,7 +4,7 @@ import logging
 
 from .errors import NotSupportedVersion
 
-log = logging.getLogger()
+log = logging.getLogger(__name__)
 
 
 class Features:
@@ -33,12 +33,16 @@ class Features:
 
         self.check_elysium = self._check(
             "Elysium requires 5.3.5+ VAST release",
-            vast_version >= (5, 3, 5))
+            vast_version >= (5, 3))  # TODO: make this validation stricter for v5.4 (beta/poc version is 5.3.0.x)
 
         self.check_zip_import = self._check(
             "Zip import requires 5.3.1+ VAST release",
             vast_version >= (5, 3, 1))
 
+        self.check_timezone = self._check(
+            "Timezone support requires 5.4+ Vast release",
+            vast_version >= (5, 4))
+
     def _check(self, msg, supported):
         log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
         if not supported:

{vastdb-1.3.8 → vastdb-1.3.10}/vastdb/table.py

@@ -1,9 +1,11 @@
 """VAST Database table."""
 
 import concurrent.futures
+import copy
 import logging
 import os
 import queue
+import sys
 from dataclasses import dataclass, field
 from math import ceil
 from threading import Event
@@ -29,6 +31,7 @@ MAX_ROWS_PER_BATCH = 512 * 1024
 MAX_INSERT_ROWS_PER_PATCH = 512 * 1024
 # in case insert has TooWideRow - need to insert in smaller batches - each cell could contain up to 128K, and our wire is limited to 5MB
 MAX_COLUMN_IN_BATCH = int(5 * 1024 / 128)
+SORTING_SCORE_BITS = 63
 
 
 @dataclass
@@ -37,7 +40,12 @@ class TableStats:
 
     num_rows: int
     size_in_bytes: int
+    sorting_score: int
+    write_amplification: int
+    acummulative_row_inserition_count: int
     is_external_rowid_alloc: bool = False
+    sorting_key_enabled: bool = False
+    sorting_done: bool = False
     endpoints: Tuple[str, ...] = ()
 
 
@@ -277,8 +285,8 @@ class Table:
                         except queue.Empty:
                             pass
                         if files_batch:
-                            log.debug("Starting import batch of %s files", len(files_batch))
-                            log.info(f"starting import of {files_batch}")
+                            log.info("Starting import batch of %s files", len(files_batch))
+                            log.debug(f"starting import of {files_batch}")
                             session.import_data(
                                 self.bucket.name, self.schema.name, self.name, files_batch, txid=self.tx.txid,
                                 key_names=key_names)
@@ -327,7 +335,8 @@ class Table:
                predicate: Union[ibis.expr.types.BooleanColumn, ibis.common.deferred.Deferred] = None,
                config: Optional[QueryConfig] = None,
                *,
-               internal_row_id: bool = False) -> pa.RecordBatchReader:
+               internal_row_id: bool = False,
+               limit_rows: Optional[int] = None) -> pa.RecordBatchReader:
         """Execute a query over this table.
 
         To read a subset of the columns, specify their names via `columns` argument. Otherwise, all columns will be read.
@@ -336,8 +345,10 @@ class Table:
 
         Query-execution configuration options can be specified via the optional `config` argument.
         """
-        if config is None:
-            config = QueryConfig()
+        config = copy.deepcopy(config) if config else QueryConfig()
+
+        if limit_rows:
+            config.limit_rows_per_sub_split = limit_rows
 
         stats = None
         # Retrieve snapshots only if needed
@@ -372,7 +383,7 @@ class Table:
             num_rows = 0
             if self.sorted_table:
                 num_rows = self._get_row_estimate(columns, predicate, query_schema)
-                log.info(f'sorted estimate: {num_rows}')
+                log.debug(f'sorted estimate: {num_rows}')
             if num_rows == 0:
                 if stats is None:
                     stats = self.get_stats()
@@ -396,7 +407,7 @@ class Table:
         for split in range(config.num_splits):
             splits_queue.put(split)
 
-        # this queue shouldn't be large it is marely a pipe through which the results
+        # this queue shouldn't be large it is merely a pipe through which the results
         # are sent to the main thread. Most of the pages actually held in the
         # threads that fetch the pages.
         record_batches_queue: queue.Queue[pa.RecordBatch] = queue.Queue(maxsize=2)
@@ -452,6 +463,7 @@ class Table:
             if config.query_id:
                 threads_prefix = threads_prefix + "-" + config.query_id
 
+            total_num_rows = limit_rows if limit_rows else sys.maxsize
             with concurrent.futures.ThreadPoolExecutor(max_workers=len(endpoints), thread_name_prefix=threads_prefix) as tp:  # TODO: concurrency == enpoints is just a heuristic
                 futures = [tp.submit(single_endpoint_worker, endpoint) for endpoint in endpoints]
                 tasks_running = len(futures)
@@ -461,7 +473,14 @@ class Table:
 
                         batch = record_batches_queue.get()
                         if batch is not None:
-                            yield batch
+                            if batch.num_rows < total_num_rows:
+                                yield batch
+                                total_num_rows -= batch.num_rows
+                            else:
+                                yield batch.slice(length=total_num_rows)
+                                log.info("reached limit rows per query: %d - stop query", limit_rows)
+                                stop_event.set()
+                                break
                         else:
                             tasks_running -= 1
                             log.debug("one worker thread finished, remaining: %d", tasks_running)
@@ -590,7 +609,7 @@ class Table:
         self.name = new_name
 
     def add_sorting_key(self, sorting_key: list) -> None:
-        """Ads a sorting key to a table that doesn't have any."""
+        """Add a sorting key to a table that doesn't have any."""
         self.tx._rpc.features.check_elysium()
         self.tx._rpc.api.alter_table(
             self.bucket.name, self.schema.name, self.name, txid=self.tx.txid, sorting_key=sorting_key)
@@ -653,6 +672,20 @@ class Table:
         """
         return self._ibis_table[col_name]
 
+    def sorting_done(self) -> int:
+        """Sorting done indicator for the table.  Always False for unsorted tables."""
+        if not self.sorted_table:
+            return False
+        raw_sorting_score = self.tx._rpc.api.raw_sorting_score(self.schema.bucket.name, self.schema.name, self.schema.tx.txid, self.name)
+        return bool(raw_sorting_score >> SORTING_SCORE_BITS)
+
+    def sorting_score(self) -> int:
+        """Sorting score for the table.  Always 0 for unsorted tables."""
+        if not self.sorted_table:
+            return 0
+        raw_sorting_score = self.tx._rpc.api.raw_sorting_score(self.schema.bucket.name, self.schema.name, self.schema.tx.txid, self.name)
+        return raw_sorting_score & ((1 << SORTING_SCORE_BITS) - 1)
+
 
 @dataclass
 class Projection:
@@ -710,7 +743,8 @@ class Projection:
 
 def _parse_projection_info(projection_info, table: "Table"):
     log.info("Projection info %s", str(projection_info))
-    stats = TableStats(num_rows=projection_info.num_rows, size_in_bytes=projection_info.size_in_bytes)
+    stats = TableStats(num_rows=projection_info.num_rows, size_in_bytes=projection_info.size_in_bytes,
+                       sorting_score=0, write_amplification=0, acummulative_row_inserition_count=0)
     return Projection(name=projection_info.name, table=table, stats=stats, handle=int(projection_info.handle))