vastdb 0.1.7__tar.gz → 0.1.9__tar.gz

{vastdb-0.1.7 → vastdb-0.1.9}/CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.1.9] (2024-07-01)
+[0.1.9]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.8...v0.1.9
+
+## Fixed
+ - Don't drop timezone information when creating a table
+ - Clarify the catalog example
+
+## [0.1.8] (2024-06-26)
+[0.1.8]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.7...v0.1.8
+
+## Fixed
+ - Allow retries also during query response parsing
+ - Don't retrieve stats during Table.select() if not needed
+ - Log successful server-side file import
+
 ## [0.1.7] (2024-06-03)
 [0.1.7]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.6...v0.1.7
 

{vastdb-0.1.7/vastdb.egg-info → vastdb-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.7
+Version: 0.1.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.1.7 → vastdb-0.1.9}/README.md

@@ -65,7 +65,7 @@ with session.transaction() as tx:
 
     # run `SELECT * FROM t`
     reader = table.select()  # return a `pyarrow.RecordBatchReader`
-    result = pa.Table.from_batches(reader)  # build an PyArrow Table from the `pyarrow.RecordBatch` objects read from VAST
+    result = reader.read_all()  # build an PyArrow Table from the `pyarrow.RecordBatch` objects read from VAST
     assert result == arrow_table
 
     # the transaction is automatically committed when exiting the context
@@ -176,18 +176,27 @@ batches = snaps[0].schema('schema-name').table('table-name').select()
 [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database) can be queried as a regular table:
 
 ```python
-table = pa.Table.from_batches(tx.catalog.select(['element_type']))
-df = table.to_pandas()
+import pyarrow as pa
+import vastdb
+
+session = vastdb.connect(
+    endpoint='http://vip-pool.v123-xy.VastENG.lab',
+    access=AWS_ACCESS_KEY_ID,
+    secret=AWS_SECRET_ACCESS_KEY)
+
+with session.transaction() as tx:
+    table = tx.catalog().select(['element_type']).read_all()
+    df = table.to_pandas()
 
-total_elements = len(df)
-print(f"Total elements in the catalog: {total_elements}")
+    total_elements = len(df)
+    print(f"Total elements in the catalog: {total_elements}")
 
-file_count = (df['element_type'] == 'FILE').sum()
-print(f"Number of files/objects: {file_count}")
+    file_count = (df['element_type'] == 'FILE').sum()
+    print(f"Number of files/objects: {file_count}")
 
-distinct_elements = df['element_type'].unique()
-print("Distinct element types on the system:")
-print(distinct_elements)
+    distinct_elements = df['element_type'].unique()
+    print("Distinct element types on the system:")
+    print(distinct_elements)
 ```
 
 See the following blog posts for more examples:

{vastdb-0.1.7 → vastdb-0.1.9}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='0.1.7' + suffix,
+    version='0.1.9' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/_internal.py

@@ -722,23 +722,30 @@ def _parse_table_info(obj):
 TableStatsResult = namedtuple("TableStatsResult", ["num_rows", "size_in_bytes", "is_external_rowid_alloc", "endpoints"])
 
 
+_RETRIABLE_EXCEPTIONS = (
+    errors.ConnectionError,  # only if 'may_retry' is True
+    errors.Slowdown,
+)
+
+
 def _backoff_giveup(exc: Exception) -> bool:
+    """Exception types below MUST be part of `_RETRIABLE_EXCEPTIONS` above."""
 
+    _logger.info("Backoff giveup: %r", exc)
     if isinstance(exc, errors.Slowdown):
-        # the server is overloaded, retry later
-        return False
+        return False  # the server is overloaded, don't give up
 
-    if isinstance(exc, requests.exceptions.ConnectionError):
-        if exc.request.method == "GET":
-            # low-level connection issue, it is safe to retry only read-only requests
-            return False
+    if isinstance(exc, errors.ConnectionError):
+        if exc.may_retry:
+            return False  # don't give up of retriable connection errors
 
-    return True  # giveup in case of other exceptions
+    return True  # give up in case of other exceptions
 
 
 @dataclass
 class BackoffConfig:
     wait_gen: Callable = field(default=backoff.expo)
+    max_value: Optional[float] = None  # max duration for a single wait period
     max_tries: int = 10
     max_time: float = 60.0  # in seconds
     backoff_log_level: int = logging.DEBUG
@@ -767,14 +774,15 @@ class VastdbApi:
         self._session.headers['user-agent'] = self.client_sdk_version
 
         backoff_config = backoff_config or BackoffConfig()
-        backoff_decorator = backoff.on_exception(
+        self._backoff_decorator = backoff.on_exception(
             wait_gen=backoff_config.wait_gen,
-            exception=(requests.exceptions.ConnectionError, errors.Slowdown),
+            exception=_RETRIABLE_EXCEPTIONS,
             giveup=_backoff_giveup,
             max_tries=backoff_config.max_tries,
             max_time=backoff_config.max_time,
+            max_value=backoff_config.max_value,  # passed to `backoff_config.wait_gen`
             backoff_log_level=backoff_config.backoff_log_level)
-        self._request = backoff_decorator(self._single_request)
+        self._request = self._backoff_decorator(self._single_request)
 
         if url.port in {80, 443, None}:
             self.aws_host = f'{url.host}'
@@ -812,7 +820,14 @@ class VastdbApi:
         raise NotImplementedError(msg)
 
     def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
-        res = self._session.request(method=method, url=url, **kwargs)
+        _logger.debug("Sending request: %s %s %s", method, url, kwargs)
+        try:
+            res = self._session.request(method=method, url=url, **kwargs)
+        except requests.exceptions.ConnectionError as err:
+            # low-level connection issue, it is safe to retry only read-only requests
+            may_retry = (method == "GET")
+            raise errors.ConnectionError(cause=err, may_retry=may_retry) from err
+
         if not skip_status_check:
             if exc := errors.from_response(res):
                 raise exc  # application-level error
@@ -1380,7 +1395,8 @@ class VastdbApi:
 
         url_params = self._build_query_data_url_params(projection, query_imports_table)
 
-        return self._request(
+        # The retries will be done during SelectSplitState processing:
+        return self._single_request(
             method="GET",
             url=self._url(bucket=bucket, schema=schema, table=table, command="data", url_params=url_params),
             data=params, headers=headers, stream=True)
@@ -1460,6 +1476,10 @@ class VastdbApi:
                 else:
                     _logger.debug("import_data of object name '%s' is in progress. "
                                   "status: %s", chunk_dict['object_name'], chunk_dict['res'])
+                    if chunk_dict['res'] == 'Success':
+                        _logger.info("imported /%s/%s into table=/%s/%s/%s",
+                                     chunk_dict['bucket_name'], chunk_dict['object_name'],
+                                     bucket, schema, table)
             return response
 
         headers = self._fill_common_headers(txid=txid, client_tags=client_tags)
@@ -1792,15 +1812,12 @@ def _iter_query_data_response_columns(fileobj, stream_ids=None):
             yield (stream_id, next_row_id, table)
 
 
-def parse_query_data_response(conn, schema, stream_ids=None, start_row_ids=None, debug=False, parser: Optional[QueryDataParser] = None):
+def parse_query_data_response(conn, schema, stream_ids=None, debug=False, parser: Optional[QueryDataParser] = None):
     """
     Generates pyarrow.Table objects from QueryData API response stream.
 
     A pyarrow.Table is a helper class that combines a Schema with multiple RecordBatches and allows easy data access.
     """
-    if start_row_ids is None:
-        start_row_ids = {}
-
     is_empty_projection = (len(schema) == 0)
     if parser is None:
         parser = QueryDataParser(schema, debug=debug)
@@ -1819,8 +1836,7 @@ def parse_query_data_response(conn, schema, stream_ids=None, start_row_ids=None,
 
             _logger.debug("stream_id=%d rows=%d next_row_id=%d table=%s",
                           stream_id, len(parsed_table), next_row_id, parsed_table)
-            start_row_ids[stream_id] = next_row_id
-            yield parsed_table  # the result of a single "select_rows()" cycle
+            yield stream_id, next_row_id, parsed_table
 
     if states:
         raise EOFError(f'all streams should be done before EOF. {states}')

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/conftest.py

@@ -9,11 +9,15 @@ import vastdb
 
 def pytest_addoption(parser):
     parser.addoption("--tabular-bucket-name", help="Name of the S3 bucket with Tabular enabled", default="vastdb")
-    parser.addoption("--tabular-access-key", help="Access key with Tabular permissions (AWS_ACCESS_KEY_ID)", default=os.environ.get("AWS_ACCESS_KEY_ID", None))
-    parser.addoption("--tabular-secret-key", help="Secret key with Tabular permissions (AWS_SECRET_ACCESS_KEY)", default=os.environ.get("AWS_SECRET_ACCESS_KEY", None))
+    parser.addoption("--tabular-access-key", help="Access key with Tabular permissions (AWS_ACCESS_KEY_ID)",
+                     default=os.environ.get("AWS_ACCESS_KEY_ID", None))
+    parser.addoption("--tabular-secret-key", help="Secret key with Tabular permissions (AWS_SECRET_ACCESS_KEY)",
+                     default=os.environ.get("AWS_SECRET_ACCESS_KEY", None))
     parser.addoption("--tabular-endpoint-url", help="Tabular server endpoint", default="http://localhost:9090")
     parser.addoption("--data-path", help="Data files location", default=None)
     parser.addoption("--crater-path", help="Save benchmark results in a dedicated location", default=None)
+    parser.addoption("--schema-name", help="Name of schema for the test to operate on", default=None)
+    parser.addoption("--table-name", help="Name of table for the test to operate on", default=None)
 
 
 @pytest.fixture(scope="session")
@@ -67,3 +71,13 @@ def parquets_path(request):
 @pytest.fixture(scope="function")
 def crater_path(request):
     return request.config.getoption("--crater-path")
+
+
+@pytest.fixture(scope="function")
+def schema_name(request):
+    return request.config.getoption("--schema-name")
+
+
+@pytest.fixture(scope="function")
+def table_name(request):
+    return request.config.getoption("--table-name")

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/errors.py

@@ -3,6 +3,7 @@ import xml.etree.ElementTree
 from dataclasses import dataclass
 from enum import Enum
 
+import pyarrow as pa
 import requests
 
 
@@ -169,12 +170,23 @@ class NotSupportedCommand(NotSupported):
     table: str
 
 
+@dataclass
+class NotSupportedType(NotSupported):
+    field: pa.Field
+
+
 @dataclass
 class NotSupportedVersion(NotSupported):
     err_msg: str
     version: str
 
 
+@dataclass
+class ConnectionError(Exception):
+    cause: Exception
+    may_retry: bool
+
+
 def handle_unavailable(**kwargs):
     if kwargs['code'] == 'SlowDown':
         raise Slowdown(**kwargs)

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/schema.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
 
 import pyarrow as pa
 
-from . import bucket, errors, schema, table
+from . import bucket, errors, schema, table, util
 
 if TYPE_CHECKING:
     from .table import Table
@@ -86,6 +86,7 @@ class Schema:
         if use_external_row_ids_allocation:
             self.tx._rpc.features.check_external_row_ids_allocation()
 
+        util.check_supported_types(columns)
         self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
                                       use_external_row_ids_allocation=use_external_row_ids_allocation)
         log.info("Created table: %s", table_name)

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/table.py

@@ -7,12 +7,11 @@ import queue
 from dataclasses import dataclass, field
 from math import ceil
 from threading import Event
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 
-import backoff
 import ibis
 import pyarrow as pa
-import requests
+import urllib3
 
 from . import _internal, errors, schema, util
 
@@ -40,12 +39,6 @@ class TableStats:
     endpoints: Tuple[str, ...] = ()
 
 
-RETRIABLE_ERRORS = (
-    errors.Slowdown,
-    requests.exceptions.ConnectionError,
-)
-
-
 @dataclass
 class QueryConfig:
     """Query execution configiration."""
@@ -85,9 +78,6 @@ class QueryConfig:
     # - if unset, the request will be added to the queue's end.
     queue_priority: Optional[int] = None
 
-    # DEPRECATED: will be removed in a future release
-    backoff_func: Any = field(default=backoff.on_exception(backoff.expo, RETRIABLE_ERRORS, max_tries=10))
-
 
 @dataclass
 class ImportConfig:
@@ -107,37 +97,53 @@ class SelectSplitState:
         self.query_data_request = query_data_request
         self.table = table
 
-    def batches(self, api: _internal.VastdbApi):
-        """Execute QueryData request, and yield parsed RecordBatch objects.
+    def process_split(self, api: _internal.VastdbApi, record_batches_queue: queue.Queue[pa.RecordBatch], check_stop: Callable):
+        """Execute a sequence of QueryData requests, and queue the parsed RecordBatch objects.
 
-        Can be called repeatedly, to allow pagination.
+        Can be called repeatedly, to support resuming the query after a disconnection / retriable error.
         """
-        while not self.done:
-            response = api.query_data(
-                            bucket=self.table.bucket.name,
-                            schema=self.table.schema.name,
-                            table=self.table.name,
-                            params=self.query_data_request.serialized,
-                            split=(self.split_id, self.config.num_splits, self.config.num_row_groups_per_sub_split),
-                            num_sub_splits=self.config.num_sub_splits,
-                            response_row_id=False,
-                            txid=self.table.tx.txid,
-                            limit_rows=self.config.limit_rows_per_sub_split,
-                            sub_split_start_row_ids=self.subsplits_state.items(),
-                            schedule_id=self.config.queue_priority,
-                            enable_sorted_projections=self.config.use_semi_sorted_projections,
-                            query_imports_table=self.table._imports_table,
-                            projection=self.config.semi_sorted_projection_name)
-            pages_iter = _internal.parse_query_data_response(
-                conn=response.raw,
-                schema=self.query_data_request.response_schema,
-                start_row_ids=self.subsplits_state,
-                parser=self.query_data_request.response_parser)
-
-            for page in pages_iter:
-                for batch in page.to_batches():
-                    if len(batch) > 0:
-                        yield batch
+        try:
+            # contains RecordBatch parts received from the server, must be re-created in case of a retry
+            while not self.done:
+                # raises if request parsing fails or throttled due to server load, and will be externally retried
+                response = api.query_data(
+                                bucket=self.table.bucket.name,
+                                schema=self.table.schema.name,
+                                table=self.table.name,
+                                params=self.query_data_request.serialized,
+                                split=(self.split_id, self.config.num_splits, self.config.num_row_groups_per_sub_split),
+                                num_sub_splits=self.config.num_sub_splits,
+                                response_row_id=False,
+                                txid=self.table.tx.txid,
+                                limit_rows=self.config.limit_rows_per_sub_split,
+                                sub_split_start_row_ids=self.subsplits_state.items(),
+                                schedule_id=self.config.queue_priority,
+                                enable_sorted_projections=self.config.use_semi_sorted_projections,
+                                query_imports_table=self.table._imports_table,
+                                projection=self.config.semi_sorted_projection_name)
+
+                # can raise during response parsing (e.g. due to disconnections), and will be externally retried
+                # the pagination state is stored in `self.subsplits_state` and must be correct in case of a reconnection
+                # the partial RecordBatch chunks are managed internally in `parse_query_data_response`
+                response_iter = _internal.parse_query_data_response(
+                    conn=response.raw,
+                    schema=self.query_data_request.response_schema,
+                    parser=self.query_data_request.response_parser)
+
+                for stream_id, next_row_id, table_chunk in response_iter:
+                    # in case of I/O error, `response_iter` will be closed and an appropriate exception will be thrown.
+                    self.subsplits_state[stream_id] = next_row_id
+                    # we have parsed a pyarrow.Table successfully, self.subsplits_state is now correctly updated
+                    # if the below loop fails, the query is not retried
+                    for batch in table_chunk.to_batches():
+                        check_stop()  # may raise StoppedException to early-exit the query (without retries)
+                        if batch:
+                            record_batches_queue.put(batch)
+        except urllib3.exceptions.ProtocolError as err:
+            log.warning("Failed parsing QueryData response table=%r split=%s/%s offsets=%s cause=%s",
+                        self.table, self.split_id, self.config.num_splits, self.subsplits_state, err)
+            # since this is a read-only idempotent operation, it is safe to retry
+            raise errors.ConnectionError(cause=err, may_retry=True)
 
     @property
     def done(self):
@@ -321,10 +327,15 @@ class Table:
         if config is None:
             config = QueryConfig()
 
-        # Take a snapshot of enpoints
-        stats = self.get_stats()
-        log.debug("stats: %s", stats)
-        endpoints = stats.endpoints if config.data_endpoints is None else config.data_endpoints
+        # Retrieve snapshots only if needed
+        if config.data_endpoints is None or config.num_splits is None:
+            stats = self.get_stats()
+            log.debug("stats: %s", stats)
+
+        if config.data_endpoints is None:
+            endpoints = stats.endpoints
+        else:
+            endpoints = tuple(config.data_endpoints)
         log.debug("endpoints: %s", endpoints)
 
         if config.num_splits is None:
@@ -382,6 +393,7 @@ class Table:
         def single_endpoint_worker(endpoint: str):
             try:
                 host_api = _internal.VastdbApi(endpoint=endpoint, access_key=self.tx._rpc.api.access_key, secret_key=self.tx._rpc.api.secret_key)
+                backoff_decorator = self.tx._rpc.api._backoff_decorator
                 while True:
                     check_stop()
                     try:
@@ -395,9 +407,9 @@ class Table:
                                                    split_id=split,
                                                    config=config)
 
-                    for batch in split_state.batches(host_api):
-                        check_stop()
-                        record_batches_queue.put(batch)
+                    process_with_retries = backoff_decorator(split_state.process_split)
+                    process_with_retries(host_api, record_batches_queue, check_stop)
+
             except StoppedException:
                 log.debug("stop signal.", exc_info=True)
                 return
@@ -474,6 +486,7 @@ class Table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         try:
             row_ids = []
+            util.check_supported_types(rows.schema)
             serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
             for slice in serialized_slices:
                 res = self.tx._rpc.api.insert_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -516,6 +529,7 @@ class Table:
 
         update_rows_rb = util.sort_record_batch_if_needed(update_rows_rb, INTERNAL_ROW_ID)
 
+        util.check_supported_types(update_rows_rb.schema)
         serialized_slices = util.iter_serialized_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.update_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -560,6 +574,7 @@ class Table:
         """Add a new column."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
+        util.check_supported_types(new_column)
         self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
         log.info("Added column(s): %s", new_column)
         self.arrow_schema = self.columns()

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/tests/test_projections.py

@@ -105,7 +105,11 @@ def test_query_data_with_projection(session, clean_bucket_name):
         t = s.table(table_name)
         projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))
         # no projection supply - need to be with p1 projeciton
-        assert expected_projection_p1 == projection_actual
+        # doing this since we also run this test against production clusters
+        if expected_projection_p1 != projection_actual:
+            config.num_row_groups_per_sub_split = 8
+            projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))
+            assert expected_projection_p1 == projection_actual
 
         config.semi_sorted_projection_name = 'p1'
         projection_actual = pa.Table.from_batches(t.select(columns=['a', 'b', 's'], predicate=(t['b'] < 5), config=config))

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/tests/test_sanity.py

@@ -5,9 +5,8 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 from itertools import cycle
 
 import pytest
-import requests
 
-import vastdb
+import vastdb.errors
 
 log = logging.getLogger(__name__)
 
@@ -26,7 +25,7 @@ def test_bad_credentials(session):
 
 def test_bad_endpoint(session):
     backoff_config = vastdb.session.BackoffConfig(max_tries=3)
-    with pytest.raises(requests.exceptions.ConnectionError):
+    with pytest.raises(vastdb.errors.ConnectionError):
         vastdb.connect(access='BAD', secret='BAD', endpoint='http://invalid-host-name-for-tests:12345', backoff_config=backoff_config)
 
 

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/tests/test_tables.py

@@ -311,6 +311,46 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
+TIMESTAMP_UNITS = ['s', 'ms', 'us', 'ns']
+
+
+def test_unsupported_timezone(session, clean_bucket_name):
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema('s1')
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+            with pytest.raises(errors.NotSupportedType):
+                s.create_table('t1', pa.schema([('ts', col_type)]))
+            assert s.tables() == []
+
+        cols = [('c', pa.int64())]
+        t1 = s.create_table('t1', pa.schema(cols))
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+            with pytest.raises(errors.NotSupportedType):
+                t1.add_column(pa.schema([('ts', col_type)]))
+
+        cols = [(f'c_{unit}', pa.timestamp(unit)) for unit in TIMESTAMP_UNITS]
+        t2 = s.create_table('t2', pa.schema(cols))
+
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+
+            rb = pa.record_batch(
+                data=[[None]],
+                schema=pa.schema([(f'c_{unit}', col_type)]))
+            with pytest.raises(errors.NotSupportedType):
+                t2.insert(rb)
+
+            rb = pa.record_batch(
+                data=[[0], [None]],
+                schema=pa.schema([
+                    (INTERNAL_ROW_ID, pa.uint64()),
+                    (f'c_{unit}', col_type)]))
+            with pytest.raises(errors.NotSupportedType):
+                t2.update(rb)
+
+
 def test_filters(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int32()),

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb/util.py

@@ -6,7 +6,7 @@ import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument, TooWideRow
+from .errors import InvalidArgument, NotSupportedType, TooWideRow
 
 log = logging.getLogger(__name__)
 
@@ -152,3 +152,10 @@ def sort_record_batch_if_needed(record_batch, sort_column):
         return record_batch.sort_by(sort_column)
     else:
         return record_batch
+
+
+def check_supported_types(fields: pa.Schema):
+    for f in fields:
+        if isinstance(f.type, pa.TimestampType):
+            if f.type.tz:
+                raise NotSupportedType(f)

vastdb-0.1.9/vastdb/vast_tests/test_ha.py

@@ -0,0 +1,29 @@
+import logging
+import time
+
+import pyarrow as pa
+import pytest
+
+from vastdb.table import QueryConfig
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.ha
+def test_ha_query(session, test_bucket_name, schema_name, table_name):
+    # runs in parallel to ha scenario
+    times_to_query, records_in_table = 50, 100_000_000
+    arrow_array = pa.array(range(0, records_in_table), type=pa.int64())
+
+    with session.transaction() as tx:
+        config = QueryConfig(num_splits=1, num_sub_splits=1)
+        start = time.time()
+        for i in range(times_to_query):
+            table = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+            s = time.time()
+            res = table.select(config=config).read_all()
+            e = time.time()
+            logger.info(f'query number {i} took {e - s} sec')
+            assert arrow_array == pa.concat_arrays(res.column('a').chunks)
+        end = time.time()
+        logger.info(f'SDK test took {end - start}')

{vastdb-0.1.7 → vastdb-0.1.9/vastdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.7
+Version: 0.1.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.1.7 → vastdb-0.1.9}/vastdb.egg-info/SOURCES.txt

@@ -180,4 +180,6 @@ vastdb/tests/test_sanity.py
 vastdb/tests/test_schemas.py
 vastdb/tests/test_tables.py
 vastdb/tests/test_util.py
-vastdb/tests/util.py
+vastdb/tests/util.py
+vastdb/vast_tests/__init__.py
+vastdb/vast_tests/test_ha.py