vastdb 0.1.10__py3-none-any.whl → 1.0.0__py3-none-any.whl

vastdb/__init__.py

@@ -1,9 +1,6 @@
 """VAST Database Python SDK."""
 
 import functools
-import importlib.metadata
-
-__version__ = importlib.metadata.distribution(__package__).version
 
 from . import session
 
@@ -12,3 +9,9 @@ from . import session
 @functools.wraps(session.Session)
 def connect(*args, **kwargs):  # noqa: D103
     return session.Session(*args, **kwargs)
+
+
+def version():
+    """Return VAST DB SDK version."""
+    import importlib
+    return importlib.metadata.distribution(__package__).version

vastdb/_internal.py

@@ -5,9 +5,8 @@ import re
 import struct
 import urllib.parse
 from collections import defaultdict, namedtuple
-from dataclasses import dataclass, field
 from enum import Enum
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import backoff
 import flatbuffers
@@ -104,6 +103,7 @@ from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list
 from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
 
 from . import errors
+from .config import BackoffConfig
 
 UINT64_MAX = 18446744073709551615
 
@@ -742,46 +742,38 @@ def _backoff_giveup(exc: Exception) -> bool:
     return True  # give up in case of other exceptions
 
 
-@dataclass
-class BackoffConfig:
-    wait_gen: Callable = field(default=backoff.expo)
-    max_value: Optional[float] = None  # max duration for a single wait period
-    max_tries: int = 10
-    max_time: float = 60.0  # in seconds
-    backoff_log_level: int = logging.DEBUG
-
-
 class VastdbApi:
     # we expect the vast version to be <major>.<minor>.<patch>.<protocol>
     VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
 
     def __init__(self, endpoint, access_key, secret_key,
             *,
-            auth_type=AuthType.SIGV4,
             ssl_verify=True,
+            timeout=None,
             backoff_config: Optional[BackoffConfig] = None):
 
-        from . import __version__  # import lazily here (to avoid circular dependencies)
-        self.client_sdk_version = f"VAST Database Python SDK {__version__} - 2024 (c)"
+        from . import version  # import lazily here (to avoid circular dependencies)
+        self.client_sdk_version = f"VAST Database Python SDK {version()} - 2024 (c)"
 
         url = urllib3.util.parse_url(endpoint)
         self.access_key = access_key
         self.secret_key = secret_key
 
+        self.timeout = timeout
         self.default_max_list_columns_page_size = 1000
         self._session = requests.Session()
         self._session.verify = ssl_verify
         self._session.headers['user-agent'] = self.client_sdk_version
 
-        backoff_config = backoff_config or BackoffConfig()
+        self.backoff_config = backoff_config or BackoffConfig()
         self._backoff_decorator = backoff.on_exception(
-            wait_gen=backoff_config.wait_gen,
+            wait_gen=self.backoff_config.wait_gen,
             exception=_RETRIABLE_EXCEPTIONS,
             giveup=_backoff_giveup,
-            max_tries=backoff_config.max_tries,
-            max_time=backoff_config.max_time,
-            max_value=backoff_config.max_value,  # passed to `backoff_config.wait_gen`
-            backoff_log_level=backoff_config.backoff_log_level)
+            max_tries=self.backoff_config.max_tries,
+            max_time=self.backoff_config.max_time,
+            max_value=self.backoff_config.max_value,  # passed to `self.backoff_config.wait_gen`
+            backoff_log_level=self.backoff_config.backoff_log_level)
         self._request = self._backoff_decorator(self._single_request)
 
         if url.port in {80, 443, None}:
@@ -819,10 +811,18 @@ class VastdbApi:
         _logger.critical(msg)
         raise NotImplementedError(msg)
 
+    def with_endpoint(self, endpoint):
+        return VastdbApi(endpoint=endpoint,
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            ssl_verify=self._session.verify,
+            timeout=self.timeout,
+            backoff_config=self.backoff_config)
+
     def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
-        _logger.debug("Sending request: %s %s %s", method, url, kwargs)
+        _logger.debug("Sending request: %s %s %s timeout=%s", method, url, kwargs, self.timeout)
         try:
-            res = self._session.request(method=method, url=url, **kwargs)
+            res = self._session.request(method=method, url=url, timeout=self.timeout, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # low-level connection issue, it is safe to retry only read-only requests
             may_retry = (method == "GET")

vastdb/bench/test_perf.py

@@ -10,11 +10,11 @@ log = logging.getLogger(__name__)
 
 
 @pytest.mark.benchmark
-def test_bench(session, clean_bucket_name, parquets_path, crater_path):
+def test_bench(session, bucket_name, parquets_path, crater_path):
     files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
 
     with session.transaction() as tx:
-        b = tx.bucket(clean_bucket_name)
+        b = tx.bucket(bucket_name)
         s = b.create_schema('s1')
         t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
         config = QueryConfig(num_splits=8, num_sub_splits=4)

vastdb/bench/test_sample.py

@@ -22,7 +22,8 @@ logging.basicConfig(
 
 log = logging.getLogger()
 
-log.info("Python SDK version: %s", vastdb.util.version())
+sdk_version = vastdb.version()
+log.info("Python SDK version: %s", sdk_version)
 
 NUM_COLUMNS = 10_000
 COLUMNS_BATCH = 10
@@ -63,6 +64,8 @@ def load_batch(bucket, session_kwargs, offset, limit):
             # skip already loaded rows
             log.info('skipping [%d..%d)', offset, limit)
 
+        pid = os.getpid()
+        tid = threading.get_native_id()
         total_nbytes = 0
         calls = 0
         t0 = time.time()
@@ -89,7 +92,7 @@ def load_batch(bucket, session_kwargs, offset, limit):
             metrics_rows.append(metrics.Row(
                 start=start, finish=finish, table_path=table.path, op=op,
                 nbytes=nbytes, rows=len(chunk), cols=len(cols_batch),
-                pid=os.getpid(), tid=threading.get_native_id()))
+                pid=pid, tid=tid, sdk_version=sdk_version))
 
             total_nbytes += nbytes
             calls += 1
@@ -104,12 +107,12 @@ def load_batch(bucket, session_kwargs, offset, limit):
     return metrics_rows
 
 
-def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
+def test_ingest(test_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
     session = vastdb.connect(**session_kwargs)
     metrics_table = metrics.Table(perf_metrics_db, "ingest")
 
     with session.transaction() as tx:
-        b = tx.bucket(clean_bucket_name)
+        b = tx.bucket(test_bucket_name)
         try:
             s = b.schema(SCHEMA)
         except vastdb.errors.MissingSchema:
@@ -127,7 +130,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
 
     with ProcessPoolExecutor(max_workers=num_workers) as executor:
         futures = [
-            executor.submit(load_batch, clean_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
+            executor.submit(load_batch, test_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
             for (offset, limit), url in zip(ranges, itertools.cycle(tabular_endpoint_urls))
         ]
         log.info("spawned %d futures", len(futures))
@@ -135,7 +138,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
             metrics_table.insert(future.result())
 
     with session.transaction() as tx:
-        t = tx.bucket(clean_bucket_name).schema(SCHEMA).table(TABLE)
+        t = tx.bucket(test_bucket_name).schema(SCHEMA).table(TABLE)
         count = sum(len(rb) for rb in t.select([]))
         log.info("%s has %d rows: %s", t, count, t.stats)
 
@@ -154,6 +157,10 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
     r = random.Random(i)
     r.shuffle(row_group_indices)
 
+    pid = os.getpid()
+    tid = threading.get_native_id()
+    metrics_rows = []
+
     session = vastdb.connect(**(session_kwargs | {"endpoint": endpoint_url}))
     with session.transaction() as tx:
         t = tx.bucket(bucket_name).schema(SCHEMA).table(TABLE)
@@ -174,23 +181,37 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
 
         for j, pred in enumerate(preds):
             log.info("%d) starting query #%d on %s", i, j, endpoint_url)
-            t0 = time.time()
+
+            start = time.perf_counter()
             res = t.select(columns=cols, predicate=pred, config=config)
             rows = 0
             data = 0
             for rb in res:
                 rows += len(rb)
                 data += rb.nbytes
-                dt = time.time() - t0
+                dt = time.perf_counter() - start
                 log.info("%d) got query #%d batch %.3f[s], %.3f[GB] %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
 
-            dt = time.time() - t0
+            finish = time.perf_counter()
+            dt = finish - start
             log.info("%d) finished query #%d %.3f[s], %.3f[GB], %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
 
+            metrics_rows.append(metrics.Row(
+                start=start, finish=finish, table_path=t.path, op="select",
+                nbytes=data, rows=rows, cols=len(cols),
+                pid=pid, tid=tid, sdk_version=sdk_version))
+
+
+def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls, perf_metrics_db):
+    metrics_table = metrics.Table(perf_metrics_db, "query")
 
-def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls):
     log.info("starting %d workers, endpoints=%s", num_workers, tabular_endpoint_urls)
     with ProcessPoolExecutor(max_workers=num_workers) as executor:
-        for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls)):
+        futures = [
             executor.submit(run_query, session_kwargs, i, test_bucket_name, url)
+            for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls))
+        ]
+        for future in as_completed(futures):
+            metrics_table.insert(future.result())
+
     log.info("finished %d workers", num_workers)

vastdb/config.py

@@ -0,0 +1,65 @@
+"""Configuration-related dataclasses."""
+
+import logging
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional
+
+import backoff
+
+
+@dataclass
+class BackoffConfig:
+    """Retry configuration."""
+
+    wait_gen: Callable = field(default=backoff.expo)
+    max_value: Optional[float] = None  # max duration for a single wait period
+    max_tries: int = 10
+    max_time: float = 60.0  # in seconds
+    backoff_log_level: int = logging.DEBUG
+
+
+@dataclass
+class QueryConfig:
+    """Query execution configiration."""
+
+    # allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
+    num_sub_splits: int = 4
+
+    # used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
+    # will be estimated from the table's row count, if not explicitly set
+    num_splits: Optional[int] = None
+
+    # each endpoint will be handled by a separate worker thread
+    # a single endpoint can be specified more than once to benefit from multithreaded execution
+    data_endpoints: Optional[List[str]] = None
+
+    # a subsplit fiber will finish after sending this number of rows back to the client
+    limit_rows_per_sub_split: int = 128 * 1024
+
+    # each fiber will read the following number of rowgroups coninuously before skipping
+    # in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
+    num_row_groups_per_sub_split: int = 8
+
+    # can be disabled for benchmarking purposes
+    use_semi_sorted_projections: bool = True
+
+    # enforce using a specific semi-sorted projection (if enabled above)
+    semi_sorted_projection_name: Optional[str] = None
+
+    # used to estimate the number of splits, given the table rows' count
+    rows_per_split: int = 4000000
+
+    # used for worker threads' naming
+    query_id: str = ""
+
+    # non-negative integer, used for server-side prioritization of queued requests:
+    # - requests with lower values will be served before requests with higher values.
+    # - if unset, the request will be added to the queue's end.
+    queue_priority: Optional[int] = None
+
+
+@dataclass
+class ImportConfig:
+    """Import execution configiration."""
+
+    import_concurrency: int = 2

vastdb/errors.py

@@ -3,7 +3,6 @@ import xml.etree.ElementTree
 from dataclasses import dataclass
 from enum import Enum
 
-import pyarrow as pa
 import requests
 
 
@@ -170,11 +169,6 @@ class NotSupportedCommand(NotSupported):
     table: str
 
 
-@dataclass
-class NotSupportedType(NotSupported):
-    field: pa.Field
-
-
 @dataclass
 class NotSupportedVersion(NotSupported):
     err_msg: str

vastdb/features.py

@@ -0,0 +1,42 @@
+"""Version-dependent features."""
+
+import logging
+
+from .errors import NotSupportedVersion
+
+log = logging.getLogger()
+
+
+class Features:
+    """VAST database features - check if server is already support a feature."""
+
+    def __init__(self, vast_version):
+        """Save the server version."""
+        self.vast_version = vast_version
+
+        self.check_imports_table = self._check(
+            "Imported objects' table feature requires 5.2+ VAST release",
+            vast_version >= (5, 2))
+
+        self.check_return_row_ids = self._check(
+            "Returning row IDs requires 5.1+ VAST release",
+            vast_version >= (5, 1))
+
+        self.check_enforce_semisorted_projection = self._check(
+            "Semi-sorted projection enforcement requires 5.1+ VAST release",
+            vast_version >= (5, 1))
+
+        self.check_external_row_ids_allocation = self._check(
+            "External row IDs allocation requires 5.1+ VAST release",
+            vast_version >= (5, 1))
+
+    def _check(self, msg, supported):
+        log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
+        if not supported:
+            def fail():
+                raise NotSupportedVersion(msg, self.vast_version)
+            return fail
+
+        def noop():
+            pass
+        return noop

vastdb/schema.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
 
 import pyarrow as pa
 
-from . import bucket, errors, schema, table, util
+from . import bucket, errors, schema, table
 
 if TYPE_CHECKING:
     from .table import Table
@@ -86,7 +86,6 @@ class Schema:
         if use_external_row_ids_allocation:
             self.tx._rpc.features.check_external_row_ids_allocation()
 
-        util.check_supported_types(columns)
         self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
                                       use_external_row_ids_allocation=use_external_row_ids_allocation)
         log.info("Created table: %s", table_name)

vastdb/session.py

@@ -7,51 +7,11 @@ For more details see:
 - [Tabular identity policy with the proper permissions](https://support.vastdata.com/s/article/UUID-14322b60-d6a2-89ac-3df0-3dfbb6974182)
 """
 
-import logging
 import os
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-import boto3
-
-from . import _internal, errors, transaction
-from ._internal import BackoffConfig
-
-log = logging.getLogger()
-
-
-class Features:
-    """VAST database features - check if server is already support a feature."""
-
-    def __init__(self, vast_version):
-        """Save the server version."""
-        self.vast_version = vast_version
-
-        self.check_imports_table = self._check(
-            "Imported objects' table feature requires 5.2+ VAST release",
-            vast_version >= (5, 2))
-
-        self.check_return_row_ids = self._check(
-            "Returning row IDs requires 5.1+ VAST release",
-            vast_version >= (5, 1))
-
-        self.check_enforce_semisorted_projection = self._check(
-            "Semi-sorted projection enforcement requires 5.1+ VAST release",
-            vast_version >= (5, 1))
-
-        self.check_external_row_ids_allocation = self._check(
-            "External row IDs allocation requires 5.1+ VAST release",
-            vast_version >= (5, 1))
-
-    def _check(self, msg, supported):
-        log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
-        if not supported:
-            def fail():
-                raise errors.NotSupportedVersion(msg, self.vast_version)
-            return fail
-
-        def noop():
-            pass
-        return noop
+if TYPE_CHECKING:
+    from .config import BackoffConfig
 
 
 class Session:
@@ -60,8 +20,13 @@ class Session:
     def __init__(self, access=None, secret=None, endpoint=None,
                  *,
                  ssl_verify=True,
-                 backoff_config: Optional[BackoffConfig] = None):
+                 timeout=None,
+                 backoff_config: Optional["BackoffConfig"] = None):
         """Connect to a VAST Database endpoint, using specified credentials."""
+        import boto3
+
+        from . import _internal, features
+
         if access is None:
             access = os.environ['AWS_ACCESS_KEY_ID']
         if secret is None:
@@ -74,8 +39,9 @@ class Session:
             access_key=access,
             secret_key=secret,
             ssl_verify=ssl_verify,
+            timeout=timeout,
             backoff_config=backoff_config)
-        self.features = Features(self.api.vast_version)
+        self.features = features.Features(self.api.vast_version)
         self.s3 = boto3.client('s3',
             aws_access_key_id=access,
             aws_secret_access_key=secret,
@@ -93,4 +59,5 @@ class Session:
             with session.transaction() as tx:
                 tx.bucket("bucket").create_schema("schema")
         """
+        from . import transaction
         return transaction.Transaction(self)

vastdb/table.py

@@ -14,6 +14,7 @@ import pyarrow as pa
 import urllib3
 
 from . import _internal, errors, schema, util
+from .config import ImportConfig, QueryConfig
 
 log = logging.getLogger(__name__)
 
@@ -39,53 +40,6 @@ class TableStats:
     endpoints: Tuple[str, ...] = ()
 
 
-@dataclass
-class QueryConfig:
-    """Query execution configiration."""
-
-    # allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
-    num_sub_splits: int = 4
-
-    # used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
-    # will be estimated from the table's row count, if not explicitly set
-    num_splits: Optional[int] = None
-
-    # each endpoint will be handled by a separate worker thread
-    # a single endpoint can be specified more than once to benefit from multithreaded execution
-    data_endpoints: Optional[List[str]] = None
-
-    # a subsplit fiber will finish after sending this number of rows back to the client
-    limit_rows_per_sub_split: int = 128 * 1024
-
-    # each fiber will read the following number of rowgroups coninuously before skipping
-    # in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
-    num_row_groups_per_sub_split: int = 8
-
-    # can be disabled for benchmarking purposes
-    use_semi_sorted_projections: bool = True
-
-    # enforce using a specific semi-sorted projection (if enabled above)
-    semi_sorted_projection_name: Optional[str] = None
-
-    # used to estimate the number of splits, given the table rows' count
-    rows_per_split: int = 4000000
-
-    # used for worker threads' naming
-    query_id: str = ""
-
-    # non-negative integer, used for server-side prioritization of queued requests:
-    # - requests with lower values will be served before requests with higher values.
-    # - if unset, the request will be added to the queue's end.
-    queue_priority: Optional[int] = None
-
-
-@dataclass
-class ImportConfig:
-    """Import execution configiration."""
-
-    import_concurrency: int = 2
-
-
 class SelectSplitState:
     """State of a specific query split execution."""
 
@@ -299,7 +253,7 @@ class Table:
                 max_workers=config.import_concurrency, thread_name_prefix='import_thread') as pool:
             try:
                 for endpoint in endpoints:
-                    session = _internal.VastdbApi(endpoint, self.tx._rpc.api.access_key, self.tx._rpc.api.secret_key)
+                    session = self.tx._rpc.api.with_endpoint(endpoint)
                     futures.append(pool.submit(import_worker, files_queue, session))
 
                 log.debug("Waiting for import workers to finish")
@@ -397,7 +351,7 @@ class Table:
 
         def single_endpoint_worker(endpoint: str):
             try:
-                host_api = _internal.VastdbApi(endpoint=endpoint, access_key=self.tx._rpc.api.access_key, secret_key=self.tx._rpc.api.secret_key)
+                host_api = self.tx._rpc.api.with_endpoint(endpoint)
                 backoff_decorator = self.tx._rpc.api._backoff_decorator
                 while True:
                     check_stop()
@@ -491,7 +445,6 @@ class Table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         try:
             row_ids = []
-            util.check_supported_types(rows.schema)
             serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
             for slice in serialized_slices:
                 res = self.tx._rpc.api.insert_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -534,7 +487,6 @@ class Table:
 
         update_rows_rb = util.sort_record_batch_if_needed(update_rows_rb, INTERNAL_ROW_ID)
 
-        util.check_supported_types(update_rows_rb.schema)
         serialized_slices = util.iter_serialized_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.update_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -579,7 +531,6 @@ class Table:
         """Add a new column."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
-        util.check_supported_types(new_column)
         self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
         log.info("Added column(s): %s", new_column)
         self.arrow_schema = self.columns()

vastdb/tests/metrics.py

@@ -20,6 +20,7 @@ class Row:
     cols: int
     pid: int
     tid: int
+    sdk_version: str
 
 
 class Table:

vastdb/tests/test_sanity.py

@@ -24,7 +24,7 @@ def test_bad_credentials(session):
 
 
 def test_bad_endpoint(session):
-    backoff_config = vastdb.session.BackoffConfig(max_tries=3)
+    backoff_config = vastdb.config.BackoffConfig(max_tries=3)
     with pytest.raises(vastdb.errors.ConnectionError):
         vastdb.connect(access='BAD', secret='BAD', endpoint='http://invalid-host-name-for-tests:12345', backoff_config=backoff_config)
 

vastdb/tests/test_tables.py

@@ -227,6 +227,35 @@ def test_select_with_priority(session, clean_bucket_name):
             t.select(config=config).read_all()
 
 
+def test_timezones(session, clean_bucket_name):
+    columns_with_tz = pa.schema([
+        ('ts0', pa.timestamp('s', tz='+00:00')),
+        ('ts3', pa.timestamp('ms', tz='UTC')),
+        ('ts6', pa.timestamp('us', tz='GMT')),
+        ('ts9', pa.timestamp('ns', tz='Universal')),
+    ])
+
+    # currently timezone information is not stored
+    columns_without_tz = pa.schema([
+        ('ts0', pa.timestamp('s')),
+        ('ts3', pa.timestamp('ms')),
+        ('ts6', pa.timestamp('us')),
+        ('ts9', pa.timestamp('ns')),
+    ])
+
+    data = [
+        [dt.datetime(2024, 4, 10, 12, 34, 56), dt.datetime(2025, 4, 10, 12, 34, 56), dt.datetime(2026, 4, 10, 12, 34, 56)],
+        [dt.datetime(2024, 4, 10, 12, 34, 56, 789000), dt.datetime(2025, 4, 10, 12, 34, 56, 789000), dt.datetime(2026, 4, 10, 12, 34, 56, 789000)],
+        [dt.datetime(2024, 4, 10, 12, 34, 56, 789789), dt.datetime(2025, 4, 10, 12, 34, 56, 789789), dt.datetime(2026, 4, 10, 12, 34, 56, 789789)],
+        [dt.datetime(2024, 4, 10, 12, 34, 56, 789789), dt.datetime(2025, 4, 10, 12, 34, 56, 789789), dt.datetime(2026, 4, 10, 12, 34, 56, 789789)],
+    ]
+
+    inserted = pa.table(schema=columns_with_tz, data=data)
+    with prepare_data(session, clean_bucket_name, 's', 't', inserted) as table:
+        assert table.arrow_schema == columns_without_tz
+        assert table.select().read_all() == pa.table(schema=columns_without_tz, data=data)
+
+
 def test_types(session, clean_bucket_name):
     columns = pa.schema([
         ('tb', pa.bool_()),
@@ -311,46 +340,6 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
-TIMESTAMP_UNITS = ['s', 'ms', 'us', 'ns']
-
-
-def test_unsupported_timezone(session, clean_bucket_name):
-    with session.transaction() as tx:
-        s = tx.bucket(clean_bucket_name).create_schema('s1')
-        for unit in TIMESTAMP_UNITS:
-            col_type = pa.timestamp(unit, 'UTC')
-            with pytest.raises(errors.NotSupportedType):
-                s.create_table('t1', pa.schema([('ts', col_type)]))
-            assert s.tables() == []
-
-        cols = [('c', pa.int64())]
-        t1 = s.create_table('t1', pa.schema(cols))
-        for unit in TIMESTAMP_UNITS:
-            col_type = pa.timestamp(unit, 'UTC')
-            with pytest.raises(errors.NotSupportedType):
-                t1.add_column(pa.schema([('ts', col_type)]))
-
-        cols = [(f'c_{unit}', pa.timestamp(unit)) for unit in TIMESTAMP_UNITS]
-        t2 = s.create_table('t2', pa.schema(cols))
-
-        for unit in TIMESTAMP_UNITS:
-            col_type = pa.timestamp(unit, 'UTC')
-
-            rb = pa.record_batch(
-                data=[[None]],
-                schema=pa.schema([(f'c_{unit}', col_type)]))
-            with pytest.raises(errors.NotSupportedType):
-                t2.insert(rb)
-
-            rb = pa.record_batch(
-                data=[[0], [None]],
-                schema=pa.schema([
-                    (INTERNAL_ROW_ID, pa.uint64()),
-                    (f'c_{unit}', col_type)]))
-            with pytest.raises(errors.NotSupportedType):
-                t2.update(rb)
-
-
 def test_filters(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int32()),

vastdb/util.py

@@ -1,4 +1,3 @@
-import importlib
 import logging
 import re
 from typing import TYPE_CHECKING, Callable, List, Optional, Union
@@ -7,7 +6,7 @@ import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument, NotSupportedType, TooWideRow
+from .errors import InvalidArgument, TooWideRow
 
 log = logging.getLogger(__name__)
 
@@ -153,14 +152,3 @@ def sort_record_batch_if_needed(record_batch, sort_column):
         return record_batch.sort_by(sort_column)
     else:
         return record_batch
-
-
-def check_supported_types(fields: pa.Schema):
-    for f in fields:
-        if isinstance(f.type, pa.TimestampType):
-            if f.type.tz:
-                raise NotSupportedType(f)
-
-
-def version():
-    return importlib.metadata.distribution('vastdb').version

vastdb/vast_tests/test_scale.py

@@ -0,0 +1,68 @@
+import logging
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import pyarrow as pa
+
+from vastdb.table import QueryConfig
+
+logger = logging.getLogger(__name__)
+
+
+def test_concurrent_query(session, test_bucket_name, schema_name, table_name):
+    """
+    This test runs several selective queries in parallel. It is used to check various internal VAST scenarios.
+    """
+    amount_of_queries_in_parallel = 10  # due to limit on requests connection-pool
+    config = QueryConfig(num_splits=1, num_sub_splits=1)
+
+    def _execute_single_query():
+        with session.transaction() as tx:
+            t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+            pred = (t["a"] == 0)  # 0 is in the min-max range
+            s = time.time()
+            t.select(config=config, predicate=pred).read_all()
+            e = time.time()
+            logger.info(f"Query took {e - s}")
+
+    logger.info(f"about to submit {amount_of_queries_in_parallel} queries in parallel")
+    with ThreadPoolExecutor() as executor:
+        futures = [executor.submit(_execute_single_query) for _ in range(amount_of_queries_in_parallel)]
+        for future in futures:
+            future.result()
+    logger.info(f"finished running {amount_of_queries_in_parallel} queries")
+
+
+def test_table_stats(session, test_bucket_name, schema_name, table_name):
+    """
+    Testing stats integrity while altering table
+    """
+    NUM_TIMES_TO_INSERT = 1000
+    seed = random.randint(0, 10)
+    logger.info(f"random seed is {seed}")
+    r = random.Random(seed)
+
+    with session.transaction() as tx:
+        t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+        initial_stat = t.get_stats()
+        table_fields = t.columns()
+
+    rand_values = {}  # create a dict with a random value from each column
+    with session.transaction() as tx:
+        t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+        for col in table_fields:
+            res = t.select(columns=[col.name]).read_all().column(col.name)
+            rand_values[col.name] = res[int(r.uniform(0, len(res)))].as_py()
+
+    logger.info(f"rand row to insert to the table - {rand_values}, {NUM_TIMES_TO_INSERT} times")
+    rb = pa.RecordBatch.from_pylist([rand_values] * NUM_TIMES_TO_INSERT)
+    with session.transaction() as tx:
+        t = tx.bucket(test_bucket_name).schema(schema_name).table(table_name)
+        t.insert(rb)
+        time.sleep(2)  # waiting for stats to get updated
+        new_stat = t.get_stats()
+
+    logger.info("inserted to table")
+    assert new_stat.size_in_bytes != initial_stat.size_in_bytes
+    assert new_stat.num_rows - NUM_TIMES_TO_INSERT == initial_stat.num_rows

{vastdb-0.1.10.dist-info → vastdb-1.0.0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.10
+Version: 1.0.0
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9

{vastdb-0.1.10.dist-info → vastdb-1.0.0.dist-info}/RECORD

@@ -148,34 +148,37 @@ vast_flatbuf/tabular/ObjectDetails.py,sha256=qW0WtbkCYYE_L-Kw6VNRDCLYaRm5lKvTbLN
 vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI0,4450
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/__init__.py,sha256=8PLcZowy_vM0zuiYSQPXuxIEMcwHD7IRFpgcPK-03bk,386
-vastdb/_internal.py,sha256=4vi6KgkfHnDOSZUrU3oQcNKKdeMH3alODLGk7Yt59Gk,90001
+vastdb/__init__.py,sha256=J1JjKiFkKC95BHowfh9kJfQFTjRce-QMsc6zF_FfxC0,432
+vastdb/_internal.py,sha256=jVQuXsqk7CDuzNsR4cmxBTYOYE09og9EUYPcJ0UA_7s,90096
 vastdb/bucket.py,sha256=5KuKhPjZOevznZqWHDVVocejvAy7dcwobPuV6BJCfPc,2544
+vastdb/config.py,sha256=1tMYtzKXerGcIUjH4tIGEvZNWvO4fviCEdcNCnELJZo,2269
 vastdb/conftest.py,sha256=ePzQiEQmlNGcM2T4GZevE4XuvcnFWfnTSzr8IVZpVKk,3438
-vastdb/errors.py,sha256=jER5RQYsBRlQsjym1ItQYRukggMypATOo_sKvsJtMbo,4278
-vastdb/schema.py,sha256=yaueil92MSMYJf6bWseov_8fXTdW5zaKLXNjP5uuyzI,5963
-vastdb/session.py,sha256=3YHhG7IamFOKuy-Fkq_IVtPNriSfI6IN_4z4arBFbDU,3349
-vastdb/table.py,sha256=C0kgV8CJVgoRxVx83SPTn75mgbTz9OWgYwK_RzLPZ5Q,32994
+vastdb/errors.py,sha256=2XR1ko7J5nkfiHSAgwuVAADw0SsyqxOwSeFaGgKZEXM,4186
+vastdb/features.py,sha256=DxV746LSkORwVSD6MP2hdXRfnyoLkJwtOwGmp1dnquo,1322
+vastdb/schema.py,sha256=X7IRrogXH7Z0kes-DsDh1bRqIhvjH6owlFigGBXy7XQ,5913
+vastdb/session.py,sha256=ZrQf8cecVIHIBUOPNg4ed8ZCnEEu0QW1OBxQgz_ia80,2241
+vastdb/table.py,sha256=7U4UbooR1DUSpnZo8t8hzAMP78hF7IVvZpqZRrDDAQw,30953
 vastdb/transaction.py,sha256=qu2rOlR7AS1ojMOzgWapQMpcorrutelZZLH1mLmTHxk,3186
-vastdb/util.py,sha256=KQ2CjML-ipWxrJjwiaBbj4bxpTCtL24Pr2Co9woyw3Y,5983
+vastdb/util.py,sha256=4LTYBBR13na376AmDm5lQILJzLcfelIKdkNPy0IqI0o,5684
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/bench/test_perf.py,sha256=yn5gE7t_nzmJHBl9bCs1hxQOgzhvFphuYElsWGko8ts,1084
-vastdb/bench/test_sample.py,sha256=0qsKPj3i88J-YTrOrGvsP19xsyWGZy_-ptIt3oXBbSw,7181
+vastdb/bench/test_perf.py,sha256=gZIqfHva6lNFpD-9bHAe7M8COBjUyrPkHu3E7F8J2L0,1072
+vastdb/bench/test_sample.py,sha256=bFmw7BOCr5FoGn4TY9pQGd6_cVNK4uBeSRi33tTubyk,7847
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/tests/metrics.py,sha256=f1oOPKDsu-BzBLin0IQvjG-ueRDHTY-Hzl357TuoxCQ,989
+vastdb/tests/metrics.py,sha256=N6ELJUmZubhAMmUtDbisXr6TFhSDgVCTTU05gBVxHRA,1010
 vastdb/tests/test_duckdb.py,sha256=STw_1PwTQR8Naz6s0p6lQTV1ZTKKhe3LPBUbhqzTCu0,1880
 vastdb/tests/test_imports.py,sha256=xKub3-bisFjH0BsZM8COfiUWuMrtoOoQKprF6VQT9RI,5669
 vastdb/tests/test_nested.py,sha256=22NAxBTm7Aq-Vn6AIYbi5Cb1ET8W0XeLK3pp4D8BYWI,3448
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
-vastdb/tests/test_sanity.py,sha256=V6dO5Y44B6pG8Eet6atTTGGH1yPz75_k0ZybHY-IiF8,3039
+vastdb/tests/test_sanity.py,sha256=oiV2gb05aPyG5RMNUQZlyjNlg3T7Fig1_8OJzpAgcsk,3038
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=RlwVfzs2hjfs2gchiRY0hnWoOAu4MV_9NbQCeHR6_us,31590
+vastdb/tests/test_tables.py,sha256=qWicD0BYuhrh1kRVqkHMJNsxcHxDcCprbEXuZJm1wm4,31529
 vastdb/tests/test_util.py,sha256=Ok_sAEBJsRGF5Voa_v5eu3eAd52GWu8jMjjQbadwW-s,1260
 vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
-vastdb-0.1.10.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.1.10.dist-info/METADATA,sha256=Pjw1EZvwnKhfEjuRVVMR0DBOSkmVo5wcHftqddxqRNY,1351
-vastdb-0.1.10.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-vastdb-0.1.10.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.1.10.dist-info/RECORD,,
+vastdb/vast_tests/test_scale.py,sha256=EpjCJmVAQrNBxVnHGJ-KHCoxevhqOcyqYFPMIIY9s60,2714
+vastdb-1.0.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.0.0.dist-info/METADATA,sha256=iEjwUBDXLAZ3RC64iEHRiQkz0afSR5PPnHGmsthuzRI,1363
+vastdb-1.0.0.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+vastdb-1.0.0.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-1.0.0.dist-info/RECORD,,

{vastdb-0.1.10.dist-info → vastdb-1.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: setuptools (70.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any