vastdb 0.1.8__tar.gz → 0.1.9__tar.gz

{vastdb-0.1.8 → vastdb-0.1.9}/CHANGELOG.md

@@ -4,9 +4,17 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.1.9] (2024-07-01)
+[0.1.9]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.8...v0.1.9
+
+## Fixed
+ - Don't drop timezone information when creating a table
+ - Clarify the catalog example
+
 ## [0.1.8] (2024-06-26)
 [0.1.8]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.7...v0.1.8
 
+## Fixed
  - Allow retries also during query response parsing
  - Don't retrieve stats during Table.select() if not needed
  - Log successful server-side file import

{vastdb-0.1.8/vastdb.egg-info → vastdb-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.8
+Version: 0.1.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.1.8 → vastdb-0.1.9}/README.md

@@ -176,18 +176,27 @@ batches = snaps[0].schema('schema-name').table('table-name').select()
 [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database) can be queried as a regular table:
 
 ```python
-table = tx.catalog().select(['element_type']).read_all()
-df = table.to_pandas()
+import pyarrow as pa
+import vastdb
+
+session = vastdb.connect(
+    endpoint='http://vip-pool.v123-xy.VastENG.lab',
+    access=AWS_ACCESS_KEY_ID,
+    secret=AWS_SECRET_ACCESS_KEY)
+
+with session.transaction() as tx:
+    table = tx.catalog().select(['element_type']).read_all()
+    df = table.to_pandas()
 
-total_elements = len(df)
-print(f"Total elements in the catalog: {total_elements}")
+    total_elements = len(df)
+    print(f"Total elements in the catalog: {total_elements}")
 
-file_count = (df['element_type'] == 'FILE').sum()
-print(f"Number of files/objects: {file_count}")
+    file_count = (df['element_type'] == 'FILE').sum()
+    print(f"Number of files/objects: {file_count}")
 
-distinct_elements = df['element_type'].unique()
-print("Distinct element types on the system:")
-print(distinct_elements)
+    distinct_elements = df['element_type'].unique()
+    print("Distinct element types on the system:")
+    print(distinct_elements)
 ```
 
 See the following blog posts for more examples:

{vastdb-0.1.8 → vastdb-0.1.9}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='0.1.8' + suffix,
+    version='0.1.9' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/errors.py

@@ -3,6 +3,7 @@ import xml.etree.ElementTree
 from dataclasses import dataclass
 from enum import Enum
 
+import pyarrow as pa
 import requests
 
 
@@ -169,6 +170,11 @@ class NotSupportedCommand(NotSupported):
     table: str
 
 
+@dataclass
+class NotSupportedType(NotSupported):
+    field: pa.Field
+
+
 @dataclass
 class NotSupportedVersion(NotSupported):
     err_msg: str

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/schema.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
 
 import pyarrow as pa
 
-from . import bucket, errors, schema, table
+from . import bucket, errors, schema, table, util
 
 if TYPE_CHECKING:
     from .table import Table
@@ -86,6 +86,7 @@ class Schema:
         if use_external_row_ids_allocation:
             self.tx._rpc.features.check_external_row_ids_allocation()
 
+        util.check_supported_types(columns)
         self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
                                       use_external_row_ids_allocation=use_external_row_ids_allocation)
         log.info("Created table: %s", table_name)

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/table.py

@@ -486,6 +486,7 @@ class Table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         try:
             row_ids = []
+            util.check_supported_types(rows.schema)
             serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
             for slice in serialized_slices:
                 res = self.tx._rpc.api.insert_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -528,6 +529,7 @@ class Table:
 
         update_rows_rb = util.sort_record_batch_if_needed(update_rows_rb, INTERNAL_ROW_ID)
 
+        util.check_supported_types(update_rows_rb.schema)
         serialized_slices = util.iter_serialized_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.update_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
@@ -572,6 +574,7 @@ class Table:
         """Add a new column."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
+        util.check_supported_types(new_column)
         self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
         log.info("Added column(s): %s", new_column)
         self.arrow_schema = self.columns()

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/tests/test_tables.py

@@ -311,6 +311,46 @@ def test_types(session, clean_bucket_name):
             assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
 
 
+TIMESTAMP_UNITS = ['s', 'ms', 'us', 'ns']
+
+
+def test_unsupported_timezone(session, clean_bucket_name):
+    with session.transaction() as tx:
+        s = tx.bucket(clean_bucket_name).create_schema('s1')
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+            with pytest.raises(errors.NotSupportedType):
+                s.create_table('t1', pa.schema([('ts', col_type)]))
+            assert s.tables() == []
+
+        cols = [('c', pa.int64())]
+        t1 = s.create_table('t1', pa.schema(cols))
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+            with pytest.raises(errors.NotSupportedType):
+                t1.add_column(pa.schema([('ts', col_type)]))
+
+        cols = [(f'c_{unit}', pa.timestamp(unit)) for unit in TIMESTAMP_UNITS]
+        t2 = s.create_table('t2', pa.schema(cols))
+
+        for unit in TIMESTAMP_UNITS:
+            col_type = pa.timestamp(unit, 'UTC')
+
+            rb = pa.record_batch(
+                data=[[None]],
+                schema=pa.schema([(f'c_{unit}', col_type)]))
+            with pytest.raises(errors.NotSupportedType):
+                t2.insert(rb)
+
+            rb = pa.record_batch(
+                data=[[0], [None]],
+                schema=pa.schema([
+                    (INTERNAL_ROW_ID, pa.uint64()),
+                    (f'c_{unit}', col_type)]))
+            with pytest.raises(errors.NotSupportedType):
+                t2.update(rb)
+
+
 def test_filters(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int32()),

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/util.py

@@ -6,7 +6,7 @@ import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument, TooWideRow
+from .errors import InvalidArgument, NotSupportedType, TooWideRow
 
 log = logging.getLogger(__name__)
 
@@ -152,3 +152,10 @@ def sort_record_batch_if_needed(record_batch, sort_column):
         return record_batch.sort_by(sort_column)
     else:
         return record_batch
+
+
+def check_supported_types(fields: pa.Schema):
+    for f in fields:
+        if isinstance(f.type, pa.TimestampType):
+            if f.type.tz:
+                raise NotSupportedType(f)

{vastdb-0.1.8 → vastdb-0.1.9}/vastdb/vast_tests/test_ha.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.ha
-def test_ha_query(session, crater_path, test_bucket_name, schema_name, table_name):
+def test_ha_query(session, test_bucket_name, schema_name, table_name):
     # runs in parallel to ha scenario
     times_to_query, records_in_table = 50, 100_000_000
     arrow_array = pa.array(range(0, records_in_table), type=pa.int64())

{vastdb-0.1.8 → vastdb-0.1.9/vastdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.8
+Version: 0.1.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA