vastdb 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

vastdb/tests/test_tables.py

@@ -6,7 +6,6 @@ import threading
 from contextlib import closing
 from tempfile import NamedTemporaryFile
 
-import duckdb
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
@@ -91,7 +90,6 @@ def test_exists(session, clean_bucket_name):
         assert s.tables() == [t]
 
 
-
 def test_update_table(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int64()),
@@ -147,12 +145,13 @@ def test_update_table(session, clean_bucket_name):
             'b': [0.5, 1.5, 2.5]
         }
 
+
 def test_select_with_multisplits(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int32())
     ])
 
-    data = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     data = data * 1000
     expected = pa.table(schema=columns, data=[data])
 
@@ -179,15 +178,15 @@ def test_types(session, clean_bucket_name):
         ('t3', pa.time32('ms')),
         ('t6', pa.time64('us')),
         ('t9', pa.time64('ns')),
-        ('ts0' ,pa.timestamp('s')),
-        ('ts3' ,pa.timestamp('ms')),
-        ('ts6' ,pa.timestamp('us')),
-        ('ts9' ,pa.timestamp('ns')),
+        ('ts0', pa.timestamp('s')),
+        ('ts3', pa.timestamp('ms')),
+        ('ts6', pa.timestamp('us')),
+        ('ts9', pa.timestamp('ns')),
     ])
 
     expected = pa.table(schema=columns, data=[
         [True, True, False],
-        [1 , 2, 4],
+        [1, 2, 4],
         [1999, 2000, 2001],
         [11122221, 222111122, 333333],
         [0.5, 1.5, 2.5],
@@ -295,7 +294,7 @@ def test_filters(session, clean_bucket_name):
         assert select(((t['a'] > 111) | (t['a'] < 333)) & (t['b'] < 2.5)) == expected.filter(((pc.field('a') > 111) | (pc.field('a') < 333)) & (pc.field('b') < 2.5))
         with pytest.raises(NotImplementedError):
             assert select((t['a'] > 111) | (t['b'] > 0) | (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) | (pc.field('b') > 0) | (pc.field('s') < 'ccc'))
-        assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777) ) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
+        assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
 
         assert select(t['s'].isnull()) == expected.filter(pc.field('s').is_null())
         assert select((t['s'].isnull()) | (t['s'] == 'bb'))  == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
@@ -306,26 +305,6 @@ def test_filters(session, clean_bucket_name):
         assert select(t['s'].contains('y')) == expected.filter(pc.field('s') == 'xyz')
 
 
-def test_duckdb(session, clean_bucket_name):
-    columns = pa.schema([
-        ('a', pa.int32()),
-        ('b', pa.float64()),
-    ])
-    data = pa.table(schema=columns, data=[
-        [111, 222, 333],
-        [0.5, 1.5, 2.5],
-    ])
-    with prepare_data(session, clean_bucket_name, 's', 't', data) as t:
-        conn = duckdb.connect()
-        batches = t.select(columns=['a'], predicate=(t['b'] < 2))  # noqa: F841
-        actual = conn.execute('SELECT max(a) as "a_max" FROM batches').arrow()
-        expected = (data
-            .filter(pc.field('b') < 2)
-            .group_by([])
-            .aggregate([('a', 'max')]))
-        assert actual == expected
-
-
 def test_parquet_export(session, clean_bucket_name):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s1')
@@ -344,8 +323,7 @@ def test_parquet_export(session, clean_bucket_name):
             ['a', 'b'],
         ])
         expected = pa.Table.from_batches([rb])
-        rb = t.insert(rb)
-        assert rb.to_pylist() == [0, 1]
+        t.insert(rb)
         actual = pa.Table.from_batches(t.select())
         assert actual == expected
 
@@ -359,6 +337,7 @@ def test_parquet_export(session, clean_bucket_name):
 
             assert expected == pq.read_table(parquet_file.name)
 
+
 def test_errors(session, clean_bucket_name):
     with pytest.raises(errors.MissingSchema):
         with session.transaction() as tx:
@@ -378,7 +357,8 @@ def test_errors(session, clean_bucket_name):
                 ('s', pa.utf8()),
             ])
             s.create_table('t1', columns)
-            s.drop() # cannot drop schema without dropping its tables first
+            s.drop()  # cannot drop schema without dropping its tables first
+
 
 def test_rename_schema(session, clean_bucket_name):
 
@@ -436,20 +416,21 @@ def test_rename_table(session, clean_bucket_name):
             s.table('t')
         t = s.table('t2')
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         with pytest.raises(errors.MissingTable):
             tx2.bucket(clean_bucket_name).schema('s').table('t2')
         tx2.bucket(clean_bucket_name).schema('s').table('t')
 
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         with pytest.raises(errors.MissingTable):
             s.table('t')
         t = s.table('t2')
         t.drop()
         s.drop()
 
+
 def test_add_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
@@ -472,18 +453,18 @@ def test_add_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         assert t.arrow_schema == new_schema
         t.drop()
         s.drop()
 
+
 def test_drop_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
@@ -507,31 +488,32 @@ def test_drop_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         assert t.arrow_schema == new_schema
         t.drop()
         s.drop()
 
+
 def test_rename_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
             ('b', pa.float32()),
             ('s', pa.utf8()),
         ])
-    def prepare_rename_column(schema : pa.Schema, old_name : str, new_name : str) -> pa.Schema:
+
+    def prepare_rename_column(schema: pa.Schema, old_name: str, new_name: str) -> pa.Schema:
         field_idx = schema.get_field_index(old_name)
         column_to_rename = schema.field(field_idx)
         renamed_column = column_to_rename.with_name(new_name)
         return schema.set(field_idx, renamed_column)
 
-    new_schema = prepare_rename_column(columns,'a','aaa')
+    new_schema = prepare_rename_column(columns, 'a', 'aaa')
 
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s')
@@ -546,10 +528,10 @@ def test_rename_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-    #assert that new transactions see the change
+    # assert that new transactions see the change
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
@@ -564,7 +546,7 @@ def test_rename_column(session, clean_bucket_name):
             t1 = tx1.bucket(clean_bucket_name).schema('s').table('t')
             t2 = tx2.bucket(clean_bucket_name).schema('s').table('t')
             t1.rename_column('b', 'bb')
-            with pytest.raises(HTTPError, match = '409 Client Error: Conflict'):
+            with pytest.raises(HTTPError, match='409 Client Error: Conflict'):
                 t2.rename_column('b', 'bbb')
 
     with session.transaction() as tx:
@@ -580,6 +562,7 @@ def test_rename_column(session, clean_bucket_name):
         t.drop()
         s.drop()
 
+
 def test_select_stop(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.uint8()),
@@ -602,15 +585,16 @@ def test_select_stop(session, clean_bucket_name):
     qc = QueryConfig(num_sub_splits=2, num_splits=4, num_row_groups_per_sub_split=1)
     with session.transaction() as tx:
         t = tx.bucket(clean_bucket_name).schema('s').table('t')
-        t.refresh_stats()
-        qc.data_endpoints = list(t.stats.endpoints) * 2
+        qc.data_endpoints = list(t.get_stats().endpoints) * 2
 
     # Duplicate the table until it is large enough to generate enough batches
     while num_rows < (qc.num_sub_splits * qc.num_splits) * ROWS_PER_GROUP:
+        # We need two separate transactions to prevent an infinite loop that may happen
+        # while appending and reading the same table using a single transaction.
         with session.transaction() as tx_read, session.transaction() as tx_write:
             t_read = tx_read.bucket(clean_bucket_name).schema('s').table('t')
             t_write = tx_write.bucket(clean_bucket_name).schema('s').table('t')
-            for batch in t_read.select(['a'],config=qc):
+            for batch in t_read.select(['a'], config=qc):
                 t_write.insert(batch)
         num_rows = num_rows * 2
         log.info("Num rows: %d", num_rows)
@@ -627,11 +611,12 @@ def test_select_stop(session, clean_bucket_name):
     # If this assert triggers it just means that the test assumptions about how
     # the tabular server splits the batches is not true anymore and we need to
     # rewrite the test.
-    assert read_batches == qc.num_splits*qc.num_sub_splits
-    qc.query_id = str(random.randint(0,2**32))
+    assert read_batches == qc.num_splits * qc.num_sub_splits
+    qc.query_id = str(random.randint(0, 2**32))
     log.info("query id is: %s", qc.query_id)
+
     def active_threads():
-        log.debug("%s",[t.getName() for t in threading.enumerate() if t.is_alive()])
+        log.debug("%s", [t.getName() for t in threading.enumerate() if t.is_alive()])
         return sum([1 if t.is_alive() and qc.query_id in t.getName() else 0 for t in threading.enumerate()])
 
     assert active_threads() == 0

vastdb/tests/util.py

@@ -9,10 +9,7 @@ def prepare_data(session, clean_bucket_name, schema_name, table_name, arrow_tabl
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema(schema_name)
         t = s.create_table(table_name, arrow_table.schema)
-        row_ids_array = t.insert(arrow_table)
-        row_ids = row_ids_array.to_pylist()
-        log.debug("row_ids=%s" % row_ids)
-        assert row_ids == list(range(arrow_table.num_rows))
+        t.insert(arrow_table)
         yield t
         t.drop()
         s.drop()

vastdb/transaction.py

@@ -8,19 +8,21 @@ A transcation is used as a context manager, since every Database-related operati
 
 import logging
 from dataclasses import dataclass
+from typing import Optional
 
 import botocore
 
-from . import bucket, errors, session
+from . import bucket, errors, schema, session, table
 
 log = logging.getLogger(__name__)
 
+
 @dataclass
 class Transaction:
     """A holder of a single VAST transaction."""
 
     _rpc: "session.Session"
-    txid: int = None
+    txid: Optional[int] = None
 
     def __enter__(self):
         """Create a transaction and store its ID."""
@@ -31,12 +33,14 @@ class Transaction:
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
         """On success, the transaction is committed. Otherwise, it is rolled back."""
+        txid = self.txid
+        self.txid = None
         if (exc_type, exc_value, exc_traceback) == (None, None, None):
-            log.debug("committing txid=%016x", self.txid)
-            self._rpc.api.commit_transaction(self.txid)
+            log.debug("committing txid=%016x", txid)
+            self._rpc.api.commit_transaction(txid)
         else:
-            log.exception("rolling back txid=%016x due to:", self.txid)
-            self._rpc.api.rollback_transaction(self.txid)
+            log.exception("rolling back txid=%016x due to:", txid)
+            self._rpc.api.rollback_transaction(txid)
 
     def __repr__(self):
         """Don't show the session details."""
@@ -52,3 +56,9 @@ class Transaction:
                 raise errors.MissingBucket(name)
             raise
         return bucket.Bucket(name, self)
+
+    def catalog(self, fail_if_missing=True) -> Optional["table.Table"]:
+        """Return VAST Catalog table."""
+        b = bucket.Bucket("vast-big-catalog-bucket", self)
+        s = schema.Schema("vast_big_catalog_schema", b)
+        return s.table(name="vast_big_catalog_table", fail_if_missing=fail_if_missing)

vastdb/util.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Callable
+from typing import Callable, List, Optional
 
 import pyarrow as pa
 import pyarrow.parquet as pq
@@ -12,8 +12,9 @@ log = logging.getLogger(__name__)
 
 
 def create_table_from_files(
-        schema: Schema, table_name: str, parquet_files: [str], schema_merge_func: Callable = None,
-        config: ImportConfig = None) -> Table:
+        schema: Schema, table_name: str, parquet_files: List[str],
+        schema_merge_func: Optional[Callable] = None,
+        config: Optional[ImportConfig] = None) -> Table:
     if not schema_merge_func:
         schema_merge_func = default_schema_merge
     else:

{vastdb-0.1.1.dist-info → vastdb-0.1.2.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.1
+Version: 0.1.2
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -34,5 +33,3 @@ and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system
 enabling schema and table management, efficient ingest, query and modification of columnar data.
 
 For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
-
-

{vastdb-0.1.1.dist-info → vastdb-0.1.2.dist-info}/RECORD

@@ -148,28 +148,29 @@ vast_flatbuf/tabular/ObjectDetails.py,sha256=qW0WtbkCYYE_L-Kw6VNRDCLYaRm5lKvTbLN
 vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI0,4450
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/__init__.py,sha256=GY30IfZQApfl7HfcFmfTzFpx48oHgQIrDcUQCiTnxpo,206
-vastdb/bucket.py,sha256=5J8KBdRViaz5bZ8WEPggQj7DfJaIhY7CqpoWP6reaDo,2854
+vastdb/__init__.py,sha256=cMJtZuJ0IL9aKyM3DUWqTCzuP1H1MXXVivKKE1-q0DY,292
+vastdb/bucket.py,sha256=xtKs7S4w0jmI4MujDWH3HDI-iEgbq5Xqqsod-tw4zSo,2991
 vastdb/conftest.py,sha256=pKpo_46Vq4QHzTDQAFxasrVhnZ2V2L-y6IMLxojxaFM,2132
-vastdb/errors.py,sha256=wCJp70QyBW8US2KMwhB6e5ZnKRft4GiN8jyJ36f1Yuo,3315
-vastdb/internal_commands.py,sha256=rmxOjIq229gsxFFZ4nKXwVIFJcu8WR1DVsE-91w4-BY,101564
-vastdb/schema.py,sha256=x9Yn4tFTFkSpJbQqpqlecKUSOK214XsRLdOUrNW0jzM,3192
+vastdb/errors.py,sha256=fxpKSxjEgoJZuBtEGWzTW9lpDlEjuzgpgXwAQc1W6BQ,3436
+vastdb/internal_commands.py,sha256=3F6FiYu-Ama1zBO7hENPxCaQYJT8mcZP6rSQvtI7Sks,101273
+vastdb/schema.py,sha256=MrQr-WIrES8KcQ0V6cJkRRp_-9jj9FboyrBnkNBsw-8,3324
 vastdb/session.py,sha256=VZOFGZbAdr5Tl4cp88VRQYnR4Q16UNuYjSmX_QPW1II,1718
-vastdb/table.py,sha256=eALN5YpUfDFqZNF_lp6lZD5RJkBKqp5Mlc6hpwGI8Rg,20443
-vastdb/transaction.py,sha256=2I5k81YvcgDsp07BrAWkmXf41qUP6m88Y40rFfuIKvI,1796
-vastdb/util.py,sha256=VR0UJ1D0WUpqS5edG_mkxDZYZJ_qqce8y7iJOvqeyWE,2974
+vastdb/table.py,sha256=bdx3C1iWiFivKmtifH7MyG7TMqnVVIU91as-_hMn1rE,20532
+vastdb/transaction.py,sha256=1uCSHXqWcwsMJv6DuNx4WyQMGUm8P-RCCqYdBdUGusI,2196
+vastdb/util.py,sha256=Tjj6p4gqabK5G21uWuCiuYM9FaaR04_Zk5X8NWtcdj8,3022
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/bench/test_perf.py,sha256=X7BIo60L5Oj7H-56e8pDFtXY9rNLerkywKexXWiqvrY,1111
+vastdb/bench/test_perf.py,sha256=iHE3E60fvyU5SBDHPi4h03Dj6QcY6VI9l9mMhgNMtPc,1117
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vastdb/tests/test_duckdb.py,sha256=KDuv4PrjGEwChCGHG36xNT2JiFlBOt6K3DQ3L06Kq-A,1913
 vastdb/tests/test_imports.py,sha256=fDUjO5U-5i4QTIMoNnSSW4X_ZnOStLbx0mJkNq2pj9Q,5033
 vastdb/tests/test_nested.py,sha256=3kejEvtSqV0LrUgb1QglRjrlxnKI4_AXTFw2nE7Q520,951
-vastdb/tests/test_projections.py,sha256=0ZiFya6rzGvnKOrdb1xxxv-BEerNmiK_ymfZM6eIvvw,1254
-vastdb/tests/test_sanity.py,sha256=kaOmZWDGBc-XhZ8eFQ3sks2Mo9De8q41Z5pqYWzJsHM,2958
-vastdb/tests/test_schemas.py,sha256=8ZlEvnU7Fyg-TDQDxD65GAql4rU8R2_SFWVGrdv564o,1721
-vastdb/tests/test_tables.py,sha256=o_JPqr2GX1DDpPB4Zq4E1YPFgmlsiXyVe1S3TcCjF-w,26226
-vastdb/tests/util.py,sha256=_euE3fKJqgNssT9gVxlcHjdE61mnsNQcwDPzn1tTe9g,597
-vastdb-0.1.1.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.1.1.dist-info/METADATA,sha256=e84OEOXS09DEXniHJAU2aeK80-1h2rIZmYNBCMLa1AM,1331
-vastdb-0.1.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-vastdb-0.1.1.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.1.1.dist-info/RECORD,,
+vastdb/tests/test_projections.py,sha256=_cDNfD5zTwbCXLk6uGpPUWGN0P-4HElu5OjubWu-Jg0,1255
+vastdb/tests/test_sanity.py,sha256=ixx0QPo73hLHjAa7bByFXjS1XST0WvmSwLEpgnHh_JY,2960
+vastdb/tests/test_schemas.py,sha256=b-JpYHOFYVTdE570_La7O2RWf8BGN-q8KDXNXeC8CSg,1724
+vastdb/tests/test_tables.py,sha256=TXM4LSBvPb3EEu7XScZ5iEiu_zhHClq61W18EQodxw8,25667
+vastdb/tests/util.py,sha256=NaCzKymEGy1xuiyMxyt2_0frKVfVk9iGrFwLf3GHjTI,435
+vastdb-0.1.2.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-0.1.2.dist-info/METADATA,sha256=edJPdDWmHj6tRHRR97eSppfN9_4ARfIr0jS9HMjHfSQ,1311
+vastdb-0.1.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+vastdb-0.1.2.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-0.1.2.dist-info/RECORD,,

{vastdb-0.1.1.dist-info → vastdb-0.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any