vastdb 1.3.11__py3-none-any.whl → 1.4.0__py3-none-any.whl

vastdb/table.py

@@ -532,19 +532,24 @@ class Table:
             columns_name_chunk = columns_names[start:end]
             columns_chunks = columns[start:end]
             arrays_chunks = arrays[start:end]
-            columns_chunks.append(INTERNAL_ROW_ID_FIELD)
+            columns_chunks.append(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD)
             arrays_chunks.append(row_ids.to_pylist())
             column_record_batch = pa.RecordBatch.from_arrays(arrays_chunks, schema=pa.schema(columns_chunks))
             self.update(rows=column_record_batch, columns=columns_name_chunk)
         return row_ids
 
-    def insert(self, rows: Union[pa.RecordBatch, pa.Table]):
+    def insert(self, rows: Union[pa.RecordBatch, pa.Table], by_columns: bool = False):
         """Insert a RecordBatch into this table."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         if 0 == rows.num_rows:
             log.debug("Ignoring empty insert into %s", self.name)
-            return pa.chunked_array([], type=INTERNAL_ROW_ID_FIELD.type)
+            return pa.chunked_array([], type=(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD).type)
+
+        if by_columns:
+            self.tx._rpc.features.check_return_row_ids()
+            return self.insert_in_column_batches(rows)
+
         try:
             row_ids = []
             serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
@@ -557,7 +562,7 @@ class Table:
                 self.tx._rpc.features.check_return_row_ids()
             except errors.NotSupportedVersion:
                 return  # type: ignore
-            return pa.chunked_array(row_ids, type=INTERNAL_ROW_ID_FIELD.type)
+            return pa.chunked_array(row_ids, type=(INTERNAL_ROW_ID_SORTED_FIELD if self.sorted_table else INTERNAL_ROW_ID_FIELD).type)
         except errors.TooWideRow:
             self.tx._rpc.features.check_return_row_ids()
             return self.insert_in_column_batches(rows)

vastdb/tests/test_tables.py

@@ -15,10 +15,11 @@ import pytest
 from requests.exceptions import HTTPError
 
 from vastdb.errors import BadRequest
+from vastdb.session import Session
 
 from .. import errors
 from ..table import INTERNAL_ROW_ID, QueryConfig
-from .util import prepare_data
+from .util import assert_row_ids_ascending_on_first_insertion_to_table, prepare_data
 
 log = logging.getLogger(__name__)
 
@@ -1150,7 +1151,7 @@ def test_elysium_tx(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert len(sorted_columns) == 0
         t.add_sorting_key(sorting)
@@ -1186,7 +1187,7 @@ def test_elysium_double_enable(elysium_session, clean_bucket_name):
             t.add_sorting_key(sorting)
 
 
-def test_elysium_update_table_tx(elysium_session, clean_bucket_name):
+def test_elysium_update_table_tx(elysium_session: Session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int64()),
         ('b', pa.float32()),
@@ -1205,7 +1206,7 @@ def test_elysium_update_table_tx(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert sorted_columns[0].name == 's'
         assert sorted_columns[1].name == 'b'
@@ -1278,7 +1279,7 @@ def test_elysium_splits(elysium_session, clean_bucket_name):
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         sorted_columns = t.sorted_columns()
         assert sorted_columns[0].name == 'a'
 

vastdb/tests/util.py

@@ -3,17 +3,27 @@ from contextlib import contextmanager
 
 import pyarrow as pa
 
+from vastdb.session import Session
+
 log = logging.getLogger(__name__)
 
 
+def assert_row_ids_ascending_on_first_insertion_to_table(row_ids, expected_num_rows, sorted_table):
+    adjusted_row_ids = [
+        int(row_id) & 0xFFFFFFFFFFFFFF for row_id in row_ids
+        ] if sorted_table else row_ids
+
+    assert adjusted_row_ids == list(range(expected_num_rows))
+
+
 @contextmanager
-def prepare_data(session, clean_bucket_name, schema_name, table_name, arrow_table, sorting_key=[]):
+def prepare_data(session: Session, clean_bucket_name, schema_name, table_name, arrow_table, sorting_key=[]):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema(schema_name)
         t = s.create_table(table_name, arrow_table.schema, sorting_key=sorting_key)
         row_ids_array = t.insert(arrow_table)
         row_ids = row_ids_array.to_pylist()
-        assert row_ids == list(range(arrow_table.num_rows))
+        assert_row_ids_ascending_on_first_insertion_to_table(row_ids, arrow_table.num_rows, t.sorted_table)
         yield t
         t.drop()
         s.drop()

{vastdb-1.3.11.dist-info → vastdb-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.11
+Version: 1.4.0
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.3.11.dist-info → vastdb-1.4.0.dist-info}/RECORD

@@ -7,7 +7,7 @@ vastdb/errors.py,sha256=NiKdwbfVsWJIixP2Tf3JgiBoEt8rRaZ0VeCyD9mXnoM,5645
 vastdb/features.py,sha256=ivYbvhiGA858B00vhs_CNzlVV9QDUe53yW6V3J5EoxM,1874
 vastdb/schema.py,sha256=5BZ0f3b_c-fGRKAaBBL6B3avHel5EDwwxte7t17WeTw,6718
 vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
-vastdb/table.py,sha256=fUAmOaVdMSzas1XHldRgT0UbHEG-hYJjuWkDIz3hchs,36936
+vastdb/table.py,sha256=rM-xJgpSuvowUH3oilVVuW-7JsPDmMJ4rbHnru1EA0Y,37268
 vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
 vastdb/util.py,sha256=8CUnVRsJukC3uNHNoB5D0qPf0FxS8OSdVB84nNoLJKc,6290
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -47,9 +47,9 @@ vastdb/tests/test_nested.py,sha256=c7q9a3MsyDymqAtShPC4cMHlzjCr18kbu_Db3u_c4IQ,6
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
 vastdb/tests/test_sanity.py,sha256=bv1ypGDzvOgmMvGbucDYiLQu8krQLlE6NB3M__q87x8,3303
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=Bl0ldJApPCFEL94jRNycsCi8M_2jTHWDXiA-7JHVlzk,51260
+vastdb/tests/test_tables.py,sha256=2m7Ao97mLwvCgzGShxTHMxFbt3vnNfb6InNe96DTJMA,51502
 vastdb/tests/test_util.py,sha256=n7gvT5Wg6b6bxgqkFXkYqvFd_W1GlUdVfmPv66XYXyA,1956
-vastdb/tests/util.py,sha256=YsCBCcx7n1QOH-IPDpCsl6KEaUQQJRZwGPeayijHNb4,1307
+vastdb/tests/util.py,sha256=j8WSYOsScz7uvJljIcI9ZQWz3cA3WPykurwymnDd3s0,1690
 vastdb/vast_flatbuf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_flatbuf/org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_flatbuf/org/apache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -210,8 +210,8 @@ vastdb/vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
 vastdb/vast_tests/test_scale.py,sha256=5jGwOdZH6Tv5tPdZYPWoqcxOceI2jA5i2D1zNKZHER4,3958
-vastdb-1.3.11.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.3.11.dist-info/METADATA,sha256=KwME3q5diBEN8GTagURPqGDDBv41TS7xFN348qr26lw,1341
-vastdb-1.3.11.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vastdb-1.3.11.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
-vastdb-1.3.11.dist-info/RECORD,,
+vastdb-1.4.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.4.0.dist-info/METADATA,sha256=7bga0VnmmT_RYdlB7abV_l-SEDegSuLMIRf7rB8UGpQ,1340
+vastdb-1.4.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.4.0.dist-info/top_level.txt,sha256=nnKAaZaQa8GFbYpWAexr_B9HrhonZbUlX6hL6AC--yA,7
+vastdb-1.4.0.dist-info/RECORD,,