vastdb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

vastdb/tests/test_duckdb.py

@@ -0,0 +1,61 @@
+import logging
+
+import duckdb
+import pyarrow as pa
+import pyarrow.compute as pc
+import pytest
+
+from ..table import QueryConfig
+from .util import prepare_data
+
+log = logging.getLogger(__name__)
+
+
+def test_duckdb(session, clean_bucket_name):
+    columns = pa.schema([
+        ('a', pa.int32()),
+        ('b', pa.float64()),
+    ])
+    data = pa.table(schema=columns, data=[
+        [111, 222, 333],
+        [0.5, 1.5, 2.5],
+    ])
+    with prepare_data(session, clean_bucket_name, 's', 't', data) as t:
+        conn = duckdb.connect()
+        batches = t.select(columns=['a'], predicate=(t['b'] < 2))  # noqa: F841
+        actual = conn.execute('SELECT max(a) as "a_max" FROM batches').arrow()
+        expected = (data
+            .filter(pc.field('b') < 2)
+            .group_by([])
+            .aggregate([('a', 'max')]))
+        assert actual == expected
+
+
+def test_closed_tx(session, clean_bucket_name):
+    columns = pa.schema([
+        ('a', pa.int64()),
+    ])
+    data = pa.table(schema=columns, data=[
+        list(range(10000)),
+    ])
+
+    with session.transaction() as tx:
+        t = tx.bucket(clean_bucket_name).create_schema("s1").create_table("t1", columns)
+        t.insert(data)
+
+        config = QueryConfig(
+            num_sub_splits=1,
+            num_splits=1,
+            num_row_groups_per_sub_split=1,
+            limit_rows_per_sub_split=100)
+        batches = t.select(config=config)  # noqa: F841
+        first = next(batches)  # make sure that HTTP response processing has started
+        assert first['a'].to_pylist() == list(range(100))
+
+        conn = duckdb.connect()
+        res = conn.execute('SELECT a FROM batches')
+        log.debug("closing tx=%s after first batch=%s", t.tx, first)
+
+    # transaction is closed, collecting the result should fail
+    with pytest.raises(duckdb.InvalidInputException, match="Detail: Python exception: MissingTransaction"):
+        res.arrow()

vastdb/tests/test_imports.py

@@ -6,7 +6,7 @@ import pyarrow.parquet as pq
 import pytest
 
 from vastdb import util
-from vastdb.errors import ImportFilesError, InvalidArgument
+from vastdb.errors import ImportFilesError, InternalServerError, InvalidArgument
 
 log = logging.getLogger(__name__)
 
@@ -34,12 +34,24 @@ def test_parallel_imports(session, clean_bucket_name, s3):
         b = tx.bucket(clean_bucket_name)
         s = b.create_schema('s1')
         t = s.create_table('t1', pa.schema([('num', pa.int64())]))
+        with pytest.raises(InternalServerError):
+            t.create_imports_table()
         log.info("Starting import of %d files", num_files)
         t.import_files(files)
         arrow_table = pa.Table.from_batches(t.select(columns=['num']))
         assert arrow_table.num_rows == num_rows * num_files
         arrow_table = pa.Table.from_batches(t.select(columns=['num'], predicate=t['num'] == 100))
         assert arrow_table.num_rows == num_files
+        import_table = t.imports_table()
+        # checking all imports are on the imports table:
+        objects_name = pa.Table.from_batches(import_table.select(columns=["ObjectName"]))
+        objects_name = objects_name.to_pydict()
+        object_names = set(objects_name['ObjectName'])
+        prefix = 'prq'
+        numbers = set(range(53))
+        assert all(name.startswith(prefix) for name in object_names)
+        numbers.issubset(int(name.replace(prefix, '')) for name in object_names)
+        assert len(object_names) == len(objects_name['ObjectName'])
 
 
 def test_create_table_from_files(session, clean_bucket_name, s3):

vastdb/tests/test_projections.py

@@ -4,6 +4,7 @@ import pyarrow as pa
 
 log = logging.getLogger(__name__)
 
+
 def test_basic_projections(session, clean_bucket_name):
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s1')

vastdb/tests/test_sanity.py

@@ -57,10 +57,10 @@ def test_version_extraction():
             return f"vast {version}" if version else "vast"
 
         def log_message(self, format, *args):
-            log.debug(format,*args)
+            log.debug(format, *args)
 
     # start the server on localhost on some available port port
-    server_address =('localhost', 0)
+    server_address = ('localhost', 0)
     httpd = HTTPServer(server_address, MockOptionsHandler)
 
     def start_http_server_in_thread():

vastdb/tests/test_schemas.py

@@ -50,14 +50,14 @@ def test_commits_and_rollbacks(session, clean_bucket_name):
             b = tx.bucket(clean_bucket_name)
             b.schema("s3").drop()
             assert b.schemas() == []
-            1/0  # rollback schema dropping
+            1 / 0  # rollback schema dropping
 
     with session.transaction() as tx:
         b = tx.bucket(clean_bucket_name)
         assert b.schemas() != []
 
+
 def test_list_snapshots(session, clean_bucket_name):
     with session.transaction() as tx:
         b = tx.bucket(clean_bucket_name)
-        s = b.snapshots()
-        assert s == []
+        b.snapshots()  # VAST Catalog may create some snapshots

vastdb/tests/test_tables.py

@@ -3,10 +3,10 @@ import decimal
 import logging
 import random
 import threading
+import time
 from contextlib import closing
 from tempfile import NamedTemporaryFile
 
-import duckdb
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
@@ -91,7 +91,6 @@ def test_exists(session, clean_bucket_name):
         assert s.tables() == [t]
 
 
-
 def test_update_table(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int64()),
@@ -147,12 +146,13 @@ def test_update_table(session, clean_bucket_name):
             'b': [0.5, 1.5, 2.5]
         }
 
+
 def test_select_with_multisplits(session, clean_bucket_name):
     columns = pa.schema([
         ('a', pa.int32())
     ])
 
-    data = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     data = data * 1000
     expected = pa.table(schema=columns, data=[data])
 
@@ -179,15 +179,15 @@ def test_types(session, clean_bucket_name):
         ('t3', pa.time32('ms')),
         ('t6', pa.time64('us')),
         ('t9', pa.time64('ns')),
-        ('ts0' ,pa.timestamp('s')),
-        ('ts3' ,pa.timestamp('ms')),
-        ('ts6' ,pa.timestamp('us')),
-        ('ts9' ,pa.timestamp('ns')),
+        ('ts0', pa.timestamp('s')),
+        ('ts3', pa.timestamp('ms')),
+        ('ts6', pa.timestamp('us')),
+        ('ts9', pa.timestamp('ns')),
     ])
 
     expected = pa.table(schema=columns, data=[
         [True, True, False],
-        [1 , 2, 4],
+        [1, 2, 4],
         [1999, 2000, 2001],
         [11122221, 222111122, 333333],
         [0.5, 1.5, 2.5],
@@ -262,7 +262,7 @@ def test_filters(session, clean_bucket_name):
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
         def select(predicate):
-            return pa.Table.from_batches(t.select(predicate=predicate))
+            return pa.Table.from_batches(t.select(predicate=predicate), t.arrow_schema)
 
         assert select(None) == expected
 
@@ -295,7 +295,7 @@ def test_filters(session, clean_bucket_name):
         assert select(((t['a'] > 111) | (t['a'] < 333)) & (t['b'] < 2.5)) == expected.filter(((pc.field('a') > 111) | (pc.field('a') < 333)) & (pc.field('b') < 2.5))
         with pytest.raises(NotImplementedError):
             assert select((t['a'] > 111) | (t['b'] > 0) | (t['s'] < 'ccc')) == expected.filter((pc.field('a') > 111) | (pc.field('b') > 0) | (pc.field('s') < 'ccc'))
-        assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777) ) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
+        assert select((t['a'] > 111) | (t['a'] < 333) | (t['a'] == 777)) == expected.filter((pc.field('a') > 111) | (pc.field('a') < 333) | (pc.field('a') == 777))
 
         assert select(t['s'].isnull()) == expected.filter(pc.field('s').is_null())
         assert select((t['s'].isnull()) | (t['s'] == 'bb'))  == expected.filter((pc.field('s').is_null()) | (pc.field('s') == 'bb'))
@@ -305,25 +305,12 @@ def test_filters(session, clean_bucket_name):
         assert select(t['s'].contains('b')) == expected.filter(pc.field('s') == 'bb')
         assert select(t['s'].contains('y')) == expected.filter(pc.field('s') == 'xyz')
 
+        assert select(t['a'].isin([555])) == expected.filter(pc.field('a').isin([555]))
+        assert select(t['a'].isin([111, 222, 999])) == expected.filter(pc.field('a').isin([111, 222, 999]))
+        assert select((t['a'] == 111) | t['a'].isin([333, 444]) | (t['a'] > 600)) == expected.filter((pc.field('a') == 111) | pc.field('a').isin([333, 444]) | (pc.field('a') > 600))
 
-def test_duckdb(session, clean_bucket_name):
-    columns = pa.schema([
-        ('a', pa.int32()),
-        ('b', pa.float64()),
-    ])
-    data = pa.table(schema=columns, data=[
-        [111, 222, 333],
-        [0.5, 1.5, 2.5],
-    ])
-    with prepare_data(session, clean_bucket_name, 's', 't', data) as t:
-        conn = duckdb.connect()
-        batches = t.select(columns=['a'], predicate=(t['b'] < 2))  # noqa: F841
-        actual = conn.execute('SELECT max(a) as "a_max" FROM batches').arrow()
-        expected = (data
-            .filter(pc.field('b') < 2)
-            .group_by([])
-            .aggregate([('a', 'max')]))
-        assert actual == expected
+        with pytest.raises(NotImplementedError):
+            select(t['a'].isin([]))
 
 
 def test_parquet_export(session, clean_bucket_name):
@@ -344,8 +331,7 @@ def test_parquet_export(session, clean_bucket_name):
             ['a', 'b'],
         ])
         expected = pa.Table.from_batches([rb])
-        rb = t.insert(rb)
-        assert rb.to_pylist() == [0, 1]
+        t.insert(rb)
         actual = pa.Table.from_batches(t.select())
         assert actual == expected
 
@@ -359,6 +345,7 @@ def test_parquet_export(session, clean_bucket_name):
 
             assert expected == pq.read_table(parquet_file.name)
 
+
 def test_errors(session, clean_bucket_name):
     with pytest.raises(errors.MissingSchema):
         with session.transaction() as tx:
@@ -378,7 +365,8 @@ def test_errors(session, clean_bucket_name):
                 ('s', pa.utf8()),
             ])
             s.create_table('t1', columns)
-            s.drop() # cannot drop schema without dropping its tables first
+            s.drop()  # cannot drop schema without dropping its tables first
+
 
 def test_rename_schema(session, clean_bucket_name):
 
@@ -436,20 +424,21 @@ def test_rename_table(session, clean_bucket_name):
             s.table('t')
         t = s.table('t2')
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         with pytest.raises(errors.MissingTable):
             tx2.bucket(clean_bucket_name).schema('s').table('t2')
         tx2.bucket(clean_bucket_name).schema('s').table('t')
 
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         with pytest.raises(errors.MissingTable):
             s.table('t')
         t = s.table('t2')
         t.drop()
         s.drop()
 
+
 def test_add_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
@@ -472,18 +461,18 @@ def test_add_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         assert t.arrow_schema == new_schema
         t.drop()
         s.drop()
 
+
 def test_drop_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
@@ -507,31 +496,32 @@ def test_drop_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
-        #assert that new transactions see the change
+        # assert that new transactions see the change
         assert t.arrow_schema == new_schema
         t.drop()
         s.drop()
 
+
 def test_rename_column(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.int16()),
             ('b', pa.float32()),
             ('s', pa.utf8()),
         ])
-    def prepare_rename_column(schema : pa.Schema, old_name : str, new_name : str) -> pa.Schema:
+
+    def prepare_rename_column(schema: pa.Schema, old_name: str, new_name: str) -> pa.Schema:
         field_idx = schema.get_field_index(old_name)
         column_to_rename = schema.field(field_idx)
         renamed_column = column_to_rename.with_name(new_name)
         return schema.set(field_idx, renamed_column)
 
-    new_schema = prepare_rename_column(columns,'a','aaa')
+    new_schema = prepare_rename_column(columns, 'a', 'aaa')
 
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema('s')
@@ -546,10 +536,10 @@ def test_rename_column(session, clean_bucket_name):
         # in which it was added
         assert t.arrow_schema == new_schema
 
-        #assert that other transactions are isolated
+        # assert that other transactions are isolated
         assert tx2.bucket(clean_bucket_name).schema('s').table('t').arrow_schema == columns
 
-    #assert that new transactions see the change
+    # assert that new transactions see the change
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).schema('s')
         t = s.table('t')
@@ -564,7 +554,7 @@ def test_rename_column(session, clean_bucket_name):
             t1 = tx1.bucket(clean_bucket_name).schema('s').table('t')
             t2 = tx2.bucket(clean_bucket_name).schema('s').table('t')
             t1.rename_column('b', 'bb')
-            with pytest.raises(HTTPError, match = '409 Client Error: Conflict'):
+            with pytest.raises(HTTPError, match='409 Client Error: Conflict'):
                 t2.rename_column('b', 'bbb')
 
     with session.transaction() as tx:
@@ -580,6 +570,7 @@ def test_rename_column(session, clean_bucket_name):
         t.drop()
         s.drop()
 
+
 def test_select_stop(session, clean_bucket_name):
     columns = pa.schema([
             ('a', pa.uint8()),
@@ -602,15 +593,16 @@ def test_select_stop(session, clean_bucket_name):
     qc = QueryConfig(num_sub_splits=2, num_splits=4, num_row_groups_per_sub_split=1)
     with session.transaction() as tx:
         t = tx.bucket(clean_bucket_name).schema('s').table('t')
-        t.refresh_stats()
-        qc.data_endpoints = list(t.stats.endpoints) * 2
+        qc.data_endpoints = list(t.get_stats().endpoints) * 2
 
     # Duplicate the table until it is large enough to generate enough batches
     while num_rows < (qc.num_sub_splits * qc.num_splits) * ROWS_PER_GROUP:
+        # We need two separate transactions to prevent an infinite loop that may happen
+        # while appending and reading the same table using a single transaction.
         with session.transaction() as tx_read, session.transaction() as tx_write:
             t_read = tx_read.bucket(clean_bucket_name).schema('s').table('t')
             t_write = tx_write.bucket(clean_bucket_name).schema('s').table('t')
-            for batch in t_read.select(['a'],config=qc):
+            for batch in t_read.select(['a'], config=qc):
                 t_write.insert(batch)
         num_rows = num_rows * 2
         log.info("Num rows: %d", num_rows)
@@ -627,11 +619,12 @@ def test_select_stop(session, clean_bucket_name):
     # If this assert triggers it just means that the test assumptions about how
     # the tabular server splits the batches is not true anymore and we need to
     # rewrite the test.
-    assert read_batches == qc.num_splits*qc.num_sub_splits
-    qc.query_id = str(random.randint(0,2**32))
+    assert read_batches == qc.num_splits * qc.num_sub_splits
+    qc.query_id = str(random.randint(0, 2**32))
     log.info("query id is: %s", qc.query_id)
+
     def active_threads():
-        log.debug("%s",[t.getName() for t in threading.enumerate() if t.is_alive()])
+        log.debug("%s", [t.getName() for t in threading.enumerate() if t.is_alive()])
         return sum([1 if t.is_alive() and qc.query_id in t.getName() else 0 for t in threading.enumerate()])
 
     assert active_threads() == 0
@@ -653,3 +646,20 @@ def test_select_stop(session, clean_bucket_name):
 
     # validate that all query threads were killed.
     assert active_threads() == 0
+
+
+def test_big_catalog_select(session, clean_bucket_name):
+    with session.transaction() as tx:
+        bc = tx.catalog()
+        actual = pa.Table.from_batches(bc.select(['name']))
+        assert actual
+        log.info("actual=%s", actual)
+
+
+def test_audit_log_select(session, clean_bucket_name):
+    with session.transaction() as tx:
+        a = tx.audit_log()
+        a.columns()
+        time.sleep(1)
+        actual = pa.Table.from_batches(a.select(), a.arrow_schema)
+        log.info("actual=%s", actual)

vastdb/tests/test_util.py

@@ -0,0 +1,39 @@
+import pyarrow as pa
+import pytest
+
+from .. import errors, util
+
+
+def test_slices():
+    ROWS = 1 << 20
+    t = pa.table({"x": range(ROWS), "y": [i / 1000 for i in range(ROWS)]})
+
+    chunks = list(util.iter_serialized_slices(t))
+    assert len(chunks) > 1
+    sizes = [len(c) for c in chunks]
+
+    assert max(sizes) < util.MAX_RECORD_BATCH_SLICE_SIZE
+    assert t == pa.Table.from_batches(_parse(chunks))
+
+    chunks = list(util.iter_serialized_slices(t, 1000))
+    assert len(chunks) > 1
+    sizes = [len(c) for c in chunks]
+
+    assert max(sizes) < util.MAX_RECORD_BATCH_SLICE_SIZE
+    assert t == pa.Table.from_batches(_parse(chunks))
+
+
+def test_wide_row():
+    cols = [pa.field(f"x{i}", pa.utf8()) for i in range(1000)]
+    values = [['a' * 10000]] * len(cols)
+    t = pa.table(values, schema=pa.schema(cols))
+    assert len(t) == 1
+
+    with pytest.raises(errors.TooWideRow):
+        list(util.iter_serialized_slices(t))
+
+
+def _parse(bufs):
+    for buf in bufs:
+        with pa.ipc.open_stream(buf) as reader:
+            yield from reader

vastdb/tests/util.py

@@ -9,10 +9,7 @@ def prepare_data(session, clean_bucket_name, schema_name, table_name, arrow_tabl
     with session.transaction() as tx:
         s = tx.bucket(clean_bucket_name).create_schema(schema_name)
         t = s.create_table(table_name, arrow_table.schema)
-        row_ids_array = t.insert(arrow_table)
-        row_ids = row_ids_array.to_pylist()
-        log.debug("row_ids=%s" % row_ids)
-        assert row_ids == list(range(arrow_table.num_rows))
+        t.insert(arrow_table)
         yield t
         t.drop()
         s.drop()

vastdb/transaction.py

@@ -8,19 +8,29 @@ A transcation is used as a context manager, since every Database-related operati
 
 import logging
 from dataclasses import dataclass
+from typing import Optional
 
 import botocore
 
-from . import bucket, errors, session
+from . import bucket, errors, schema, session, table
 
 log = logging.getLogger(__name__)
 
+TABULAR_BC_BUCKET = "vast-big-catalog-bucket"
+VAST_CATALOG_SCHEMA_NAME = 'vast_big_catalog_schema'
+VAST_CATALOG_TABLE_NAME = 'vast_big_catalog_table'
+
+TABULAR_AUDERY_BUCKET = "vast-audit-log-bucket"
+AUDERY_SCHEMA_NAME = 'vast_audit_log_schema'
+AUDERY_TABLE_NAME = 'vast_audit_log_table'
+
+
 @dataclass
 class Transaction:
     """A holder of a single VAST transaction."""
 
     _rpc: "session.Session"
-    txid: int = None
+    txid: Optional[int] = None
 
     def __enter__(self):
         """Create a transaction and store its ID."""
@@ -31,15 +41,19 @@ class Transaction:
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
         """On success, the transaction is committed. Otherwise, it is rolled back."""
+        txid = self.txid
+        self.txid = None
         if (exc_type, exc_value, exc_traceback) == (None, None, None):
-            log.debug("committing txid=%016x", self.txid)
-            self._rpc.api.commit_transaction(self.txid)
+            log.debug("committing txid=%016x", txid)
+            self._rpc.api.commit_transaction(txid)
         else:
-            log.exception("rolling back txid=%016x due to:", self.txid)
-            self._rpc.api.rollback_transaction(self.txid)
+            log.exception("rolling back txid=%016x due to:", txid)
+            self._rpc.api.rollback_transaction(txid)
 
     def __repr__(self):
         """Don't show the session details."""
+        if self.txid is None:
+            return 'InvalidTransaction'
         return f'Transaction(id=0x{self.txid:016x})'
 
     def bucket(self, name: str) -> "bucket.Bucket":
@@ -52,3 +66,15 @@ class Transaction:
                 raise errors.MissingBucket(name)
             raise
         return bucket.Bucket(name, self)
+
+    def catalog(self, fail_if_missing=True) -> Optional["table.Table"]:
+        """Return VAST Catalog table."""
+        b = bucket.Bucket(TABULAR_BC_BUCKET, self)
+        s = schema.Schema(VAST_CATALOG_SCHEMA_NAME, b)
+        return s.table(name=VAST_CATALOG_TABLE_NAME, fail_if_missing=fail_if_missing)
+
+    def audit_log(self, fail_if_missing=True) -> Optional["table.Table"]:
+        """Return VAST AuditLog table."""
+        b = bucket.Bucket(TABULAR_AUDERY_BUCKET, self)
+        s = schema.Schema(AUDERY_SCHEMA_NAME, b)
+        return s.table(name=AUDERY_TABLE_NAME, fail_if_missing=fail_if_missing)

vastdb/util.py

@@ -1,19 +1,22 @@
 import logging
-from typing import Callable
+from typing import TYPE_CHECKING, Callable, List, Optional, Union
 
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument
-from .schema import Schema
-from .table import ImportConfig, Table
+from .errors import InvalidArgument, TooWideRow
 
 log = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from .schema import Schema
+    from .table import ImportConfig, Table
+
 
 def create_table_from_files(
-        schema: Schema, table_name: str, parquet_files: [str], schema_merge_func: Callable = None,
-        config: ImportConfig = None) -> Table:
+        schema: "Schema", table_name: str, parquet_files: List[str],
+        schema_merge_func: Optional[Callable] = None,
+        config: Optional["ImportConfig"] = None) -> "Table":
     if not schema_merge_func:
         schema_merge_func = default_schema_merge
     else:
@@ -76,3 +79,36 @@ def union_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.S
     This function returns a unified schema from potentially two different schemas.
     """
     return pa.unify_schemas([current_schema, new_schema])
+
+
+MAX_TABULAR_REQUEST_SIZE = 5 << 20  # in bytes
+MAX_RECORD_BATCH_SLICE_SIZE = int(0.9 * MAX_TABULAR_REQUEST_SIZE)
+
+
+def iter_serialized_slices(batch: Union[pa.RecordBatch, pa.Table], max_rows_per_slice=None):
+    """Iterate over a list of record batch slices."""
+
+    rows_per_slice = int(0.9 * len(batch) * MAX_RECORD_BATCH_SLICE_SIZE / batch.nbytes)
+    if max_rows_per_slice is not None:
+        rows_per_slice = min(rows_per_slice, max_rows_per_slice)
+
+    offset = 0
+    while offset < len(batch):
+        if rows_per_slice < 1:
+            raise TooWideRow(batch)
+
+        batch_slice = batch.slice(offset, rows_per_slice)
+        serialized_slice_batch = serialize_record_batch(batch_slice)
+        if len(serialized_slice_batch) <= MAX_RECORD_BATCH_SLICE_SIZE:
+            yield serialized_slice_batch
+            offset += rows_per_slice
+        else:
+            rows_per_slice = rows_per_slice // 2
+
+
+def serialize_record_batch(batch: Union[pa.RecordBatch, pa.Table]):
+    """Serialize a RecordBatch using Arrow IPC format."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, batch.schema) as writer:
+        writer.write(batch)
+    return sink.getvalue()

{vastdb-0.1.1.dist-info → vastdb-0.1.3.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.1
+Version: 0.1.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -22,7 +21,7 @@ License-File: LICENSE
 Requires-Dist: aws-requests-auth
 Requires-Dist: boto3
 Requires-Dist: flatbuffers
-Requires-Dist: ibis-framework
+Requires-Dist: ibis-framework ==8.0.0
 Requires-Dist: pyarrow
 Requires-Dist: requests
 Requires-Dist: xmltodict
@@ -34,5 +33,3 @@ and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system
 enabling schema and table management, efficient ingest, query and modification of columnar data.
 
 For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
-
-