vastdb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

vastdb/transaction.py

@@ -16,6 +16,14 @@ from . import bucket, errors, schema, session, table
 
 log = logging.getLogger(__name__)
 
+TABULAR_BC_BUCKET = "vast-big-catalog-bucket"
+VAST_CATALOG_SCHEMA_NAME = 'vast_big_catalog_schema'
+VAST_CATALOG_TABLE_NAME = 'vast_big_catalog_table'
+
+TABULAR_AUDERY_BUCKET = "vast-audit-log-bucket"
+AUDERY_SCHEMA_NAME = 'vast_audit_log_schema'
+AUDERY_TABLE_NAME = 'vast_audit_log_table'
+
 
 @dataclass
 class Transaction:
@@ -44,6 +52,8 @@ class Transaction:
 
     def __repr__(self):
         """Don't show the session details."""
+        if self.txid is None:
+            return 'InvalidTransaction'
         return f'Transaction(id=0x{self.txid:016x})'
 
     def bucket(self, name: str) -> "bucket.Bucket":
@@ -59,6 +69,12 @@ class Transaction:
 
     def catalog(self, fail_if_missing=True) -> Optional["table.Table"]:
         """Return VAST Catalog table."""
-        b = bucket.Bucket("vast-big-catalog-bucket", self)
-        s = schema.Schema("vast_big_catalog_schema", b)
-        return s.table(name="vast_big_catalog_table", fail_if_missing=fail_if_missing)
+        b = bucket.Bucket(TABULAR_BC_BUCKET, self)
+        s = schema.Schema(VAST_CATALOG_SCHEMA_NAME, b)
+        return s.table(name=VAST_CATALOG_TABLE_NAME, fail_if_missing=fail_if_missing)
+
+    def audit_log(self, fail_if_missing=True) -> Optional["table.Table"]:
+        """Return VAST AuditLog table."""
+        b = bucket.Bucket(TABULAR_AUDERY_BUCKET, self)
+        s = schema.Schema(AUDERY_SCHEMA_NAME, b)
+        return s.table(name=AUDERY_TABLE_NAME, fail_if_missing=fail_if_missing)

vastdb/util.py

@@ -1,20 +1,22 @@
 import logging
-from typing import Callable, List, Optional
+from typing import TYPE_CHECKING, Callable, List, Optional, Union
 
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument
-from .schema import Schema
-from .table import ImportConfig, Table
+from .errors import InvalidArgument, TooWideRow
 
 log = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from .schema import Schema
+    from .table import ImportConfig, Table
+
 
 def create_table_from_files(
-        schema: Schema, table_name: str, parquet_files: List[str],
+        schema: "Schema", table_name: str, parquet_files: List[str],
         schema_merge_func: Optional[Callable] = None,
-        config: Optional[ImportConfig] = None) -> Table:
+        config: Optional["ImportConfig"] = None) -> "Table":
     if not schema_merge_func:
         schema_merge_func = default_schema_merge
     else:
@@ -77,3 +79,36 @@ def union_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.S
     This function returns a unified schema from potentially two different schemas.
     """
     return pa.unify_schemas([current_schema, new_schema])
+
+
+MAX_TABULAR_REQUEST_SIZE = 5 << 20  # in bytes
+MAX_RECORD_BATCH_SLICE_SIZE = int(0.9 * MAX_TABULAR_REQUEST_SIZE)
+
+
+def iter_serialized_slices(batch: Union[pa.RecordBatch, pa.Table], max_rows_per_slice=None):
+    """Iterate over a list of record batch slices."""
+
+    rows_per_slice = int(0.9 * len(batch) * MAX_RECORD_BATCH_SLICE_SIZE / batch.nbytes)
+    if max_rows_per_slice is not None:
+        rows_per_slice = min(rows_per_slice, max_rows_per_slice)
+
+    offset = 0
+    while offset < len(batch):
+        if rows_per_slice < 1:
+            raise TooWideRow(batch)
+
+        batch_slice = batch.slice(offset, rows_per_slice)
+        serialized_slice_batch = serialize_record_batch(batch_slice)
+        if len(serialized_slice_batch) <= MAX_RECORD_BATCH_SLICE_SIZE:
+            yield serialized_slice_batch
+            offset += rows_per_slice
+        else:
+            rows_per_slice = rows_per_slice // 2
+
+
+def serialize_record_batch(batch: Union[pa.RecordBatch, pa.Table]):
+    """Serialize a RecordBatch using Arrow IPC format."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, batch.schema) as writer:
+        writer.write(batch)
+    return sink.getvalue()

{vastdb-0.1.2.dist-info → vastdb-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.2
+Version: 0.1.4
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
@@ -21,10 +21,11 @@ License-File: LICENSE
 Requires-Dist: aws-requests-auth
 Requires-Dist: boto3
 Requires-Dist: flatbuffers
-Requires-Dist: ibis-framework
+Requires-Dist: ibis-framework ==8.0.0
 Requires-Dist: pyarrow
 Requires-Dist: requests
 Requires-Dist: xmltodict
+Requires-Dist: backoff ==2.2.1
 
 
 `vastdb` is a Python-based SDK designed for interacting

{vastdb-0.1.2.dist-info → vastdb-0.1.4.dist-info}/RECORD

@@ -149,28 +149,29 @@ vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=cMJtZuJ0IL9aKyM3DUWqTCzuP1H1MXXVivKKE1-q0DY,292
-vastdb/bucket.py,sha256=xtKs7S4w0jmI4MujDWH3HDI-iEgbq5Xqqsod-tw4zSo,2991
+vastdb/bucket.py,sha256=4rPEm9qlPTg7ccWO6VGmd4LKb8w-BDhJYwzXGjn03sc,3566
 vastdb/conftest.py,sha256=pKpo_46Vq4QHzTDQAFxasrVhnZ2V2L-y6IMLxojxaFM,2132
-vastdb/errors.py,sha256=fxpKSxjEgoJZuBtEGWzTW9lpDlEjuzgpgXwAQc1W6BQ,3436
-vastdb/internal_commands.py,sha256=3F6FiYu-Ama1zBO7hENPxCaQYJT8mcZP6rSQvtI7Sks,101273
-vastdb/schema.py,sha256=MrQr-WIrES8KcQ0V6cJkRRp_-9jj9FboyrBnkNBsw-8,3324
-vastdb/session.py,sha256=VZOFGZbAdr5Tl4cp88VRQYnR4Q16UNuYjSmX_QPW1II,1718
-vastdb/table.py,sha256=bdx3C1iWiFivKmtifH7MyG7TMqnVVIU91as-_hMn1rE,20532
-vastdb/transaction.py,sha256=1uCSHXqWcwsMJv6DuNx4WyQMGUm8P-RCCqYdBdUGusI,2196
-vastdb/util.py,sha256=Tjj6p4gqabK5G21uWuCiuYM9FaaR04_Zk5X8NWtcdj8,3022
+vastdb/errors.py,sha256=fj8IlPnGi1lbJWIl1-8MSjLavL9bYQ-YUoboWbXCo54,4047
+vastdb/internal_commands.py,sha256=yS6ylyuJjaAwAm4OqVGX4tq-Un5cvM-LXp7F4eYOUDw,100414
+vastdb/schema.py,sha256=ql4TPB1W_FQ_BHov3CKHI8JX3krXMlcKWz7dTrjpQ1w,3346
+vastdb/session.py,sha256=2tu5cp7xG28ynyQfEl9_HM2dtNcLM2AoJmm3bfNLC0o,2563
+vastdb/table.py,sha256=apRXCrglg6_glozJXu8D7q6du5seP7NMi42PNjyGcTM,28891
+vastdb/transaction.py,sha256=g8YTcYnsNPIhB2udbHyT5RIFB5kHnBLJcvV2CWRICwI,2845
+vastdb/util.py,sha256=rs7nLL2Qz-OVEZDSVIqAvS-uETMq-zxQs5jBksB5-JA,4276
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/test_perf.py,sha256=iHE3E60fvyU5SBDHPi4h03Dj6QcY6VI9l9mMhgNMtPc,1117
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/tests/test_duckdb.py,sha256=KDuv4PrjGEwChCGHG36xNT2JiFlBOt6K3DQ3L06Kq-A,1913
-vastdb/tests/test_imports.py,sha256=fDUjO5U-5i4QTIMoNnSSW4X_ZnOStLbx0mJkNq2pj9Q,5033
+vastdb/tests/test_imports.py,sha256=48kbJKsa_MrEXcBYQUbUDr1e9wzjG4FHQ7C3wUEQfXA,5705
 vastdb/tests/test_nested.py,sha256=3kejEvtSqV0LrUgb1QglRjrlxnKI4_AXTFw2nE7Q520,951
 vastdb/tests/test_projections.py,sha256=_cDNfD5zTwbCXLk6uGpPUWGN0P-4HElu5OjubWu-Jg0,1255
 vastdb/tests/test_sanity.py,sha256=ixx0QPo73hLHjAa7bByFXjS1XST0WvmSwLEpgnHh_JY,2960
-vastdb/tests/test_schemas.py,sha256=b-JpYHOFYVTdE570_La7O2RWf8BGN-q8KDXNXeC8CSg,1724
-vastdb/tests/test_tables.py,sha256=TXM4LSBvPb3EEu7XScZ5iEiu_zhHClq61W18EQodxw8,25667
-vastdb/tests/util.py,sha256=NaCzKymEGy1xuiyMxyt2_0frKVfVk9iGrFwLf3GHjTI,435
-vastdb-0.1.2.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.1.2.dist-info/METADATA,sha256=edJPdDWmHj6tRHRR97eSppfN9_4ARfIr0jS9HMjHfSQ,1311
-vastdb-0.1.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-vastdb-0.1.2.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.1.2.dist-info/RECORD,,
+vastdb/tests/test_schemas.py,sha256=qoHTLX51D-0S4bMxdCpRh9gaYQd-BkZdT_agGOwFwTM,1739
+vastdb/tests/test_tables.py,sha256=pfQx0OZm6oVJj1-CziPWUoEn3l2-OET4Bpl8M9Z4mws,27499
+vastdb/tests/test_util.py,sha256=owRAU3TCKMq-kz54NRdA5wX2O_bZIHqG5ucUR77jm5k,1046
+vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
+vastdb-0.1.4.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-0.1.4.dist-info/METADATA,sha256=SyZkyjQSwklzsq3oub8m8w9lY-HuI4XOG72y8trKvf4,1350
+vastdb-0.1.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+vastdb-0.1.4.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-0.1.4.dist-info/RECORD,,