vastdb 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

vastdb/_internal.py

@@ -1298,6 +1298,15 @@ class VastdbApi:
 
         return columns, next_key, is_truncated, count
 
+    def head_bucket(self, bucket_name):
+        """
+        Reimplemented, instead of depending on boto3 for checking the existence of a bucket.
+        https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html
+        """
+        return self._request(
+            method="HEAD",
+            url=self._url(bucket=bucket_name))
+
     def begin_transaction(self, client_tags=[], expected_retvals=[]):
         """
         POST /?transaction HTTP/1.1

vastdb/bench/test_sample.py

@@ -109,7 +109,7 @@ def load_batch(bucket, session_kwargs, offset, limit):
 
 def test_ingest(test_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
     session = vastdb.connect(**session_kwargs)
-    metrics_table = metrics.Table(perf_metrics_db, "ingest")
+    metrics_table = metrics.Table(perf_metrics_db, "test_ingest")
 
     with session.transaction() as tx:
         b = tx.bucket(test_bucket_name)
@@ -201,9 +201,11 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
                 nbytes=data, rows=rows, cols=len(cols),
                 pid=pid, tid=tid, sdk_version=sdk_version))
 
+    return metrics_rows
+
 
 def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls, perf_metrics_db):
-    metrics_table = metrics.Table(perf_metrics_db, "query")
+    metrics_table = metrics.Table(perf_metrics_db, "test_scan")
 
     log.info("starting %d workers, endpoints=%s", num_workers, tabular_endpoint_urls)
     with ProcessPoolExecutor(max_workers=num_workers) as executor:

vastdb/conftest.py

@@ -102,4 +102,4 @@ def table_name(request):
 
 @pytest.fixture(scope="function")
 def perf_metrics_db(crater_path):
-    return sqlite3.connect(f"{crater_path}/metrics.sqlite")
+    return sqlite3.connect(f"{crater_path}/metrics.sqlite" if crater_path else ":memory:")

vastdb/session.py

@@ -23,8 +23,6 @@ class Session:
                  timeout=None,
                  backoff_config: Optional["BackoffConfig"] = None):
         """Connect to a VAST Database endpoint, using specified credentials."""
-        import boto3
-
         from . import _internal, features
 
         if access is None:
@@ -42,10 +40,6 @@ class Session:
             timeout=timeout,
             backoff_config=backoff_config)
         self.features = features.Features(self.api.vast_version)
-        self.s3 = boto3.client('s3',
-            aws_access_key_id=access,
-            aws_secret_access_key=secret,
-            endpoint_url=endpoint)
 
     def __repr__(self):
         """Don't show the secret key."""

vastdb/table.py

@@ -439,7 +439,7 @@ class Table:
             self.update(rows=column_record_batch, columns=columns_name_chunk)
         return row_ids
 
-    def insert(self, rows: pa.RecordBatch):
+    def insert(self, rows: Union[pa.RecordBatch, pa.Table]):
         """Insert a RecordBatch into this table."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)

vastdb/tests/test_tables.py

@@ -81,6 +81,19 @@ def test_insert_wide_row(session, clean_bucket_name):
         assert actual == expected
 
 
+def test_multi_batch_table(session, clean_bucket_name):
+    columns = pa.schema([pa.field('s', pa.utf8())])
+    expected = pa.Table.from_batches([
+        pa.record_batch(schema=columns, data=[['a']]),
+        pa.record_batch(schema=columns, data=[['b']]),
+        pa.record_batch(schema=columns, data=[['c']]),
+    ])
+
+    with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
+        actual = t.select().read_all()
+        assert actual == expected
+
+
 def test_insert_empty(session, clean_bucket_name):
     columns = pa.schema([('a', pa.int8()), ('b', pa.float32())])
     data = [[None] * 5, [None] * 5]

vastdb/transaction.py

@@ -10,8 +10,6 @@ import logging
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Iterable, Optional
 
-import botocore
-
 from . import bucket, errors, schema, session
 
 if TYPE_CHECKING:
@@ -64,12 +62,10 @@ class Transaction:
     def bucket(self, name: str) -> "Bucket":
         """Return a VAST Bucket, if exists."""
         try:
-            self._rpc.s3.head_bucket(Bucket=name)
-        except botocore.exceptions.ClientError as e:
-            log.warning("res: %s", e.response)
-            if e.response['Error']['Code'] == '404':
-                raise errors.MissingBucket(name) from e
-            raise
+            self._rpc.api.head_bucket(name)
+        except errors.NotFound as e:
+            raise errors.MissingBucket(name) from e
+
         return bucket.Bucket(name, self)
 
     def catalog_snapshots(self) -> Iterable["Bucket"]:

vastdb/util.py

@@ -114,6 +114,11 @@ def iter_serialized_slices(batch: Union[pa.RecordBatch, pa.Table], max_rows_per_
 
 def serialize_record_batch(batch: Union[pa.RecordBatch, pa.Table]):
     """Serialize a RecordBatch using Arrow IPC format."""
+    if isinstance(batch, pa.Table):
+        if len(batch.to_batches()) > 1:
+            # the server expects a single RecordBatch per request
+            batch = batch.combine_chunks()
+
     sink = pa.BufferOutputStream()
     with pa.ipc.new_stream(sink, batch.schema) as writer:
         writer.write(batch)

{vastdb-1.1.0.dist-info → vastdb-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.1.0
+Version: 1.1.1
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
@@ -19,13 +19,12 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aws-requests-auth
-Requires-Dist: boto3
 Requires-Dist: flatbuffers
-Requires-Dist: ibis-framework ==9.0.0
+Requires-Dist: ibis-framework==9.0.0
 Requires-Dist: pyarrow
 Requires-Dist: requests
 Requires-Dist: xmltodict
-Requires-Dist: backoff ==2.2.1
+Requires-Dist: backoff==2.2.1
 
 
 `vastdb` is a Python-based SDK designed for interacting

{vastdb-1.1.0.dist-info → vastdb-1.1.1.dist-info}/RECORD

@@ -149,20 +149,20 @@ vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=J1JjKiFkKC95BHowfh9kJfQFTjRce-QMsc6zF_FfxC0,432
-vastdb/_internal.py,sha256=lTzkF_N5YSfyVxu1q3F3dxpBriLC-pyS04a46eevssQ,90917
+vastdb/_internal.py,sha256=KRFNlB25Ckj4nowVUneC5tBSYXpnIv-MDdw4Vn6KYnc,91255
 vastdb/bucket.py,sha256=5KuKhPjZOevznZqWHDVVocejvAy7dcwobPuV6BJCfPc,2544
 vastdb/config.py,sha256=1tMYtzKXerGcIUjH4tIGEvZNWvO4fviCEdcNCnELJZo,2269
-vastdb/conftest.py,sha256=ePzQiEQmlNGcM2T4GZevE4XuvcnFWfnTSzr8IVZpVKk,3438
+vastdb/conftest.py,sha256=X2kVveySPQYZlVBXUMoo7Oea5IsvmJzjdqq3fpH2kVw,3469
 vastdb/errors.py,sha256=2XR1ko7J5nkfiHSAgwuVAADw0SsyqxOwSeFaGgKZEXM,4186
 vastdb/features.py,sha256=DxV746LSkORwVSD6MP2hdXRfnyoLkJwtOwGmp1dnquo,1322
 vastdb/schema.py,sha256=X7IRrogXH7Z0kes-DsDh1bRqIhvjH6owlFigGBXy7XQ,5913
-vastdb/session.py,sha256=ZrQf8cecVIHIBUOPNg4ed8ZCnEEu0QW1OBxQgz_ia80,2241
-vastdb/table.py,sha256=OYdwB-RYr14OfvMRM4BPKLhAWThCympQq5zTwsMIMOg,31077
-vastdb/transaction.py,sha256=qu2rOlR7AS1ojMOzgWapQMpcorrutelZZLH1mLmTHxk,3186
-vastdb/util.py,sha256=4LTYBBR13na376AmDm5lQILJzLcfelIKdkNPy0IqI0o,5684
+vastdb/session.py,sha256=toMR0BXwTaECdWDKnIZky1F3MA1SmelRBiqCrqQ3GCM,2067
+vastdb/table.py,sha256=XLNAwlRjT9sE8nPzYoBC2ehdYpjbit0gLyG0ieNvVfs,31094
+vastdb/transaction.py,sha256=NlVkEowJ_pmtffjWBBDaKExYDKPekjSZyj_fK_bZPJE,3026
+vastdb/util.py,sha256=eunfTuqbCrqQEFZEO9T15N-Bu8Fqpw7Zlqp2TAGfYaY,5870
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/test_perf.py,sha256=gZIqfHva6lNFpD-9bHAe7M8COBjUyrPkHu3E7F8J2L0,1072
-vastdb/bench/test_sample.py,sha256=bFmw7BOCr5FoGn4TY9pQGd6_cVNK4uBeSRi33tTubyk,7847
+vastdb/bench/test_sample.py,sha256=LgF4syzij09sH3Noiv1EyCAJ9pvrUE5bxR4RJTVEYag,7881
 vastdb/bench/perf_bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/perf_bench/cli.py,sha256=NtaPEBTDI6PWgEtwI1wVbwmUeA5bwGqAj_Z_2lDJ28I,5931
 vastdb/bench/perf_bench/run.py,sha256=2dKooybt7e8VqM7FzY1jrIu6wZYQuoAF9o7AHVh-WVQ,2632
@@ -196,14 +196,14 @@ vastdb/tests/test_nested.py,sha256=LPU6uV3Ri23dBzAEMFQqRPbqapV5LfmiHSHkhILPIY0,6
 vastdb/tests/test_projections.py,sha256=3y1kubwVrzO-xoR0hyps7zrjOJI8niCYspaFTN16Q9w,4540
 vastdb/tests/test_sanity.py,sha256=oiV2gb05aPyG5RMNUQZlyjNlg3T7Fig1_8OJzpAgcsk,3038
 vastdb/tests/test_schemas.py,sha256=l70YQMlx2UL1KRQhApriiG2ZM7GJF-IzWU31H3Yqn1U,3312
-vastdb/tests/test_tables.py,sha256=qWicD0BYuhrh1kRVqkHMJNsxcHxDcCprbEXuZJm1wm4,31529
+vastdb/tests/test_tables.py,sha256=3YYytrcPZiflFCM6yZPOBI3pz5iXKy0sQbmUDFRVmdA,32000
 vastdb/tests/test_util.py,sha256=Ok_sAEBJsRGF5Voa_v5eu3eAd52GWu8jMjjQbadwW-s,1260
 vastdb/tests/util.py,sha256=dpRJYbboDnlqL4qIdvScpp8--5fxRUBIcIYitrfcj9o,555
 vastdb/vast_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/vast_tests/test_ha.py,sha256=744P4G6VJ09RIkHhMQL4wlipCBJWQVMhyvUrSc4k1HQ,975
 vastdb/vast_tests/test_scale.py,sha256=EpjCJmVAQrNBxVnHGJ-KHCoxevhqOcyqYFPMIIY9s60,2714
-vastdb-1.1.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-1.1.0.dist-info/METADATA,sha256=N78CWW546GDDjVafhYRYYU4fM1xaCTeCHob0XF0p9EI,1363
-vastdb-1.1.0.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-vastdb-1.1.0.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-1.1.0.dist-info/RECORD,,
+vastdb-1.1.1.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-1.1.1.dist-info/METADATA,sha256=C6KtrB0iOL3E8noTM0UqjYvurcant8lSNjwukCXnXTE,1340
+vastdb-1.1.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vastdb-1.1.1.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-1.1.1.dist-info/RECORD,,

{vastdb-1.1.0.dist-info → vastdb-1.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any