vastdb 0.0.5.2__tar.gz → 0.0.5.3__tar.gz

{vastdb-0.0.5.2 → vastdb-0.0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.2
+Version: 0.0.5.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.0.5.2 → vastdb-0.0.5.3}/setup.py

@@ -46,7 +46,7 @@ if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
 setup(
     name='vastdb',
     description='VAST Data SDK',
-    version='0.0.5.2' + suffix,
+    version='0.0.5.3' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.0.5.2 → vastdb-0.0.5.3}/vastdb/api.py

@@ -757,6 +757,7 @@ class VastdbApi:
         if not port:
             port = 443 if secure else 80
 
+        self.default_max_list_columns_page_size = 1000
         self.session = requests.Session()
         self.session.verify = False
         self.session.headers['user-agent'] = "VastData Tabular API 1.0 - 2022 (c)"
@@ -1207,7 +1208,7 @@ class VastdbApi:
                                 data=serialized_schema, headers=headers)
         return self._check_res(res, "drop_columns", expected_retvals)
 
-    def list_columns(self, bucket, schema, table, *, txid=0, client_tags=None, max_keys=1000, next_key=0,
+    def list_columns(self, bucket, schema, table, *, txid=0, client_tags=None, max_keys=None, next_key=0,
                      count_only=False, name_prefix="", exact_match=False,
                      expected_retvals=None, bc_list_internals=False):
         """
@@ -1218,6 +1219,7 @@ class VastdbApi:
         tabular-max-keys: 1000
         tabular-next-key: NextColumnId
         """
+        max_keys = max_keys or self.default_max_list_columns_page_size
         client_tags = client_tags or []
         expected_retvals = expected_retvals or []
 
@@ -2007,7 +2009,7 @@ class VastdbApi:
         txid, created_txid = self._begin_tx_if_necessary(txid)
 
         if rows:
-            columns = self._list_table_columns(bucket, schema, table, field_names=rows.keys())
+            columns = self._list_table_columns(bucket, schema, table, field_names=rows.keys(), txid=txid)
             columns_dict = dict([(column[0], column[1]) for column in columns])
             arrow_schema = pa.schema([])
             arrays = []

vastdb-0.0.5.3/vastdb/bench_scan.py

@@ -0,0 +1,45 @@
+from vastdb import api
+
+from logbook import Logger, StreamHandler
+import sys
+import time
+import pprint
+
+StreamHandler(sys.stdout).push_application()
+log = Logger('Logbook')
+
+# access_key_id=F3YUMQZDQB60ZZJ1PBAZ
+# secret_access_key=9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p
+
+a = api.VastdbApi(
+    access_key='F3YUMQZDQB60ZZJ1PBAZ', 
+    secret_key='9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p', 
+    host='172.19.111.1:172.19.111.16')
+
+kwargs = dict(
+    bucket='tabular-slothful-jocular-jack',
+    schema='tpcds_schema_create_as_select',
+    table='store_sales',
+    field_names=['ss_sold_date_sk', 'ss_sold_time_sk', 'ss_item_sk'],
+    filters={'ss_item_sk': ['le 1']},
+    num_sub_splits=8)
+
+pprint.pprint(kwargs)
+
+res = a.query_iterator(**kwargs)
+
+total_bytes = 0
+total_rows = 0
+start = time.time()
+last_log = None
+
+for b in res: 
+    total_bytes += b.get_total_buffer_size()
+    total_rows += len(b)
+    dt = time.time() - start
+    if last_log != int(dt):
+        log.info("{:.3f} Mrow/s, {:.3f} MB/s", (total_rows/dt) / 1e6, (total_bytes/dt) / 1e6)
+        last_log = int(dt)
+
+dt = time.time() - start
+log.info("Done after {:.3f} seconds, {:.3f} Mrows, {:.3f} MB", dt, total_rows / 1e6, total_bytes / 1e6)

{vastdb-0.0.5.2 → vastdb-0.0.5.3}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.2
+Version: 0.0.5.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.0.5.2 → vastdb-0.0.5.3}/vastdb.egg-info/SOURCES.txt

@@ -152,6 +152,7 @@ vast_flatbuf/tabular/S3File.py
 vast_flatbuf/tabular/__init__.py
 vastdb/__init__.py
 vastdb/api.py
+vastdb/bench_scan.py
 vastdb.egg-info/PKG-INFO
 vastdb.egg-info/SOURCES.txt
 vastdb.egg-info/dependency_links.txt