vastdb 0.0.5.3__tar.gz → 0.0.5.4__tar.gz

{vastdb-0.0.5.3 → vastdb-0.0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.3
+Version: 0.0.5.4
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.0.5.3 → vastdb-0.0.5.4}/setup.py

@@ -46,7 +46,7 @@ if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
 setup(
     name='vastdb',
     description='VAST Data SDK',
-    version='0.0.5.3' + suffix,
+    version='0.0.5.4' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.0.5.3 → vastdb-0.0.5.4}/vastdb/api.py

@@ -1296,7 +1296,7 @@ class VastdbApi:
         return self._check_res(res, "get_transaction", expected_retvals)
 
     def select_row_ids(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[],
-                       retry_count=0, enable_sorted_projections=False):
+                       retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=SelectRowIds HTTP/1.1
         """
@@ -1313,7 +1313,7 @@ class VastdbApi:
         return self._check_res(res, "query_data", expected_retvals)
 
     def read_columns_data(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[], tenant_guid=None,
-                          retry_count=0, enable_sorted_projections=False):
+                          retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=ReadColumns HTTP/1.1
         """
@@ -1329,7 +1329,7 @@ class VastdbApi:
         return self._check_res(res, "query_data", expected_retvals)
 
     def count_rows(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[], tenant_guid=None,
-                   retry_count=0, enable_sorted_projections=False):
+                   retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=CountRows HTTP/1.1
         """
@@ -1485,10 +1485,11 @@ class VastdbApi:
         return False
 
     def _query_page(self, bucket, schema, table, query_data_request, split=(0, 1, 8), num_sub_splits=1, response_row_id=False,
-                   txid=0, limit_rows=0, sub_split_start_row_ids=[], filters=None, field_names=None):
+                   txid=0, limit_rows=0, sub_split_start_row_ids=[], filters=None, field_names=None, enable_sorted_projections=True):
         res = self.query_data(bucket=bucket, schema=schema, table=table, params=query_data_request.serialized, split=split,
                               num_sub_splits=num_sub_splits, response_row_id=response_row_id, txid=txid,
-                              limit_rows=limit_rows, sub_split_start_row_ids=sub_split_start_row_ids)
+                              limit_rows=limit_rows, sub_split_start_row_ids=sub_split_start_row_ids,
+                              enable_sorted_projections=enable_sorted_projections)
         start_row_ids = {}
         sub_split_tables = parse_query_data_response(res.raw, query_data_request.response_schema,
                                                     start_row_ids=start_row_ids)
@@ -1499,10 +1500,11 @@ class VastdbApi:
         return table_page, start_row_ids
 
     def _query_page_iterator(self, bucket, schema, table, query_data_request, split=(0, 1, 8), num_sub_splits=1, response_row_id=False,
-                             txid=0, limit_rows=0, start_row_ids={}, filters=None, field_names=None):
+                             txid=0, limit_rows=0, start_row_ids={}, filters=None, field_names=None, enable_sorted_projections=True):
         res = self.query_data(bucket=bucket, schema=schema, table=table, params=query_data_request.serialized, split=split,
                               num_sub_splits=num_sub_splits, response_row_id=response_row_id, txid=txid,
-                              limit_rows=limit_rows, sub_split_start_row_ids=start_row_ids.items())
+                              limit_rows=limit_rows, sub_split_start_row_ids=start_row_ids.items(),
+                              enable_sorted_projections=enable_sorted_projections)
         for sub_split_table in parse_query_data_response(res.raw, query_data_request.response_schema,
                                                         start_row_ids=start_row_ids):
             for record_batch in sub_split_table.to_batches():
@@ -1510,7 +1512,8 @@ class VastdbApi:
         _logger.info(f"query_page_iterator: start_row_ids={start_row_ids}")
 
     def query_iterator(self, bucket, schema, table, num_sub_splits=1, num_row_groups_per_sub_split=8,
-                       response_row_id=False, txid=0, limit_per_sub_split=128*1024, filters=None, field_names=None):
+                       response_row_id=False, txid=0, limit_per_sub_split=128*1024, filters=None, field_names=None,
+                       enable_sorted_projections=True):
         """
         query rows into a table.
 
@@ -1588,7 +1591,8 @@ class VastdbApi:
                                                                          split=(split_id, num_splits, num_row_groups_per_sub_split),
                                                                          num_sub_splits=num_sub_splits, response_row_id=response_row_id,
                                                                          txid=txid, limit_rows=limit_per_sub_split,
-                                                                         start_row_ids=start_row_ids):
+                                                                         start_row_ids=start_row_ids,
+                                                                         enable_sorted_projections=enable_sorted_projections):
                             output_queue.put((split_id, record_batch))
                             while not next_sems[split_id].acquire(timeout=1): # wait for the main thread to request the next record batch
                                 if killall:
@@ -1682,7 +1686,7 @@ class VastdbApi:
 
     def query(self, bucket, schema, table, num_sub_splits=1, num_row_groups_per_sub_split=8,
               response_row_id=False, txid=0, limit=0, limit_per_sub_split=131072, filters=None, field_names=None,
-              queried_columns=None):
+              queried_columns=None, enable_sorted_projections=True):
         """
         query rows into a table.
 
@@ -1769,7 +1773,8 @@ class VastdbApi:
                                                                         split=(split_id, num_splits, num_row_groups_per_sub_split),
                                                                         num_sub_splits=num_sub_splits, response_row_id=response_row_id,
                                                                         txid=txid, limit_rows=limit_rows,
-                                                                        sub_split_start_row_ids=start_row_ids.items())
+                                                                        sub_split_start_row_ids=start_row_ids.items(),
+                                                                        enable_sorted_projections=enable_sorted_projections)
                         with lock:
                             table_pages.append(table_page)
                             row_counts[split_id] += len(table_page)

{vastdb-0.0.5.3 → vastdb-0.0.5.4}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.3
+Version: 0.0.5.4
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.0.5.3 → vastdb-0.0.5.4}/vastdb.egg-info/SOURCES.txt

@@ -152,7 +152,6 @@ vast_flatbuf/tabular/S3File.py
 vast_flatbuf/tabular/__init__.py
 vastdb/__init__.py
 vastdb/api.py
-vastdb/bench_scan.py
 vastdb.egg-info/PKG-INFO
 vastdb.egg-info/SOURCES.txt
 vastdb.egg-info/dependency_links.txt

vastdb-0.0.5.3/vastdb/bench_scan.py

@@ -1,45 +0,0 @@
-from vastdb import api
-
-from logbook import Logger, StreamHandler
-import sys
-import time
-import pprint
-
-StreamHandler(sys.stdout).push_application()
-log = Logger('Logbook')
-
-# access_key_id=F3YUMQZDQB60ZZJ1PBAZ
-# secret_access_key=9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p
-
-a = api.VastdbApi(
-    access_key='F3YUMQZDQB60ZZJ1PBAZ', 
-    secret_key='9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p', 
-    host='172.19.111.1:172.19.111.16')
-
-kwargs = dict(
-    bucket='tabular-slothful-jocular-jack',
-    schema='tpcds_schema_create_as_select',
-    table='store_sales',
-    field_names=['ss_sold_date_sk', 'ss_sold_time_sk', 'ss_item_sk'],
-    filters={'ss_item_sk': ['le 1']},
-    num_sub_splits=8)
-
-pprint.pprint(kwargs)
-
-res = a.query_iterator(**kwargs)
-
-total_bytes = 0
-total_rows = 0
-start = time.time()
-last_log = None
-
-for b in res: 
-    total_bytes += b.get_total_buffer_size()
-    total_rows += len(b)
-    dt = time.time() - start
-    if last_log != int(dt):
-        log.info("{:.3f} Mrow/s, {:.3f} MB/s", (total_rows/dt) / 1e6, (total_bytes/dt) / 1e6)
-        last_log = int(dt)
-
-dt = time.time() - start
-log.info("Done after {:.3f} seconds, {:.3f} Mrows, {:.3f} MB", dt, total_rows / 1e6, total_bytes / 1e6)