vastdb 0.0.5.3__tar.gz → 0.0.5.5__tar.gz

{vastdb-0.0.5.3 → vastdb-0.0.5.5}/PKG-INFO

@@ -1,14 +1,18 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.3
+Version: 0.0.5.5
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Platform: UNKNOWN
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: flatbuffers
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: aws-requests-auth
+Requires-Dist: xmltodict
 
 
 `VastdbApi` is a Python based API designed for interacting with *VastDB* & *Vast Catalog*, enabling operations such as schema and table management, data querying, and transaction handling.
@@ -38,5 +42,3 @@ vastdb_session = create_vastdb_session(access_key, secret_key)
 ```
 #### For the complete Guide for the SDK please go to VastData github: https://github.com/vast-data/vastdb_sdk
 
-
-

{vastdb-0.0.5.3 → vastdb-0.0.5.5}/setup.py

@@ -46,7 +46,7 @@ if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
 setup(
     name='vastdb',
     description='VAST Data SDK',
-    version='0.0.5.3' + suffix,
+    version='0.0.5.5' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.0.5.3 → vastdb-0.0.5.5}/vastdb/api.py

@@ -974,8 +974,17 @@ class VastdbApi:
 
             return snapshots, is_truncated, marker
 
+    @staticmethod
+    def encode_table_props(**kwargs):
+        if all(v is None for v in kwargs.values()):
+            return None
+        else:
+            return "$".join([f"{k}={v}" for k, v in kwargs.items()])
 
-    def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[], topic_partitions=0):
+    def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[],
+                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False):
+                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False,
+                     src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
         """
         Create a table, use the following request
         POST /bucket/schema/table?table HTTP/1.1
@@ -997,6 +1006,10 @@ class VastdbApi:
         headers['Content-Length'] = str(len(serialized_schema))
         url_params = {'topic_partitions': str(topic_partitions)} if topic_partitions else {}
 
+        if topic_partitions > 0:
+            table_props = self.encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
+            url_params['table-props'] = table_props
+
         res = self.session.post(self._api_prefix(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
                                 data=serialized_schema, headers=headers)
         return self._check_res(res, "create_table", expected_retvals)
@@ -1048,7 +1061,8 @@ class VastdbApi:
         return self._check_res(res, "get_table_stats", expected_retvals)
 
     def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
-                    new_name="", expected_retvals=[]):
+                    new_name="", expected_retvals=[],
+                    src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
         """
         PUT /mybucket/myschema/mytable?table HTTP/1.1
         Content-Length: ContentLength
@@ -1060,6 +1074,9 @@ class VastdbApi:
         """
         builder = flatbuffers.Builder(1024)
 
+        if src_timestamp is not None or retention_in_mins is not None or past_threshold_ts is not None or future_threshold_ts is not None:
+            table_properties = self.encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
+
         properties = builder.CreateString(table_properties)
         tabular_alter_table.Start(builder)
         if len(table_properties):
@@ -1296,7 +1313,7 @@ class VastdbApi:
         return self._check_res(res, "get_transaction", expected_retvals)
 
     def select_row_ids(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[],
-                       retry_count=0, enable_sorted_projections=False):
+                       retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=SelectRowIds HTTP/1.1
         """
@@ -1313,7 +1330,7 @@ class VastdbApi:
         return self._check_res(res, "query_data", expected_retvals)
 
     def read_columns_data(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[], tenant_guid=None,
-                          retry_count=0, enable_sorted_projections=False):
+                          retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=ReadColumns HTTP/1.1
         """
@@ -1329,7 +1346,7 @@ class VastdbApi:
         return self._check_res(res, "query_data", expected_retvals)
 
     def count_rows(self, bucket, schema, table, params, txid=0, client_tags=[], expected_retvals=[], tenant_guid=None,
-                   retry_count=0, enable_sorted_projections=False):
+                   retry_count=0, enable_sorted_projections=True):
         """
         POST /mybucket/myschema/mytable?query-data=CountRows HTTP/1.1
         """
@@ -1485,10 +1502,11 @@ class VastdbApi:
         return False
 
     def _query_page(self, bucket, schema, table, query_data_request, split=(0, 1, 8), num_sub_splits=1, response_row_id=False,
-                   txid=0, limit_rows=0, sub_split_start_row_ids=[], filters=None, field_names=None):
+                   txid=0, limit_rows=0, sub_split_start_row_ids=[], filters=None, field_names=None, enable_sorted_projections=True):
         res = self.query_data(bucket=bucket, schema=schema, table=table, params=query_data_request.serialized, split=split,
                               num_sub_splits=num_sub_splits, response_row_id=response_row_id, txid=txid,
-                              limit_rows=limit_rows, sub_split_start_row_ids=sub_split_start_row_ids)
+                              limit_rows=limit_rows, sub_split_start_row_ids=sub_split_start_row_ids,
+                              enable_sorted_projections=enable_sorted_projections)
         start_row_ids = {}
         sub_split_tables = parse_query_data_response(res.raw, query_data_request.response_schema,
                                                     start_row_ids=start_row_ids)
@@ -1499,10 +1517,11 @@ class VastdbApi:
         return table_page, start_row_ids
 
     def _query_page_iterator(self, bucket, schema, table, query_data_request, split=(0, 1, 8), num_sub_splits=1, response_row_id=False,
-                             txid=0, limit_rows=0, start_row_ids={}, filters=None, field_names=None):
+                             txid=0, limit_rows=0, start_row_ids={}, filters=None, field_names=None, enable_sorted_projections=True):
         res = self.query_data(bucket=bucket, schema=schema, table=table, params=query_data_request.serialized, split=split,
                               num_sub_splits=num_sub_splits, response_row_id=response_row_id, txid=txid,
-                              limit_rows=limit_rows, sub_split_start_row_ids=start_row_ids.items())
+                              limit_rows=limit_rows, sub_split_start_row_ids=start_row_ids.items(),
+                              enable_sorted_projections=enable_sorted_projections)
         for sub_split_table in parse_query_data_response(res.raw, query_data_request.response_schema,
                                                         start_row_ids=start_row_ids):
             for record_batch in sub_split_table.to_batches():
@@ -1510,7 +1529,8 @@ class VastdbApi:
         _logger.info(f"query_page_iterator: start_row_ids={start_row_ids}")
 
     def query_iterator(self, bucket, schema, table, num_sub_splits=1, num_row_groups_per_sub_split=8,
-                       response_row_id=False, txid=0, limit_per_sub_split=128*1024, filters=None, field_names=None):
+                       response_row_id=False, txid=0, limit_per_sub_split=128*1024, filters=None, field_names=None,
+                       enable_sorted_projections=True):
         """
         query rows into a table.
 
@@ -1588,7 +1608,8 @@ class VastdbApi:
                                                                          split=(split_id, num_splits, num_row_groups_per_sub_split),
                                                                          num_sub_splits=num_sub_splits, response_row_id=response_row_id,
                                                                          txid=txid, limit_rows=limit_per_sub_split,
-                                                                         start_row_ids=start_row_ids):
+                                                                         start_row_ids=start_row_ids,
+                                                                         enable_sorted_projections=enable_sorted_projections):
                             output_queue.put((split_id, record_batch))
                             while not next_sems[split_id].acquire(timeout=1): # wait for the main thread to request the next record batch
                                 if killall:
@@ -1682,7 +1703,7 @@ class VastdbApi:
 
     def query(self, bucket, schema, table, num_sub_splits=1, num_row_groups_per_sub_split=8,
               response_row_id=False, txid=0, limit=0, limit_per_sub_split=131072, filters=None, field_names=None,
-              queried_columns=None):
+              queried_columns=None, enable_sorted_projections=True):
         """
         query rows into a table.
 
@@ -1769,7 +1790,8 @@ class VastdbApi:
                                                                         split=(split_id, num_splits, num_row_groups_per_sub_split),
                                                                         num_sub_splits=num_sub_splits, response_row_id=response_row_id,
                                                                         txid=txid, limit_rows=limit_rows,
-                                                                        sub_split_start_row_ids=start_row_ids.items())
+                                                                        sub_split_start_row_ids=start_row_ids.items(),
+                                                                        enable_sorted_projections=enable_sorted_projections)
                         with lock:
                             table_pages.append(table_page)
                             row_counts[split_id] += len(table_page)

{vastdb-0.0.5.3 → vastdb-0.0.5.5}/vastdb.egg-info/PKG-INFO

@@ -1,14 +1,18 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.3
+Version: 0.0.5.5
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Platform: UNKNOWN
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: flatbuffers
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: aws-requests-auth
+Requires-Dist: xmltodict
 
 
 `VastdbApi` is a Python based API designed for interacting with *VastDB* & *Vast Catalog*, enabling operations such as schema and table management, data querying, and transaction handling.
@@ -38,5 +42,3 @@ vastdb_session = create_vastdb_session(access_key, secret_key)
 ```
 #### For the complete Guide for the SDK please go to VastData github: https://github.com/vast-data/vastdb_sdk
 
-
-

{vastdb-0.0.5.3 → vastdb-0.0.5.5}/vastdb.egg-info/SOURCES.txt

@@ -152,7 +152,6 @@ vast_flatbuf/tabular/S3File.py
 vast_flatbuf/tabular/__init__.py
 vastdb/__init__.py
 vastdb/api.py
-vastdb/bench_scan.py
 vastdb.egg-info/PKG-INFO
 vastdb.egg-info/SOURCES.txt
 vastdb.egg-info/dependency_links.txt

vastdb-0.0.5.3/vastdb/bench_scan.py

@@ -1,45 +0,0 @@
-from vastdb import api
-
-from logbook import Logger, StreamHandler
-import sys
-import time
-import pprint
-
-StreamHandler(sys.stdout).push_application()
-log = Logger('Logbook')
-
-# access_key_id=F3YUMQZDQB60ZZJ1PBAZ
-# secret_access_key=9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p
-
-a = api.VastdbApi(
-    access_key='F3YUMQZDQB60ZZJ1PBAZ', 
-    secret_key='9a9Q3if6IC5LjUexly/nXFv1UCANBnhGxi++Sw6p', 
-    host='172.19.111.1:172.19.111.16')
-
-kwargs = dict(
-    bucket='tabular-slothful-jocular-jack',
-    schema='tpcds_schema_create_as_select',
-    table='store_sales',
-    field_names=['ss_sold_date_sk', 'ss_sold_time_sk', 'ss_item_sk'],
-    filters={'ss_item_sk': ['le 1']},
-    num_sub_splits=8)
-
-pprint.pprint(kwargs)
-
-res = a.query_iterator(**kwargs)
-
-total_bytes = 0
-total_rows = 0
-start = time.time()
-last_log = None
-
-for b in res: 
-    total_bytes += b.get_total_buffer_size()
-    total_rows += len(b)
-    dt = time.time() - start
-    if last_log != int(dt):
-        log.info("{:.3f} Mrow/s, {:.3f} MB/s", (total_rows/dt) / 1e6, (total_bytes/dt) / 1e6)
-        last_log = int(dt)
-
-dt = time.time() - start
-log.info("Done after {:.3f} seconds, {:.3f} Mrows, {:.3f} MB", dt, total_rows / 1e6, total_bytes / 1e6)