vastdb 0.0.5.8__tar.gz → 0.0.5.10__tar.gz

{vastdb-0.0.5.8 → vastdb-0.0.5.10}/PKG-INFO

@@ -1,18 +1,14 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.8
+Version: 0.0.5.10
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
+Platform: UNKNOWN
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flatbuffers
-Requires-Dist: pyarrow
-Requires-Dist: requests
-Requires-Dist: aws-requests-auth
-Requires-Dist: xmltodict
 
 
 `VastdbApi` is a Python based API designed for interacting with *VastDB* & *Vast Catalog*, enabling operations such as schema and table management, data querying, and transaction handling.
@@ -42,3 +38,5 @@ vastdb_session = create_vastdb_session(access_key, secret_key)
 ```
 #### For the complete Guide for the SDK please go to VastData github: https://github.com/vast-data/vastdb_sdk
 
+
+

{vastdb-0.0.5.8 → vastdb-0.0.5.10}/setup.py

@@ -46,7 +46,7 @@ if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
 setup(
     name='vastdb',
     description='VAST Data SDK',
-    version='0.0.5.8' + suffix,
+    version='0.0.5.10' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.0.5.8 → vastdb-0.0.5.10}/vastdb/api.py

@@ -92,6 +92,8 @@ TABULAR_QUERY_DATA_FAILED_STREAM_ID = 0xFFFFFFFF - 2
 TABULAR_INVALID_ROW_ID = 0xFFFFFFFFFFFF # (1<<48)-1
 ESTORE_INVALID_EHANDLE = UINT64_MAX
 
+KAFKA_TOPICS_SCHEMA_NAME = 'kafka_topics'
+
 """
 S3 Tabular API
 """
@@ -681,32 +683,55 @@ def _iter_nested_arrays(column: pa.Array) -> Iterator[pa.Array]:
     elif isinstance(column.type, pa.ListType):
         yield from _iter_nested_arrays(column.values)  # Note: Map is serialized in VAST as a List<Struct<K, V>>
 
+class ValidateInList:
+    def __init__(self, *args):
+        self.candidates = [x.strip() for x in args]
+
+    def __call__(self, x):
+        x = x.strip()
+        if x not in self.candidates:
+            raise Exception(f'{x} is not in {self.candidates}')
+        return x
+
+    def __getitem__(self, x):
+        return self
+
+
+_int_coding = (lambda x: str(int(x)), lambda x: int(x))
+_prop_coding = {
+        "message.timestamp.type": ValidateInList('CreateTime', 'LogAppendTime'),
+        "retention.ms": _int_coding,
+        "message.timestamp.after.max.ms": _int_coding,
+        "message.timestamp.before.max.ms": _int_coding
+        }
+
+
 def _encode_table_props(**kwargs):
     if all([v is None for v in kwargs.values()]):
         return None
     else:
-        return "$".join([f"{k}={v}" for k, v in kwargs.items()])
+        pairs = [(k.replace("_", ".").strip(), v) for k, v in kwargs.items() if v is not None]
+        return "$".join([f"{k}={_prop_coding[k][0](v)}" for k, v in pairs])
 
 
 def _decode_table_props(s):
     if s.strip() == '':
         return {}
-    else:
-        return {x[0]: int(x[1]) for x in [y.split('=') for y in s.strip().split("$")]}
+    triplets = [(x.strip(), x.strip().replace(".", "_"), y.strip()) for x, y in [z.split('=') for z in s.strip().split("$")]]
+    return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
 
 TableInfo = namedtuple('table_info', 'name properties handle num_rows size_in_bytes num_partitions')
 
 
-def _parse_table_info(obj):
+def _parse_table_info(obj, parse_properties):
     name = obj.Name().decode()
     properties = obj.Properties().decode()
     handle = obj.Handle().decode()
     num_rows = obj.NumRows()
     used_bytes = obj.SizeInBytes()
     num_partitions = obj.NumPartitions()
-    if num_partitions != 0:
-        properties = _decode_table_props(properties)
+    properties = parse_properties(properties)
     return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions)
 
 def build_record_batch(column_info, column_values):
@@ -991,9 +1016,28 @@ class VastdbApi:
 
             return snapshots, is_truncated, marker
 
-    def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[],
-                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False,
-                     src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
+    def create_table(self, bucket, schema, name, arrow_schema=None, txid=0, client_tags=[], expected_retvals=[],
+                     create_imports_table=False, use_external_row_ids_allocation=False):
+        return self._create_table_internal(bucket=bucket, schema=schema, name=name, arrow_schema=arrow_schema,
+                                           txid=txid, client_tags=client_tags, expected_retvals=expected_retvals,
+                                           create_imports_table=create_imports_table,
+                                           use_external_row_ids_allocation=use_external_row_ids_allocation)
+
+    def create_topic(self, bucket, name, topic_partitions, expected_retvals=[],
+                     message_timestamp_type=None, retention_ms=None,
+                     message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
+        table_props = _encode_table_props(message_timestamp_type=message_timestamp_type,
+                                          retention_ms=retention_ms,
+                                          message_timestamp_after_max_ms=message_timestamp_after_max_ms,
+                                          message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+        return self._create_table_internal(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name, arrow_schema=None,
+                                           expected_retvals=expected_retvals, topic_partitions=topic_partitions,
+                                           table_props=table_props)
+
+    def _create_table_internal(self, bucket, schema, name, arrow_schema=None,
+                               txid=0, client_tags=[], expected_retvals=[], topic_partitions=0,
+                               create_imports_table=False, use_external_row_ids_allocation=False,
+                               table_props=None):
         """
         Create a table, use the following request
         POST /bucket/schema/table?table HTTP/1.1
@@ -1011,14 +1055,15 @@ class VastdbApi:
         """
         headers = self._fill_common_headers(txid=txid, client_tags=client_tags)
 
+        if arrow_schema is None:
+            arrow_schema = pa.schema([])
+
         serialized_schema = arrow_schema.serialize()
         headers['Content-Length'] = str(len(serialized_schema))
         url_params = {'topic_partitions': str(topic_partitions)} if topic_partitions else {}
 
-        if topic_partitions > 0:
-            table_props = _encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
-            if table_props is not None:
-                url_params['table-props'] = table_props
+        if table_props is not None:
+            url_params['table-props'] = table_props
 
         res = self.session.post(self._api_prefix(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
                                 data=serialized_schema, headers=headers)
@@ -1070,9 +1115,22 @@ class VastdbApi:
             return num_rows, size_in_bytes, is_external_rowid_alloc
         return self._check_res(res, "get_table_stats", expected_retvals)
 
-    def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
+    def alter_topic(self, bucket, name,
                     new_name="", expected_retvals=[],
-                    src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
+                    message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None,
+                    message_timestamp_before_max_ms=None):
+        table_properties = _encode_table_props(message_timestamp_type=message_timestamp_type,
+                                               retention_ms=retention_ms,
+                                               message_timestamp_after_max_ms=message_timestamp_after_max_ms,
+                                               message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+        if table_properties is None:
+            table_properties = ""
+
+        return self.alter_table(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name,
+                                table_properties=table_properties, expected_retvals=expected_retvals)
+
+    def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
+                    new_name="", expected_retvals=[]):
         """
         PUT /mybucket/myschema/mytable?table HTTP/1.1
         Content-Length: ContentLength
@@ -1084,10 +1142,8 @@ class VastdbApi:
         """
         builder = flatbuffers.Builder(1024)
 
-        if src_timestamp is not None or retention_in_mins is not None or past_threshold_ts is not None or future_threshold_ts is not None:
-            table_properties = _encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
-            if table_properties is None:
-                table_properties = ""
+        if table_properties is None:
+            table_properties = ""
 
         properties = builder.CreateString(table_properties)
         tabular_alter_table.Start(builder)
@@ -1107,6 +1163,10 @@ class VastdbApi:
 
         return self._check_res(res, "alter_table", expected_retvals)
 
+    def drop_topic(self, bucket, name, expected_retvals=[]):
+        return self.drop_table(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name,
+                               expected_retvals=expected_retvals)
+
     def drop_table(self, bucket, schema, name, txid=0, client_tags=[], expected_retvals=[]):
         """
         DELETE /mybucket/schema_path/mytable?table HTTP/1.1
@@ -1118,8 +1178,25 @@ class VastdbApi:
         res = self.session.delete(self._api_prefix(bucket=bucket, schema=schema, table=name, command="table"), headers=headers)
         return self._check_res(res, "drop_table", expected_retvals)
 
+    def list_topics(self, bucket, max_keys=1000, next_key=0, name_prefix="",
+                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+        return self._list_tables_internal(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME,
+                                          parse_properties=_decode_table_props, max_keys=max_keys,
+                                          next_key=next_key, name_prefix=name_prefix, exact_match=exact_match,
+                                          expected_retvals=expected_retvals,
+                                          include_list_stats=include_list_stats, count_only=count_only)
+
     def list_tables(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
                     exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+        parse_properties = lambda x: x
+        return self._list_tables_internal(bucket=bucket, schema=schema, txid=txid, client_tags=client_tags,
+                                          parse_properties=parse_properties, max_keys=max_keys, next_key=next_key,
+                                          name_prefix=name_prefix, exact_match=exact_match,
+                                          expected_retvals=expected_retvals,
+                                          include_list_stats=include_list_stats, count_only=count_only)
+
+    def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
+                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
         """
         GET /mybucket/schema_path?table HTTP/1.1
         tabular-txid: TransactionId
@@ -1155,7 +1232,7 @@ class VastdbApi:
             tables_length = lists.TablesLength()
             count = int(res_headers['tabular-list-count']) if 'tabular-list-count' in res_headers else tables_length
             for i in range(tables_length):
-                tables.append(_parse_table_info(lists.Tables(i)))
+                tables.append(_parse_table_info(lists.Tables(i), parse_properties))
 
             return bucket_name, schema_name, tables, next_key, is_truncated, count
 
@@ -2323,7 +2400,7 @@ class VastdbApi:
                 raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
             projections_length = lists.ProjectionsLength()
             for i in range(projections_length):
-                projections.append(_parse_table_info(lists.Projections(i)))
+                projections.append(_parse_table_info(lists.Projections(i), lambda x:x))
 
             return bucket_name, schema_name, table_name, projections, next_key, is_truncated, count
 

{vastdb-0.0.5.8 → vastdb-0.0.5.10}/vastdb.egg-info/PKG-INFO

@@ -1,18 +1,14 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.8
+Version: 0.0.5.10
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
+Platform: UNKNOWN
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flatbuffers
-Requires-Dist: pyarrow
-Requires-Dist: requests
-Requires-Dist: aws-requests-auth
-Requires-Dist: xmltodict
 
 
 `VastdbApi` is a Python based API designed for interacting with *VastDB* & *Vast Catalog*, enabling operations such as schema and table management, data querying, and transaction handling.
@@ -42,3 +38,5 @@ vastdb_session = create_vastdb_session(access_key, secret_key)
 ```
 #### For the complete Guide for the SDK please go to VastData github: https://github.com/vast-data/vastdb_sdk
 
+
+

{vastdb-0.0.5.8 → vastdb-0.0.5.10}/vastdb.egg-info/requires.txt

@@ -1,5 +1,5 @@
+aws-requests-auth
 flatbuffers
 pyarrow
 requests
-aws-requests-auth
 xmltodict