vastdb 0.0.5.7__tar.gz → 0.0.5.9__tar.gz

{vastdb-0.0.5.7 → vastdb-0.0.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.7
+Version: 0.0.5.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.0.5.7 → vastdb-0.0.5.9}/setup.py

@@ -46,7 +46,7 @@ if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
 setup(
     name='vastdb',
     description='VAST Data SDK',
-    version='0.0.5.7' + suffix,
+    version='0.0.5.9' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.0.5.7 → vastdb-0.0.5.9}/vast_flatbuf/tabular/ObjectDetails.py

@@ -60,7 +60,14 @@ class ObjectDetails(object):
             return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos)
         return 0
 
-def Start(builder): builder.StartObject(5)
+    # ObjectDetails
+    def NumPartitions(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos)
+        return 0
+
+def Start(builder): builder.StartObject(6)
 def ObjectDetailsStart(builder):
     """This method is deprecated. Please switch to Start."""
     return Start(builder)
@@ -84,6 +91,10 @@ def AddSizeInBytes(builder, sizeInBytes): builder.PrependUint64Slot(4, sizeInByt
 def ObjectDetailsAddSizeInBytes(builder, sizeInBytes):
     """This method is deprecated. Please switch to AddSizeInBytes."""
     return AddSizeInBytes(builder, sizeInBytes)
+def AddNumPartitions(builder, numPartitions): builder.PrependUint64Slot(5, numPartitions, 0)
+def ObjectDetailsAddNumPartitions(builder, numPartitions):
+    """This method is deprecated. Please switch to AddNumPartitions."""
+    return AddNumPartitions(builder, numPartitions)
 def End(builder): return builder.EndObject()
 def ObjectDetailsEnd(builder):
     """This method is deprecated. Please switch to End."""

{vastdb-0.0.5.7 → vastdb-0.0.5.9}/vastdb/api.py

@@ -681,16 +681,57 @@ def _iter_nested_arrays(column: pa.Array) -> Iterator[pa.Array]:
     elif isinstance(column.type, pa.ListType):
         yield from _iter_nested_arrays(column.values)  # Note: Map is serialized in VAST as a List<Struct<K, V>>
 
+class ValidateInList:
+    def __init__(self, *args):
+        self.candidates = [x.strip() for x in args]
+
+    def __call__(self, x):
+        x = x.strip()
+        if x not in self.candidates:
+            raise Exception(f'{x} is not in {self.candidates}')
+        return x
+
+    def __getitem__(self, x):
+        return self
+
+
+_int_coding = (lambda x: str(int(x)), lambda x: int(x))
+_prop_coding = {
+        "message.timestamp.type": ValidateInList('CreateTime', 'LogAppendTime'),
+        "retention.ms": _int_coding,
+        "message.timestamp.after.max.ms": _int_coding,
+        "message.timestamp.before.max.ms": _int_coding
+        }
+
+
+def _encode_table_props(**kwargs):
+    if all([v is None for v in kwargs.values()]):
+        return None
+    else:
+        pairs = [(k.replace("_", ".").strip(), v) for k, v in kwargs.items() if v is not None]
+        return "$".join([f"{k}={_prop_coding[k][0](v)}" for k, v in pairs])
+
+
+def _decode_table_props(s):
+    if s.strip() == '':
+        return {}
+    triplets = [(x.strip(), x.strip().replace(".", "_"), y.strip()) for x, y in [z.split('=') for z in s.strip().split("$")]]
+    return {y: _prop_coding[x][1](z) for x, y, z in triplets if z != ''}
 
-TableInfo = namedtuple('table_info', 'name properties handle num_rows size_in_bytes')
-def _parse_table_info(obj):
 
+TableInfo = namedtuple('table_info', 'name properties handle num_rows size_in_bytes num_partitions')
+
+
+def _parse_table_info(obj):
     name = obj.Name().decode()
     properties = obj.Properties().decode()
     handle = obj.Handle().decode()
     num_rows = obj.NumRows()
     used_bytes = obj.SizeInBytes()
-    return TableInfo(name, properties, handle, num_rows, used_bytes)
+    num_partitions = obj.NumPartitions()
+    if num_partitions != 0:
+        properties = _decode_table_props(properties)
+    return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions)
 
 def build_record_batch(column_info, column_values):
     _logger.info(f"column_info={column_info}")
@@ -974,16 +1015,9 @@ class VastdbApi:
 
             return snapshots, is_truncated, marker
 
-    @staticmethod
-    def encode_table_props(**kwargs):
-        if all(v is None for v in kwargs.values()):
-            return None
-        else:
-            return "$".join([f"{k}={v}" for k, v in kwargs.items()])
-
-    def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[],
+    def create_table(self, bucket, schema, name, arrow_schema=None, txid=0, client_tags=[], expected_retvals=[],
                      topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False,
-                     src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
+                     message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
         """
         Create a table, use the following request
         POST /bucket/schema/table?table HTTP/1.1
@@ -1001,12 +1035,15 @@ class VastdbApi:
         """
         headers = self._fill_common_headers(txid=txid, client_tags=client_tags)
 
+        if arrow_schema is None:
+            arrow_schema = pa.schema([])
+
         serialized_schema = arrow_schema.serialize()
         headers['Content-Length'] = str(len(serialized_schema))
         url_params = {'topic_partitions': str(topic_partitions)} if topic_partitions else {}
 
         if topic_partitions > 0:
-            table_props = self.encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
+            table_props = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
             if table_props is not None:
                 url_params['table-props'] = table_props
 
@@ -1062,7 +1099,7 @@ class VastdbApi:
 
     def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
                     new_name="", expected_retvals=[],
-                    src_timestamp=None, retention_in_mins=None, past_threshold_ts=None, future_threshold_ts=None):
+                    message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
         """
         PUT /mybucket/myschema/mytable?table HTTP/1.1
         Content-Length: ContentLength
@@ -1074,8 +1111,8 @@ class VastdbApi:
         """
         builder = flatbuffers.Builder(1024)
 
-        if src_timestamp is not None or retention_in_mins is not None or past_threshold_ts is not None or future_threshold_ts is not None:
-            table_properties = self.encode_table_props(src_timestamp=src_timestamp, retention_in_mins=retention_in_mins, past_threshold_ts=past_threshold_ts, future_threshold_ts=future_threshold_ts)
+        if message_timestamp_type is not None or retention_ms is not None or message_timestamp_after_max_ms is not None or message_timestamp_before_max_ms is not None:
+            table_properties = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
             if table_properties is None:
                 table_properties = ""
 

{vastdb-0.0.5.7 → vastdb-0.0.5.9}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.0.5.7
+Version: 0.0.5.9
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA