vastdb 1.3.4__tar.gz → 1.3.6__tar.gz

{vastdb-1.3.4 → vastdb-1.3.6}/CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.3.6] (2024-12-23)
+[1.3.2]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.1...v1.3.2
+
+### Fixed
+ - Fix missing api call.
+
+## [1.3.5] (2024-12-16)
+[1.3.5]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.4...v1.3.5
+
+### Added
+ - Separated Kafka Topic specific APIs
+
 ## [1.3.4] (2024-12-11)
 [1.3.4]: https://github.com/vast-data/vastdb_sdk/compare/v1.3.3...v1.3.4
 

{vastdb-1.3.4 → vastdb-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.4
+Version: 1.3.6
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-1.3.4 → vastdb-1.3.6}/README.md

@@ -78,6 +78,8 @@ with session.transaction() as tx:
 
 For configuration examples, see [here](docs/config.md).
 
+The list of supported data types can be found [here](docs/types.md).
+
 Note: the transaction must be remain open while the returned [pyarrow.RecordBatchReader](https://arrow.apache.org/docs/python/generated/pyarrow.RecordBatchReader.html) generator is being used.
 
 ## Use Cases

{vastdb-1.3.4 → vastdb-1.3.6}/setup.py

@@ -29,7 +29,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='1.3.4' + suffix,
+    version='1.3.6' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-1.3.4 → vastdb-1.3.6}/vastdb/_internal.py

@@ -119,6 +119,8 @@ TABULAR_INVALID_ROW_ID = 0xFFFFFFFFFFFF  # (1<<48)-1
 ESTORE_INVALID_EHANDLE = UINT64_MAX
 IMPORTED_OBJECTS_TABLE_NAME = "vastdb-imported-objects"
 
+KAFKA_TOPICS_SCHEMA_NAME = 'kafka_topics'
+
 """
 S3 Tabular API
 """
@@ -787,15 +789,14 @@ def _decode_table_props(s):
 TableInfo = namedtuple('TableInfo', 'name properties handle num_rows size_in_bytes num_partitions')
 
 
-def _parse_table_info(obj):
+def _parse_table_info(obj, parse_properties):
     name = obj.Name().decode()
     properties = obj.Properties().decode()
     handle = obj.Handle().decode()
     num_rows = obj.NumRows()
     used_bytes = obj.SizeInBytes()
     num_partitions = obj.NumPartitions()
-    if num_partitions != 0:
-        properties = _decode_table_props(properties)
+    properties = parse_properties(properties)
     return TableInfo(name, properties, handle, num_rows, used_bytes, num_partitions)
 
 
@@ -1109,9 +1110,29 @@ class VastdbApi:
 
         return snapshots, is_truncated, marker
 
-    def create_table(self, bucket, schema, name, arrow_schema=None, txid=0, client_tags=[], expected_retvals=[],
-                     topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False,
-                     message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
+    def create_table(self, bucket, schema, name, arrow_schema=None,
+                     txid=0, client_tags=[], expected_retvals=[],
+                     create_imports_table=False, use_external_row_ids_allocation=False, table_props=None):
+        self._create_table_internal(bucket=bucket, schema=schema, name=name, arrow_schema=arrow_schema,
+                                    txid=txid, client_tags=client_tags, expected_retvals=expected_retvals,
+                                    create_imports_table=create_imports_table, use_external_row_ids_allocation=use_external_row_ids_allocation,
+                                    table_props=table_props)
+
+    def create_topic(self, bucket, name, topic_partitions, expected_retvals=[],
+                     message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None,
+                     message_timestamp_before_max_ms=None):
+        table_props = _encode_table_props(message_timestamp_type=message_timestamp_type,
+                                          retention_ms=retention_ms,
+                                          message_timestamp_after_max_ms=message_timestamp_after_max_ms,
+                                          message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+
+        self._create_table_internal(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name, arrow_schema=None,
+                                    expected_retvals=expected_retvals, topic_partitions=topic_partitions,
+                                    table_props=table_props)
+
+    def _create_table_internal(self, bucket, schema, name, arrow_schema=None,
+                               txid=0, client_tags=[], expected_retvals=[], topic_partitions=0,
+                               create_imports_table=False, use_external_row_ids_allocation=False, table_props=None):
         """
         Create a table, use the following request
         POST /bucket/schema/table?table HTTP/1.1
@@ -1142,16 +1163,17 @@ class VastdbApi:
         if create_imports_table:
             url_params['sub-table'] = IMPORTED_OBJECTS_TABLE_NAME
 
-        if topic_partitions > 0:
-            table_props = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
-            if table_props is not None:
-                url_params['table-props'] = table_props
+        if table_props is not None:
+            url_params['table-props'] = table_props
 
         self._request(
             method="POST",
             url=self._url(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
             data=serialized_schema, headers=headers)
 
+    def get_topic_stats(self, bucket, name, expected_retvals=[]):
+        return self.get_table_stats(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name, expected_retvals=expected_retvals)
+
     def get_table_stats(self, bucket, schema, name, txid=0, client_tags=[], expected_retvals=[], imports_table_stats=False):
         """
         GET /mybucket/myschema/mytable?stats HTTP/1.1
@@ -1174,9 +1196,22 @@ class VastdbApi:
         endpoints = [self.url]  # we cannot replace the host by a VIP address in HTTPS-based URLs
         return TableStatsResult(num_rows, size_in_bytes, is_external_rowid_alloc, tuple(endpoints))
 
-    def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
+    def alter_topic(self, bucket, name,
                     new_name="", expected_retvals=[],
-                    message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None, message_timestamp_before_max_ms=None):
+                    message_timestamp_type=None, retention_ms=None, message_timestamp_after_max_ms=None,
+                    message_timestamp_before_max_ms=None):
+        table_properties = _encode_table_props(message_timestamp_type=message_timestamp_type,
+                                               retention_ms=retention_ms,
+                                               message_timestamp_after_max_ms=message_timestamp_after_max_ms,
+                                               message_timestamp_before_max_ms=message_timestamp_before_max_ms)
+        if table_properties is None:
+            table_properties = ""
+
+        self.alter_table(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name,
+                         table_properties=table_properties, new_name=new_name, expected_retvals=expected_retvals)
+
+    def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
+                    new_name="", expected_retvals=[]):
         """
         PUT /mybucket/myschema/mytable?table HTTP/1.1
         Content-Length: ContentLength
@@ -1188,10 +1223,8 @@ class VastdbApi:
         """
         builder = flatbuffers.Builder(1024)
 
-        if message_timestamp_type is not None or retention_ms is not None or message_timestamp_after_max_ms is not None or message_timestamp_before_max_ms is not None:
-            table_properties = _encode_table_props(message_timestamp_type=message_timestamp_type, retention_ms=retention_ms, message_timestamp_after_max_ms=message_timestamp_after_max_ms, message_timestamp_before_max_ms=message_timestamp_before_max_ms)
-            if table_properties is None:
-                table_properties = ""
+        if table_properties is None:
+            table_properties = ""
 
         properties = builder.CreateString(table_properties)
         tabular_alter_table.Start(builder)
@@ -1211,6 +1244,10 @@ class VastdbApi:
             url=self._url(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
             data=alter_table_req, headers=headers)
 
+    def drop_topic(self, bucket, name, expected_retvals=[]):
+        self.drop_table(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME, name=name,
+                        expected_retvals=expected_retvals)
+
     def drop_table(self, bucket, schema, name, txid=0, client_tags=[], expected_retvals=[], remove_imports_table=False):
         """
         DELETE /mybucket/schema_path/mytable?table HTTP/1.1
@@ -1227,8 +1264,26 @@ class VastdbApi:
             url=self._url(bucket=bucket, schema=schema, table=name, command="table", url_params=url_params),
             headers=headers)
 
+    def list_topics(self, bucket, max_keys=1000, next_key=0, name_prefix="",
+                    exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+        return self._list_tables_internal(bucket=bucket, schema=KAFKA_TOPICS_SCHEMA_NAME,
+                                          parse_properties=_decode_table_props, max_keys=max_keys,
+                                          next_key=next_key, name_prefix=name_prefix, exact_match=exact_match,
+                                          expected_retvals=expected_retvals,
+                                          include_list_stats=include_list_stats, count_only=count_only)
+
     def list_tables(self, bucket, schema, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
                     exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
+        def parse_properties(x):
+            return x
+        return self._list_tables_internal(bucket=bucket, schema=schema, txid=txid, client_tags=client_tags,
+                                          parse_properties=parse_properties, max_keys=max_keys, next_key=next_key,
+                                          name_prefix=name_prefix, exact_match=exact_match,
+                                          expected_retvals=expected_retvals,
+                                          include_list_stats=include_list_stats, count_only=count_only)
+
+    def _list_tables_internal(self, bucket, schema, parse_properties, txid=0, client_tags=[], max_keys=1000, next_key=0, name_prefix="",
+                              exact_match=False, expected_retvals=[], include_list_stats=False, count_only=False):
         """
         GET /mybucket/schema_path?table HTTP/1.1
         tabular-txid: TransactionId
@@ -1265,7 +1320,7 @@ class VastdbApi:
         tables_length = lists.TablesLength()
         count = int(res_headers['tabular-list-count']) if 'tabular-list-count' in res_headers else tables_length
         for i in range(tables_length):
-            tables.append(_parse_table_info(lists.Tables(i)))
+            tables.append(_parse_table_info(lists.Tables(i), parse_properties))
 
         return bucket_name, schema_name, tables, next_key, is_truncated, count
 
@@ -1837,7 +1892,7 @@ class VastdbApi:
             raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
         projections_length = lists.ProjectionsLength()
         for i in range(projections_length):
-            projections.append(_parse_table_info(lists.Projections(i)))
+            projections.append(_parse_table_info(lists.Projections(i), lambda x: x))
 
         return bucket_name, schema_name, table_name, projections, next_key, is_truncated, count
 

{vastdb-1.3.4 → vastdb-1.3.6}/vastdb/bucket.py

@@ -40,7 +40,7 @@ class Bucket:
         """List bucket's schemas."""
         return self._root_schema.schemas(batch_size=batch_size)
 
-    def snapshot(self, name, fail_if_missing=True) -> Optional["Bucket"]:
+    def snapshot(self, name: str, fail_if_missing=True) -> Optional["Bucket"]:
         """Get snapshot by name (if exists)."""
         snapshots, _is_truncated, _next_key = \
             self.tx._rpc.api.list_snapshots(bucket=self.name, name_prefix=name, max_keys=1)

{vastdb-1.3.4 → vastdb-1.3.6}/vastdb/schema.py

@@ -122,8 +122,11 @@ class Schema:
             if not is_truncated:
                 break
 
-    def tables(self, table_name=None) -> List["Table"]:
-        """List all tables under this schema."""
+    def tables(self, table_name: str = "") -> List["Table"]:
+        """List all tables under this schema if `table_name` is empty.
+
+        Otherwise, list only the specific table (if exists).
+        """
         return [
             _parse_table_info(table_info, self)
             for table_info in self._iter_tables(table_name=table_name)

{vastdb-1.3.4 → vastdb-1.3.6}/vastdb/table.py

@@ -168,8 +168,11 @@ class Table:
         log.debug("Found projection: %s", projs[0])
         return projs[0]
 
-    def projections(self, projection_name=None) -> Iterable["Projection"]:
-        """List all semi-sorted projections of this table."""
+    def projections(self, projection_name: str = "") -> Iterable["Projection"]:
+        """List all semi-sorted projections of this table if `projection_name` is empty.
+
+        Otherwise, list only the specific projection (if exists).
+        """
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         projections = []
@@ -523,7 +526,7 @@ class Table:
         self.tx._rpc.api.drop_table(self.bucket.name, self.schema.name, self.name, txid=self.tx.txid, remove_imports_table=self._imports_table)
         log.info("Dropped table: %s", self.name)
 
-    def rename(self, new_name) -> None:
+    def rename(self, new_name: str) -> None:
         """Rename this table."""
         if self._imports_table:
             raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
@@ -582,7 +585,7 @@ class Table:
         self.tx._rpc.features.check_imports_table()
         return Table(name=self.name, schema=self.schema, handle=int(self.handle), _imports_table=True)
 
-    def __getitem__(self, col_name):
+    def __getitem__(self, col_name: str):
         """Allow constructing ibis-like column expressions from this table.
 
         It is useful for constructing expressions for predicate pushdown in `Table.select()` method.
@@ -630,7 +633,7 @@ class Projection:
         self.arrow_schema = pa.schema([(col[0], col[1]) for col in columns])
         return self.arrow_schema
 
-    def rename(self, new_name) -> None:
+    def rename(self, new_name: str) -> None:
         """Rename this projection."""
         self.tx._rpc.api.alter_projection(self.bucket.name, self.schema.name,
                                                 self.table.name, self.name, txid=self.tx.txid, new_name=new_name)

{vastdb-1.3.4 → vastdb-1.3.6}/vastdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 1.3.4
+Version: 1.3.6
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA