vastdb 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

vastdb/bucket.py

@@ -16,14 +16,6 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
-@dataclass
-class Snapshot:
-    """VAST bucket-level snapshot."""
-
-    name: str
-    bucket: "Bucket"
-
-
 @dataclass
 class Bucket:
     """VAST bucket."""
@@ -73,7 +65,22 @@ class Bucket:
 
         return [schema.Schema(name=name, bucket=self) for name, *_ in schemas]
 
-    def snapshots(self) -> List[Snapshot]:
+    def snapshot(self, name, fail_if_missing=True) -> Optional["Bucket"]:
+        """Get snapshot by name (if exists)."""
+        snapshots, _is_truncated, _next_key = \
+            self.tx._rpc.api.list_snapshots(bucket=self.name, name_prefix=name, max_keys=1)
+
+        expected_name = f".snapshot/{name}"
+        exists = snapshots and snapshots[0] == expected_name + "/"
+        if not exists:
+            if fail_if_missing:
+                raise errors.MissingSnapshot(self.name, expected_name)
+            else:
+                return None
+
+        return Bucket(name=f'{self.name}/{expected_name}', tx=self.tx)
+
+    def snapshots(self) -> List["Bucket"]:
         """List bucket's snapshots."""
         snapshots = []
         next_key = 0
@@ -86,4 +93,7 @@ class Bucket:
             if not is_truncated:
                 break
 
-        return [Snapshot(name=snapshot, bucket=self) for snapshot in snapshots]
+        return [
+            Bucket(name=f'{self.name}/{snapshot.strip("/")}', tx=self.tx)
+            for snapshot in snapshots
+        ]

vastdb/errors.py

@@ -85,6 +85,10 @@ class InvalidArgument(Exception):
     pass
 
 
+class TooWideRow(InvalidArgument):
+    pass
+
+
 class Missing(Exception):
     pass
 
@@ -93,11 +97,21 @@ class MissingTransaction(Missing):
     pass
 
 
+class NotSupported(Exception):
+    pass
+
+
 @dataclass
 class MissingBucket(Missing):
     bucket: str
 
 
+@dataclass
+class MissingSnapshot(Missing):
+    bucket: str
+    snapshot: str
+
+
 @dataclass
 class MissingSchema(Missing):
     bucket: str
@@ -136,6 +150,19 @@ class TableExists(Exists):
     table: str
 
 
+@dataclass
+class NotSupportedCommand(NotSupported):
+    bucket: str
+    schema: str
+    table: str
+
+
+@dataclass
+class NotSupportedVersion(NotSupported):
+    err_msg: str
+    version: str
+
+
 ERROR_TYPES_MAP = {
     HttpStatus.BAD_REQUEST: BadRequest,
     HttpStatus.FOBIDDEN: Forbidden,
@@ -178,4 +205,4 @@ def from_response(res: requests.Response):
     log.warning("RPC failed: %s", kwargs)
     status = HttpStatus(res.status_code)
     error_type = ERROR_TYPES_MAP.get(status, UnexpectedError)
-    raise error_type(**kwargs)
+    return error_type(**kwargs)

vastdb/internal_commands.py

@@ -1,7 +1,6 @@
 import itertools
 import json
 import logging
-import math
 import re
 import struct
 import urllib.parse
@@ -182,6 +181,7 @@ class Predicate:
             Equals,
             Greater,
             GreaterEqual,
+            InValues,
             Less,
             LessEqual,
             Not,
@@ -219,40 +219,54 @@ class Predicate:
                 prev_field_name = None
                 for inner_op in or_args:
                     _logger.debug('inner_op %s', inner_op)
-                    builder_func: Any = builder_map.get(type(inner_op))
+                    op_type = type(inner_op)
+                    builder_func: Any = builder_map.get(op_type)
                     if not builder_func:
-                        raise NotImplementedError(inner_op.name)
+                        if op_type == InValues:
+                            builder_func = self.build_equal
+                        else:
+                            raise NotImplementedError(self.expr)
 
                     if builder_func == self.build_is_null:
                         column, = inner_op.args
-                        literal = None
+                        literals = (None,)
                     elif builder_func == self.build_is_not_null:
                         not_arg, = inner_op.args
                         # currently we only support not is_null, checking we really got is_null under the not:
                         if not builder_map.get(type(not_arg)) == self.build_is_null:
-                            raise NotImplementedError(not_arg.args[0].name)
+                            raise NotImplementedError(self.expr)
                         column, = not_arg.args
-                        literal = None
+                        literals = (None,)
                     else:
-                        column, literal = inner_op.args
-                        if not isinstance(literal, Literal):
-                            raise NotImplementedError(inner_op.name)
+                        column, arg = inner_op.args
+                        if isinstance(arg, tuple):
+                            literals = arg
+                        else:
+                            literals = (arg,)
+                        for literal in literals:
+                            if not isinstance(literal, Literal):
+                                raise NotImplementedError(self.expr)
 
                     if not isinstance(column, TableColumn):
-                        raise NotImplementedError(inner_op.name)
+                        raise NotImplementedError(self.expr)
 
                     field_name = column.name
                     if prev_field_name is None:
                         prev_field_name = field_name
                     elif prev_field_name != field_name:
-                        raise NotImplementedError(op.name)
+                        raise NotImplementedError(self.expr)
 
-                    args_offsets = [self.build_column(position=positions_map[field_name])]
-                    if literal:
-                        field = self.schema.field(field_name)
-                        args_offsets.append(self.build_literal(field=field, value=literal.value))
+                    column_offset = self.build_column(position=positions_map[field_name])
+                    field = self.schema.field(field_name)
+                    for literal in literals:
+                        args_offsets = [column_offset]
+                        if literal is not None:
+                            args_offsets.append(self.build_literal(field=field, value=literal.value))
 
-                    inner_offsets.append(builder_func(*args_offsets))
+                        inner_offsets.append(builder_func(*args_offsets))
+
+                if not inner_offsets:
+                    raise NotImplementedError(self.expr)  # an empty OR is equivalent to a 'FALSE' literal
 
                 domain_offset = self.build_or(inner_offsets)
                 offsets.append(domain_offset)
@@ -719,20 +733,6 @@ def _parse_table_info(obj):
     return TableInfo(name, properties, handle, num_rows, used_bytes)
 
 
-def build_record_batch(column_info, column_values):
-    fields = [pa.field(column_name, column_type) for column_type, column_name in column_info]
-    schema = pa.schema(fields)
-    arrays = [pa.array(column_values[column_type], type=column_type) for column_type, _ in column_info]
-    batch = pa.record_batch(arrays, schema)
-    return serialize_record_batch(batch)
-
-
-def serialize_record_batch(batch):
-    sink = pa.BufferOutputStream()
-    with pa.ipc.new_stream(sink, batch.schema) as writer:
-        writer.write(batch)
-    return sink.getvalue()
-
 # Results that returns from tablestats
 
 
@@ -952,26 +952,27 @@ class VastdbApi:
 
             return bucket_name, schemas, next_key, is_truncated, count
 
-    def list_snapshots(self, bucket, max_keys=1000, next_token=None, expected_retvals=None):
+    def list_snapshots(self, bucket, max_keys=1000, next_token=None, name_prefix=''):
         next_token = next_token or ''
-        expected_retvals = expected_retvals or []
-        url_params = {'list_type': '2', 'prefix': '.snapshot/', 'delimiter': '/', 'max_keys': str(max_keys)}
+        url_params = {'list_type': '2', 'prefix': '.snapshot/' + name_prefix, 'delimiter': '/', 'max_keys': str(max_keys)}
         if next_token:
             url_params['continuation-token'] = next_token
 
         res = self.session.get(self._api_prefix(bucket=bucket, command="list", url_params=url_params), headers={}, stream=True)
-        self._check_res(res, "list_snapshots", expected_retvals)
-        if res.status_code == 200:
-            out = b''.join(res.iter_content(chunk_size=128))
-            xml_str = out.decode()
-            xml_dict = xmltodict.parse(xml_str)
-            list_res = xml_dict['ListBucketResult']
-            is_truncated = list_res['IsTruncated'] == 'true'
-            marker = list_res['Marker']
-            common_prefixes = list_res['CommonPrefixes'] if 'CommonPrefixes' in list_res else []
-            snapshots = [v['Prefix'] for v in common_prefixes]
+        self._check_res(res, "list_snapshots")
+
+        out = b''.join(res.iter_content(chunk_size=128))
+        xml_str = out.decode()
+        xml_dict = xmltodict.parse(xml_str)
+        list_res = xml_dict['ListBucketResult']
+        is_truncated = list_res['IsTruncated'] == 'true'
+        marker = list_res['Marker']
+        common_prefixes = list_res.get('CommonPrefixes', [])
+        if isinstance(common_prefixes, dict):  # in case there is a single snapshot
+            common_prefixes = [common_prefixes]
+        snapshots = [v['Prefix'] for v in common_prefixes]
 
-            return snapshots, is_truncated, marker
+        return snapshots, is_truncated, marker
 
     def create_table(self, bucket, schema, name, arrow_schema, txid=0, client_tags=[], expected_retvals=[],
                      topic_partitions=0, create_imports_table=False, use_external_row_ids_allocation=False):
@@ -1030,7 +1031,7 @@ class VastdbApi:
         # create the table
         return self.create_table(bucket, schema, name, arrow_schema, txid, client_tags, expected_retvals)
 
-    def get_table_stats(self, bucket, schema, name, txid=0, client_tags=[], expected_retvals=[]):
+    def get_table_stats(self, bucket, schema, name, txid=0, client_tags=[], expected_retvals=[], imports_table_stats=False):
         """
         GET /mybucket/myschema/mytable?stats HTTP/1.1
         tabular-txid: TransactionId
@@ -1039,7 +1040,8 @@ class VastdbApi:
         The Command will return the statistics in flatbuf format
         """
         headers = self._fill_common_headers(txid=txid, client_tags=client_tags)
-        res = self.session.get(self._api_prefix(bucket=bucket, schema=schema, table=name, command="stats"), headers=headers)
+        url_params = {'sub-table': IMPORTED_OBJECTS_TABLE_NAME} if imports_table_stats else {}
+        res = self.session.get(self._api_prefix(bucket=bucket, schema=schema, table=name, command="stats", url_params=url_params), headers=headers)
         self._check_res(res, "get_table_stats", expected_retvals)
 
         flatbuf = b''.join(res.iter_content(chunk_size=128))
@@ -1527,11 +1529,18 @@ class VastdbApi:
             if response.status_code != 200:
                 return response
 
+            ALLOWED_IMPORT_STATES = {
+                'Success',
+                'TabularInProgress',
+                'TabularAlreadyImported',
+                'TabularImportNotStarted',
+            }
+
             chunk_size = 1024
             for chunk in response.iter_content(chunk_size=chunk_size):
                 chunk_dict = json.loads(chunk)
                 _logger.debug("import data chunk=%s, result: %s", chunk_dict, chunk_dict['res'])
-                if chunk_dict['res'] != 'Success' and chunk_dict['res'] != 'TabularInProgress' and chunk_dict['res'] != 'TabularAlreadyImported':
+                if chunk_dict['res'] not in ALLOWED_IMPORT_STATES:
                     raise errors.ImportFilesError(
                         f"Encountered an error during import_data. status: {chunk_dict['res']}, "
                         f"error message: {chunk_dict['err_msg'] or 'Unexpected error'} during import of "
@@ -1555,48 +1564,6 @@ class VastdbApi:
 
         return self._check_res(res, "import_data", expected_retvals)
 
-    def _record_batch_slices(self, batch, rows_per_slice=None):
-        max_slice_size_in_bytes = int(0.9 * 5 * 1024 * 1024)  # 0.9 * 5MB
-        batch_len = len(batch)
-        serialized_batch = serialize_record_batch(batch)
-        batch_size_in_bytes = len(serialized_batch)
-        _logger.debug('max_slice_size_in_bytes=%d batch_len=%d batch_size_in_bytes=%d',
-                      max_slice_size_in_bytes, batch_len, batch_size_in_bytes)
-
-        if not rows_per_slice:
-            if batch_size_in_bytes < max_slice_size_in_bytes:
-                rows_per_slice = batch_len
-            else:
-                rows_per_slice = int(0.9 * batch_len * max_slice_size_in_bytes / batch_size_in_bytes)
-
-        done_slicing = False
-        while not done_slicing:
-            # Attempt slicing according to the current rows_per_slice
-            offset = 0
-            serialized_slices = []
-            for i in range(math.ceil(batch_len / rows_per_slice)):
-                offset = rows_per_slice * i
-                if offset >= batch_len:
-                    done_slicing = True
-                    break
-                slice_batch = batch.slice(offset, rows_per_slice)
-                serialized_slice_batch = serialize_record_batch(slice_batch)
-                sizeof_serialized_slice_batch = len(serialized_slice_batch)
-
-                if sizeof_serialized_slice_batch <= max_slice_size_in_bytes:
-                    serialized_slices.append(serialized_slice_batch)
-                else:
-                    _logger.info(f'Using rows_per_slice {rows_per_slice} slice {i} size {sizeof_serialized_slice_batch} exceeds {max_slice_size_in_bytes} bytes, trying smaller rows_per_slice')
-                    # We have a slice that is too large
-                    rows_per_slice = int(rows_per_slice / 2)
-                    if rows_per_slice < 1:
-                        raise ValueError('cannot decrease batch size below 1 row')
-                    break
-            else:
-                done_slicing = True
-
-        return serialized_slices
-
     def insert_rows(self, bucket, schema, table, record_batch, txid=0, client_tags=[], expected_retvals=[]):
         """
         POST /mybucket/myschema/mytable?rows HTTP/1.1

vastdb/schema.py

@@ -87,4 +87,4 @@ class Schema:
 
 def _parse_table_info(table_info, schema: "schema.Schema"):
     stats = table.TableStats(num_rows=table_info.num_rows, size_in_bytes=table_info.size_in_bytes)
-    return table.Table(name=table_info.name, schema=schema, handle=int(table_info.handle), stats=stats)
+    return table.Table(name=table_info.name, schema=schema, handle=int(table_info.handle), stats=stats, _imports_table=False)

vastdb/session.py

@@ -11,7 +11,20 @@ import os
 
 import boto3
 
-from . import internal_commands, transaction
+from . import errors, internal_commands, transaction
+
+
+class Features:
+    """VAST database features - check if server is already support a feature."""
+
+    def __init__(self, vast_version):
+        """Save the server version."""
+        self.vast_version = vast_version
+
+    def check_imports_table(self):
+        """Check if the feature that support imports table is supported."""
+        if self.vast_version < (5, 2):
+            raise errors.NotSupportedVersion("import_table requires 5.2+", self.vast_version)
 
 
 class Session:
@@ -27,6 +40,8 @@ class Session:
             endpoint = os.environ['AWS_S3_ENDPOINT_URL']
 
         self.api = internal_commands.VastdbApi(endpoint, access, secret)
+        version_tuple = tuple(int(part) for part in self.api.vast_version.split('.'))
+        self.features = Features(version_tuple)
         self.s3 = boto3.client('s3',
             aws_access_key_id=access,
             aws_secret_access_key=secret,

vastdb/table.py

@@ -1,3 +1,5 @@
+"""VAST Database table."""
+
 import concurrent.futures
 import logging
 import os
@@ -10,7 +12,7 @@ from typing import Dict, List, Optional, Tuple, Union
 import ibis
 import pyarrow as pa
 
-from . import errors, internal_commands, schema
+from . import errors, internal_commands, schema, util
 
 log = logging.getLogger(__name__)
 
@@ -24,6 +26,8 @@ MAX_INSERT_ROWS_PER_PATCH = 512 * 1024
 
 @dataclass
 class TableStats:
+    """Table-related information."""
+
     num_rows: int
     size_in_bytes: int
     is_external_rowid_alloc: bool = False
@@ -32,6 +36,8 @@ class TableStats:
 
 @dataclass
 class QueryConfig:
+    """Query execution configiration."""
+
     num_sub_splits: int = 4
     num_splits: int = 1
     data_endpoints: Optional[List[str]] = None
@@ -44,11 +50,16 @@ class QueryConfig:
 
 @dataclass
 class ImportConfig:
+    """Import execution configiration."""
+
     import_concurrency: int = 2
 
 
-class SelectSplitState():
+class SelectSplitState:
+    """State of a specific query split execution."""
+
     def __init__(self, query_data_request, table: "Table", split_id: int, config: QueryConfig) -> None:
+        """Initialize query split state."""
         self.split_id = split_id
         self.subsplits_state = {i: 0 for i in range(config.num_sub_splits)}
         self.config = config
@@ -56,6 +67,10 @@ class SelectSplitState():
         self.table = table
 
     def batches(self, api: internal_commands.VastdbApi):
+        """Execute QueryData request, and yield parsed RecordBatch objects.
+
+        Can be called repeatedly, to allow pagination.
+        """
         while not self.done:
             response = api.query_data(
                             bucket=self.table.bucket.name,
@@ -68,7 +83,8 @@ class SelectSplitState():
                             txid=self.table.tx.txid,
                             limit_rows=self.config.limit_rows_per_sub_split,
                             sub_split_start_row_ids=self.subsplits_state.items(),
-                            enable_sorted_projections=self.config.use_semi_sorted_projections)
+                            enable_sorted_projections=self.config.use_semi_sorted_projections,
+                            query_imports_table=self.table._imports_table)
             pages_iter = internal_commands.parse_query_data_response(
                 conn=response.raw,
                 schema=self.query_data_request.response_schema,
@@ -82,19 +98,24 @@ class SelectSplitState():
 
     @property
     def done(self):
+        """Returns true iff the pagination over."""
         return all(row_id == internal_commands.TABULAR_INVALID_ROW_ID for row_id in self.subsplits_state.values())
 
 
 @dataclass
 class Table:
+    """VAST Table."""
+
     name: str
     schema: "schema.Schema"
     handle: int
     stats: TableStats
-    arrow_schema: pa.Schema = field(init=False, compare=False)
-    _ibis_table: ibis.Schema = field(init=False, compare=False)
+    arrow_schema: pa.Schema = field(init=False, compare=False, repr=False)
+    _ibis_table: ibis.Schema = field(init=False, compare=False, repr=False)
+    _imports_table: bool
 
     def __post_init__(self):
+        """Also, load columns' metadata."""
         self.arrow_schema = self.columns()
 
         table_path = f'{self.schema.bucket.name}/{self.schema.name}/{self.name}'
@@ -102,21 +123,21 @@ class Table:
 
     @property
     def tx(self):
+        """Return transaction."""
         return self.schema.tx
 
     @property
     def bucket(self):
+        """Return bucket."""
         return self.schema.bucket
 
-    def __repr__(self):
-        return f"{type(self).__name__}(name={self.name})"
-
     def columns(self) -> pa.Schema:
+        """Return columns' metadata."""
         fields = []
         next_key = 0
         while True:
             cur_columns, next_key, is_truncated, _count = self.tx._rpc.api.list_columns(
-                bucket=self.bucket.name, schema=self.schema.name, table=self.name, next_key=next_key, txid=self.tx.txid)
+                bucket=self.bucket.name, schema=self.schema.name, table=self.name, next_key=next_key, txid=self.tx.txid, list_imports_table=self._imports_table)
             fields.extend(cur_columns)
             if not is_truncated:
                 break
@@ -125,6 +146,9 @@ class Table:
         return self.arrow_schema
 
     def projection(self, name: str) -> "Projection":
+        """Get a specific semi-sorted projection of this table."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         projs = self.projections(projection_name=name)
         if not projs:
             raise errors.MissingProjection(self.bucket.name, self.schema.name, self.name, name)
@@ -133,6 +157,9 @@ class Table:
         return projs[0]
 
     def projections(self, projection_name=None) -> List["Projection"]:
+        """List all semi-sorted projections of this table."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         projections = []
         next_key = 0
         name_prefix = projection_name if projection_name else ""
@@ -150,6 +177,12 @@ class Table:
         return [_parse_projection_info(projection, self) for projection in projections]
 
     def import_files(self, files_to_import: List[str], config: Optional[ImportConfig] = None) -> None:
+        """Import a list of Parquet files into this table.
+
+        The files must be on VAST S3 server and be accessible using current credentials.
+        """
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         source_files = {}
         for f in files_to_import:
             bucket_name, object_path = _parse_bucket_and_object_names(f)
@@ -158,6 +191,13 @@ class Table:
         self._execute_import(source_files, config=config)
 
     def import_partitioned_files(self, files_and_partitions: Dict[str, pa.RecordBatch], config: Optional[ImportConfig] = None) -> None:
+        """Import a list of Parquet files into this table.
+
+        The files must be on VAST S3 server and be accessible using current credentials.
+        Each file must have its own partition values defined as an Arrow RecordBatch.
+        """
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         source_files = {}
         for f, record_batch in files_and_partitions.items():
             bucket_name, object_path = _parse_bucket_and_object_names(f)
@@ -216,8 +256,10 @@ class Table:
                 # ThreadPoolExecutor will be joined at the end of the context
 
     def get_stats(self) -> TableStats:
+        """Get the statistics of this table."""
         stats_tuple = self.tx._rpc.api.get_table_stats(
-            bucket=self.bucket.name, schema=self.schema.name, name=self.name, txid=self.tx.txid)
+            bucket=self.bucket.name, schema=self.schema.name, name=self.name, txid=self.tx.txid,
+            imports_table_stats=self._imports_table)
         return TableStats(**stats_tuple._asdict())
 
     def select(self, columns: Optional[List[str]] = None,
@@ -225,6 +267,14 @@ class Table:
                config: Optional[QueryConfig] = None,
                *,
                internal_row_id: bool = False) -> pa.RecordBatchReader:
+        """Execute a query over this table.
+
+        To read a subset of the columns, specify their names via `columns` argument. Otherwise, all columns will be read.
+
+        In order to apply a filter, a predicate can be specified. See https://github.com/vast-data/vastdb_sdk/blob/main/README.md#filters-and-projections for more details.
+
+        Query-execution configuration options can be specified via the optional `config` argument.
+        """
         if config is None:
             config = QueryConfig()
 
@@ -335,82 +385,129 @@ class Table:
 
         return pa.RecordBatchReader.from_batches(query_data_request.response_schema, batches_iterator())
 
-    def _combine_chunks(self, col):
-        if hasattr(col, "combine_chunks"):
-            return col.combine_chunks()
-        else:
-            return col
-
     def insert(self, rows: pa.RecordBatch) -> pa.RecordBatch:
-        serialized_slices = self.tx._rpc.api._record_batch_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
+        """Insert a RecordBatch into this table."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
+        serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.insert_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
                                                txid=self.tx.txid)
 
     def update(self, rows: Union[pa.RecordBatch, pa.Table], columns: Optional[List[str]] = None) -> None:
+        """Update a subset of cells in this table.
+
+        Row IDs are specified using a special field (named "$row_id" of uint64 type).
+
+        A subset of columns to be updated can be specified via the `columns` argument.
+        """
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         if columns is not None:
             update_fields = [(INTERNAL_ROW_ID, pa.uint64())]
-            update_values = [self._combine_chunks(rows[INTERNAL_ROW_ID])]
+            update_values = [_combine_chunks(rows[INTERNAL_ROW_ID])]
             for col in columns:
                 update_fields.append(rows.field(col))
-                update_values.append(self._combine_chunks(rows[col]))
+                update_values.append(_combine_chunks(rows[col]))
 
             update_rows_rb = pa.record_batch(schema=pa.schema(update_fields), data=update_values)
         else:
             update_rows_rb = rows
 
-        serialized_slices = self.tx._rpc.api._record_batch_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
+        serialized_slices = util.iter_serialized_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.update_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
                                          txid=self.tx.txid)
 
     def delete(self, rows: Union[pa.RecordBatch, pa.Table]) -> None:
+        """Delete a subset of rows in this table.
+
+        Row IDs are specified using a special field (named "$row_id" of uint64 type).
+        """
         delete_rows_rb = pa.record_batch(schema=pa.schema([(INTERNAL_ROW_ID, pa.uint64())]),
-                                         data=[self._combine_chunks(rows[INTERNAL_ROW_ID])])
+                                         data=[_combine_chunks(rows[INTERNAL_ROW_ID])])
 
-        serialized_slices = self.tx._rpc.api._record_batch_slices(delete_rows_rb, MAX_ROWS_PER_BATCH)
+        serialized_slices = util.iter_serialized_slices(delete_rows_rb, MAX_ROWS_PER_BATCH)
         for slice in serialized_slices:
             self.tx._rpc.api.delete_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
-                                         txid=self.tx.txid)
+                                         txid=self.tx.txid, delete_from_imports_table=self._imports_table)
 
     def drop(self) -> None:
-        self.tx._rpc.api.drop_table(self.bucket.name, self.schema.name, self.name, txid=self.tx.txid)
+        """Drop this table."""
+        self.tx._rpc.api.drop_table(self.bucket.name, self.schema.name, self.name, txid=self.tx.txid, remove_imports_table=self._imports_table)
         log.info("Dropped table: %s", self.name)
 
     def rename(self, new_name) -> None:
+        """Rename this table."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         self.tx._rpc.api.alter_table(
             self.bucket.name, self.schema.name, self.name, txid=self.tx.txid, new_name=new_name)
         log.info("Renamed table from %s to %s ", self.name, new_name)
         self.name = new_name
 
     def add_column(self, new_column: pa.Schema) -> None:
+        """Add a new column."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
         log.info("Added column(s): %s", new_column)
         self.arrow_schema = self.columns()
 
     def drop_column(self, column_to_drop: pa.Schema) -> None:
+        """Drop an existing column."""
+        if self._imports_table:
+            raise errors.NotSupported(self.bucket.name, self.schema.name, self.name)
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         self.tx._rpc.api.drop_columns(self.bucket.name, self.schema.name, self.name, column_to_drop, txid=self.tx.txid)
         log.info("Dropped column(s): %s", column_to_drop)
         self.arrow_schema = self.columns()
 
     def rename_column(self, current_column_name: str, new_column_name: str) -> None:
+        """Rename an existing column."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         self.tx._rpc.api.alter_column(self.bucket.name, self.schema.name, self.name, name=current_column_name,
                                        new_name=new_column_name, txid=self.tx.txid)
         log.info("Renamed column: %s to %s", current_column_name, new_column_name)
         self.arrow_schema = self.columns()
 
     def create_projection(self, projection_name: str, sorted_columns: List[str], unsorted_columns: List[str]) -> "Projection":
+        """Create a new semi-sorted projection."""
+        if self._imports_table:
+            raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
         columns = [(sorted_column, "Sorted") for sorted_column in sorted_columns] + [(unsorted_column, "Unorted") for unsorted_column in unsorted_columns]
         self.tx._rpc.api.create_projection(self.bucket.name, self.schema.name, self.name, projection_name, columns=columns, txid=self.tx.txid)
         log.info("Created projection: %s", projection_name)
         return self.projection(projection_name)
 
+    def create_imports_table(self, fail_if_exists=True) -> "Table":
+        """Create imports table."""
+        self.tx._rpc.features.check_imports_table()
+        empty_schema = pa.schema([])
+        self.tx._rpc.api.create_table(self.bucket.name, self.schema.name, self.name, empty_schema, txid=self.tx.txid,
+                                        create_imports_table=True)
+        log.info("Created imports table for table: %s", self.name)
+        return self.imports_table()  # type: ignore[return-value]
+
+    def imports_table(self) -> Optional["Table"]:
+        """Get the imports table under of this table."""
+        self.tx._rpc.features.check_imports_table()
+        return Table(name=self.name, schema=self.schema, handle=int(self.handle), stats=self.stats, _imports_table=True)
+
     def __getitem__(self, col_name):
+        """Allow constructing ibis-like column expressions from this table.
+
+        It is useful for constructing expressions for predicate pushdown in `Table.select()` method.
+        """
         return self._ibis_table[col_name]
 
 
 @dataclass
 class Projection:
+    """VAST semi-sorted projection."""
+
     name: str
     table: Table
     handle: int
@@ -418,20 +515,21 @@ class Projection:
 
     @property
     def bucket(self):
+        """Return bucket."""
         return self.table.schema.bucket
 
     @property
     def schema(self):
+        """Return schema."""
         return self.table.schema
 
     @property
     def tx(self):
+        """Return transaction."""
         return self.table.schema.tx
 
-    def __repr__(self):
-        return f"{type(self).__name__}(name={self.name})"
-
     def columns(self) -> pa.Schema:
+        """Return this projections' columns as an Arrow schema."""
         columns = []
         next_key = 0
         while True:
@@ -447,12 +545,14 @@ class Projection:
         return self.arrow_schema
 
     def rename(self, new_name) -> None:
+        """Rename this projection."""
         self.tx._rpc.api.alter_projection(self.bucket.name, self.schema.name,
                                                 self.table.name, self.name, txid=self.tx.txid, new_name=new_name)
         log.info("Renamed projection from %s to %s ", self.name, new_name)
         self.name = new_name
 
     def drop(self) -> None:
+        """Drop this projection."""
         self.tx._rpc.api.drop_projection(self.bucket.name, self.schema.name, self.table.name,
                                          self.name, txid=self.tx.txid)
         log.info("Dropped projection: %s", self.name)
@@ -478,3 +578,10 @@ def _serialize_record_batch(record_batch: pa.RecordBatch) -> pa.lib.Buffer:
     with pa.ipc.new_stream(sink, record_batch.schema) as writer:
         writer.write(record_batch)
     return sink.getvalue()
+
+
+def _combine_chunks(col):
+    if hasattr(col, "combine_chunks"):
+        return col.combine_chunks()
+    else:
+        return col

vastdb/tests/test_imports.py

@@ -6,7 +6,7 @@ import pyarrow.parquet as pq
 import pytest
 
 from vastdb import util
-from vastdb.errors import ImportFilesError, InvalidArgument
+from vastdb.errors import ImportFilesError, InternalServerError, InvalidArgument
 
 log = logging.getLogger(__name__)
 
@@ -34,12 +34,24 @@ def test_parallel_imports(session, clean_bucket_name, s3):
         b = tx.bucket(clean_bucket_name)
         s = b.create_schema('s1')
         t = s.create_table('t1', pa.schema([('num', pa.int64())]))
+        with pytest.raises(InternalServerError):
+            t.create_imports_table()
         log.info("Starting import of %d files", num_files)
         t.import_files(files)
         arrow_table = pa.Table.from_batches(t.select(columns=['num']))
         assert arrow_table.num_rows == num_rows * num_files
         arrow_table = pa.Table.from_batches(t.select(columns=['num'], predicate=t['num'] == 100))
         assert arrow_table.num_rows == num_files
+        import_table = t.imports_table()
+        # checking all imports are on the imports table:
+        objects_name = pa.Table.from_batches(import_table.select(columns=["ObjectName"]))
+        objects_name = objects_name.to_pydict()
+        object_names = set(objects_name['ObjectName'])
+        prefix = 'prq'
+        numbers = set(range(53))
+        assert all(name.startswith(prefix) for name in object_names)
+        numbers.issubset(int(name.replace(prefix, '')) for name in object_names)
+        assert len(object_names) == len(objects_name['ObjectName'])
 
 
 def test_create_table_from_files(session, clean_bucket_name, s3):

vastdb/tests/test_schemas.py

@@ -60,5 +60,4 @@ def test_commits_and_rollbacks(session, clean_bucket_name):
 def test_list_snapshots(session, clean_bucket_name):
     with session.transaction() as tx:
         b = tx.bucket(clean_bucket_name)
-        s = b.snapshots()
-        assert s == []
+        b.snapshots()  # VAST Catalog may create some snapshots

vastdb/tests/test_tables.py

@@ -3,6 +3,7 @@ import decimal
 import logging
 import random
 import threading
+import time
 from contextlib import closing
 from tempfile import NamedTemporaryFile
 
@@ -261,7 +262,7 @@ def test_filters(session, clean_bucket_name):
 
     with prepare_data(session, clean_bucket_name, 's', 't', expected) as t:
         def select(predicate):
-            return pa.Table.from_batches(t.select(predicate=predicate))
+            return pa.Table.from_batches(t.select(predicate=predicate), t.arrow_schema)
 
         assert select(None) == expected
 
@@ -304,6 +305,13 @@ def test_filters(session, clean_bucket_name):
         assert select(t['s'].contains('b')) == expected.filter(pc.field('s') == 'bb')
         assert select(t['s'].contains('y')) == expected.filter(pc.field('s') == 'xyz')
 
+        assert select(t['a'].isin([555])) == expected.filter(pc.field('a').isin([555]))
+        assert select(t['a'].isin([111, 222, 999])) == expected.filter(pc.field('a').isin([111, 222, 999]))
+        assert select((t['a'] == 111) | t['a'].isin([333, 444]) | (t['a'] > 600)) == expected.filter((pc.field('a') == 111) | pc.field('a').isin([333, 444]) | (pc.field('a') > 600))
+
+        with pytest.raises(NotImplementedError):
+            select(t['a'].isin([]))
+
 
 def test_parquet_export(session, clean_bucket_name):
     with session.transaction() as tx:
@@ -638,3 +646,20 @@ def test_select_stop(session, clean_bucket_name):
 
     # validate that all query threads were killed.
     assert active_threads() == 0
+
+
+def test_big_catalog_select(session, clean_bucket_name):
+    with session.transaction() as tx:
+        bc = tx.catalog()
+        actual = pa.Table.from_batches(bc.select(['name']))
+        assert actual
+        log.info("actual=%s", actual)
+
+
+def test_audit_log_select(session, clean_bucket_name):
+    with session.transaction() as tx:
+        a = tx.audit_log()
+        a.columns()
+        time.sleep(1)
+        actual = pa.Table.from_batches(a.select(), a.arrow_schema)
+        log.info("actual=%s", actual)

vastdb/tests/test_util.py

@@ -0,0 +1,39 @@
+import pyarrow as pa
+import pytest
+
+from .. import errors, util
+
+
+def test_slices():
+    ROWS = 1 << 20
+    t = pa.table({"x": range(ROWS), "y": [i / 1000 for i in range(ROWS)]})
+
+    chunks = list(util.iter_serialized_slices(t))
+    assert len(chunks) > 1
+    sizes = [len(c) for c in chunks]
+
+    assert max(sizes) < util.MAX_RECORD_BATCH_SLICE_SIZE
+    assert t == pa.Table.from_batches(_parse(chunks))
+
+    chunks = list(util.iter_serialized_slices(t, 1000))
+    assert len(chunks) > 1
+    sizes = [len(c) for c in chunks]
+
+    assert max(sizes) < util.MAX_RECORD_BATCH_SLICE_SIZE
+    assert t == pa.Table.from_batches(_parse(chunks))
+
+
+def test_wide_row():
+    cols = [pa.field(f"x{i}", pa.utf8()) for i in range(1000)]
+    values = [['a' * 10000]] * len(cols)
+    t = pa.table(values, schema=pa.schema(cols))
+    assert len(t) == 1
+
+    with pytest.raises(errors.TooWideRow):
+        list(util.iter_serialized_slices(t))
+
+
+def _parse(bufs):
+    for buf in bufs:
+        with pa.ipc.open_stream(buf) as reader:
+            yield from reader

vastdb/transaction.py

@@ -16,6 +16,14 @@ from . import bucket, errors, schema, session, table
 
 log = logging.getLogger(__name__)
 
+TABULAR_BC_BUCKET = "vast-big-catalog-bucket"
+VAST_CATALOG_SCHEMA_NAME = 'vast_big_catalog_schema'
+VAST_CATALOG_TABLE_NAME = 'vast_big_catalog_table'
+
+TABULAR_AUDERY_BUCKET = "vast-audit-log-bucket"
+AUDERY_SCHEMA_NAME = 'vast_audit_log_schema'
+AUDERY_TABLE_NAME = 'vast_audit_log_table'
+
 
 @dataclass
 class Transaction:
@@ -44,6 +52,8 @@ class Transaction:
 
     def __repr__(self):
         """Don't show the session details."""
+        if self.txid is None:
+            return 'InvalidTransaction'
         return f'Transaction(id=0x{self.txid:016x})'
 
     def bucket(self, name: str) -> "bucket.Bucket":
@@ -59,6 +69,12 @@ class Transaction:
 
     def catalog(self, fail_if_missing=True) -> Optional["table.Table"]:
         """Return VAST Catalog table."""
-        b = bucket.Bucket("vast-big-catalog-bucket", self)
-        s = schema.Schema("vast_big_catalog_schema", b)
-        return s.table(name="vast_big_catalog_table", fail_if_missing=fail_if_missing)
+        b = bucket.Bucket(TABULAR_BC_BUCKET, self)
+        s = schema.Schema(VAST_CATALOG_SCHEMA_NAME, b)
+        return s.table(name=VAST_CATALOG_TABLE_NAME, fail_if_missing=fail_if_missing)
+
+    def audit_log(self, fail_if_missing=True) -> Optional["table.Table"]:
+        """Return VAST AuditLog table."""
+        b = bucket.Bucket(TABULAR_AUDERY_BUCKET, self)
+        s = schema.Schema(AUDERY_SCHEMA_NAME, b)
+        return s.table(name=AUDERY_TABLE_NAME, fail_if_missing=fail_if_missing)

vastdb/util.py

@@ -1,20 +1,22 @@
 import logging
-from typing import Callable, List, Optional
+from typing import TYPE_CHECKING, Callable, List, Optional, Union
 
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-from .errors import InvalidArgument
-from .schema import Schema
-from .table import ImportConfig, Table
+from .errors import InvalidArgument, TooWideRow
 
 log = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from .schema import Schema
+    from .table import ImportConfig, Table
+
 
 def create_table_from_files(
-        schema: Schema, table_name: str, parquet_files: List[str],
+        schema: "Schema", table_name: str, parquet_files: List[str],
         schema_merge_func: Optional[Callable] = None,
-        config: Optional[ImportConfig] = None) -> Table:
+        config: Optional["ImportConfig"] = None) -> "Table":
     if not schema_merge_func:
         schema_merge_func = default_schema_merge
     else:
@@ -77,3 +79,36 @@ def union_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.S
     This function returns a unified schema from potentially two different schemas.
     """
     return pa.unify_schemas([current_schema, new_schema])
+
+
+MAX_TABULAR_REQUEST_SIZE = 5 << 20  # in bytes
+MAX_RECORD_BATCH_SLICE_SIZE = int(0.9 * MAX_TABULAR_REQUEST_SIZE)
+
+
+def iter_serialized_slices(batch: Union[pa.RecordBatch, pa.Table], max_rows_per_slice=None):
+    """Iterate over a list of record batch slices."""
+
+    rows_per_slice = int(0.9 * len(batch) * MAX_RECORD_BATCH_SLICE_SIZE / batch.nbytes)
+    if max_rows_per_slice is not None:
+        rows_per_slice = min(rows_per_slice, max_rows_per_slice)
+
+    offset = 0
+    while offset < len(batch):
+        if rows_per_slice < 1:
+            raise TooWideRow(batch)
+
+        batch_slice = batch.slice(offset, rows_per_slice)
+        serialized_slice_batch = serialize_record_batch(batch_slice)
+        if len(serialized_slice_batch) <= MAX_RECORD_BATCH_SLICE_SIZE:
+            yield serialized_slice_batch
+            offset += rows_per_slice
+        else:
+            rows_per_slice = rows_per_slice // 2
+
+
+def serialize_record_batch(batch: Union[pa.RecordBatch, pa.Table]):
+    """Serialize a RecordBatch using Arrow IPC format."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, batch.schema) as writer:
+        writer.write(batch)
+    return sink.getvalue()

{vastdb-0.1.2.dist-info → vastdb-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.2
+Version: 0.1.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
@@ -21,7 +21,7 @@ License-File: LICENSE
 Requires-Dist: aws-requests-auth
 Requires-Dist: boto3
 Requires-Dist: flatbuffers
-Requires-Dist: ibis-framework
+Requires-Dist: ibis-framework ==8.0.0
 Requires-Dist: pyarrow
 Requires-Dist: requests
 Requires-Dist: xmltodict

{vastdb-0.1.2.dist-info → vastdb-0.1.3.dist-info}/RECORD

@@ -149,28 +149,29 @@ vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI
 vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/__init__.py,sha256=cMJtZuJ0IL9aKyM3DUWqTCzuP1H1MXXVivKKE1-q0DY,292
-vastdb/bucket.py,sha256=xtKs7S4w0jmI4MujDWH3HDI-iEgbq5Xqqsod-tw4zSo,2991
+vastdb/bucket.py,sha256=4rPEm9qlPTg7ccWO6VGmd4LKb8w-BDhJYwzXGjn03sc,3566
 vastdb/conftest.py,sha256=pKpo_46Vq4QHzTDQAFxasrVhnZ2V2L-y6IMLxojxaFM,2132
-vastdb/errors.py,sha256=fxpKSxjEgoJZuBtEGWzTW9lpDlEjuzgpgXwAQc1W6BQ,3436
-vastdb/internal_commands.py,sha256=3F6FiYu-Ama1zBO7hENPxCaQYJT8mcZP6rSQvtI7Sks,101273
-vastdb/schema.py,sha256=MrQr-WIrES8KcQ0V6cJkRRp_-9jj9FboyrBnkNBsw-8,3324
-vastdb/session.py,sha256=VZOFGZbAdr5Tl4cp88VRQYnR4Q16UNuYjSmX_QPW1II,1718
-vastdb/table.py,sha256=bdx3C1iWiFivKmtifH7MyG7TMqnVVIU91as-_hMn1rE,20532
-vastdb/transaction.py,sha256=1uCSHXqWcwsMJv6DuNx4WyQMGUm8P-RCCqYdBdUGusI,2196
-vastdb/util.py,sha256=Tjj6p4gqabK5G21uWuCiuYM9FaaR04_Zk5X8NWtcdj8,3022
+vastdb/errors.py,sha256=vKWoq1yXrHyafMWwJgW_sQkSxQYxlI1JbTVCLz5Xi9Y,3793
+vastdb/internal_commands.py,sha256=ZD2YXYvZ3lJWYzZU0oHtv8G3lNtDQUF0e8yg8813Xt4,99575
+vastdb/schema.py,sha256=ql4TPB1W_FQ_BHov3CKHI8JX3krXMlcKWz7dTrjpQ1w,3346
+vastdb/session.py,sha256=ciYS8Je2cRpuaAEE6Wjk79VsW0KAPdnRB2cqfxFCjis,2323
+vastdb/table.py,sha256=xnSTWUUa0QHzXC5MUQWsGT1fsG8yAgMLy3nrgSH4j5Q,25661
+vastdb/transaction.py,sha256=g8YTcYnsNPIhB2udbHyT5RIFB5kHnBLJcvV2CWRICwI,2845
+vastdb/util.py,sha256=rs7nLL2Qz-OVEZDSVIqAvS-uETMq-zxQs5jBksB5-JA,4276
 vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/bench/test_perf.py,sha256=iHE3E60fvyU5SBDHPi4h03Dj6QcY6VI9l9mMhgNMtPc,1117
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vastdb/tests/test_duckdb.py,sha256=KDuv4PrjGEwChCGHG36xNT2JiFlBOt6K3DQ3L06Kq-A,1913
-vastdb/tests/test_imports.py,sha256=fDUjO5U-5i4QTIMoNnSSW4X_ZnOStLbx0mJkNq2pj9Q,5033
+vastdb/tests/test_imports.py,sha256=48kbJKsa_MrEXcBYQUbUDr1e9wzjG4FHQ7C3wUEQfXA,5705
 vastdb/tests/test_nested.py,sha256=3kejEvtSqV0LrUgb1QglRjrlxnKI4_AXTFw2nE7Q520,951
 vastdb/tests/test_projections.py,sha256=_cDNfD5zTwbCXLk6uGpPUWGN0P-4HElu5OjubWu-Jg0,1255
 vastdb/tests/test_sanity.py,sha256=ixx0QPo73hLHjAa7bByFXjS1XST0WvmSwLEpgnHh_JY,2960
-vastdb/tests/test_schemas.py,sha256=b-JpYHOFYVTdE570_La7O2RWf8BGN-q8KDXNXeC8CSg,1724
-vastdb/tests/test_tables.py,sha256=TXM4LSBvPb3EEu7XScZ5iEiu_zhHClq61W18EQodxw8,25667
+vastdb/tests/test_schemas.py,sha256=qoHTLX51D-0S4bMxdCpRh9gaYQd-BkZdT_agGOwFwTM,1739
+vastdb/tests/test_tables.py,sha256=joeEQ30TwKBQc-2N_qGIdviZVnQr4rs6thlNsy5s_og,26672
+vastdb/tests/test_util.py,sha256=owRAU3TCKMq-kz54NRdA5wX2O_bZIHqG5ucUR77jm5k,1046
 vastdb/tests/util.py,sha256=NaCzKymEGy1xuiyMxyt2_0frKVfVk9iGrFwLf3GHjTI,435
-vastdb-0.1.2.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.1.2.dist-info/METADATA,sha256=edJPdDWmHj6tRHRR97eSppfN9_4ARfIr0jS9HMjHfSQ,1311
-vastdb-0.1.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-vastdb-0.1.2.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.1.2.dist-info/RECORD,,
+vastdb-0.1.3.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-0.1.3.dist-info/METADATA,sha256=3h3JttUxw9oMMsxV_CVG_LMYwhgegsS9-b4gZkihrM0,1319
+vastdb-0.1.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+vastdb-0.1.3.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-0.1.3.dist-info/RECORD,,