vastdb 0.1.4__tar.gz → 0.1.6__tar.gz

{vastdb-0.1.4 → vastdb-0.1.6}/CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.1.6] (2024-05-23)
+[0.1.6]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.5...v0.1.6
+
+### Added
+ - Allow listing and querying Catalog snapshots
+ - Support nested schemas
+ - Use automatic split estimation
+
+### Fixed
+ - Allow using specific semi-sorted projection (needs VAST 5.1+ release)
+ - Add a client-side check for too large requests
+
+## [0.1.5] (2024-05-16)
+[0.1.5]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.4...v0.1.5
+
+### Added
+ - Allow passing `ssl_verify` via `Session` c-tor
+ - Retry `requests.exceptions.ConnectionError`
+ - Document `QueryConfig` properties
+ - Support "deferred" `ibis` predicates using [Underscore (`_`) API](https://github.com/ibis-project/ibis/pull/3804)
+
+### Fixed
+ - Fix predicate pushdown when nested columns are present
+ - Attach original traceback to `MissingBucket` exception
+
 ## [0.1.4] (2024-05-13)
 [0.1.4]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.3...v0.1.4
 

{vastdb-0.1.4/vastdb.egg-info → vastdb-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.4
+Version: 0.1.6
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.1.4 → vastdb-0.1.6}/README.md

@@ -74,16 +74,18 @@ with session.transaction() as tx:
 Our SDK supports predicate and projection pushdown:
 
 ```python
+    from ibis import _
+
     # SELECT c1 FROM t WHERE (c2 > 2) AND (c3 IS NULL)
     table.select(columns=['c1'],
-                 predicate=(table['c2'] > 2) & table['c3'].isnull())
+                 predicate=(_.c2 > 2) & _.c3.isnull())
 
     # SELECT c2, c3 FROM t WHERE (c2 BETWEEN 0 AND 1) OR (c2 > 10)
     table.select(columns=['c2', 'c3'],
-                 predicate=table['c2'].between(0, 1) | (table['c2'] > 10))
+                 predicate=(_.c2.between(0, 1) | (table['c2'] > 10))
 
     # SELECT * FROM t WHERE c3 LIKE '%substring%'
-    table.select(predicate=table['c3'].contains('substring'))
+    table.select(predicate=_.c3.contains('substring'))
 ```
 
 See [here for more details](docs/predicate.md).
@@ -132,10 +134,12 @@ with contextlib.closing(pa.parquet.ParquetWriter('/path/to/file.parquet', batche
 We can use [DuckDB](https://duckdb.org/docs/guides/python/sql_on_arrow.html) to post-process the resulting stream of [PyArrow record batches](https://arrow.apache.org/docs/python/data.html#record-batches):
 
 ```python
+from ibis import _
+
 import duckdb
 conn = duckdb.connect()
 
-batches = table.select(columns=['c1'], predicate=(table['c2'] > 2))
+batches = table.select(columns=['c1'], predicate=(_.c2 > 2))
 print(conn.execute("SELECT sum(c1) FROM batches").arrow())
 ```
 

{vastdb-0.1.4 → vastdb-0.1.6}/setup.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from setuptools import find_packages, setup
 
@@ -28,13 +29,13 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='0.1.4' + suffix,
+    version='0.1.6' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',
     license='Copyright (C) VAST Data Ltd.',
     packages=find_packages(),
-    install_requires=open('requirements.txt').read().strip().split('\n'),
+    install_requires=Path('requirements.txt').read_text().strip().split(),
     long_description=long_description,
     long_description_content_type='text/markdown',
     classifiers=[

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/bench/test_perf.py

@@ -1,7 +1,6 @@
 import logging
 import time
 
-import pyarrow as pa
 import pytest
 
 from vastdb import util
@@ -20,7 +19,7 @@ def test_bench(session, clean_bucket_name, parquets_path, crater_path):
         t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
         config = QueryConfig(num_splits=8, num_sub_splits=4)
         s = time.time()
-        pa_table = pa.Table.from_batches(t.select(columns=['sid'], predicate=t['sid'] == 10033007, config=config))
+        pa_table = t.select(columns=['sid'], predicate=t['sid'] == 10033007, config=config).read_all()
         e = time.time()
         log.info("'SELECT sid from TABLE WHERE sid = 10033007' returned in %s seconds.", e - s)
         if crater_path:

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/bucket.py

@@ -5,7 +5,7 @@ It is possible to list and access VAST snapshots generated over a bucket.
 """
 
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List, Optional
 
 from . import errors, schema, transaction
@@ -22,48 +22,23 @@ class Bucket:
 
     name: str
     tx: "transaction.Transaction"
+    _root_schema: "Schema" = field(init=False, compare=False, repr=False)
 
-    def create_schema(self, path: str, fail_if_exists=True) -> "Schema":
+    def __post_init__(self):
+        """Root schema is empty."""
+        self._root_schema = schema.Schema(name="", bucket=self)
+
+    def create_schema(self, name: str, fail_if_exists=True) -> "Schema":
         """Create a new schema (a container of tables) under this bucket."""
-        if current := self.schema(path, fail_if_missing=False):
-            if fail_if_exists:
-                raise errors.SchemaExists(self.name, path)
-            else:
-                return current
-        self.tx._rpc.api.create_schema(self.name, path, txid=self.tx.txid)
-        log.info("Created schema: %s", path)
-        return self.schema(path)  # type: ignore[return-value]
+        return self._root_schema.create_schema(name=name, fail_if_exists=fail_if_exists)
 
-    def schema(self, path: str, fail_if_missing=True) -> Optional["Schema"]:
+    def schema(self, name: str, fail_if_missing=True) -> Optional["Schema"]:
         """Get a specific schema (a container of tables) under this bucket."""
-        s = self.schemas(path)
-        log.debug("schema: %s", s)
-        if not s:
-            if fail_if_missing:
-                raise errors.MissingSchema(self.name, path)
-            else:
-                return None
-        assert len(s) == 1, f"Expected to receive only a single schema, but got: {len(s)}. ({s})"
-        log.debug("Found schema: %s", s[0].name)
-        return s[0]
+        return self._root_schema.schema(name=name, fail_if_missing=fail_if_missing)
 
-    def schemas(self, name: Optional[str] = None) -> List["Schema"]:
+    def schemas(self, batch_size=None):
         """List bucket's schemas."""
-        schemas = []
-        next_key = 0
-        exact_match = bool(name)
-        log.debug("list schemas param: schema=%s, exact_match=%s", name, exact_match)
-        while True:
-            _bucket_name, curr_schemas, next_key, is_truncated, _ = \
-                self.tx._rpc.api.list_schemas(bucket=self.name, next_key=next_key, txid=self.tx.txid,
-                                               name_prefix=name, exact_match=exact_match)
-            if not curr_schemas:
-                break
-            schemas.extend(curr_schemas)
-            if not is_truncated:
-                break
-
-        return [schema.Schema(name=name, bucket=self) for name, *_ in schemas]
+        return self._root_schema.schemas(batch_size=batch_size)
 
     def snapshot(self, name, fail_if_missing=True) -> Optional["Bucket"]:
         """Get snapshot by name (if exists)."""

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/conftest.py

@@ -30,14 +30,23 @@ def test_bucket_name(request):
     return request.config.getoption("--tabular-bucket-name")
 
 
+def iter_schemas(s):
+    """Recusively scan all schemas."""
+    children = s.schemas()
+    for c in children:
+        yield from iter_schemas(c)
+    yield s
+
+
 @pytest.fixture(scope="function")
 def clean_bucket_name(request, test_bucket_name, session):
     with session.transaction() as tx:
         b = tx.bucket(test_bucket_name)
-        for s in b.schemas():
-            for t in s.tables():
-                t.drop()
-            s.drop()
+        for top_schema in b.schemas():
+            for s in iter_schemas(top_schema):
+                for t in s.tables():
+                    t.drop()
+                s.drop()
     return test_bucket_name
 
 

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/errors.py

@@ -89,7 +89,11 @@ class InvalidArgument(Exception):
     pass
 
 
-class TooWideRow(InvalidArgument):
+class TooLargeRequest(InvalidArgument):
+    pass
+
+
+class TooWideRow(TooLargeRequest):
     pass
 
 

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/internal_commands.py

@@ -6,7 +6,6 @@ import struct
 import urllib.parse
 from collections import defaultdict, namedtuple
 from enum import Enum
-from ipaddress import IPv4Address, IPv6Address
 from typing import Any, Dict, Iterator, List, Optional, Union
 
 import flatbuffers
@@ -130,46 +129,13 @@ def get_unit_to_flatbuff_time_unit(type):
 class Predicate:
     def __init__(self, schema: 'pa.Schema', expr: ibis.expr.types.BooleanColumn):
         self.schema = schema
+        index = itertools.count()  # used to generate leaf column positions for VAST QueryData RPC
+        # Arrow schema contains the top-level columns, where each column may include multiple subfields
+        # We use DFS is used to enumerate all the sub-columns, using `index` as an ID allocator
+        nodes = [FieldNode(field, index) for field in schema]
+        self.nodes_map = {node.field.name: node for node in nodes}
         self.expr = expr
 
-    def get_field_indexes(self, field: 'pa.Field', field_name_per_index: list) -> None:
-        field_name_per_index.append(field.name)
-
-        if isinstance(field.type, pa.StructType):
-            flat_fields = field.flatten()
-        elif isinstance(field.type, pa.MapType):
-            flat_fields = [pa.field(f'{field.name}.entries', pa.struct([field.type.key_field, field.type.item_field]))]
-        elif isinstance(field.type, pa.ListType):
-            flat_fields = [pa.field(f'{field.name}.{field.type.value_field.name}', field.type.value_field.type)]
-        else:
-            return
-
-        for flat_field in flat_fields:
-            self.get_field_indexes(flat_field, field_name_per_index)
-
-    @property
-    def field_name_per_index(self):
-        if self._field_name_per_index is None:
-            _field_name_per_index = []
-            for field in self.schema:
-                self.get_field_indexes(field, _field_name_per_index)
-            self._field_name_per_index = {field: index for index, field in enumerate(_field_name_per_index)}
-        return self._field_name_per_index
-
-    def get_projections(self, builder: 'flatbuffers.builder.Builder', field_names: Optional[List[str]] = None):
-        if field_names is None:
-            field_names = self.field_name_per_index.keys()
-        projection_fields = []
-        for field_name in field_names:
-            fb_field_index.Start(builder)
-            fb_field_index.AddPosition(builder, self.field_name_per_index[field_name])
-            offset = fb_field_index.End(builder)
-            projection_fields.append(offset)
-        fb_source.StartProjectionVector(builder, len(projection_fields))
-        for offset in reversed(projection_fields):
-            builder.PrependUOffsetTRelative(offset)
-        return builder.EndVector()
-
     def serialize(self, builder: 'flatbuffers.builder.Builder'):
         from ibis.expr.operations.generic import (
             IsNull,
@@ -204,8 +170,6 @@ class Predicate:
             Between: self.build_between,
         }
 
-        positions_map = dict((f.name, index) for index, f in enumerate(self.schema))  # TODO: BFS
-
         self.builder = builder
 
         offsets = []
@@ -261,7 +225,11 @@ class Predicate:
                     elif prev_field_name != field_name:
                         raise NotImplementedError(self.expr)
 
-                    column_offset = self.build_column(position=positions_map[field_name])
+                    node = self.nodes_map[field_name]
+                    # TODO: support predicate pushdown for leaf nodes (ORION-160338)
+                    if node.children:
+                        raise NotImplementedError(node.field)  # no predicate pushdown for nested columns
+                    column_offset = self.build_column(position=node.index)
                     field = self.schema.field(field_name)
                     for literal in literals:
                         args_offsets = [column_offset]
@@ -839,12 +807,13 @@ class VastdbApi:
         return prefix
 
     def _fill_common_headers(self, txid=0, client_tags=[], version_id=1):
-        common_headers = {'tabular-txid': str(txid), 'tabular-api-version-id': str(version_id),
-                          'tabular-client-name': 'tabular-api'}
-        for tag in client_tags:
-            common_headers['tabular-client-tags-%d' % client_tags.index(tag)] = tag
+        common_headers = {
+            'tabular-txid': str(txid),
+            'tabular-api-version-id': str(version_id),
+            'tabular-client-name': 'tabular-api'
+        }
 
-        return common_headers
+        return common_headers | {f'tabular-client-tags-{index}': tag for index, tag in enumerate(client_tags)}
 
     def _check_res(self, res, cmd="", expected_retvals=[]):
         if exc := errors.from_response(res):
@@ -952,8 +921,7 @@ class VastdbApi:
             res_headers = res.headers
             next_key = int(res_headers['tabular-next-key'])
             is_truncated = res_headers['tabular-is-truncated'] == 'true'
-            flatbuf = b''.join(res.iter_content(chunk_size=128))
-            lists = list_schemas.GetRootAs(flatbuf)
+            lists = list_schemas.GetRootAs(res.content)
             bucket_name = lists.BucketName().decode()
             if not bucket.startswith(bucket_name):
                 raise ValueError(f'bucket: {bucket} did not start from {bucket_name}')
@@ -976,8 +944,7 @@ class VastdbApi:
         res = self.session.get(self._api_prefix(bucket=bucket, command="list", url_params=url_params), headers={}, stream=True)
         self._check_res(res, "list_snapshots")
 
-        out = b''.join(res.iter_content(chunk_size=128))
-        xml_str = out.decode()
+        xml_str = res.content.decode()
         xml_dict = xmltodict.parse(xml_str)
         list_res = xml_dict['ListBucketResult']
         is_truncated = list_res['IsTruncated'] == 'true'
@@ -1059,30 +1026,11 @@ class VastdbApi:
         res = self.session.get(self._api_prefix(bucket=bucket, schema=schema, table=name, command="stats", url_params=url_params), headers=headers)
         self._check_res(res, "get_table_stats", expected_retvals)
 
-        flatbuf = b''.join(res.iter_content(chunk_size=128))
-        stats = get_table_stats.GetRootAs(flatbuf)
+        stats = get_table_stats.GetRootAs(res.content)
         num_rows = stats.NumRows()
         size_in_bytes = stats.SizeInBytes()
         is_external_rowid_alloc = stats.IsExternalRowidAlloc()
-        endpoints = []
-        if stats.VipsLength() == 0:
-            endpoints.append(self.url)
-        else:
-            url = urllib3.util.parse_url(self.url)
-
-            ip_cls = IPv6Address if (stats.AddressType() == "ipv6") else IPv4Address
-            vips = [stats.Vips(i) for i in range(stats.VipsLength())]
-            ips = []
-            # extract the vips into list of IPs
-            for vip in vips:
-                start_ip = int(ip_cls(vip.StartAddress().decode()))
-                ips.extend(ip_cls(start_ip + i) for i in range(vip.AddressCount()))
-            # build a list of endpoint URLs, reusing schema and port (if specified when constructing the session).
-            # it is assumed that the client can access the returned IPs (e.g. if they are part of the VIP pool).
-            for ip in ips:
-                d = url._asdict()
-                d['host'] = str(ip)
-                endpoints.append(str(urllib3.util.Url(**d)))
+        endpoints = [self.url]  # we cannot replace the host by a VIP address in HTTPS-based URLs
         return TableStatsResult(num_rows, size_in_bytes, is_external_rowid_alloc, tuple(endpoints))
 
     def alter_table(self, bucket, schema, name, txid=0, client_tags=[], table_properties="",
@@ -1159,8 +1107,7 @@ class VastdbApi:
             res_headers = res.headers
             next_key = int(res_headers['tabular-next-key'])
             is_truncated = res_headers['tabular-is-truncated'] == 'true'
-            flatbuf = b''.join(res.iter_content(chunk_size=128))
-            lists = list_tables.GetRootAs(flatbuf)
+            lists = list_tables.GetRootAs(res.content)
             bucket_name = lists.BucketName().decode()
             schema_name = lists.SchemaName().decode()
             if not bucket.startswith(bucket_name):  # ignore snapshot name
@@ -1288,11 +1235,7 @@ class VastdbApi:
             next_key = int(res_headers['tabular-next-key'])
             is_truncated = res_headers['tabular-is-truncated'] == 'true'
             count = int(res_headers['tabular-list-count'])
-            columns = []
-            if not count_only:
-                schema_buf = b''.join(res.iter_content(chunk_size=128))
-                schema_out = pa.ipc.open_stream(schema_buf).schema
-                columns = schema_out
+            columns = [] if count_only else pa.ipc.open_stream(res.content).schema
 
             return columns, next_key, is_truncated, count
 
@@ -1692,8 +1635,7 @@ class VastdbApi:
         res = self.session.get(self._api_prefix(bucket=bucket, schema=schema, table=table, command="projection-stats", url_params=url_params),
                                headers=headers)
         if res.status_code == 200:
-            flatbuf = b''.join(res.iter_content(chunk_size=128))
-            stats = get_projection_table_stats.GetRootAs(flatbuf)
+            stats = get_projection_table_stats.GetRootAs(res.content)
             num_rows = stats.NumRows()
             size_in_bytes = stats.SizeInBytes()
             dirty_blocks_percentage = stats.DirtyBlocksPercentage()
@@ -1779,8 +1721,7 @@ class VastdbApi:
             next_key = int(res_headers['tabular-next-key'])
             is_truncated = res_headers['tabular-is-truncated'] == 'true'
             count = int(res_headers['tabular-list-count'])
-            flatbuf = b''.join(res.iter_content(chunk_size=128))
-            lists = list_projections.GetRootAs(flatbuf)
+            lists = list_projections.GetRootAs(res.content)
             bucket_name = lists.BucketName().decode()
             schema_name = lists.SchemaName().decode()
             table_name = lists.TableName().decode()
@@ -1827,13 +1768,8 @@ class VastdbApi:
             next_key = int(res_headers['tabular-next-key'])
             is_truncated = res_headers['tabular-is-truncated'] == 'true'
             count = int(res_headers['tabular-list-count'])
-            columns = []
-            if not count_only:
-                schema_buf = b''.join(res.iter_content(chunk_size=128))
-                schema_out = pa.ipc.open_stream(schema_buf).schema
-                for f in schema_out:
-                    columns.append([f.name, f.type, f.metadata])
-                #   sort_type = f.metadata[b'VAST:sort_type'].decode()
+            columns = [] if count_only else [[f.name, f.type, f.metadata] for f in
+                                             pa.ipc.open_stream(res.content).schema]
 
             return columns, next_key, is_truncated, count
 

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/schema.py

@@ -31,6 +31,51 @@ class Schema:
         """VAST transaction used for this schema."""
         return self.bucket.tx
 
+    def _subschema_full_name(self, name: str) -> str:
+        return f"{self.name}/{name}" if self.name else name
+
+    def create_schema(self, name: str, fail_if_exists=True) -> "Schema":
+        """Create a new schema (a container of tables) under this schema."""
+        if current := self.schema(name, fail_if_missing=False):
+            if fail_if_exists:
+                raise errors.SchemaExists(self.bucket.name, name)
+            else:
+                return current
+        full_name = self._subschema_full_name(name)
+        self.tx._rpc.api.create_schema(self.bucket.name, full_name, txid=self.tx.txid)
+        log.info("Created schema: %s", full_name)
+        return self.schema(name)  # type: ignore[return-value]
+
+    def schema(self, name: str, fail_if_missing=True) -> Optional["Schema"]:
+        """Get a specific schema (a container of tables) under this schema."""
+        _bucket_name, schemas, _next_key, _is_truncated, _ = \
+            self.tx._rpc.api.list_schemas(bucket=self.bucket.name, schema=self.name, next_key=0, txid=self.tx.txid,
+                                          name_prefix=name, exact_match=True, max_keys=1)
+        names = [name for name, *_ in schemas]
+        log.debug("Found schemas: %s", names)
+        if not names:
+            if fail_if_missing:
+                raise errors.MissingSchema(self.bucket.name, self._subschema_full_name(name))
+            else:
+                return None
+
+        assert len(names) == 1, f"Expected to receive only a single schema, but got {len(schemas)}: ({schemas})"
+        return schema.Schema(name=self._subschema_full_name(names[0]), bucket=self.bucket)
+
+    def schemas(self, batch_size=None) -> List["Schema"]:
+        """List child schemas."""
+        next_key = 0
+        if not batch_size:
+            batch_size = 1000
+        result: List["Schema"] = []
+        while True:
+            _bucket_name, curr_schemas, next_key, is_truncated, _ = \
+                self.tx._rpc.api.list_schemas(bucket=self.bucket.name, schema=self.name, next_key=next_key, max_keys=batch_size, txid=self.tx.txid)
+            result.extend(schema.Schema(name=self._subschema_full_name(name), bucket=self.bucket) for name, *_ in curr_schemas)
+            if not is_truncated:
+                break
+        return result
+
     def create_table(self, table_name: str, columns: pa.Schema, fail_if_exists=True) -> "Table":
         """Create a new table under this schema."""
         if current := self.table(table_name, fail_if_missing=False):

{vastdb-0.1.4 → vastdb-0.1.6}/vastdb/session.py

@@ -7,12 +7,15 @@ For more details see:
 - [Tabular identity policy with the proper permissions](https://support.vastdata.com/s/article/UUID-14322b60-d6a2-89ac-3df0-3dfbb6974182)
 """
 
+import logging
 import os
 
 import boto3
 
 from . import errors, internal_commands, transaction
 
+log = logging.getLogger()
+
 
 class Features:
     """VAST database features - check if server is already support a feature."""
@@ -21,21 +24,34 @@ class Features:
         """Save the server version."""
         self.vast_version = vast_version
 
-    def check_imports_table(self):
-        """Check if the feature that support imports table is supported."""
-        if self.vast_version < (5, 2):
-            raise errors.NotSupportedVersion("import_table requires 5.2+", self.vast_version)
+        self.check_imports_table = self._check(
+            "Imported objects' table feature requires 5.2+ VAST release",
+            vast_version >= (5, 2))
+
+        self.check_return_row_ids = self._check(
+            "Returning row IDs requires 5.1+ VAST release",
+            vast_version >= (5, 1))
+
+        self.check_enforce_semisorted_projection = self._check(
+            "Semi-sorted projection enforcement requires 5.1+ VAST release",
+            vast_version >= (5, 1))
+
+    def _check(self, msg, supported):
+        log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
+        if not supported:
+            def fail():
+                raise errors.NotSupportedVersion(msg, self.vast_version)
+            return fail
 
-    def check_return_row_ids(self):
-        """Check if insert/update/delete can return the row_ids."""
-        if self.vast_version < (5, 1):
-            raise errors.NotSupportedVersion("return_row_ids requires 5.1+", self.vast_version)
+        def noop():
+            pass
+        return noop
 
 
 class Session:
     """VAST database session."""
 
-    def __init__(self, access=None, secret=None, endpoint=None):
+    def __init__(self, access=None, secret=None, endpoint=None, ssl_verify=True):
         """Connect to a VAST Database endpoint, using specified credentials."""
         if access is None:
             access = os.environ['AWS_ACCESS_KEY_ID']
@@ -44,7 +60,7 @@ class Session:
         if endpoint is None:
             endpoint = os.environ['AWS_S3_ENDPOINT_URL']
 
-        self.api = internal_commands.VastdbApi(endpoint, access, secret)
+        self.api = internal_commands.VastdbApi(endpoint, access, secret, ssl_verify=ssl_verify)
         version_tuple = tuple(int(part) for part in self.api.vast_version.split('.'))
         self.features = Features(version_tuple)
         self.s3 = boto3.client('s3',