vastdb 0.1.1__tar.gz → 0.1.3__tar.gz

{vastdb-0.1.1 → vastdb-0.1.3}/CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.1.3] (2024-05-05)
+[0.1.3]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.2...v0.1.3
+
+### Added
+ - Document predicate pushdown support
+ - Access imports' table (for VAST 5.2+)
+ - Support `is_in` predicate pushdown
+ - Document `table.py`
+
+### Fixed
+ - Freeze `ibis` dependency at 8.0.0
+ - Support snapshot-based access
+ - Optimize RecordBatch slicing
+
+## [0.1.2] (2024-04-25)
+[0.1.2]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.1...v0.1.2
+
+### Added
+ - Allow querying [VAST Catalog](https://www.vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database)
+ - Use [mypy](https://mypy-lang.org/) for type annotation checking
+
+### Fixed
+ - Fix handling HTTPS endpoints [#50]
+ - Optimize `Table.select()` performance
+ - Use `GET` request to retrieve cluster version
+ - Enable `ruff` preview mode
+ - Make sure DuckDB integration fails gracefully when transaction is closed prematurely
+
+### Removed
+ - Don't return row IDs from `Table.insert()`
+
 ## [0.1.1] (2024-04-15)
 [0.1.1]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.0...v0.1.1
 
@@ -93,3 +124,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 [#42]: https://github.com/vast-data/vastdb_sdk/pull/42
 [#43]: https://github.com/vast-data/vastdb_sdk/pull/43
 [#47]: https://github.com/vast-data/vastdb_sdk/pull/47
+[#50]: https://github.com/vast-data/vastdb_sdk/pull/50

{vastdb-0.1.1/vastdb.egg-info → vastdb-0.1.3}/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.1
+Version: 0.1.3
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA
 Author-email: hello@vastdata.com
 License: Copyright (C) VAST Data Ltd.
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -19,6 +18,13 @@ Classifier: Topic :: Database :: Front-Ends
 Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aws-requests-auth
+Requires-Dist: boto3
+Requires-Dist: flatbuffers
+Requires-Dist: ibis-framework==8.0.0
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: xmltodict
 
 
 `vastdb` is a Python-based SDK designed for interacting
@@ -27,5 +33,3 @@ and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system
 enabling schema and table management, efficient ingest, query and modification of columnar data.
 
 For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
-
-

{vastdb-0.1.1 → vastdb-0.1.3}/README.md

@@ -86,6 +86,8 @@ Our SDK supports predicate and projection pushdown:
     table.select(predicate=table['c3'].contains('substring'))
 ```
 
+See [here for more details](docs/predicate.md).
+
 ## Import a single Parquet file via S3 protocol
 
 It is possible to efficiently create a table from a Parquet file (without copying it via the client):
@@ -150,10 +152,11 @@ p.drop()
 
 ## Snapshots
 
-It is possible to list [available snapshots](https://vastdata.com/blog/bringing-snapshots-to-vasts-element-store):
+It is possible to use [snapshots](https://vastdata.com/blog/bringing-snapshots-to-vasts-element-store) for accessing the Database:
 
 ```python
-print(bucket.list_snapshots())
+snaps = bucket.list_snapshots()
+batches = snaps[0].schema('schema-name').table('table-name').select()
 ```
 
 ## VAST Catalog

{vastdb-0.1.1 → vastdb-0.1.3}/requirements.txt

@@ -1,7 +1,7 @@
 aws-requests-auth
 boto3
 flatbuffers
-ibis-framework
+ibis-framework==8.0.0
 pyarrow
 requests
 xmltodict

{vastdb-0.1.1 → vastdb-0.1.3}/setup.py

@@ -11,6 +11,7 @@ enabling schema and table management, efficient ingest, query and modification o
 For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
 """
 
+
 def _get_version_suffix():
     import subprocess
 
@@ -18,6 +19,7 @@ def _get_version_suffix():
     print(f"Git commit: {commit}")
     return f".dev1+vast.{commit.decode()[:16]}"
 
+
 suffix = ''
 if os.environ.get('VASTDB_APPEND_VERSION_SUFFIX'):
     suffix = _get_version_suffix()
@@ -26,7 +28,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='0.1.1' + suffix,
+    version='0.1.3' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

{vastdb-0.1.1 → vastdb-0.1.3}/vastdb/__init__.py

@@ -1,7 +1,11 @@
 """VAST Database Python SDK."""
 
+import functools
+
 from . import session
 
+
 # A helper function, useful as a short-hand for Session c-tor: `session = vastdb.connect(...)`
-connect = session.Session
-connect.__name__ = 'connect'
+@functools.wraps(session.Session)
+def connect(*args, **kwargs):  # noqa: D103
+    return session.Session(*args, **kwargs)

{vastdb-0.1.1 → vastdb-0.1.3}/vastdb/bench/test_perf.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 
 @pytest.mark.benchmark
 def test_bench(session, clean_bucket_name, parquets_path, crater_path):
-    files = [str(parquets_path/f) for f in (parquets_path.glob('**/*.pq'))]
+    files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
 
     with session.transaction() as tx:
         b = tx.bucket(clean_bucket_name)
@@ -22,8 +22,8 @@ def test_bench(session, clean_bucket_name, parquets_path, crater_path):
         s = time.time()
         pa_table = pa.Table.from_batches(t.select(columns=['sid'], predicate=t['sid'] == 10033007, config=config))
         e = time.time()
-        log.info("'SELECT sid from TABLE WHERE sid = 10033007' returned in %s seconds.", e-s)
+        log.info("'SELECT sid from TABLE WHERE sid = 10033007' returned in %s seconds.", e - s)
         if crater_path:
             with open(f'{crater_path}/bench_results', 'a') as f:
-                f.write(f"'SELECT sid FROM TABLE WHERE sid = 10033007' returned in {e-s} seconds")
+                f.write(f"'SELECT sid FROM TABLE WHERE sid = 10033007' returned in {e - s} seconds")
         assert pa_table.num_rows == 255_075

{vastdb-0.1.1 → vastdb-0.1.3}/vastdb/bucket.py

@@ -6,18 +6,14 @@ It is possible to list and access VAST snapshots generated over a bucket.
 
 import logging
 from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional
 
 from . import errors, schema, transaction
 
-log = logging.getLogger(__name__)
-
-
-@dataclass
-class Snapshot:
-    """VAST bucket-level snapshot."""
+if TYPE_CHECKING:
+    from .schema import Schema
 
-    name: str
-    bucket: "Bucket"
+log = logging.getLogger(__name__)
 
 
 @dataclass
@@ -27,7 +23,7 @@ class Bucket:
     name: str
     tx: "transaction.Transaction"
 
-    def create_schema(self, path: str, fail_if_exists=True) -> "schema.Schema":
+    def create_schema(self, path: str, fail_if_exists=True) -> "Schema":
         """Create a new schema (a container of tables) under this bucket."""
         if current := self.schema(path, fail_if_missing=False):
             if fail_if_exists:
@@ -36,9 +32,9 @@ class Bucket:
                 return current
         self.tx._rpc.api.create_schema(self.name, path, txid=self.tx.txid)
         log.info("Created schema: %s", path)
-        return self.schema(path)
+        return self.schema(path)  # type: ignore[return-value]
 
-    def schema(self, path: str, fail_if_missing=True) -> "schema.Schema":
+    def schema(self, path: str, fail_if_missing=True) -> Optional["Schema"]:
         """Get a specific schema (a container of tables) under this bucket."""
         s = self.schemas(path)
         log.debug("schema: %s", s)
@@ -51,14 +47,14 @@ class Bucket:
         log.debug("Found schema: %s", s[0].name)
         return s[0]
 
-    def schemas(self, name: str = None) -> ["schema.Schema"]:
+    def schemas(self, name: Optional[str] = None) -> List["Schema"]:
         """List bucket's schemas."""
         schemas = []
         next_key = 0
         exact_match = bool(name)
         log.debug("list schemas param: schema=%s, exact_match=%s", name, exact_match)
         while True:
-            bucket_name, curr_schemas, next_key, is_truncated, _ = \
+            _bucket_name, curr_schemas, next_key, is_truncated, _ = \
                 self.tx._rpc.api.list_schemas(bucket=self.name, next_key=next_key, txid=self.tx.txid,
                                                name_prefix=name, exact_match=exact_match)
             if not curr_schemas:
@@ -69,7 +65,22 @@ class Bucket:
 
         return [schema.Schema(name=name, bucket=self) for name, *_ in schemas]
 
-    def snapshots(self) -> [Snapshot]:
+    def snapshot(self, name, fail_if_missing=True) -> Optional["Bucket"]:
+        """Get snapshot by name (if exists)."""
+        snapshots, _is_truncated, _next_key = \
+            self.tx._rpc.api.list_snapshots(bucket=self.name, name_prefix=name, max_keys=1)
+
+        expected_name = f".snapshot/{name}"
+        exists = snapshots and snapshots[0] == expected_name + "/"
+        if not exists:
+            if fail_if_missing:
+                raise errors.MissingSnapshot(self.name, expected_name)
+            else:
+                return None
+
+        return Bucket(name=f'{self.name}/{expected_name}', tx=self.tx)
+
+    def snapshots(self) -> List["Bucket"]:
         """List bucket's snapshots."""
         snapshots = []
         next_key = 0
@@ -82,4 +93,7 @@ class Bucket:
             if not is_truncated:
                 break
 
-        return [Snapshot(name=snapshot, bucket=self) for snapshot in snapshots]
+        return [
+            Bucket(name=f'{self.name}/{snapshot.strip("/")}', tx=self.tx)
+            for snapshot in snapshots
+        ]

{vastdb-0.1.1 → vastdb-0.1.3}/vastdb/errors.py

@@ -26,6 +26,7 @@ log = logging.getLogger(__name__)
 class HttpError(Exception):
     code: str
     message: str
+    method: str
     url: str
     status: int  # HTTP status
     headers: requests.structures.CaseInsensitiveDict  # HTTP response headers
@@ -84,15 +85,33 @@ class InvalidArgument(Exception):
     pass
 
 
+class TooWideRow(InvalidArgument):
+    pass
+
+
 class Missing(Exception):
     pass
 
 
+class MissingTransaction(Missing):
+    pass
+
+
+class NotSupported(Exception):
+    pass
+
+
 @dataclass
 class MissingBucket(Missing):
     bucket: str
 
 
+@dataclass
+class MissingSnapshot(Missing):
+    bucket: str
+    snapshot: str
+
+
 @dataclass
 class MissingSchema(Missing):
     bucket: str
@@ -131,6 +150,19 @@ class TableExists(Exists):
     table: str
 
 
+@dataclass
+class NotSupportedCommand(NotSupported):
+    bucket: str
+    schema: str
+    table: str
+
+
+@dataclass
+class NotSupportedVersion(NotSupported):
+    err_msg: str
+    version: str
+
+
 ERROR_TYPES_MAP = {
     HttpStatus.BAD_REQUEST: BadRequest,
     HttpStatus.FOBIDDEN: Forbidden,
@@ -150,21 +182,22 @@ def from_response(res: requests.Response):
 
     log.debug("response: url='%s', code=%s, headers=%s, body='%s'", res.request.url, res.status_code, res.headers, res.text)
     # try to parse S3 XML response for the error details:
-    code = None
-    message = None
+    code_str = None
+    message_str = None
     if res.text:
         try:
             root = xml.etree.ElementTree.fromstring(res.text)
             code = root.find('Code')
-            code = code.text if code is not None else None
+            code_str = code.text if code is not None else None
             message = root.find('Message')
-            message = message.text if message is not None else None
+            message_str = message.text if message is not None else None
         except xml.etree.ElementTree.ParseError:
             log.debug("invalid XML: %r", res.text)
 
     kwargs = dict(
-        code=code,
-        message=message,
+        code=code_str,
+        message=message_str,
+        method=res.request.method,
         url=res.request.url,
         status=res.status_code,
         headers=res.headers,
@@ -172,4 +205,4 @@ def from_response(res: requests.Response):
     log.warning("RPC failed: %s", kwargs)
     status = HttpStatus(res.status_code)
     error_type = ERROR_TYPES_MAP.get(status, UnexpectedError)
-    raise error_type(**kwargs)
+    return error_type(**kwargs)