vastdb 0.1.0__tar.gz → 0.1.1__tar.gz

vastdb-0.1.1/CHANGELOG.md

@@ -0,0 +1,95 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [0.1.1] (2024-04-15)
+[0.1.1]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.0...v0.1.1
+
+### Added
+ - Support date & timestamp predicate pushdown [#25]
+
+### Fixed
+ - Add missing `requirements.txt` file to released tarball [#47]
+
+## [0.1.0] (2024-04-11)
+[0.1.0]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.5.3...v0.1.0
+
+### Added
+- Transactions are implemented using [context managers](https://docs.python.org/3/library/stdtypes.html#typecontextmanager)
+- Database entities are derived from a specific transaction
+- Predicate pushdown is using [ibis](https://ibis-project.org/)-like expressions
+- Public SDK [test suite](vastdb/tests/)
+- Internal CI with Python 3.9 - 3.12
+- Using [`ruff`](https://github.com/astral-sh/ruff) for linting and code formatting
+- Improved [`README.md`](README.md) with new API examples
+
+### Removed
+
+- Old SDK classes and methods are deprecated
+
+## [0.0.5.3] (2024-03-24)
+[0.0.5.3]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.5.2...v0.0.5.3
+
+### Added
+- Allow changing default max list_columns page size [#43]
+
+### Fixed
+- Fixing insert function to use the correct tx while listing columns [#42]
+
+## [0.0.5.2] (2024-03-19)
+[0.0.5.2]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.5.1...v0.0.5.2
+
+### Added
+- Support retrieving row IDs via `VastdbApi.query()` [#31]
+
+### Fixed
+- Fixed several functions that didn't use the correct txid [#30]
+- Fix graceful cancellation for `VastdbApi.query_iterator()` [#33]
+
+### Removed
+- Remove `vast_protobuf` package and `protobuf` dependency [#29]
+
+
+## [0.0.5.1] (2024-03-01)
+[0.0.5.1]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.5.0...v0.0.5.1
+
+### Fixed
+- Support latest Python (3.11+) by not requiring too old dependencies. [#16]
+
+
+## [0.0.5.0] (2024-02-25)
+[0.0.5.0]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.4.0...v0.0.5.0
+
+### Added
+- Created a CHANGELOG
+
+### Fixed
+- Wait until all files are processed during import
+- Sync with internal `api.py` code
+
+
+## [0.0.4.0] (2024-02-14)
+[0.0.4.0]: https://github.com/vast-data/vastdb_sdk/compare/v0.0.2...v0.0.4.0
+
+### Added
+- Allow creating a table using the schema of an existing Parquet file via `create_table_from_parquet_schema()`
+- Allow inserting RecordBatch into DB via `insert()`
+
+### Changed
+- Rename `vastdb_api` module to `api`
+
+### Fixed
+- Fix `concurrent.futures` import error
+
+
+[#16]: https://github.com/vast-data/vastdb_sdk/pull/16
+[#25]: https://github.com/vast-data/vastdb_sdk/pull/25
+[#29]: https://github.com/vast-data/vastdb_sdk/pull/29
+[#30]: https://github.com/vast-data/vastdb_sdk/pull/30
+[#31]: https://github.com/vast-data/vastdb_sdk/pull/31
+[#33]: https://github.com/vast-data/vastdb_sdk/pull/33
+[#42]: https://github.com/vast-data/vastdb_sdk/pull/42
+[#43]: https://github.com/vast-data/vastdb_sdk/pull/43
+[#47]: https://github.com/vast-data/vastdb_sdk/pull/47

vastdb-0.1.1/MANIFEST.in

@@ -0,0 +1 @@
+include *.md LICENSE requirements.txt

{vastdb-0.1.0 → vastdb-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vastdb
-Version: 0.1.0
+Version: 0.1.1
 Summary: VAST Data SDK
 Home-page: https://github.com/vast-data/vastdb_sdk
 Author: VAST DATA

{vastdb-0.1.0 → vastdb-0.1.1}/setup.py

@@ -1,5 +1,6 @@
 import os
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 long_description = """
 `vastdb` is a Python-based SDK designed for interacting
@@ -25,7 +26,7 @@ setup(
     name='vastdb',
     python_requires='>=3.9.0',
     description='VAST Data SDK',
-    version='0.1.0' + suffix,
+    version='0.1.1' + suffix,
     url='https://github.com/vast-data/vastdb_sdk',
     author='VAST DATA',
     author_email='hello@vastdata.com',

vastdb-0.1.1/vastdb/bench/test_perf.py

@@ -0,0 +1,29 @@
+import logging
+import time
+
+import pyarrow as pa
+import pytest
+
+from vastdb import util
+from vastdb.table import ImportConfig, QueryConfig
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.benchmark
+def test_bench(session, clean_bucket_name, parquets_path, crater_path):
+    files = [str(parquets_path/f) for f in (parquets_path.glob('**/*.pq'))]
+
+    with session.transaction() as tx:
+        b = tx.bucket(clean_bucket_name)
+        s = b.create_schema('s1')
+        t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
+        config = QueryConfig(num_splits=8, num_sub_splits=4)
+        s = time.time()
+        pa_table = pa.Table.from_batches(t.select(columns=['sid'], predicate=t['sid'] == 10033007, config=config))
+        e = time.time()
+        log.info("'SELECT sid from TABLE WHERE sid = 10033007' returned in %s seconds.", e-s)
+        if crater_path:
+            with open(f'{crater_path}/bench_results', 'a') as f:
+                f.write(f"'SELECT sid FROM TABLE WHERE sid = 10033007' returned in {e-s} seconds")
+        assert pa_table.num_rows == 255_075

{vastdb-0.1.0 → vastdb-0.1.1}/vastdb/bucket.py

@@ -4,10 +4,10 @@ VAST S3 buckets can be used to create Database schemas and tables.
 It is possible to list and access VAST snapshots generated over a bucket.
 """
 
-from . import errors, schema, transaction
-
-from dataclasses import dataclass
 import logging
+from dataclasses import dataclass
+
+from . import errors, schema, transaction
 
 log = logging.getLogger(__name__)
 
@@ -27,18 +27,26 @@ class Bucket:
     name: str
     tx: "transaction.Transaction"
 
-    def create_schema(self, path: str) -> "schema.Schema":
+    def create_schema(self, path: str, fail_if_exists=True) -> "schema.Schema":
         """Create a new schema (a container of tables) under this bucket."""
+        if current := self.schema(path, fail_if_missing=False):
+            if fail_if_exists:
+                raise errors.SchemaExists(self.name, path)
+            else:
+                return current
         self.tx._rpc.api.create_schema(self.name, path, txid=self.tx.txid)
         log.info("Created schema: %s", path)
         return self.schema(path)
 
-    def schema(self, path: str) -> "schema.Schema":
+    def schema(self, path: str, fail_if_missing=True) -> "schema.Schema":
         """Get a specific schema (a container of tables) under this bucket."""
         s = self.schemas(path)
         log.debug("schema: %s", s)
         if not s:
-            raise errors.MissingSchema(self.name, path)
+            if fail_if_missing:
+                raise errors.MissingSchema(self.name, path)
+            else:
+                return None
         assert len(s) == 1, f"Expected to receive only a single schema, but got: {len(s)}. ({s})"
         log.debug("Found schema: %s", s[0].name)
         return s[0]

{vastdb-0.1.0/vastdb/tests → vastdb-0.1.1/vastdb}/conftest.py

@@ -1,15 +1,19 @@
-import vastdb
+import os
+from pathlib import Path
 
-import pytest
 import boto3
-import os
+import pytest
+
+import vastdb
 
 
 def pytest_addoption(parser):
-    parser.addoption("--tabular-bucket-name", help="Name of the S3 bucket with Tabular enabled", default = "vastdb")
-    parser.addoption("--tabular-access-key", help="Access key with Tabular permissions (AWS_ACCESS_KEY_ID)", default = os.environ.get("AWS_ACCESS_KEY_ID", None))
-    parser.addoption("--tabular-secret-key", help="Secret key with Tabular permissions (AWS_SECRET_ACCESS_KEY)" , default = os.environ.get("AWS_SECRET_ACCESS_KEY", None))
-    parser.addoption("--tabular-endpoint-url", help="Tabular server endpoint", default = "http://localhost:9090")
+    parser.addoption("--tabular-bucket-name", help="Name of the S3 bucket with Tabular enabled", default="vastdb")
+    parser.addoption("--tabular-access-key", help="Access key with Tabular permissions (AWS_ACCESS_KEY_ID)", default=os.environ.get("AWS_ACCESS_KEY_ID", None))
+    parser.addoption("--tabular-secret-key", help="Secret key with Tabular permissions (AWS_SECRET_ACCESS_KEY)", default=os.environ.get("AWS_SECRET_ACCESS_KEY", None))
+    parser.addoption("--tabular-endpoint-url", help="Tabular server endpoint", default="http://localhost:9090")
+    parser.addoption("--data-path", help="Data files location", default=None)
+    parser.addoption("--crater-path", help="Save benchmark results in a dedicated location", default=None)
 
 
 @pytest.fixture(scope="session")
@@ -44,3 +48,13 @@ def s3(request):
         aws_access_key_id=request.config.getoption("--tabular-access-key"),
         aws_secret_access_key=request.config.getoption("--tabular-secret-key"),
         endpoint_url=request.config.getoption("--tabular-endpoint-url"))
+
+
+@pytest.fixture(scope="function")
+def parquets_path(request):
+    return Path(request.config.getoption("--data-path"))
+
+
+@pytest.fixture(scope="function")
+def crater_path(request):
+    return request.config.getoption("--crater-path")

{vastdb-0.1.0 → vastdb-0.1.1}/vastdb/errors.py

@@ -1,9 +1,9 @@
 import logging
-import requests
 import xml.etree.ElementTree
-
-from enum import Enum
 from dataclasses import dataclass
+from enum import Enum
+
+import requests
 
 
 class HttpStatus(Enum):
@@ -114,6 +114,23 @@ class MissingProjection(Missing):
     projection: str
 
 
+class Exists(Exception):
+    pass
+
+
+@dataclass
+class SchemaExists(Exists):
+    bucket: str
+    schema: str
+
+
+@dataclass
+class TableExists(Exists):
+    bucket: str
+    schema: str
+    table: str
+
+
 ERROR_TYPES_MAP = {
     HttpStatus.BAD_REQUEST: BadRequest,
     HttpStatus.FOBIDDEN: Forbidden,