vastdb 0.0.5.3__py3-none-any.whl → 0.1.1__py3-none-any.whl

vastdb/transaction.py

@@ -0,0 +1,54 @@
+"""VAST Database transaction.
+
+A transcation is used as a context manager, since every Database-related operation in VAST requires a transaction.
+
+    with session.transaction() as tx:
+        tx.bucket("bucket").create_schema("schema")
+"""
+
+import logging
+from dataclasses import dataclass
+
+import botocore
+
+from . import bucket, errors, session
+
+log = logging.getLogger(__name__)
+
+@dataclass
+class Transaction:
+    """A holder of a single VAST transaction."""
+
+    _rpc: "session.Session"
+    txid: int = None
+
+    def __enter__(self):
+        """Create a transaction and store its ID."""
+        response = self._rpc.api.begin_transaction()
+        self.txid = int(response.headers['tabular-txid'])
+        log.debug("opened txid=%016x", self.txid)
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        """On success, the transaction is committed. Otherwise, it is rolled back."""
+        if (exc_type, exc_value, exc_traceback) == (None, None, None):
+            log.debug("committing txid=%016x", self.txid)
+            self._rpc.api.commit_transaction(self.txid)
+        else:
+            log.exception("rolling back txid=%016x due to:", self.txid)
+            self._rpc.api.rollback_transaction(self.txid)
+
+    def __repr__(self):
+        """Don't show the session details."""
+        return f'Transaction(id=0x{self.txid:016x})'
+
+    def bucket(self, name: str) -> "bucket.Bucket":
+        """Return a VAST Bucket, if exists."""
+        try:
+            self._rpc.s3.head_bucket(Bucket=name)
+        except botocore.exceptions.ClientError as e:
+            log.warning("res: %s", e.response)
+            if e.response['Error']['Code'] == '404':
+                raise errors.MissingBucket(name)
+            raise
+        return bucket.Bucket(name, self)

vastdb/util.py

@@ -4,14 +4,16 @@ from typing import Callable
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-from vastdb.v2 import InvalidArgumentError, Table, Schema
-
+from .errors import InvalidArgument
+from .schema import Schema
+from .table import ImportConfig, Table
 
 log = logging.getLogger(__name__)
 
 
 def create_table_from_files(
-        schema: Schema, table_name: str, parquet_files: [str], schema_merge_func: Callable = None) -> Table:
+        schema: Schema, table_name: str, parquet_files: [str], schema_merge_func: Callable = None,
+        config: ImportConfig = None) -> Table:
     if not schema_merge_func:
         schema_merge_func = default_schema_merge
     else:
@@ -22,17 +24,16 @@ def create_table_from_files(
         access_key=tx._rpc.api.access_key, secret_key=tx._rpc.api.secret_key, endpoint_override=tx._rpc.api.url)
     for prq_file in parquet_files:
         if not prq_file.startswith('/'):
-            raise InvalidArgumentError(f"Path {prq_file} must start with a '/'")
+            raise InvalidArgument(f"Path {prq_file} must start with a '/'")
         parquet_ds = pq.ParquetDataset(prq_file.lstrip('/'), filesystem=s3fs)
         current_schema = schema_merge_func(current_schema, parquet_ds.schema)
 
-
     log.info("Creating table %s from %d Parquet files, with columns: %s",
              table_name, len(parquet_files), list(current_schema))
     table = schema.create_table(table_name, current_schema)
 
     log.info("Starting import of %d files to table: %s", len(parquet_files), table)
-    table.import_files(parquet_files)
+    table.import_files(parquet_files, config=config)
     log.info("Finished import of %d files to table: %s", len(parquet_files), table)
     return table
 
@@ -40,7 +41,7 @@ def create_table_from_files(
 def default_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.Schema:
     """
     This function validates a schema is contained in another schema
-    Raises an InvalidArgumentError if a certain field does not exist in the target schema
+    Raises an InvalidArgument if a certain field does not exist in the target schema
     """
     if not current_schema.names:
         return new_schema
@@ -55,17 +56,17 @@ def default_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa
 
     if not s1.issubset(s2):
         log.error("Schema mismatch. schema: %s isn't contained in schema: %s.", s1, s2)
-        raise InvalidArgumentError("Found mismatch in parquet files schemas.")
+        raise InvalidArgument("Found mismatch in parquet files schemas.")
     return result
 
 
 def strict_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.Schema:
     """
     This function validates two Schemas are identical.
-    Raises an InvalidArgumentError if schemas aren't identical.
+    Raises an InvalidArgument if schemas aren't identical.
     """
     if current_schema.names and current_schema != new_schema:
-        raise InvalidArgumentError(f"Schemas are not identical. \n {current_schema} \n vs \n {new_schema}")
+        raise InvalidArgument(f"Schemas are not identical. \n {current_schema} \n vs \n {new_schema}")
 
     return new_schema
 

vastdb-0.1.1.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: vastdb
+Version: 0.1.1
+Summary: VAST Data SDK
+Home-page: https://github.com/vast-data/vastdb_sdk
+Author: VAST DATA
+Author-email: hello@vastdata.com
+License: Copyright (C) VAST Data Ltd.
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Database :: Front-Ends
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aws-requests-auth
+Requires-Dist: boto3
+Requires-Dist: flatbuffers
+Requires-Dist: ibis-framework
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: xmltodict
+
+
+`vastdb` is a Python-based SDK designed for interacting
+with [VAST Database](https://vastdata.com/database)
+and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database),
+enabling schema and table management, efficient ingest, query and modification of columnar data.
+
+For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
+
+

{vastdb-0.0.5.3.dist-info → vastdb-0.1.1.dist-info}/RECORD

@@ -139,42 +139,37 @@ vast_flatbuf/tabular/ColumnType.py,sha256=_4-jMG08VR2zdn1ZH7F4aahYPxWsBSm7adUoVf
 vast_flatbuf/tabular/CreateProjectionRequest.py,sha256=POlK1DrYMAldNJscLIRL3j4jAT0Sv_fRzfvBXwZAAMw,2516
 vast_flatbuf/tabular/CreateSchemaRequest.py,sha256=MrOfWaFu0Q1-mxLlGV8YMPajZ5kASyvowVSrKU-NPx8,1626
 vast_flatbuf/tabular/GetProjectionTableStatsResponse.py,sha256=Bp-ln-0lcZEiUvp3vWYmnCP6t2UsZ5J-lezgkUUWhzo,3474
-vast_flatbuf/tabular/GetTableStatsResponse.py,sha256=xO9xU_fwGEQst57IMCmSiwXoLA1QkAyEZXLEOG31h4g,2732
+vast_flatbuf/tabular/GetTableStatsResponse.py,sha256=_UsKj6-VAvyDZ8Eku9fegQlRKV-T_0Dsb7qjulYoZus,4655
 vast_flatbuf/tabular/ImportDataRequest.py,sha256=f1chKp5d5NUxfNjI8YI1o4MYInF8UDhIhpWkT3vG4Do,2450
 vast_flatbuf/tabular/ListProjectionsResponse.py,sha256=secqrBsJY3ydbA28j09rmxzBqj-c1JNqaP7JMuib7nE,4240
 vast_flatbuf/tabular/ListSchemasResponse.py,sha256=V8tbwcWAC96eNwuoqDNqCSb02BnMdq60TpyISuWTVMk,3036
 vast_flatbuf/tabular/ListTablesResponse.py,sha256=V7jZAS8ryKY8s6o_QyjWzgan-rsGm17zjKEmi7K6qTM,3550
 vast_flatbuf/tabular/ObjectDetails.py,sha256=qW0WtbkCYYE_L-Kw6VNRDCLYaRm5lKvTbLNkfD4zV4A,3589
 vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI0,4450
+vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/algebra_pb2.py,sha256=Y0RBz_IszyfCTgyNA5fh-tJPq6IJs3QqhYZoyFOy2Wg,100838
-vast_protobuf/substrait/capabilities_pb2.py,sha256=NDfdXUrGPNGfB11h3QI1OrYtZypfAlu9lE17BAVimMQ,2453
-vast_protobuf/substrait/ddl_pb2.py,sha256=2MDXdDznqoD6vtMSWwvkxpiZ-yPBnSXchc8jDQyoKZw,2683
-vast_protobuf/substrait/extended_expression_pb2.py,sha256=Rs8A8HmNcEevxO3jVsNlIeYqyXCaIwGb2xTK8p7_eFU,3481
-vast_protobuf/substrait/function_pb2.py,sha256=dtVctHDJC-BsofPY0ktPYLJkAWxLAKVMsOhGER36hoo,13339
-vast_protobuf/substrait/parameterized_types_pb2.py,sha256=hCTBDXbqunSuSmrxnkvFDNUjCxr8UPE8XrghpX2mqsM,15074
-vast_protobuf/substrait/plan_pb2.py,sha256=LDQFI5QE-KpVItmqzG9k-9XuiAT-eaXuOTMUaMoeODQ,3831
-vast_protobuf/substrait/type_expressions_pb2.py,sha256=hpvSwkZjmpEfh0q6pGeCt_5ARzyOoX2HVXYXNShSh3s,17633
-vast_protobuf/substrait/type_pb2.py,sha256=w-FzIb2OJNpaOCp1f1ox_CVDevB1ID5wudju8e1NkBY,20790
-vast_protobuf/substrait/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/extensions/extensions_pb2.py,sha256=I_6c6nMmMaYvVtzF-5ycqpzFYlsAVlKQDyatoU8RewQ,6110
-vast_protobuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/tabular/rpc_pb2.py,sha256=7kW2WrA2sGk6WVbD83mc_cKkZ2MxoImSO5GOVz6NbbE,23776
-vastdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/api.py,sha256=u5Cf01LeHGN7x_pcjnzfLV-lU485FGFCv7eTIKpSaB0,124883
-vastdb/bench_scan.py,sha256=95O34oHS0UehX2ad4T2mok87CKszCFLCDZASMnZp77M,1208
-vastdb/util.py,sha256=EF892Gbs08BxHVgG3FZ6QvhpKI2-eIL5bPzzrYE_Qd8,2905
-vastdb/v2.py,sha256=gWZUnhSLEvtrXPxoTpTAwNuzU9qxrCaWKXmeNBpMrGY,12601
+vastdb/__init__.py,sha256=GY30IfZQApfl7HfcFmfTzFpx48oHgQIrDcUQCiTnxpo,206
+vastdb/bucket.py,sha256=5J8KBdRViaz5bZ8WEPggQj7DfJaIhY7CqpoWP6reaDo,2854
+vastdb/conftest.py,sha256=pKpo_46Vq4QHzTDQAFxasrVhnZ2V2L-y6IMLxojxaFM,2132
+vastdb/errors.py,sha256=wCJp70QyBW8US2KMwhB6e5ZnKRft4GiN8jyJ36f1Yuo,3315
+vastdb/internal_commands.py,sha256=rmxOjIq229gsxFFZ4nKXwVIFJcu8WR1DVsE-91w4-BY,101564
+vastdb/schema.py,sha256=x9Yn4tFTFkSpJbQqpqlecKUSOK214XsRLdOUrNW0jzM,3192
+vastdb/session.py,sha256=VZOFGZbAdr5Tl4cp88VRQYnR4Q16UNuYjSmX_QPW1II,1718
+vastdb/table.py,sha256=eALN5YpUfDFqZNF_lp6lZD5RJkBKqp5Mlc6hpwGI8Rg,20443
+vastdb/transaction.py,sha256=2I5k81YvcgDsp07BrAWkmXf41qUP6m88Y40rFfuIKvI,1796
+vastdb/util.py,sha256=VR0UJ1D0WUpqS5edG_mkxDZYZJ_qqce8y7iJOvqeyWE,2974
+vastdb/bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vastdb/bench/test_perf.py,sha256=X7BIo60L5Oj7H-56e8pDFtXY9rNLerkywKexXWiqvrY,1111
 vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/tests/conftest.py,sha256=dcYFJO0Riyn687qZTwcwKbwGieg6s4yZrVFrJAX-ylU,1461
-vastdb/tests/test_create_table_from_parquets.py,sha256=dxykmvUR-vui6Z3qUvXPYJ9Nw6V_qcxKl4NDNQK4kiY,1963
-vastdb/tests/test_sanity.py,sha256=7HmCjuOmtoYnuWiPjMP6m7sYQYop1_qRCzq2ZX0rKlc,2404
-vastdb/tests/test_schemas.py,sha256=-nntn3ltBaaqSTsUvi-i9J0yr4TYvOTRyTNY039vEIk,1047
-vastdb/tests/test_tables.py,sha256=KPe0ESVGWixecTSwQ8whzSF-NZrNVZ-Kv-C4Gz-OQnQ,1225
-vastdb-0.0.5.3.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.0.5.3.dist-info/METADATA,sha256=Yd93AoZE5ZUhJUr0MhtfhcMaQUtSFZ1wbzc6vvEvclQ,1369
-vastdb-0.0.5.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-vastdb-0.0.5.3.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.0.5.3.dist-info/RECORD,,
+vastdb/tests/test_imports.py,sha256=fDUjO5U-5i4QTIMoNnSSW4X_ZnOStLbx0mJkNq2pj9Q,5033
+vastdb/tests/test_nested.py,sha256=3kejEvtSqV0LrUgb1QglRjrlxnKI4_AXTFw2nE7Q520,951
+vastdb/tests/test_projections.py,sha256=0ZiFya6rzGvnKOrdb1xxxv-BEerNmiK_ymfZM6eIvvw,1254
+vastdb/tests/test_sanity.py,sha256=kaOmZWDGBc-XhZ8eFQ3sks2Mo9De8q41Z5pqYWzJsHM,2958
+vastdb/tests/test_schemas.py,sha256=8ZlEvnU7Fyg-TDQDxD65GAql4rU8R2_SFWVGrdv564o,1721
+vastdb/tests/test_tables.py,sha256=o_JPqr2GX1DDpPB4Zq4E1YPFgmlsiXyVe1S3TcCjF-w,26226
+vastdb/tests/util.py,sha256=_euE3fKJqgNssT9gVxlcHjdE61mnsNQcwDPzn1tTe9g,597
+vastdb-0.1.1.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-0.1.1.dist-info/METADATA,sha256=e84OEOXS09DEXniHJAU2aeK80-1h2rIZmYNBCMLa1AM,1331
+vastdb-0.1.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+vastdb-0.1.1.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-0.1.1.dist-info/RECORD,,