vastdb 0.0.5.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

vastdb/util.py

@@ -0,0 +1,77 @@
+import logging
+from typing import Callable
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from .errors import InvalidArgument
+from .schema import Schema
+from .table import Table
+
+log = logging.getLogger(__name__)
+
+
+def create_table_from_files(
+        schema: Schema, table_name: str, parquet_files: [str], schema_merge_func: Callable = None) -> Table:
+    if not schema_merge_func:
+        schema_merge_func = default_schema_merge
+    else:
+        assert schema_merge_func in [default_schema_merge, strict_schema_merge, union_schema_merge]
+    tx = schema.tx
+    current_schema = pa.schema([])
+    s3fs = pa.fs.S3FileSystem(
+        access_key=tx._rpc.api.access_key, secret_key=tx._rpc.api.secret_key, endpoint_override=tx._rpc.api.url)
+    for prq_file in parquet_files:
+        if not prq_file.startswith('/'):
+            raise InvalidArgument(f"Path {prq_file} must start with a '/'")
+        parquet_ds = pq.ParquetDataset(prq_file.lstrip('/'), filesystem=s3fs)
+        current_schema = schema_merge_func(current_schema, parquet_ds.schema)
+
+    log.info("Creating table %s from %d Parquet files, with columns: %s",
+             table_name, len(parquet_files), list(current_schema))
+    table = schema.create_table(table_name, current_schema)
+
+    log.info("Starting import of %d files to table: %s", len(parquet_files), table)
+    table.import_files(parquet_files)
+    log.info("Finished import of %d files to table: %s", len(parquet_files), table)
+    return table
+
+
+def default_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.Schema:
+    """
+    This function validates a schema is contained in another schema
+    Raises an InvalidArgument if a certain field does not exist in the target schema
+    """
+    if not current_schema.names:
+        return new_schema
+    s1 = set(current_schema)
+    s2 = set(new_schema)
+
+    if len(s1) > len(s2):
+        s1, s2 = s2, s1
+        result = current_schema  # We need this variable in order to preserve the original fields order
+    else:
+        result = new_schema
+
+    if not s1.issubset(s2):
+        log.error("Schema mismatch. schema: %s isn't contained in schema: %s.", s1, s2)
+        raise InvalidArgument("Found mismatch in parquet files schemas.")
+    return result
+
+
+def strict_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.Schema:
+    """
+    This function validates two Schemas are identical.
+    Raises an InvalidArgument if schemas aren't identical.
+    """
+    if current_schema.names and current_schema != new_schema:
+        raise InvalidArgument(f"Schemas are not identical. \n {current_schema} \n vs \n {new_schema}")
+
+    return new_schema
+
+
+def union_schema_merge(current_schema: pa.Schema, new_schema: pa.Schema) -> pa.Schema:
+    """
+    This function returns a unified schema from potentially two different schemas.
+    """
+    return pa.unify_schemas([current_schema, new_schema])

vastdb-0.1.0.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: vastdb
+Version: 0.1.0
+Summary: VAST Data SDK
+Home-page: https://github.com/vast-data/vastdb_sdk
+Author: VAST DATA
+Author-email: hello@vastdata.com
+License: Copyright (C) VAST Data Ltd.
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Database :: Front-Ends
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aws-requests-auth
+Requires-Dist: boto3
+Requires-Dist: flatbuffers
+Requires-Dist: ibis-framework
+Requires-Dist: pyarrow
+Requires-Dist: requests
+Requires-Dist: xmltodict
+
+
+`vastdb` is a Python-based SDK designed for interacting
+with [VAST Database](https://vastdata.com/database)
+and [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database),
+enabling schema and table management, efficient ingest, query and modification of columnar data.
+
+For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
+
+

{vastdb-0.0.5.2.dist-info → vastdb-0.1.0.dist-info}/RECORD

@@ -139,34 +139,33 @@ vast_flatbuf/tabular/ColumnType.py,sha256=_4-jMG08VR2zdn1ZH7F4aahYPxWsBSm7adUoVf
 vast_flatbuf/tabular/CreateProjectionRequest.py,sha256=POlK1DrYMAldNJscLIRL3j4jAT0Sv_fRzfvBXwZAAMw,2516
 vast_flatbuf/tabular/CreateSchemaRequest.py,sha256=MrOfWaFu0Q1-mxLlGV8YMPajZ5kASyvowVSrKU-NPx8,1626
 vast_flatbuf/tabular/GetProjectionTableStatsResponse.py,sha256=Bp-ln-0lcZEiUvp3vWYmnCP6t2UsZ5J-lezgkUUWhzo,3474
-vast_flatbuf/tabular/GetTableStatsResponse.py,sha256=xO9xU_fwGEQst57IMCmSiwXoLA1QkAyEZXLEOG31h4g,2732
+vast_flatbuf/tabular/GetTableStatsResponse.py,sha256=_UsKj6-VAvyDZ8Eku9fegQlRKV-T_0Dsb7qjulYoZus,4655
 vast_flatbuf/tabular/ImportDataRequest.py,sha256=f1chKp5d5NUxfNjI8YI1o4MYInF8UDhIhpWkT3vG4Do,2450
 vast_flatbuf/tabular/ListProjectionsResponse.py,sha256=secqrBsJY3ydbA28j09rmxzBqj-c1JNqaP7JMuib7nE,4240
 vast_flatbuf/tabular/ListSchemasResponse.py,sha256=V8tbwcWAC96eNwuoqDNqCSb02BnMdq60TpyISuWTVMk,3036
 vast_flatbuf/tabular/ListTablesResponse.py,sha256=V7jZAS8ryKY8s6o_QyjWzgan-rsGm17zjKEmi7K6qTM,3550
 vast_flatbuf/tabular/ObjectDetails.py,sha256=qW0WtbkCYYE_L-Kw6VNRDCLYaRm5lKvTbLNkfD4zV4A,3589
 vast_flatbuf/tabular/S3File.py,sha256=KC9c2oS5-JXwTTriUVFdjOvRG0B54Cq9kviSDZY3NI0,4450
+vast_flatbuf/tabular/VipRange.py,sha256=_BJd1RRZAcK76T9vlsHzXKYVsPVaz6WTEAqStMQCAUQ,2069
 vast_flatbuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/algebra_pb2.py,sha256=Y0RBz_IszyfCTgyNA5fh-tJPq6IJs3QqhYZoyFOy2Wg,100838
-vast_protobuf/substrait/capabilities_pb2.py,sha256=NDfdXUrGPNGfB11h3QI1OrYtZypfAlu9lE17BAVimMQ,2453
-vast_protobuf/substrait/ddl_pb2.py,sha256=2MDXdDznqoD6vtMSWwvkxpiZ-yPBnSXchc8jDQyoKZw,2683
-vast_protobuf/substrait/extended_expression_pb2.py,sha256=Rs8A8HmNcEevxO3jVsNlIeYqyXCaIwGb2xTK8p7_eFU,3481
-vast_protobuf/substrait/function_pb2.py,sha256=dtVctHDJC-BsofPY0ktPYLJkAWxLAKVMsOhGER36hoo,13339
-vast_protobuf/substrait/parameterized_types_pb2.py,sha256=hCTBDXbqunSuSmrxnkvFDNUjCxr8UPE8XrghpX2mqsM,15074
-vast_protobuf/substrait/plan_pb2.py,sha256=LDQFI5QE-KpVItmqzG9k-9XuiAT-eaXuOTMUaMoeODQ,3831
-vast_protobuf/substrait/type_expressions_pb2.py,sha256=hpvSwkZjmpEfh0q6pGeCt_5ARzyOoX2HVXYXNShSh3s,17633
-vast_protobuf/substrait/type_pb2.py,sha256=w-FzIb2OJNpaOCp1f1ox_CVDevB1ID5wudju8e1NkBY,20790
-vast_protobuf/substrait/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/substrait/extensions/extensions_pb2.py,sha256=I_6c6nMmMaYvVtzF-5ycqpzFYlsAVlKQDyatoU8RewQ,6110
-vast_protobuf/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vast_protobuf/tabular/rpc_pb2.py,sha256=7kW2WrA2sGk6WVbD83mc_cKkZ2MxoImSO5GOVz6NbbE,23776
-vastdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vastdb/api.py,sha256=ZfkwDyTqaQ_j1wmp2iog9Yj3LW98S8IzLxs7gExiPG8,124746
-vastdb/v2.py,sha256=0fLulaIQGlIbVNBBFGd6iwYPuGhaaJIHTiJORyio_YQ,2438
-vastdb-0.0.5.2.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
-vastdb-0.0.5.2.dist-info/METADATA,sha256=t6yxMkCPHuy7GJU6lSmd6QjrCsGmcPvzJE0qihLjMQQ,1369
-vastdb-0.0.5.2.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-vastdb-0.0.5.2.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
-vastdb-0.0.5.2.dist-info/RECORD,,
+vastdb/__init__.py,sha256=GY30IfZQApfl7HfcFmfTzFpx48oHgQIrDcUQCiTnxpo,206
+vastdb/bucket.py,sha256=Xbgn5Ns7veBL4oKH7EaSj4SxTPDRlicl9Saaz_39ZsU,2526
+vastdb/errors.py,sha256=mveQ2O0fLKOS51V9k5Y-HwY8Y1XiYdE9aJ9j0wlltWQ,3119
+vastdb/internal_commands.py,sha256=FR4rkr-sRvqMc-Y5hW7APOPa41a8d7L8DNJ2ROHRaFM,104441
+vastdb/schema.py,sha256=TbgqaUqAraj43vKCyVQNUSOMuJdw2Y4M06svs5jpcPo,2836
+vastdb/session.py,sha256=qgjT9rt1yUr4GyHOZRsVGFj3HYBoHFAEdczi_R26O8o,1731
+vastdb/table.py,sha256=p0uE0Gw9hen7hLTx9xC_MbxTaG6ZhZEFieaenUnbyUY,20442
+vastdb/transaction.py,sha256=jleiVmg4iui2q1GqMFsPo5GZRNtIa5NzGLFVooBbzkQ,1797
+vastdb/util.py,sha256=pBw4ywNJfkvKik-T7ZKPrWBoZOqGns-WsSZkG7HHa2I,2908
+vastdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vastdb/tests/conftest.py,sha256=VLqImQ1XMr1FLCLCqZpgv8wMjNskfYAtMp-qjZFBqWo,1694
+vastdb/tests/test_imports.py,sha256=OhkbuhTeLgD4I6Vbub-B7wQo-G37TlXoHVQhGCLz9Wo,5035
+vastdb/tests/test_projections.py,sha256=PRi1Jf__95fsL9ZCQ_s2PtszWIO5FIFbniiL6FnV18M,1253
+vastdb/tests/test_sanity.py,sha256=gijOWK4ymGhVRHkf0ecHibVlaJxl92RinPdFUwWj1OQ,2959
+vastdb/tests/test_schemas.py,sha256=YX0lF8FbXzNCNVUAxq3g0L0OCjGq1OwkQaNaBtzDe4Q,1253
+vastdb/tests/test_tables.py,sha256=qnDRBtOPh9qcV4O3kB6xF4WJFj3B3WX2RAmUzkzl05g,23634
+vastdb-0.1.0.dist-info/LICENSE,sha256=obffan7LYrq7hLHNrY7vHcn2pKUTBUYXMKu-VOAvDxU,11333
+vastdb-0.1.0.dist-info/METADATA,sha256=pCY34hVZGzoB51SyIgrzrtxDaOXC_2DGOQbafZdOmQg,1331
+vastdb-0.1.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+vastdb-0.1.0.dist-info/top_level.txt,sha256=Vsj2MKtlhPg0J4so64slQtnwjhgoPmJgcG-6YcVAwVc,20
+vastdb-0.1.0.dist-info/RECORD,,