vastdb 0.1.10__tar.gz → 0.1.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vastdb-0.1.10 → vastdb-0.1.11}/CHANGELOG.md +12 -0
- {vastdb-0.1.10/vastdb.egg-info → vastdb-0.1.11}/PKG-INFO +1 -1
- {vastdb-0.1.10 → vastdb-0.1.11}/setup.py +1 -1
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/__init__.py +6 -3
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/_internal.py +8 -15
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/bench/test_perf.py +2 -2
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/bench/test_sample.py +32 -11
- vastdb-0.1.11/vastdb/config.py +65 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/errors.py +0 -6
- vastdb-0.1.11/vastdb/features.py +42 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/schema.py +1 -2
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/session.py +12 -45
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/table.py +1 -50
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/metrics.py +1 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_sanity.py +1 -1
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_tables.py +29 -40
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/util.py +1 -13
- vastdb-0.1.11/vastdb/vast_tests/test_scale.py +68 -0
- {vastdb-0.1.10 → vastdb-0.1.11/vastdb.egg-info}/PKG-INFO +1 -1
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb.egg-info/SOURCES.txt +4 -1
- {vastdb-0.1.10 → vastdb-0.1.11}/LICENSE +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/MANIFEST.in +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/README.md +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/requirements.txt +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/setup.cfg +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Aggregate.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySlice.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySubscript.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BinaryLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BooleanLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Bound.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Call.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CaseFragment.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Cast.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConcreteBoundImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConditionalCase.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CurrentRow.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DateLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DecimalLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Deref.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DurationLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Expression.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ExpressionImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldIndex.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldRef.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Filter.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FixedSizeBinaryLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Following.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Frame.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Grouping.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int8Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralDaysMilliseconds.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralMonths.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Join.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/JoinKind.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/KeyValue.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Limit.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ListLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralColumn.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralRelation.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapKey.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/OrderBy.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Ordering.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Plan.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Preceding.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Project.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelId.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Relation.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelationImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOpKind.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOperation.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SimpleCase.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SortKey.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Source.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StringLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructField.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimeLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimestampLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt8Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Unbounded.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/WindowCall.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Binary.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Block.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompression.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompressionMethod.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Bool.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Buffer.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/CompressionType.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Date.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/DateUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Decimal.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryBatch.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryEncoding.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryKind.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Duration.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Endianness.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Feature.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Field.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/FieldNode.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeBinary.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeList.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/FloatingPoint.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Footer.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Int.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Interval.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/IntervalUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/KeyValue.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/LargeBinary.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/LargeList.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/LargeUtf8.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/List.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Map.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Message.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/MessageHeader.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/MetadataVersion.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Null.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Precision.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/RecordBatch.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Schema.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixCompressedAxis.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensor.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndex.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCOO.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCSF.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Struct_.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Tensor.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/TensorDim.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Time.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/TimeUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Timestamp.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Type.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Union.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/UnionMode.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/Utf8.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/org/apache/arrow/flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/AlterColumnRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/AlterProjectionTableRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/AlterSchemaRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/AlterTableRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/Column.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ColumnType.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/CreateProjectionRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/CreateSchemaRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/GetProjectionTableStatsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/GetTableStatsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ImportDataRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ListProjectionsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ListSchemasResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ListTablesResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/ObjectDetails.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/S3File.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/VipRange.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vast_flatbuf/tabular/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/bench/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/bucket.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/conftest.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_duckdb.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_imports.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_nested.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_projections.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_schemas.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/test_util.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/tests/util.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/transaction.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/vast_tests/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb/vast_tests/test_ha.py +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb.egg-info/dependency_links.txt +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb.egg-info/requires.txt +0 -0
- {vastdb-0.1.10 → vastdb-0.1.11}/vastdb.egg-info/top_level.txt +0 -0
|
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
6
6
|
|
|
7
|
+
## [0.1.11] (2024-07-17)
|
|
8
|
+
[0.1.11]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.10...v0.1.11
|
|
9
|
+
|
|
10
|
+
## Fixed
|
|
11
|
+
- Revert timezone check to keep previous SDK behaviour
|
|
12
|
+
|
|
13
|
+
## Added
|
|
14
|
+
- Use Sphinx for building documentation
|
|
15
|
+
|
|
16
|
+
## Changed
|
|
17
|
+
- Defer imports to optimize initial import latency
|
|
18
|
+
|
|
7
19
|
## [0.1.10] (2024-07-09)
|
|
8
20
|
[0.1.10]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.9...v0.1.10
|
|
9
21
|
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
"""VAST Database Python SDK."""
|
|
2
2
|
|
|
3
3
|
import functools
|
|
4
|
-
import importlib.metadata
|
|
5
|
-
|
|
6
|
-
__version__ = importlib.metadata.distribution(__package__).version
|
|
7
4
|
|
|
8
5
|
from . import session
|
|
9
6
|
|
|
@@ -12,3 +9,9 @@ from . import session
|
|
|
12
9
|
@functools.wraps(session.Session)
|
|
13
10
|
def connect(*args, **kwargs): # noqa: D103
|
|
14
11
|
return session.Session(*args, **kwargs)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def version():
|
|
15
|
+
"""Return VAST DB SDK version."""
|
|
16
|
+
import importlib
|
|
17
|
+
return importlib.metadata.distribution(__package__).version
|
|
@@ -5,9 +5,8 @@ import re
|
|
|
5
5
|
import struct
|
|
6
6
|
import urllib.parse
|
|
7
7
|
from collections import defaultdict, namedtuple
|
|
8
|
-
from dataclasses import dataclass, field
|
|
9
8
|
from enum import Enum
|
|
10
|
-
from typing import Any,
|
|
9
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
|
11
10
|
|
|
12
11
|
import backoff
|
|
13
12
|
import flatbuffers
|
|
@@ -104,6 +103,7 @@ from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list
|
|
|
104
103
|
from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
|
|
105
104
|
|
|
106
105
|
from . import errors
|
|
106
|
+
from .config import BackoffConfig
|
|
107
107
|
|
|
108
108
|
UINT64_MAX = 18446744073709551615
|
|
109
109
|
|
|
@@ -742,15 +742,6 @@ def _backoff_giveup(exc: Exception) -> bool:
|
|
|
742
742
|
return True # give up in case of other exceptions
|
|
743
743
|
|
|
744
744
|
|
|
745
|
-
@dataclass
|
|
746
|
-
class BackoffConfig:
|
|
747
|
-
wait_gen: Callable = field(default=backoff.expo)
|
|
748
|
-
max_value: Optional[float] = None # max duration for a single wait period
|
|
749
|
-
max_tries: int = 10
|
|
750
|
-
max_time: float = 60.0 # in seconds
|
|
751
|
-
backoff_log_level: int = logging.DEBUG
|
|
752
|
-
|
|
753
|
-
|
|
754
745
|
class VastdbApi:
|
|
755
746
|
# we expect the vast version to be <major>.<minor>.<patch>.<protocol>
|
|
756
747
|
VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
|
|
@@ -759,15 +750,17 @@ class VastdbApi:
|
|
|
759
750
|
*,
|
|
760
751
|
auth_type=AuthType.SIGV4,
|
|
761
752
|
ssl_verify=True,
|
|
753
|
+
timeout=None,
|
|
762
754
|
backoff_config: Optional[BackoffConfig] = None):
|
|
763
755
|
|
|
764
|
-
from . import
|
|
765
|
-
self.client_sdk_version = f"VAST Database Python SDK {
|
|
756
|
+
from . import version # import lazily here (to avoid circular dependencies)
|
|
757
|
+
self.client_sdk_version = f"VAST Database Python SDK {version()} - 2024 (c)"
|
|
766
758
|
|
|
767
759
|
url = urllib3.util.parse_url(endpoint)
|
|
768
760
|
self.access_key = access_key
|
|
769
761
|
self.secret_key = secret_key
|
|
770
762
|
|
|
763
|
+
self.timeout = timeout
|
|
771
764
|
self.default_max_list_columns_page_size = 1000
|
|
772
765
|
self._session = requests.Session()
|
|
773
766
|
self._session.verify = ssl_verify
|
|
@@ -820,9 +813,9 @@ class VastdbApi:
|
|
|
820
813
|
raise NotImplementedError(msg)
|
|
821
814
|
|
|
822
815
|
def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
|
|
823
|
-
_logger.debug("Sending request: %s %s %s", method, url, kwargs)
|
|
816
|
+
_logger.debug("Sending request: %s %s %s timeout=%s", method, url, kwargs, self.timeout)
|
|
824
817
|
try:
|
|
825
|
-
res = self._session.request(method=method, url=url, **kwargs)
|
|
818
|
+
res = self._session.request(method=method, url=url, timeout=self.timeout, **kwargs)
|
|
826
819
|
except requests.exceptions.ConnectionError as err:
|
|
827
820
|
# low-level connection issue, it is safe to retry only read-only requests
|
|
828
821
|
may_retry = (method == "GET")
|
|
@@ -10,11 +10,11 @@ log = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@pytest.mark.benchmark
|
|
13
|
-
def test_bench(session,
|
|
13
|
+
def test_bench(session, bucket_name, parquets_path, crater_path):
|
|
14
14
|
files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
|
|
15
15
|
|
|
16
16
|
with session.transaction() as tx:
|
|
17
|
-
b = tx.bucket(
|
|
17
|
+
b = tx.bucket(bucket_name)
|
|
18
18
|
s = b.create_schema('s1')
|
|
19
19
|
t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
|
|
20
20
|
config = QueryConfig(num_splits=8, num_sub_splits=4)
|
|
@@ -22,7 +22,8 @@ logging.basicConfig(
|
|
|
22
22
|
|
|
23
23
|
log = logging.getLogger()
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
sdk_version = vastdb.version()
|
|
26
|
+
log.info("Python SDK version: %s", sdk_version)
|
|
26
27
|
|
|
27
28
|
NUM_COLUMNS = 10_000
|
|
28
29
|
COLUMNS_BATCH = 10
|
|
@@ -63,6 +64,8 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
63
64
|
# skip already loaded rows
|
|
64
65
|
log.info('skipping [%d..%d)', offset, limit)
|
|
65
66
|
|
|
67
|
+
pid = os.getpid()
|
|
68
|
+
tid = threading.get_native_id()
|
|
66
69
|
total_nbytes = 0
|
|
67
70
|
calls = 0
|
|
68
71
|
t0 = time.time()
|
|
@@ -89,7 +92,7 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
89
92
|
metrics_rows.append(metrics.Row(
|
|
90
93
|
start=start, finish=finish, table_path=table.path, op=op,
|
|
91
94
|
nbytes=nbytes, rows=len(chunk), cols=len(cols_batch),
|
|
92
|
-
pid=
|
|
95
|
+
pid=pid, tid=tid, sdk_version=sdk_version))
|
|
93
96
|
|
|
94
97
|
total_nbytes += nbytes
|
|
95
98
|
calls += 1
|
|
@@ -104,12 +107,12 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
104
107
|
return metrics_rows
|
|
105
108
|
|
|
106
109
|
|
|
107
|
-
def test_ingest(
|
|
110
|
+
def test_ingest(test_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
|
|
108
111
|
session = vastdb.connect(**session_kwargs)
|
|
109
112
|
metrics_table = metrics.Table(perf_metrics_db, "ingest")
|
|
110
113
|
|
|
111
114
|
with session.transaction() as tx:
|
|
112
|
-
b = tx.bucket(
|
|
115
|
+
b = tx.bucket(test_bucket_name)
|
|
113
116
|
try:
|
|
114
117
|
s = b.schema(SCHEMA)
|
|
115
118
|
except vastdb.errors.MissingSchema:
|
|
@@ -127,7 +130,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
|
|
|
127
130
|
|
|
128
131
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
129
132
|
futures = [
|
|
130
|
-
executor.submit(load_batch,
|
|
133
|
+
executor.submit(load_batch, test_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
|
|
131
134
|
for (offset, limit), url in zip(ranges, itertools.cycle(tabular_endpoint_urls))
|
|
132
135
|
]
|
|
133
136
|
log.info("spawned %d futures", len(futures))
|
|
@@ -135,7 +138,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
|
|
|
135
138
|
metrics_table.insert(future.result())
|
|
136
139
|
|
|
137
140
|
with session.transaction() as tx:
|
|
138
|
-
t = tx.bucket(
|
|
141
|
+
t = tx.bucket(test_bucket_name).schema(SCHEMA).table(TABLE)
|
|
139
142
|
count = sum(len(rb) for rb in t.select([]))
|
|
140
143
|
log.info("%s has %d rows: %s", t, count, t.stats)
|
|
141
144
|
|
|
@@ -154,6 +157,10 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
|
|
|
154
157
|
r = random.Random(i)
|
|
155
158
|
r.shuffle(row_group_indices)
|
|
156
159
|
|
|
160
|
+
pid = os.getpid()
|
|
161
|
+
tid = threading.get_native_id()
|
|
162
|
+
metrics_rows = []
|
|
163
|
+
|
|
157
164
|
session = vastdb.connect(**(session_kwargs | {"endpoint": endpoint_url}))
|
|
158
165
|
with session.transaction() as tx:
|
|
159
166
|
t = tx.bucket(bucket_name).schema(SCHEMA).table(TABLE)
|
|
@@ -174,23 +181,37 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
|
|
|
174
181
|
|
|
175
182
|
for j, pred in enumerate(preds):
|
|
176
183
|
log.info("%d) starting query #%d on %s", i, j, endpoint_url)
|
|
177
|
-
|
|
184
|
+
|
|
185
|
+
start = time.perf_counter()
|
|
178
186
|
res = t.select(columns=cols, predicate=pred, config=config)
|
|
179
187
|
rows = 0
|
|
180
188
|
data = 0
|
|
181
189
|
for rb in res:
|
|
182
190
|
rows += len(rb)
|
|
183
191
|
data += rb.nbytes
|
|
184
|
-
dt = time.
|
|
192
|
+
dt = time.perf_counter() - start
|
|
185
193
|
log.info("%d) got query #%d batch %.3f[s], %.3f[GB] %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
|
|
186
194
|
|
|
187
|
-
|
|
195
|
+
finish = time.perf_counter()
|
|
196
|
+
dt = finish - start
|
|
188
197
|
log.info("%d) finished query #%d %.3f[s], %.3f[GB], %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
|
|
189
198
|
|
|
199
|
+
metrics_rows.append(metrics.Row(
|
|
200
|
+
start=start, finish=finish, table_path=t.path, op="select",
|
|
201
|
+
nbytes=data, rows=rows, cols=len(cols),
|
|
202
|
+
pid=pid, tid=tid, sdk_version=sdk_version))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls, perf_metrics_db):
|
|
206
|
+
metrics_table = metrics.Table(perf_metrics_db, "query")
|
|
190
207
|
|
|
191
|
-
def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls):
|
|
192
208
|
log.info("starting %d workers, endpoints=%s", num_workers, tabular_endpoint_urls)
|
|
193
209
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
194
|
-
|
|
210
|
+
futures = [
|
|
195
211
|
executor.submit(run_query, session_kwargs, i, test_bucket_name, url)
|
|
212
|
+
for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls))
|
|
213
|
+
]
|
|
214
|
+
for future in as_completed(futures):
|
|
215
|
+
metrics_table.insert(future.result())
|
|
216
|
+
|
|
196
217
|
log.info("finished %d workers", num_workers)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Configuration-related dataclasses."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Callable, List, Optional
|
|
6
|
+
|
|
7
|
+
import backoff
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class BackoffConfig:
|
|
12
|
+
"""Retry configuration."""
|
|
13
|
+
|
|
14
|
+
wait_gen: Callable = field(default=backoff.expo)
|
|
15
|
+
max_value: Optional[float] = None # max duration for a single wait period
|
|
16
|
+
max_tries: int = 10
|
|
17
|
+
max_time: float = 60.0 # in seconds
|
|
18
|
+
backoff_log_level: int = logging.DEBUG
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class QueryConfig:
|
|
23
|
+
"""Query execution configiration."""
|
|
24
|
+
|
|
25
|
+
# allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
|
|
26
|
+
num_sub_splits: int = 4
|
|
27
|
+
|
|
28
|
+
# used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
|
|
29
|
+
# will be estimated from the table's row count, if not explicitly set
|
|
30
|
+
num_splits: Optional[int] = None
|
|
31
|
+
|
|
32
|
+
# each endpoint will be handled by a separate worker thread
|
|
33
|
+
# a single endpoint can be specified more than once to benefit from multithreaded execution
|
|
34
|
+
data_endpoints: Optional[List[str]] = None
|
|
35
|
+
|
|
36
|
+
# a subsplit fiber will finish after sending this number of rows back to the client
|
|
37
|
+
limit_rows_per_sub_split: int = 128 * 1024
|
|
38
|
+
|
|
39
|
+
# each fiber will read the following number of rowgroups coninuously before skipping
|
|
40
|
+
# in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
|
|
41
|
+
num_row_groups_per_sub_split: int = 8
|
|
42
|
+
|
|
43
|
+
# can be disabled for benchmarking purposes
|
|
44
|
+
use_semi_sorted_projections: bool = True
|
|
45
|
+
|
|
46
|
+
# enforce using a specific semi-sorted projection (if enabled above)
|
|
47
|
+
semi_sorted_projection_name: Optional[str] = None
|
|
48
|
+
|
|
49
|
+
# used to estimate the number of splits, given the table rows' count
|
|
50
|
+
rows_per_split: int = 4000000
|
|
51
|
+
|
|
52
|
+
# used for worker threads' naming
|
|
53
|
+
query_id: str = ""
|
|
54
|
+
|
|
55
|
+
# non-negative integer, used for server-side prioritization of queued requests:
|
|
56
|
+
# - requests with lower values will be served before requests with higher values.
|
|
57
|
+
# - if unset, the request will be added to the queue's end.
|
|
58
|
+
queue_priority: Optional[int] = None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class ImportConfig:
|
|
63
|
+
"""Import execution configiration."""
|
|
64
|
+
|
|
65
|
+
import_concurrency: int = 2
|
|
@@ -3,7 +3,6 @@ import xml.etree.ElementTree
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from enum import Enum
|
|
5
5
|
|
|
6
|
-
import pyarrow as pa
|
|
7
6
|
import requests
|
|
8
7
|
|
|
9
8
|
|
|
@@ -170,11 +169,6 @@ class NotSupportedCommand(NotSupported):
|
|
|
170
169
|
table: str
|
|
171
170
|
|
|
172
171
|
|
|
173
|
-
@dataclass
|
|
174
|
-
class NotSupportedType(NotSupported):
|
|
175
|
-
field: pa.Field
|
|
176
|
-
|
|
177
|
-
|
|
178
172
|
@dataclass
|
|
179
173
|
class NotSupportedVersion(NotSupported):
|
|
180
174
|
err_msg: str
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Version-dependent features."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from .errors import NotSupportedVersion
|
|
6
|
+
|
|
7
|
+
log = logging.getLogger()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Features:
|
|
11
|
+
"""VAST database features - check if server is already support a feature."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, vast_version):
|
|
14
|
+
"""Save the server version."""
|
|
15
|
+
self.vast_version = vast_version
|
|
16
|
+
|
|
17
|
+
self.check_imports_table = self._check(
|
|
18
|
+
"Imported objects' table feature requires 5.2+ VAST release",
|
|
19
|
+
vast_version >= (5, 2))
|
|
20
|
+
|
|
21
|
+
self.check_return_row_ids = self._check(
|
|
22
|
+
"Returning row IDs requires 5.1+ VAST release",
|
|
23
|
+
vast_version >= (5, 1))
|
|
24
|
+
|
|
25
|
+
self.check_enforce_semisorted_projection = self._check(
|
|
26
|
+
"Semi-sorted projection enforcement requires 5.1+ VAST release",
|
|
27
|
+
vast_version >= (5, 1))
|
|
28
|
+
|
|
29
|
+
self.check_external_row_ids_allocation = self._check(
|
|
30
|
+
"External row IDs allocation requires 5.1+ VAST release",
|
|
31
|
+
vast_version >= (5, 1))
|
|
32
|
+
|
|
33
|
+
def _check(self, msg, supported):
|
|
34
|
+
log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
|
|
35
|
+
if not supported:
|
|
36
|
+
def fail():
|
|
37
|
+
raise NotSupportedVersion(msg, self.vast_version)
|
|
38
|
+
return fail
|
|
39
|
+
|
|
40
|
+
def noop():
|
|
41
|
+
pass
|
|
42
|
+
return noop
|
|
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
|
|
|
10
10
|
|
|
11
11
|
import pyarrow as pa
|
|
12
12
|
|
|
13
|
-
from . import bucket, errors, schema, table
|
|
13
|
+
from . import bucket, errors, schema, table
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from .table import Table
|
|
@@ -86,7 +86,6 @@ class Schema:
|
|
|
86
86
|
if use_external_row_ids_allocation:
|
|
87
87
|
self.tx._rpc.features.check_external_row_ids_allocation()
|
|
88
88
|
|
|
89
|
-
util.check_supported_types(columns)
|
|
90
89
|
self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
|
|
91
90
|
use_external_row_ids_allocation=use_external_row_ids_allocation)
|
|
92
91
|
log.info("Created table: %s", table_name)
|
|
@@ -7,51 +7,11 @@ For more details see:
|
|
|
7
7
|
- [Tabular identity policy with the proper permissions](https://support.vastdata.com/s/article/UUID-14322b60-d6a2-89ac-3df0-3dfbb6974182)
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
import logging
|
|
11
10
|
import os
|
|
12
|
-
from typing import Optional
|
|
11
|
+
from typing import TYPE_CHECKING, Optional
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
from . import _internal, errors, transaction
|
|
17
|
-
from ._internal import BackoffConfig
|
|
18
|
-
|
|
19
|
-
log = logging.getLogger()
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class Features:
|
|
23
|
-
"""VAST database features - check if server is already support a feature."""
|
|
24
|
-
|
|
25
|
-
def __init__(self, vast_version):
|
|
26
|
-
"""Save the server version."""
|
|
27
|
-
self.vast_version = vast_version
|
|
28
|
-
|
|
29
|
-
self.check_imports_table = self._check(
|
|
30
|
-
"Imported objects' table feature requires 5.2+ VAST release",
|
|
31
|
-
vast_version >= (5, 2))
|
|
32
|
-
|
|
33
|
-
self.check_return_row_ids = self._check(
|
|
34
|
-
"Returning row IDs requires 5.1+ VAST release",
|
|
35
|
-
vast_version >= (5, 1))
|
|
36
|
-
|
|
37
|
-
self.check_enforce_semisorted_projection = self._check(
|
|
38
|
-
"Semi-sorted projection enforcement requires 5.1+ VAST release",
|
|
39
|
-
vast_version >= (5, 1))
|
|
40
|
-
|
|
41
|
-
self.check_external_row_ids_allocation = self._check(
|
|
42
|
-
"External row IDs allocation requires 5.1+ VAST release",
|
|
43
|
-
vast_version >= (5, 1))
|
|
44
|
-
|
|
45
|
-
def _check(self, msg, supported):
|
|
46
|
-
log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
|
|
47
|
-
if not supported:
|
|
48
|
-
def fail():
|
|
49
|
-
raise errors.NotSupportedVersion(msg, self.vast_version)
|
|
50
|
-
return fail
|
|
51
|
-
|
|
52
|
-
def noop():
|
|
53
|
-
pass
|
|
54
|
-
return noop
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from .config import BackoffConfig
|
|
55
15
|
|
|
56
16
|
|
|
57
17
|
class Session:
|
|
@@ -60,8 +20,13 @@ class Session:
|
|
|
60
20
|
def __init__(self, access=None, secret=None, endpoint=None,
|
|
61
21
|
*,
|
|
62
22
|
ssl_verify=True,
|
|
63
|
-
|
|
23
|
+
timeout=None,
|
|
24
|
+
backoff_config: Optional["BackoffConfig"] = None):
|
|
64
25
|
"""Connect to a VAST Database endpoint, using specified credentials."""
|
|
26
|
+
import boto3
|
|
27
|
+
|
|
28
|
+
from . import _internal, features
|
|
29
|
+
|
|
65
30
|
if access is None:
|
|
66
31
|
access = os.environ['AWS_ACCESS_KEY_ID']
|
|
67
32
|
if secret is None:
|
|
@@ -74,8 +39,9 @@ class Session:
|
|
|
74
39
|
access_key=access,
|
|
75
40
|
secret_key=secret,
|
|
76
41
|
ssl_verify=ssl_verify,
|
|
42
|
+
timeout=timeout,
|
|
77
43
|
backoff_config=backoff_config)
|
|
78
|
-
self.features = Features(self.api.vast_version)
|
|
44
|
+
self.features = features.Features(self.api.vast_version)
|
|
79
45
|
self.s3 = boto3.client('s3',
|
|
80
46
|
aws_access_key_id=access,
|
|
81
47
|
aws_secret_access_key=secret,
|
|
@@ -93,4 +59,5 @@ class Session:
|
|
|
93
59
|
with session.transaction() as tx:
|
|
94
60
|
tx.bucket("bucket").create_schema("schema")
|
|
95
61
|
"""
|
|
62
|
+
from . import transaction
|
|
96
63
|
return transaction.Transaction(self)
|
|
@@ -14,6 +14,7 @@ import pyarrow as pa
|
|
|
14
14
|
import urllib3
|
|
15
15
|
|
|
16
16
|
from . import _internal, errors, schema, util
|
|
17
|
+
from .config import ImportConfig, QueryConfig
|
|
17
18
|
|
|
18
19
|
log = logging.getLogger(__name__)
|
|
19
20
|
|
|
@@ -39,53 +40,6 @@ class TableStats:
|
|
|
39
40
|
endpoints: Tuple[str, ...] = ()
|
|
40
41
|
|
|
41
42
|
|
|
42
|
-
@dataclass
|
|
43
|
-
class QueryConfig:
|
|
44
|
-
"""Query execution configiration."""
|
|
45
|
-
|
|
46
|
-
# allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
|
|
47
|
-
num_sub_splits: int = 4
|
|
48
|
-
|
|
49
|
-
# used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
|
|
50
|
-
# will be estimated from the table's row count, if not explicitly set
|
|
51
|
-
num_splits: Optional[int] = None
|
|
52
|
-
|
|
53
|
-
# each endpoint will be handled by a separate worker thread
|
|
54
|
-
# a single endpoint can be specified more than once to benefit from multithreaded execution
|
|
55
|
-
data_endpoints: Optional[List[str]] = None
|
|
56
|
-
|
|
57
|
-
# a subsplit fiber will finish after sending this number of rows back to the client
|
|
58
|
-
limit_rows_per_sub_split: int = 128 * 1024
|
|
59
|
-
|
|
60
|
-
# each fiber will read the following number of rowgroups coninuously before skipping
|
|
61
|
-
# in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
|
|
62
|
-
num_row_groups_per_sub_split: int = 8
|
|
63
|
-
|
|
64
|
-
# can be disabled for benchmarking purposes
|
|
65
|
-
use_semi_sorted_projections: bool = True
|
|
66
|
-
|
|
67
|
-
# enforce using a specific semi-sorted projection (if enabled above)
|
|
68
|
-
semi_sorted_projection_name: Optional[str] = None
|
|
69
|
-
|
|
70
|
-
# used to estimate the number of splits, given the table rows' count
|
|
71
|
-
rows_per_split: int = 4000000
|
|
72
|
-
|
|
73
|
-
# used for worker threads' naming
|
|
74
|
-
query_id: str = ""
|
|
75
|
-
|
|
76
|
-
# non-negative integer, used for server-side prioritization of queued requests:
|
|
77
|
-
# - requests with lower values will be served before requests with higher values.
|
|
78
|
-
# - if unset, the request will be added to the queue's end.
|
|
79
|
-
queue_priority: Optional[int] = None
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
@dataclass
|
|
83
|
-
class ImportConfig:
|
|
84
|
-
"""Import execution configiration."""
|
|
85
|
-
|
|
86
|
-
import_concurrency: int = 2
|
|
87
|
-
|
|
88
|
-
|
|
89
43
|
class SelectSplitState:
|
|
90
44
|
"""State of a specific query split execution."""
|
|
91
45
|
|
|
@@ -491,7 +445,6 @@ class Table:
|
|
|
491
445
|
raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
|
|
492
446
|
try:
|
|
493
447
|
row_ids = []
|
|
494
|
-
util.check_supported_types(rows.schema)
|
|
495
448
|
serialized_slices = util.iter_serialized_slices(rows, MAX_INSERT_ROWS_PER_PATCH)
|
|
496
449
|
for slice in serialized_slices:
|
|
497
450
|
res = self.tx._rpc.api.insert_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
|
|
@@ -534,7 +487,6 @@ class Table:
|
|
|
534
487
|
|
|
535
488
|
update_rows_rb = util.sort_record_batch_if_needed(update_rows_rb, INTERNAL_ROW_ID)
|
|
536
489
|
|
|
537
|
-
util.check_supported_types(update_rows_rb.schema)
|
|
538
490
|
serialized_slices = util.iter_serialized_slices(update_rows_rb, MAX_ROWS_PER_BATCH)
|
|
539
491
|
for slice in serialized_slices:
|
|
540
492
|
self.tx._rpc.api.update_rows(self.bucket.name, self.schema.name, self.name, record_batch=slice,
|
|
@@ -579,7 +531,6 @@ class Table:
|
|
|
579
531
|
"""Add a new column."""
|
|
580
532
|
if self._imports_table:
|
|
581
533
|
raise errors.NotSupportedCommand(self.bucket.name, self.schema.name, self.name)
|
|
582
|
-
util.check_supported_types(new_column)
|
|
583
534
|
self.tx._rpc.api.add_columns(self.bucket.name, self.schema.name, self.name, new_column, txid=self.tx.txid)
|
|
584
535
|
log.info("Added column(s): %s", new_column)
|
|
585
536
|
self.arrow_schema = self.columns()
|
|
@@ -24,7 +24,7 @@ def test_bad_credentials(session):
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def test_bad_endpoint(session):
|
|
27
|
-
backoff_config = vastdb.
|
|
27
|
+
backoff_config = vastdb.config.BackoffConfig(max_tries=3)
|
|
28
28
|
with pytest.raises(vastdb.errors.ConnectionError):
|
|
29
29
|
vastdb.connect(access='BAD', secret='BAD', endpoint='http://invalid-host-name-for-tests:12345', backoff_config=backoff_config)
|
|
30
30
|
|
|
@@ -227,6 +227,35 @@ def test_select_with_priority(session, clean_bucket_name):
|
|
|
227
227
|
t.select(config=config).read_all()
|
|
228
228
|
|
|
229
229
|
|
|
230
|
+
def test_timezones(session, clean_bucket_name):
|
|
231
|
+
columns_with_tz = pa.schema([
|
|
232
|
+
('ts0', pa.timestamp('s', tz='+00:00')),
|
|
233
|
+
('ts3', pa.timestamp('ms', tz='UTC')),
|
|
234
|
+
('ts6', pa.timestamp('us', tz='GMT')),
|
|
235
|
+
('ts9', pa.timestamp('ns', tz='Universal')),
|
|
236
|
+
])
|
|
237
|
+
|
|
238
|
+
# currently timezone information is not stored
|
|
239
|
+
columns_without_tz = pa.schema([
|
|
240
|
+
('ts0', pa.timestamp('s')),
|
|
241
|
+
('ts3', pa.timestamp('ms')),
|
|
242
|
+
('ts6', pa.timestamp('us')),
|
|
243
|
+
('ts9', pa.timestamp('ns')),
|
|
244
|
+
])
|
|
245
|
+
|
|
246
|
+
data = [
|
|
247
|
+
[dt.datetime(2024, 4, 10, 12, 34, 56), dt.datetime(2025, 4, 10, 12, 34, 56), dt.datetime(2026, 4, 10, 12, 34, 56)],
|
|
248
|
+
[dt.datetime(2024, 4, 10, 12, 34, 56, 789000), dt.datetime(2025, 4, 10, 12, 34, 56, 789000), dt.datetime(2026, 4, 10, 12, 34, 56, 789000)],
|
|
249
|
+
[dt.datetime(2024, 4, 10, 12, 34, 56, 789789), dt.datetime(2025, 4, 10, 12, 34, 56, 789789), dt.datetime(2026, 4, 10, 12, 34, 56, 789789)],
|
|
250
|
+
[dt.datetime(2024, 4, 10, 12, 34, 56, 789789), dt.datetime(2025, 4, 10, 12, 34, 56, 789789), dt.datetime(2026, 4, 10, 12, 34, 56, 789789)],
|
|
251
|
+
]
|
|
252
|
+
|
|
253
|
+
inserted = pa.table(schema=columns_with_tz, data=data)
|
|
254
|
+
with prepare_data(session, clean_bucket_name, 's', 't', inserted) as table:
|
|
255
|
+
assert table.arrow_schema == columns_without_tz
|
|
256
|
+
assert table.select().read_all() == pa.table(schema=columns_without_tz, data=data)
|
|
257
|
+
|
|
258
|
+
|
|
230
259
|
def test_types(session, clean_bucket_name):
|
|
231
260
|
columns = pa.schema([
|
|
232
261
|
('tb', pa.bool_()),
|
|
@@ -311,46 +340,6 @@ def test_types(session, clean_bucket_name):
|
|
|
311
340
|
assert select(t['ts9'] == ts_literal) == expected.filter(pc.field('ts9') == ts_literal)
|
|
312
341
|
|
|
313
342
|
|
|
314
|
-
TIMESTAMP_UNITS = ['s', 'ms', 'us', 'ns']
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
def test_unsupported_timezone(session, clean_bucket_name):
|
|
318
|
-
with session.transaction() as tx:
|
|
319
|
-
s = tx.bucket(clean_bucket_name).create_schema('s1')
|
|
320
|
-
for unit in TIMESTAMP_UNITS:
|
|
321
|
-
col_type = pa.timestamp(unit, 'UTC')
|
|
322
|
-
with pytest.raises(errors.NotSupportedType):
|
|
323
|
-
s.create_table('t1', pa.schema([('ts', col_type)]))
|
|
324
|
-
assert s.tables() == []
|
|
325
|
-
|
|
326
|
-
cols = [('c', pa.int64())]
|
|
327
|
-
t1 = s.create_table('t1', pa.schema(cols))
|
|
328
|
-
for unit in TIMESTAMP_UNITS:
|
|
329
|
-
col_type = pa.timestamp(unit, 'UTC')
|
|
330
|
-
with pytest.raises(errors.NotSupportedType):
|
|
331
|
-
t1.add_column(pa.schema([('ts', col_type)]))
|
|
332
|
-
|
|
333
|
-
cols = [(f'c_{unit}', pa.timestamp(unit)) for unit in TIMESTAMP_UNITS]
|
|
334
|
-
t2 = s.create_table('t2', pa.schema(cols))
|
|
335
|
-
|
|
336
|
-
for unit in TIMESTAMP_UNITS:
|
|
337
|
-
col_type = pa.timestamp(unit, 'UTC')
|
|
338
|
-
|
|
339
|
-
rb = pa.record_batch(
|
|
340
|
-
data=[[None]],
|
|
341
|
-
schema=pa.schema([(f'c_{unit}', col_type)]))
|
|
342
|
-
with pytest.raises(errors.NotSupportedType):
|
|
343
|
-
t2.insert(rb)
|
|
344
|
-
|
|
345
|
-
rb = pa.record_batch(
|
|
346
|
-
data=[[0], [None]],
|
|
347
|
-
schema=pa.schema([
|
|
348
|
-
(INTERNAL_ROW_ID, pa.uint64()),
|
|
349
|
-
(f'c_{unit}', col_type)]))
|
|
350
|
-
with pytest.raises(errors.NotSupportedType):
|
|
351
|
-
t2.update(rb)
|
|
352
|
-
|
|
353
|
-
|
|
354
343
|
def test_filters(session, clean_bucket_name):
|
|
355
344
|
columns = pa.schema([
|
|
356
345
|
('a', pa.int32()),
|