vastdb 0.1.10__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vastdb-0.1.10 → vastdb-1.0.0}/CHANGELOG.md +19 -0
- vastdb-1.0.0/CONTRIBUTING.md +33 -0
- {vastdb-0.1.10/vastdb.egg-info → vastdb-1.0.0}/PKG-INFO +2 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/README.md +0 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/setup.py +2 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/__init__.py +6 -3
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/_internal.py +22 -22
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/test_perf.py +2 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/test_sample.py +32 -11
- vastdb-1.0.0/vastdb/config.py +65 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/errors.py +0 -6
- vastdb-1.0.0/vastdb/features.py +42 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/schema.py +1 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/session.py +12 -45
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/table.py +3 -52
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/metrics.py +1 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_sanity.py +1 -1
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_tables.py +29 -40
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/util.py +1 -13
- vastdb-1.0.0/vastdb/vast_tests/test_scale.py +68 -0
- {vastdb-0.1.10 → vastdb-1.0.0/vastdb.egg-info}/PKG-INFO +2 -2
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/SOURCES.txt +5 -1
- {vastdb-0.1.10 → vastdb-1.0.0}/LICENSE +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/MANIFEST.in +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/requirements.txt +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/setup.cfg +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Aggregate.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySlice.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySubscript.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BinaryLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BooleanLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Bound.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Call.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CaseFragment.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Cast.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConcreteBoundImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConditionalCase.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CurrentRow.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DateLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DecimalLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Deref.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DurationLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Expression.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ExpressionImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldIndex.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldRef.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Filter.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FixedSizeBinaryLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Following.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Frame.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Grouping.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int8Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralDaysMilliseconds.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralMonths.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Join.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/JoinKind.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/KeyValue.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Limit.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ListLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralColumn.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralRelation.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapKey.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/OrderBy.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Ordering.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Plan.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Preceding.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Project.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelId.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Relation.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelationImpl.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOpKind.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOperation.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SimpleCase.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SortKey.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Source.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StringLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructField.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimeLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimestampLiteral.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt16Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt32Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt64Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt8Literal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Unbounded.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/WindowCall.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Binary.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Block.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompression.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompressionMethod.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Bool.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Buffer.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/CompressionType.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Date.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DateUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Decimal.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryBatch.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryEncoding.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryKind.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Duration.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Endianness.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Feature.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Field.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FieldNode.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeBinary.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeList.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FloatingPoint.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Footer.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Int.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Interval.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/IntervalUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/KeyValue.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeBinary.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeList.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeUtf8.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/List.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Map.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Message.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/MessageHeader.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/MetadataVersion.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Null.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Precision.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/RecordBatch.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Schema.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixCompressedAxis.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensor.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndex.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCOO.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCSF.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Struct_.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Tensor.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/TensorDim.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Time.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/TimeUnit.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Timestamp.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Type.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Union.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/UnionMode.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Utf8.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterColumnRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterProjectionTableRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterSchemaRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterTableRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/Column.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ColumnType.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/CreateProjectionRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/CreateSchemaRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/GetProjectionTableStatsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/GetTableStatsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ImportDataRequest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListProjectionsResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListSchemasResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListTablesResponse.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ObjectDetails.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/S3File.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/VipRange.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bucket.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/conftest.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_duckdb.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_imports.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_nested.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_projections.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_schemas.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_util.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/util.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/transaction.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/vast_tests/__init__.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/vast_tests/test_ha.py +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/dependency_links.txt +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/requires.txt +0 -0
- {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/top_level.txt +0 -0
|
@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
6
6
|
|
|
7
|
+
## [1.0.0] (2024-07-21)
|
|
8
|
+
[1.0.0]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.11...v1.0.0
|
|
9
|
+
|
|
10
|
+
## Added
|
|
11
|
+
- Add CLA (`CONTRIBUTING.md`)
|
|
12
|
+
- Support setting HTTP timeouts
|
|
13
|
+
|
|
14
|
+
## [0.1.11] (2024-07-17)
|
|
15
|
+
[0.1.11]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.10...v0.1.11
|
|
16
|
+
|
|
17
|
+
## Fixed
|
|
18
|
+
- Revert timezone check to keep previous SDK behaviour
|
|
19
|
+
|
|
20
|
+
## Added
|
|
21
|
+
- Use Sphinx for building documentation
|
|
22
|
+
|
|
23
|
+
## Changed
|
|
24
|
+
- Defer imports to optimize initial import latency
|
|
25
|
+
|
|
7
26
|
## [0.1.10] (2024-07-09)
|
|
8
27
|
[0.1.10]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.9...v0.1.10
|
|
9
28
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# VAST DB SDK Contribution License Agreement
|
|
2
|
+
|
|
3
|
+
Thank you for your interest in VAST DB SDK project (“Project”). You hereby accept and agree to the following terms and conditions (“Agreement”) for Your present and future Contributions (as defined below) submitted to the Project or as part of. Except for the license granted herein to the Project originator – Vast Data Ltd. together with any of its affiliates, assignee, or successor (“Vast Data” or “us” or “we”), You reserve all right, title, and interest in and to Your Contributions.
|
|
4
|
+
|
|
5
|
+
## Definitions.
|
|
6
|
+
|
|
7
|
+
"You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Vast Data. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single contributor. For the purposes of this definition, "control" means (i) the power, directly or indirectly, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
|
|
8
|
+
|
|
9
|
+
"Contribution" shall mean the code, documentation, or any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to the Project (or to Vast Data), inclusion in, or documentation of, the Project (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Project or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Project for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
|
|
10
|
+
|
|
11
|
+
## Licenses
|
|
12
|
+
|
|
13
|
+
Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Vast Data and at Vast Data’ discretion, its affiliates, a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, sell, and distribute Your Contributions and such derivative works.
|
|
14
|
+
|
|
15
|
+
Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Vast Data and, at Vast Data’ discretion, its affiliates a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by the combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes a direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
|
|
16
|
+
|
|
17
|
+
## Representations
|
|
18
|
+
|
|
19
|
+
You represent that You are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes Your Contributions, You represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Vast Data, or that your employer has executed a separate written agreement with Vast Data.
|
|
20
|
+
|
|
21
|
+
You represent that each of Your Contributions is Your original creation (see below for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
|
|
22
|
+
|
|
23
|
+
You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all.
|
|
24
|
+
|
|
25
|
+
UNLESS REQUIRED BY APPLICABLE LAW, AGREED TO IN WRITING, OR AS STATED IN THIS CLA, YOU PROVIDE YOUR CONTRIBUTION ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
|
26
|
+
|
|
27
|
+
Should You wish to submit work that is not Your original creation, You may submit it to Vast Data separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]". Vast Data shall have the right to determine at its discretion whether or not to use such Contribution.
|
|
28
|
+
|
|
29
|
+
Code of Conduct
|
|
30
|
+
|
|
31
|
+
In Your Contribution and/or Work or otherwise in your participation in the Project, you agree to: (i) not use or include any defamatory, illegal, derogatory, abusive, harassing, destructive, malicious, or otherwise, illegitimate content, code or language; or (ii) publish others’ private or personal information without their explicit permission (“Code of Conduct”) Vast Data’ will be entitled but not obligated to enforce the Code of Conduct and may take appropriate and fair action in response to any behaviour that is reasonably deemed by us to be inappropriate, threatening, offensive, or harmful. Further Vast Data will have the right but not the obligation to remove, edit, or reject any comments, code, issues, and other Contributions or Works submitted hereunder including without limitation by reason of nonalignment with the Code of Conduct.
|
|
32
|
+
|
|
33
|
+
You agree to notify Vast Data of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: vastdb
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: VAST Data SDK
|
|
5
5
|
Home-page: https://github.com/vast-data/vastdb_sdk
|
|
6
6
|
Author: VAST DATA
|
|
7
7
|
Author-email: hello@vastdata.com
|
|
8
8
|
License: Copyright (C) VAST Data Ltd.
|
|
9
|
-
Classifier: Development Status ::
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
10
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
# VAST DB Python SDK
|
|
4
4
|
|
|
5
|
-
🚧 Please note that this package in a pre-release stage. Until version 1.x is officially released, the API should be considered unstable.
|
|
6
|
-
|
|
7
5
|
## Introduction
|
|
8
6
|
|
|
9
7
|
`vastdb` is a Python-based SDK designed for interacting with [VAST Database](https://vastdata.com/database) & [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database), enabling schema and table management, efficient ingest, query and modification of columnar data. For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
|
|
@@ -29,7 +29,7 @@ setup(
|
|
|
29
29
|
name='vastdb',
|
|
30
30
|
python_requires='>=3.9.0',
|
|
31
31
|
description='VAST Data SDK',
|
|
32
|
-
version='0.
|
|
32
|
+
version='1.0.0' + suffix,
|
|
33
33
|
url='https://github.com/vast-data/vastdb_sdk',
|
|
34
34
|
author='VAST DATA',
|
|
35
35
|
author_email='hello@vastdata.com',
|
|
@@ -39,7 +39,7 @@ setup(
|
|
|
39
39
|
long_description=long_description,
|
|
40
40
|
long_description_content_type='text/markdown',
|
|
41
41
|
classifiers=[
|
|
42
|
-
'Development Status ::
|
|
42
|
+
'Development Status :: 5 - Production/Stable',
|
|
43
43
|
'License :: OSI Approved :: Apache Software License',
|
|
44
44
|
'Programming Language :: Python :: 3',
|
|
45
45
|
'Programming Language :: Python :: 3.9',
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
"""VAST Database Python SDK."""
|
|
2
2
|
|
|
3
3
|
import functools
|
|
4
|
-
import importlib.metadata
|
|
5
|
-
|
|
6
|
-
__version__ = importlib.metadata.distribution(__package__).version
|
|
7
4
|
|
|
8
5
|
from . import session
|
|
9
6
|
|
|
@@ -12,3 +9,9 @@ from . import session
|
|
|
12
9
|
@functools.wraps(session.Session)
|
|
13
10
|
def connect(*args, **kwargs): # noqa: D103
|
|
14
11
|
return session.Session(*args, **kwargs)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def version():
|
|
15
|
+
"""Return VAST DB SDK version."""
|
|
16
|
+
import importlib
|
|
17
|
+
return importlib.metadata.distribution(__package__).version
|
|
@@ -5,9 +5,8 @@ import re
|
|
|
5
5
|
import struct
|
|
6
6
|
import urllib.parse
|
|
7
7
|
from collections import defaultdict, namedtuple
|
|
8
|
-
from dataclasses import dataclass, field
|
|
9
8
|
from enum import Enum
|
|
10
|
-
from typing import Any,
|
|
9
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
|
11
10
|
|
|
12
11
|
import backoff
|
|
13
12
|
import flatbuffers
|
|
@@ -104,6 +103,7 @@ from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list
|
|
|
104
103
|
from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
|
|
105
104
|
|
|
106
105
|
from . import errors
|
|
106
|
+
from .config import BackoffConfig
|
|
107
107
|
|
|
108
108
|
UINT64_MAX = 18446744073709551615
|
|
109
109
|
|
|
@@ -742,46 +742,38 @@ def _backoff_giveup(exc: Exception) -> bool:
|
|
|
742
742
|
return True # give up in case of other exceptions
|
|
743
743
|
|
|
744
744
|
|
|
745
|
-
@dataclass
|
|
746
|
-
class BackoffConfig:
|
|
747
|
-
wait_gen: Callable = field(default=backoff.expo)
|
|
748
|
-
max_value: Optional[float] = None # max duration for a single wait period
|
|
749
|
-
max_tries: int = 10
|
|
750
|
-
max_time: float = 60.0 # in seconds
|
|
751
|
-
backoff_log_level: int = logging.DEBUG
|
|
752
|
-
|
|
753
|
-
|
|
754
745
|
class VastdbApi:
|
|
755
746
|
# we expect the vast version to be <major>.<minor>.<patch>.<protocol>
|
|
756
747
|
VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
|
|
757
748
|
|
|
758
749
|
def __init__(self, endpoint, access_key, secret_key,
|
|
759
750
|
*,
|
|
760
|
-
auth_type=AuthType.SIGV4,
|
|
761
751
|
ssl_verify=True,
|
|
752
|
+
timeout=None,
|
|
762
753
|
backoff_config: Optional[BackoffConfig] = None):
|
|
763
754
|
|
|
764
|
-
from . import
|
|
765
|
-
self.client_sdk_version = f"VAST Database Python SDK {
|
|
755
|
+
from . import version # import lazily here (to avoid circular dependencies)
|
|
756
|
+
self.client_sdk_version = f"VAST Database Python SDK {version()} - 2024 (c)"
|
|
766
757
|
|
|
767
758
|
url = urllib3.util.parse_url(endpoint)
|
|
768
759
|
self.access_key = access_key
|
|
769
760
|
self.secret_key = secret_key
|
|
770
761
|
|
|
762
|
+
self.timeout = timeout
|
|
771
763
|
self.default_max_list_columns_page_size = 1000
|
|
772
764
|
self._session = requests.Session()
|
|
773
765
|
self._session.verify = ssl_verify
|
|
774
766
|
self._session.headers['user-agent'] = self.client_sdk_version
|
|
775
767
|
|
|
776
|
-
backoff_config = backoff_config or BackoffConfig()
|
|
768
|
+
self.backoff_config = backoff_config or BackoffConfig()
|
|
777
769
|
self._backoff_decorator = backoff.on_exception(
|
|
778
|
-
wait_gen=backoff_config.wait_gen,
|
|
770
|
+
wait_gen=self.backoff_config.wait_gen,
|
|
779
771
|
exception=_RETRIABLE_EXCEPTIONS,
|
|
780
772
|
giveup=_backoff_giveup,
|
|
781
|
-
max_tries=backoff_config.max_tries,
|
|
782
|
-
max_time=backoff_config.max_time,
|
|
783
|
-
max_value=backoff_config.max_value, # passed to `backoff_config.wait_gen`
|
|
784
|
-
backoff_log_level=backoff_config.backoff_log_level)
|
|
773
|
+
max_tries=self.backoff_config.max_tries,
|
|
774
|
+
max_time=self.backoff_config.max_time,
|
|
775
|
+
max_value=self.backoff_config.max_value, # passed to `self.backoff_config.wait_gen`
|
|
776
|
+
backoff_log_level=self.backoff_config.backoff_log_level)
|
|
785
777
|
self._request = self._backoff_decorator(self._single_request)
|
|
786
778
|
|
|
787
779
|
if url.port in {80, 443, None}:
|
|
@@ -819,10 +811,18 @@ class VastdbApi:
|
|
|
819
811
|
_logger.critical(msg)
|
|
820
812
|
raise NotImplementedError(msg)
|
|
821
813
|
|
|
814
|
+
def with_endpoint(self, endpoint):
|
|
815
|
+
return VastdbApi(endpoint=endpoint,
|
|
816
|
+
access_key=self.access_key,
|
|
817
|
+
secret_key=self.secret_key,
|
|
818
|
+
ssl_verify=self._session.verify,
|
|
819
|
+
timeout=self.timeout,
|
|
820
|
+
backoff_config=self.backoff_config)
|
|
821
|
+
|
|
822
822
|
def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
|
|
823
|
-
_logger.debug("Sending request: %s %s %s", method, url, kwargs)
|
|
823
|
+
_logger.debug("Sending request: %s %s %s timeout=%s", method, url, kwargs, self.timeout)
|
|
824
824
|
try:
|
|
825
|
-
res = self._session.request(method=method, url=url, **kwargs)
|
|
825
|
+
res = self._session.request(method=method, url=url, timeout=self.timeout, **kwargs)
|
|
826
826
|
except requests.exceptions.ConnectionError as err:
|
|
827
827
|
# low-level connection issue, it is safe to retry only read-only requests
|
|
828
828
|
may_retry = (method == "GET")
|
|
@@ -10,11 +10,11 @@ log = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@pytest.mark.benchmark
|
|
13
|
-
def test_bench(session,
|
|
13
|
+
def test_bench(session, bucket_name, parquets_path, crater_path):
|
|
14
14
|
files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
|
|
15
15
|
|
|
16
16
|
with session.transaction() as tx:
|
|
17
|
-
b = tx.bucket(
|
|
17
|
+
b = tx.bucket(bucket_name)
|
|
18
18
|
s = b.create_schema('s1')
|
|
19
19
|
t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
|
|
20
20
|
config = QueryConfig(num_splits=8, num_sub_splits=4)
|
|
@@ -22,7 +22,8 @@ logging.basicConfig(
|
|
|
22
22
|
|
|
23
23
|
log = logging.getLogger()
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
sdk_version = vastdb.version()
|
|
26
|
+
log.info("Python SDK version: %s", sdk_version)
|
|
26
27
|
|
|
27
28
|
NUM_COLUMNS = 10_000
|
|
28
29
|
COLUMNS_BATCH = 10
|
|
@@ -63,6 +64,8 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
63
64
|
# skip already loaded rows
|
|
64
65
|
log.info('skipping [%d..%d)', offset, limit)
|
|
65
66
|
|
|
67
|
+
pid = os.getpid()
|
|
68
|
+
tid = threading.get_native_id()
|
|
66
69
|
total_nbytes = 0
|
|
67
70
|
calls = 0
|
|
68
71
|
t0 = time.time()
|
|
@@ -89,7 +92,7 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
89
92
|
metrics_rows.append(metrics.Row(
|
|
90
93
|
start=start, finish=finish, table_path=table.path, op=op,
|
|
91
94
|
nbytes=nbytes, rows=len(chunk), cols=len(cols_batch),
|
|
92
|
-
pid=
|
|
95
|
+
pid=pid, tid=tid, sdk_version=sdk_version))
|
|
93
96
|
|
|
94
97
|
total_nbytes += nbytes
|
|
95
98
|
calls += 1
|
|
@@ -104,12 +107,12 @@ def load_batch(bucket, session_kwargs, offset, limit):
|
|
|
104
107
|
return metrics_rows
|
|
105
108
|
|
|
106
109
|
|
|
107
|
-
def test_ingest(
|
|
110
|
+
def test_ingest(test_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
|
|
108
111
|
session = vastdb.connect(**session_kwargs)
|
|
109
112
|
metrics_table = metrics.Table(perf_metrics_db, "ingest")
|
|
110
113
|
|
|
111
114
|
with session.transaction() as tx:
|
|
112
|
-
b = tx.bucket(
|
|
115
|
+
b = tx.bucket(test_bucket_name)
|
|
113
116
|
try:
|
|
114
117
|
s = b.schema(SCHEMA)
|
|
115
118
|
except vastdb.errors.MissingSchema:
|
|
@@ -127,7 +130,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
|
|
|
127
130
|
|
|
128
131
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
129
132
|
futures = [
|
|
130
|
-
executor.submit(load_batch,
|
|
133
|
+
executor.submit(load_batch, test_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
|
|
131
134
|
for (offset, limit), url in zip(ranges, itertools.cycle(tabular_endpoint_urls))
|
|
132
135
|
]
|
|
133
136
|
log.info("spawned %d futures", len(futures))
|
|
@@ -135,7 +138,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
|
|
|
135
138
|
metrics_table.insert(future.result())
|
|
136
139
|
|
|
137
140
|
with session.transaction() as tx:
|
|
138
|
-
t = tx.bucket(
|
|
141
|
+
t = tx.bucket(test_bucket_name).schema(SCHEMA).table(TABLE)
|
|
139
142
|
count = sum(len(rb) for rb in t.select([]))
|
|
140
143
|
log.info("%s has %d rows: %s", t, count, t.stats)
|
|
141
144
|
|
|
@@ -154,6 +157,10 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
|
|
|
154
157
|
r = random.Random(i)
|
|
155
158
|
r.shuffle(row_group_indices)
|
|
156
159
|
|
|
160
|
+
pid = os.getpid()
|
|
161
|
+
tid = threading.get_native_id()
|
|
162
|
+
metrics_rows = []
|
|
163
|
+
|
|
157
164
|
session = vastdb.connect(**(session_kwargs | {"endpoint": endpoint_url}))
|
|
158
165
|
with session.transaction() as tx:
|
|
159
166
|
t = tx.bucket(bucket_name).schema(SCHEMA).table(TABLE)
|
|
@@ -174,23 +181,37 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
|
|
|
174
181
|
|
|
175
182
|
for j, pred in enumerate(preds):
|
|
176
183
|
log.info("%d) starting query #%d on %s", i, j, endpoint_url)
|
|
177
|
-
|
|
184
|
+
|
|
185
|
+
start = time.perf_counter()
|
|
178
186
|
res = t.select(columns=cols, predicate=pred, config=config)
|
|
179
187
|
rows = 0
|
|
180
188
|
data = 0
|
|
181
189
|
for rb in res:
|
|
182
190
|
rows += len(rb)
|
|
183
191
|
data += rb.nbytes
|
|
184
|
-
dt = time.
|
|
192
|
+
dt = time.perf_counter() - start
|
|
185
193
|
log.info("%d) got query #%d batch %.3f[s], %.3f[GB] %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
|
|
186
194
|
|
|
187
|
-
|
|
195
|
+
finish = time.perf_counter()
|
|
196
|
+
dt = finish - start
|
|
188
197
|
log.info("%d) finished query #%d %.3f[s], %.3f[GB], %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
|
|
189
198
|
|
|
199
|
+
metrics_rows.append(metrics.Row(
|
|
200
|
+
start=start, finish=finish, table_path=t.path, op="select",
|
|
201
|
+
nbytes=data, rows=rows, cols=len(cols),
|
|
202
|
+
pid=pid, tid=tid, sdk_version=sdk_version))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls, perf_metrics_db):
|
|
206
|
+
metrics_table = metrics.Table(perf_metrics_db, "query")
|
|
190
207
|
|
|
191
|
-
def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls):
|
|
192
208
|
log.info("starting %d workers, endpoints=%s", num_workers, tabular_endpoint_urls)
|
|
193
209
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
194
|
-
|
|
210
|
+
futures = [
|
|
195
211
|
executor.submit(run_query, session_kwargs, i, test_bucket_name, url)
|
|
212
|
+
for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls))
|
|
213
|
+
]
|
|
214
|
+
for future in as_completed(futures):
|
|
215
|
+
metrics_table.insert(future.result())
|
|
216
|
+
|
|
196
217
|
log.info("finished %d workers", num_workers)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Configuration-related dataclasses."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Callable, List, Optional
|
|
6
|
+
|
|
7
|
+
import backoff
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class BackoffConfig:
|
|
12
|
+
"""Retry configuration."""
|
|
13
|
+
|
|
14
|
+
wait_gen: Callable = field(default=backoff.expo)
|
|
15
|
+
max_value: Optional[float] = None # max duration for a single wait period
|
|
16
|
+
max_tries: int = 10
|
|
17
|
+
max_time: float = 60.0 # in seconds
|
|
18
|
+
backoff_log_level: int = logging.DEBUG
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class QueryConfig:
|
|
23
|
+
"""Query execution configiration."""
|
|
24
|
+
|
|
25
|
+
# allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
|
|
26
|
+
num_sub_splits: int = 4
|
|
27
|
+
|
|
28
|
+
# used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
|
|
29
|
+
# will be estimated from the table's row count, if not explicitly set
|
|
30
|
+
num_splits: Optional[int] = None
|
|
31
|
+
|
|
32
|
+
# each endpoint will be handled by a separate worker thread
|
|
33
|
+
# a single endpoint can be specified more than once to benefit from multithreaded execution
|
|
34
|
+
data_endpoints: Optional[List[str]] = None
|
|
35
|
+
|
|
36
|
+
# a subsplit fiber will finish after sending this number of rows back to the client
|
|
37
|
+
limit_rows_per_sub_split: int = 128 * 1024
|
|
38
|
+
|
|
39
|
+
# each fiber will read the following number of rowgroups coninuously before skipping
|
|
40
|
+
# in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
|
|
41
|
+
num_row_groups_per_sub_split: int = 8
|
|
42
|
+
|
|
43
|
+
# can be disabled for benchmarking purposes
|
|
44
|
+
use_semi_sorted_projections: bool = True
|
|
45
|
+
|
|
46
|
+
# enforce using a specific semi-sorted projection (if enabled above)
|
|
47
|
+
semi_sorted_projection_name: Optional[str] = None
|
|
48
|
+
|
|
49
|
+
# used to estimate the number of splits, given the table rows' count
|
|
50
|
+
rows_per_split: int = 4000000
|
|
51
|
+
|
|
52
|
+
# used for worker threads' naming
|
|
53
|
+
query_id: str = ""
|
|
54
|
+
|
|
55
|
+
# non-negative integer, used for server-side prioritization of queued requests:
|
|
56
|
+
# - requests with lower values will be served before requests with higher values.
|
|
57
|
+
# - if unset, the request will be added to the queue's end.
|
|
58
|
+
queue_priority: Optional[int] = None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class ImportConfig:
|
|
63
|
+
"""Import execution configiration."""
|
|
64
|
+
|
|
65
|
+
import_concurrency: int = 2
|
|
@@ -3,7 +3,6 @@ import xml.etree.ElementTree
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from enum import Enum
|
|
5
5
|
|
|
6
|
-
import pyarrow as pa
|
|
7
6
|
import requests
|
|
8
7
|
|
|
9
8
|
|
|
@@ -170,11 +169,6 @@ class NotSupportedCommand(NotSupported):
|
|
|
170
169
|
table: str
|
|
171
170
|
|
|
172
171
|
|
|
173
|
-
@dataclass
|
|
174
|
-
class NotSupportedType(NotSupported):
|
|
175
|
-
field: pa.Field
|
|
176
|
-
|
|
177
|
-
|
|
178
172
|
@dataclass
|
|
179
173
|
class NotSupportedVersion(NotSupported):
|
|
180
174
|
err_msg: str
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Version-dependent features."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from .errors import NotSupportedVersion
|
|
6
|
+
|
|
7
|
+
log = logging.getLogger()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Features:
|
|
11
|
+
"""VAST database features - check if server is already support a feature."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, vast_version):
|
|
14
|
+
"""Save the server version."""
|
|
15
|
+
self.vast_version = vast_version
|
|
16
|
+
|
|
17
|
+
self.check_imports_table = self._check(
|
|
18
|
+
"Imported objects' table feature requires 5.2+ VAST release",
|
|
19
|
+
vast_version >= (5, 2))
|
|
20
|
+
|
|
21
|
+
self.check_return_row_ids = self._check(
|
|
22
|
+
"Returning row IDs requires 5.1+ VAST release",
|
|
23
|
+
vast_version >= (5, 1))
|
|
24
|
+
|
|
25
|
+
self.check_enforce_semisorted_projection = self._check(
|
|
26
|
+
"Semi-sorted projection enforcement requires 5.1+ VAST release",
|
|
27
|
+
vast_version >= (5, 1))
|
|
28
|
+
|
|
29
|
+
self.check_external_row_ids_allocation = self._check(
|
|
30
|
+
"External row IDs allocation requires 5.1+ VAST release",
|
|
31
|
+
vast_version >= (5, 1))
|
|
32
|
+
|
|
33
|
+
def _check(self, msg, supported):
|
|
34
|
+
log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
|
|
35
|
+
if not supported:
|
|
36
|
+
def fail():
|
|
37
|
+
raise NotSupportedVersion(msg, self.vast_version)
|
|
38
|
+
return fail
|
|
39
|
+
|
|
40
|
+
def noop():
|
|
41
|
+
pass
|
|
42
|
+
return noop
|
|
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
|
|
|
10
10
|
|
|
11
11
|
import pyarrow as pa
|
|
12
12
|
|
|
13
|
-
from . import bucket, errors, schema, table
|
|
13
|
+
from . import bucket, errors, schema, table
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from .table import Table
|
|
@@ -86,7 +86,6 @@ class Schema:
|
|
|
86
86
|
if use_external_row_ids_allocation:
|
|
87
87
|
self.tx._rpc.features.check_external_row_ids_allocation()
|
|
88
88
|
|
|
89
|
-
util.check_supported_types(columns)
|
|
90
89
|
self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
|
|
91
90
|
use_external_row_ids_allocation=use_external_row_ids_allocation)
|
|
92
91
|
log.info("Created table: %s", table_name)
|
|
@@ -7,51 +7,11 @@ For more details see:
|
|
|
7
7
|
- [Tabular identity policy with the proper permissions](https://support.vastdata.com/s/article/UUID-14322b60-d6a2-89ac-3df0-3dfbb6974182)
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
import logging
|
|
11
10
|
import os
|
|
12
|
-
from typing import Optional
|
|
11
|
+
from typing import TYPE_CHECKING, Optional
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
from . import _internal, errors, transaction
|
|
17
|
-
from ._internal import BackoffConfig
|
|
18
|
-
|
|
19
|
-
log = logging.getLogger()
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class Features:
|
|
23
|
-
"""VAST database features - check if server is already support a feature."""
|
|
24
|
-
|
|
25
|
-
def __init__(self, vast_version):
|
|
26
|
-
"""Save the server version."""
|
|
27
|
-
self.vast_version = vast_version
|
|
28
|
-
|
|
29
|
-
self.check_imports_table = self._check(
|
|
30
|
-
"Imported objects' table feature requires 5.2+ VAST release",
|
|
31
|
-
vast_version >= (5, 2))
|
|
32
|
-
|
|
33
|
-
self.check_return_row_ids = self._check(
|
|
34
|
-
"Returning row IDs requires 5.1+ VAST release",
|
|
35
|
-
vast_version >= (5, 1))
|
|
36
|
-
|
|
37
|
-
self.check_enforce_semisorted_projection = self._check(
|
|
38
|
-
"Semi-sorted projection enforcement requires 5.1+ VAST release",
|
|
39
|
-
vast_version >= (5, 1))
|
|
40
|
-
|
|
41
|
-
self.check_external_row_ids_allocation = self._check(
|
|
42
|
-
"External row IDs allocation requires 5.1+ VAST release",
|
|
43
|
-
vast_version >= (5, 1))
|
|
44
|
-
|
|
45
|
-
def _check(self, msg, supported):
|
|
46
|
-
log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
|
|
47
|
-
if not supported:
|
|
48
|
-
def fail():
|
|
49
|
-
raise errors.NotSupportedVersion(msg, self.vast_version)
|
|
50
|
-
return fail
|
|
51
|
-
|
|
52
|
-
def noop():
|
|
53
|
-
pass
|
|
54
|
-
return noop
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from .config import BackoffConfig
|
|
55
15
|
|
|
56
16
|
|
|
57
17
|
class Session:
|
|
@@ -60,8 +20,13 @@ class Session:
|
|
|
60
20
|
def __init__(self, access=None, secret=None, endpoint=None,
|
|
61
21
|
*,
|
|
62
22
|
ssl_verify=True,
|
|
63
|
-
|
|
23
|
+
timeout=None,
|
|
24
|
+
backoff_config: Optional["BackoffConfig"] = None):
|
|
64
25
|
"""Connect to a VAST Database endpoint, using specified credentials."""
|
|
26
|
+
import boto3
|
|
27
|
+
|
|
28
|
+
from . import _internal, features
|
|
29
|
+
|
|
65
30
|
if access is None:
|
|
66
31
|
access = os.environ['AWS_ACCESS_KEY_ID']
|
|
67
32
|
if secret is None:
|
|
@@ -74,8 +39,9 @@ class Session:
|
|
|
74
39
|
access_key=access,
|
|
75
40
|
secret_key=secret,
|
|
76
41
|
ssl_verify=ssl_verify,
|
|
42
|
+
timeout=timeout,
|
|
77
43
|
backoff_config=backoff_config)
|
|
78
|
-
self.features = Features(self.api.vast_version)
|
|
44
|
+
self.features = features.Features(self.api.vast_version)
|
|
79
45
|
self.s3 = boto3.client('s3',
|
|
80
46
|
aws_access_key_id=access,
|
|
81
47
|
aws_secret_access_key=secret,
|
|
@@ -93,4 +59,5 @@ class Session:
|
|
|
93
59
|
with session.transaction() as tx:
|
|
94
60
|
tx.bucket("bucket").create_schema("schema")
|
|
95
61
|
"""
|
|
62
|
+
from . import transaction
|
|
96
63
|
return transaction.Transaction(self)
|