vastdb 0.1.10__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. {vastdb-0.1.10 → vastdb-1.0.0}/CHANGELOG.md +19 -0
  2. vastdb-1.0.0/CONTRIBUTING.md +33 -0
  3. {vastdb-0.1.10/vastdb.egg-info → vastdb-1.0.0}/PKG-INFO +2 -2
  4. {vastdb-0.1.10 → vastdb-1.0.0}/README.md +0 -2
  5. {vastdb-0.1.10 → vastdb-1.0.0}/setup.py +2 -2
  6. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/__init__.py +6 -3
  7. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/_internal.py +22 -22
  8. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/test_perf.py +2 -2
  9. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/test_sample.py +32 -11
  10. vastdb-1.0.0/vastdb/config.py +65 -0
  11. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/errors.py +0 -6
  12. vastdb-1.0.0/vastdb/features.py +42 -0
  13. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/schema.py +1 -2
  14. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/session.py +12 -45
  15. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/table.py +3 -52
  16. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/metrics.py +1 -0
  17. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_sanity.py +1 -1
  18. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_tables.py +29 -40
  19. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/util.py +1 -13
  20. vastdb-1.0.0/vastdb/vast_tests/test_scale.py +68 -0
  21. {vastdb-0.1.10 → vastdb-1.0.0/vastdb.egg-info}/PKG-INFO +2 -2
  22. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/SOURCES.txt +5 -1
  23. {vastdb-0.1.10 → vastdb-1.0.0}/LICENSE +0 -0
  24. {vastdb-0.1.10 → vastdb-1.0.0}/MANIFEST.in +0 -0
  25. {vastdb-0.1.10 → vastdb-1.0.0}/requirements.txt +0 -0
  26. {vastdb-0.1.10 → vastdb-1.0.0}/setup.cfg +0 -0
  27. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/__init__.py +0 -0
  28. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/__init__.py +0 -0
  29. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/__init__.py +0 -0
  30. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/__init__.py +0 -0
  31. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/__init__.py +0 -0
  32. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Aggregate.py +0 -0
  33. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySlice.py +0 -0
  34. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ArraySubscript.py +0 -0
  35. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BinaryLiteral.py +0 -0
  36. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/BooleanLiteral.py +0 -0
  37. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Bound.py +0 -0
  38. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Call.py +0 -0
  39. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CaseFragment.py +0 -0
  40. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Cast.py +0 -0
  41. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConcreteBoundImpl.py +0 -0
  42. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ConditionalCase.py +0 -0
  43. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/CurrentRow.py +0 -0
  44. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DateLiteral.py +0 -0
  45. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DecimalLiteral.py +0 -0
  46. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Deref.py +0 -0
  47. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/DurationLiteral.py +0 -0
  48. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Expression.py +0 -0
  49. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ExpressionImpl.py +0 -0
  50. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldIndex.py +0 -0
  51. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FieldRef.py +0 -0
  52. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Filter.py +0 -0
  53. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/FixedSizeBinaryLiteral.py +0 -0
  54. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float16Literal.py +0 -0
  55. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float32Literal.py +0 -0
  56. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Float64Literal.py +0 -0
  57. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Following.py +0 -0
  58. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Frame.py +0 -0
  59. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Grouping.py +0 -0
  60. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int16Literal.py +0 -0
  61. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int32Literal.py +0 -0
  62. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int64Literal.py +0 -0
  63. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Int8Literal.py +0 -0
  64. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteral.py +0 -0
  65. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralDaysMilliseconds.py +0 -0
  66. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralImpl.py +0 -0
  67. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/IntervalLiteralMonths.py +0 -0
  68. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Join.py +0 -0
  69. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/JoinKind.py +0 -0
  70. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/KeyValue.py +0 -0
  71. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Limit.py +0 -0
  72. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/ListLiteral.py +0 -0
  73. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Literal.py +0 -0
  74. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralColumn.py +0 -0
  75. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralImpl.py +0 -0
  76. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/LiteralRelation.py +0 -0
  77. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapKey.py +0 -0
  78. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/MapLiteral.py +0 -0
  79. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/OrderBy.py +0 -0
  80. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Ordering.py +0 -0
  81. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Plan.py +0 -0
  82. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Preceding.py +0 -0
  83. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Project.py +0 -0
  84. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelId.py +0 -0
  85. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Relation.py +0 -0
  86. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/RelationImpl.py +0 -0
  87. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOpKind.py +0 -0
  88. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SetOperation.py +0 -0
  89. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SimpleCase.py +0 -0
  90. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/SortKey.py +0 -0
  91. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Source.py +0 -0
  92. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StringLiteral.py +0 -0
  93. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructField.py +0 -0
  94. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/StructLiteral.py +0 -0
  95. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimeLiteral.py +0 -0
  96. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/TimestampLiteral.py +0 -0
  97. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt16Literal.py +0 -0
  98. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt32Literal.py +0 -0
  99. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt64Literal.py +0 -0
  100. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/UInt8Literal.py +0 -0
  101. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/Unbounded.py +0 -0
  102. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/WindowCall.py +0 -0
  103. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/computeir/flatbuf/__init__.py +0 -0
  104. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Binary.py +0 -0
  105. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Block.py +0 -0
  106. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompression.py +0 -0
  107. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/BodyCompressionMethod.py +0 -0
  108. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Bool.py +0 -0
  109. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Buffer.py +0 -0
  110. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/CompressionType.py +0 -0
  111. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Date.py +0 -0
  112. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DateUnit.py +0 -0
  113. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Decimal.py +0 -0
  114. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryBatch.py +0 -0
  115. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryEncoding.py +0 -0
  116. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/DictionaryKind.py +0 -0
  117. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Duration.py +0 -0
  118. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Endianness.py +0 -0
  119. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Feature.py +0 -0
  120. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Field.py +0 -0
  121. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FieldNode.py +0 -0
  122. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeBinary.py +0 -0
  123. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FixedSizeList.py +0 -0
  124. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/FloatingPoint.py +0 -0
  125. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Footer.py +0 -0
  126. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Int.py +0 -0
  127. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Interval.py +0 -0
  128. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/IntervalUnit.py +0 -0
  129. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/KeyValue.py +0 -0
  130. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeBinary.py +0 -0
  131. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeList.py +0 -0
  132. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/LargeUtf8.py +0 -0
  133. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/List.py +0 -0
  134. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Map.py +0 -0
  135. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Message.py +0 -0
  136. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/MessageHeader.py +0 -0
  137. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/MetadataVersion.py +0 -0
  138. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Null.py +0 -0
  139. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Precision.py +0 -0
  140. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/RecordBatch.py +0 -0
  141. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Schema.py +0 -0
  142. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixCompressedAxis.py +0 -0
  143. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.py +0 -0
  144. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensor.py +0 -0
  145. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndex.py +0 -0
  146. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCOO.py +0 -0
  147. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/SparseTensorIndexCSF.py +0 -0
  148. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Struct_.py +0 -0
  149. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Tensor.py +0 -0
  150. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/TensorDim.py +0 -0
  151. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Time.py +0 -0
  152. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/TimeUnit.py +0 -0
  153. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Timestamp.py +0 -0
  154. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Type.py +0 -0
  155. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Union.py +0 -0
  156. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/UnionMode.py +0 -0
  157. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/Utf8.py +0 -0
  158. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/org/apache/arrow/flatbuf/__init__.py +0 -0
  159. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterColumnRequest.py +0 -0
  160. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterProjectionTableRequest.py +0 -0
  161. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterSchemaRequest.py +0 -0
  162. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/AlterTableRequest.py +0 -0
  163. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/Column.py +0 -0
  164. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ColumnType.py +0 -0
  165. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/CreateProjectionRequest.py +0 -0
  166. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/CreateSchemaRequest.py +0 -0
  167. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/GetProjectionTableStatsResponse.py +0 -0
  168. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/GetTableStatsResponse.py +0 -0
  169. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ImportDataRequest.py +0 -0
  170. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListProjectionsResponse.py +0 -0
  171. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListSchemasResponse.py +0 -0
  172. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ListTablesResponse.py +0 -0
  173. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/ObjectDetails.py +0 -0
  174. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/S3File.py +0 -0
  175. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/VipRange.py +0 -0
  176. {vastdb-0.1.10 → vastdb-1.0.0}/vast_flatbuf/tabular/__init__.py +0 -0
  177. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bench/__init__.py +0 -0
  178. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/bucket.py +0 -0
  179. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/conftest.py +0 -0
  180. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/__init__.py +0 -0
  181. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_duckdb.py +0 -0
  182. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_imports.py +0 -0
  183. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_nested.py +0 -0
  184. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_projections.py +0 -0
  185. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_schemas.py +0 -0
  186. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/test_util.py +0 -0
  187. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/tests/util.py +0 -0
  188. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/transaction.py +0 -0
  189. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/vast_tests/__init__.py +0 -0
  190. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb/vast_tests/test_ha.py +0 -0
  191. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/dependency_links.txt +0 -0
  192. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/requires.txt +0 -0
  193. {vastdb-0.1.10 → vastdb-1.0.0}/vastdb.egg-info/top_level.txt +0 -0
@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
6
6
 
7
+ ## [1.0.0] (2024-07-21)
8
+ [1.0.0]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.11...v1.0.0
9
+
10
+ ## Added
11
+ - Add CLA (`CONTRIBUTING.md`)
12
+ - Support setting HTTP timeouts
13
+
14
+ ## [0.1.11] (2024-07-17)
15
+ [0.1.11]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.10...v0.1.11
16
+
17
+ ## Fixed
18
+ - Revert timezone check to keep previous SDK behaviour
19
+
20
+ ## Added
21
+ - Use Sphinx for building documentation
22
+
23
+ ## Changed
24
+ - Defer imports to optimize initial import latency
25
+
7
26
  ## [0.1.10] (2024-07-09)
8
27
  [0.1.10]: https://github.com/vast-data/vastdb_sdk/compare/v0.1.9...v0.1.10
9
28
 
@@ -0,0 +1,33 @@
1
+ # VAST DB SDK Contribution License Agreement
2
+
3
+ Thank you for your interest in VAST DB SDK project (“Project”). You hereby accept and agree to the following terms and conditions (“Agreement”) for Your present and future Contributions (as defined below) submitted to the Project or as part of. Except for the license granted herein to the Project originator – Vast Data Ltd. together with any of its affiliates, assignee, or successor (“Vast Data” or “us” or “we”), You reserve all right, title, and interest in and to Your Contributions.
4
+
5
+ ## Definitions.
6
+
7
+ "You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Vast Data. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single contributor. For the purposes of this definition, "control" means (i) the power, directly or indirectly, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
8
+
9
+ "Contribution" shall mean the code, documentation, or any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to the Project (or to Vast Data), inclusion in, or documentation of, the Project (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Project or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Project for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
10
+
11
+ ## Licenses
12
+
13
+ Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Vast Data and at Vast Data’ discretion, its affiliates, a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, sell, and distribute Your Contributions and such derivative works.
14
+
15
+ Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Vast Data and, at Vast Data’ discretion, its affiliates a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by the combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes a direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
16
+
17
+ ## Representations
18
+
19
+ You represent that You are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes Your Contributions, You represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Vast Data, or that your employer has executed a separate written agreement with Vast Data.
20
+
21
+ You represent that each of Your Contributions is Your original creation (see below for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
22
+
23
+ You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all.
24
+
25
+ UNLESS REQUIRED BY APPLICABLE LAW, AGREED TO IN WRITING, OR AS STATED IN THIS CLA, YOU PROVIDE YOUR CONTRIBUTION ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
26
+
27
+ Should You wish to submit work that is not Your original creation, You may submit it to Vast Data separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]". Vast Data shall have the right to determine at its discretion whether or not to use such Contribution.
28
+
29
+ Code of Conduct
30
+
31
+ In Your Contribution and/or Work or otherwise in your participation in the Project, you agree to: (i) not use or include any defamatory, illegal, derogatory, abusive, harassing, destructive, malicious, or otherwise, illegitimate content, code or language; or (ii) publish others’ private or personal information without their explicit permission (“Code of Conduct”) Vast Data’ will be entitled but not obligated to enforce the Code of Conduct and may take appropriate and fair action in response to any behaviour that is reasonably deemed by us to be inappropriate, threatening, offensive, or harmful. Further Vast Data will have the right but not the obligation to remove, edit, or reject any comments, code, issues, and other Contributions or Works submitted hereunder including without limitation by reason of nonalignment with the Code of Conduct.
32
+
33
+ You agree to notify Vast Data of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: vastdb
3
- Version: 0.1.10
3
+ Version: 1.0.0
4
4
  Summary: VAST Data SDK
5
5
  Home-page: https://github.com/vast-data/vastdb_sdk
6
6
  Author: VAST DATA
7
7
  Author-email: hello@vastdata.com
8
8
  License: Copyright (C) VAST Data Ltd.
9
- Classifier: Development Status :: 4 - Beta
9
+ Classifier: Development Status :: 5 - Production/Stable
10
10
  Classifier: License :: OSI Approved :: Apache Software License
11
11
  Classifier: Programming Language :: Python :: 3
12
12
  Classifier: Programming Language :: Python :: 3.9
@@ -2,8 +2,6 @@
2
2
 
3
3
  # VAST DB Python SDK
4
4
 
5
- 🚧 Please note that this package in a pre-release stage. Until version 1.x is officially released, the API should be considered unstable.
6
-
7
5
  ## Introduction
8
6
 
9
7
  `vastdb` is a Python-based SDK designed for interacting with [VAST Database](https://vastdata.com/database) & [VAST Catalog](https://vastdata.com/blog/vast-catalog-treat-your-file-system-like-a-database), enabling schema and table management, efficient ingest, query and modification of columnar data. For more details, see [our whitepaper](https://vastdata.com/whitepaper/#TheVASTDataBase).
@@ -29,7 +29,7 @@ setup(
29
29
  name='vastdb',
30
30
  python_requires='>=3.9.0',
31
31
  description='VAST Data SDK',
32
- version='0.1.10' + suffix,
32
+ version='1.0.0' + suffix,
33
33
  url='https://github.com/vast-data/vastdb_sdk',
34
34
  author='VAST DATA',
35
35
  author_email='hello@vastdata.com',
@@ -39,7 +39,7 @@ setup(
39
39
  long_description=long_description,
40
40
  long_description_content_type='text/markdown',
41
41
  classifiers=[
42
- 'Development Status :: 4 - Beta',
42
+ 'Development Status :: 5 - Production/Stable',
43
43
  'License :: OSI Approved :: Apache Software License',
44
44
  'Programming Language :: Python :: 3',
45
45
  'Programming Language :: Python :: 3.9',
@@ -1,9 +1,6 @@
1
1
  """VAST Database Python SDK."""
2
2
 
3
3
  import functools
4
- import importlib.metadata
5
-
6
- __version__ = importlib.metadata.distribution(__package__).version
7
4
 
8
5
  from . import session
9
6
 
@@ -12,3 +9,9 @@ from . import session
12
9
  @functools.wraps(session.Session)
13
10
  def connect(*args, **kwargs): # noqa: D103
14
11
  return session.Session(*args, **kwargs)
12
+
13
+
14
+ def version():
15
+ """Return VAST DB SDK version."""
16
+ import importlib
17
+ return importlib.metadata.distribution(__package__).version
@@ -5,9 +5,8 @@ import re
5
5
  import struct
6
6
  import urllib.parse
7
7
  from collections import defaultdict, namedtuple
8
- from dataclasses import dataclass, field
9
8
  from enum import Enum
10
- from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
9
+ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
11
10
 
12
11
  import backoff
13
12
  import flatbuffers
@@ -104,6 +103,7 @@ from vast_flatbuf.tabular.ListSchemasResponse import ListSchemasResponse as list
104
103
  from vast_flatbuf.tabular.ListTablesResponse import ListTablesResponse as list_tables
105
104
 
106
105
  from . import errors
106
+ from .config import BackoffConfig
107
107
 
108
108
  UINT64_MAX = 18446744073709551615
109
109
 
@@ -742,46 +742,38 @@ def _backoff_giveup(exc: Exception) -> bool:
742
742
  return True # give up in case of other exceptions
743
743
 
744
744
 
745
- @dataclass
746
- class BackoffConfig:
747
- wait_gen: Callable = field(default=backoff.expo)
748
- max_value: Optional[float] = None # max duration for a single wait period
749
- max_tries: int = 10
750
- max_time: float = 60.0 # in seconds
751
- backoff_log_level: int = logging.DEBUG
752
-
753
-
754
745
  class VastdbApi:
755
746
  # we expect the vast version to be <major>.<minor>.<patch>.<protocol>
756
747
  VAST_VERSION_REGEX = re.compile(r'^vast (\d+\.\d+\.\d+\.\d+)$')
757
748
 
758
749
  def __init__(self, endpoint, access_key, secret_key,
759
750
  *,
760
- auth_type=AuthType.SIGV4,
761
751
  ssl_verify=True,
752
+ timeout=None,
762
753
  backoff_config: Optional[BackoffConfig] = None):
763
754
 
764
- from . import __version__ # import lazily here (to avoid circular dependencies)
765
- self.client_sdk_version = f"VAST Database Python SDK {__version__} - 2024 (c)"
755
+ from . import version # import lazily here (to avoid circular dependencies)
756
+ self.client_sdk_version = f"VAST Database Python SDK {version()} - 2024 (c)"
766
757
 
767
758
  url = urllib3.util.parse_url(endpoint)
768
759
  self.access_key = access_key
769
760
  self.secret_key = secret_key
770
761
 
762
+ self.timeout = timeout
771
763
  self.default_max_list_columns_page_size = 1000
772
764
  self._session = requests.Session()
773
765
  self._session.verify = ssl_verify
774
766
  self._session.headers['user-agent'] = self.client_sdk_version
775
767
 
776
- backoff_config = backoff_config or BackoffConfig()
768
+ self.backoff_config = backoff_config or BackoffConfig()
777
769
  self._backoff_decorator = backoff.on_exception(
778
- wait_gen=backoff_config.wait_gen,
770
+ wait_gen=self.backoff_config.wait_gen,
779
771
  exception=_RETRIABLE_EXCEPTIONS,
780
772
  giveup=_backoff_giveup,
781
- max_tries=backoff_config.max_tries,
782
- max_time=backoff_config.max_time,
783
- max_value=backoff_config.max_value, # passed to `backoff_config.wait_gen`
784
- backoff_log_level=backoff_config.backoff_log_level)
773
+ max_tries=self.backoff_config.max_tries,
774
+ max_time=self.backoff_config.max_time,
775
+ max_value=self.backoff_config.max_value, # passed to `self.backoff_config.wait_gen`
776
+ backoff_log_level=self.backoff_config.backoff_log_level)
785
777
  self._request = self._backoff_decorator(self._single_request)
786
778
 
787
779
  if url.port in {80, 443, None}:
@@ -819,10 +811,18 @@ class VastdbApi:
819
811
  _logger.critical(msg)
820
812
  raise NotImplementedError(msg)
821
813
 
814
+ def with_endpoint(self, endpoint):
815
+ return VastdbApi(endpoint=endpoint,
816
+ access_key=self.access_key,
817
+ secret_key=self.secret_key,
818
+ ssl_verify=self._session.verify,
819
+ timeout=self.timeout,
820
+ backoff_config=self.backoff_config)
821
+
822
822
  def _single_request(self, *, method, url, skip_status_check=False, **kwargs):
823
- _logger.debug("Sending request: %s %s %s", method, url, kwargs)
823
+ _logger.debug("Sending request: %s %s %s timeout=%s", method, url, kwargs, self.timeout)
824
824
  try:
825
- res = self._session.request(method=method, url=url, **kwargs)
825
+ res = self._session.request(method=method, url=url, timeout=self.timeout, **kwargs)
826
826
  except requests.exceptions.ConnectionError as err:
827
827
  # low-level connection issue, it is safe to retry only read-only requests
828
828
  may_retry = (method == "GET")
@@ -10,11 +10,11 @@ log = logging.getLogger(__name__)
10
10
 
11
11
 
12
12
  @pytest.mark.benchmark
13
- def test_bench(session, clean_bucket_name, parquets_path, crater_path):
13
+ def test_bench(session, bucket_name, parquets_path, crater_path):
14
14
  files = [str(parquets_path / f) for f in (parquets_path.glob('**/*.pq'))]
15
15
 
16
16
  with session.transaction() as tx:
17
- b = tx.bucket(clean_bucket_name)
17
+ b = tx.bucket(bucket_name)
18
18
  s = b.create_schema('s1')
19
19
  t = util.create_table_from_files(s, 't1', files, config=ImportConfig(import_concurrency=8))
20
20
  config = QueryConfig(num_splits=8, num_sub_splits=4)
@@ -22,7 +22,8 @@ logging.basicConfig(
22
22
 
23
23
  log = logging.getLogger()
24
24
 
25
- log.info("Python SDK version: %s", vastdb.util.version())
25
+ sdk_version = vastdb.version()
26
+ log.info("Python SDK version: %s", sdk_version)
26
27
 
27
28
  NUM_COLUMNS = 10_000
28
29
  COLUMNS_BATCH = 10
@@ -63,6 +64,8 @@ def load_batch(bucket, session_kwargs, offset, limit):
63
64
  # skip already loaded rows
64
65
  log.info('skipping [%d..%d)', offset, limit)
65
66
 
67
+ pid = os.getpid()
68
+ tid = threading.get_native_id()
66
69
  total_nbytes = 0
67
70
  calls = 0
68
71
  t0 = time.time()
@@ -89,7 +92,7 @@ def load_batch(bucket, session_kwargs, offset, limit):
89
92
  metrics_rows.append(metrics.Row(
90
93
  start=start, finish=finish, table_path=table.path, op=op,
91
94
  nbytes=nbytes, rows=len(chunk), cols=len(cols_batch),
92
- pid=os.getpid(), tid=threading.get_native_id()))
95
+ pid=pid, tid=tid, sdk_version=sdk_version))
93
96
 
94
97
  total_nbytes += nbytes
95
98
  calls += 1
@@ -104,12 +107,12 @@ def load_batch(bucket, session_kwargs, offset, limit):
104
107
  return metrics_rows
105
108
 
106
109
 
107
- def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
110
+ def test_ingest(test_bucket_name, session_kwargs, tabular_endpoint_urls, num_workers, perf_metrics_db):
108
111
  session = vastdb.connect(**session_kwargs)
109
112
  metrics_table = metrics.Table(perf_metrics_db, "ingest")
110
113
 
111
114
  with session.transaction() as tx:
112
- b = tx.bucket(clean_bucket_name)
115
+ b = tx.bucket(test_bucket_name)
113
116
  try:
114
117
  s = b.schema(SCHEMA)
115
118
  except vastdb.errors.MissingSchema:
@@ -127,7 +130,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
127
130
 
128
131
  with ProcessPoolExecutor(max_workers=num_workers) as executor:
129
132
  futures = [
130
- executor.submit(load_batch, clean_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
133
+ executor.submit(load_batch, test_bucket_name, session_kwargs | {'endpoint': url}, offset, limit)
131
134
  for (offset, limit), url in zip(ranges, itertools.cycle(tabular_endpoint_urls))
132
135
  ]
133
136
  log.info("spawned %d futures", len(futures))
@@ -135,7 +138,7 @@ def test_ingest(clean_bucket_name, session_kwargs, tabular_endpoint_urls, num_wo
135
138
  metrics_table.insert(future.result())
136
139
 
137
140
  with session.transaction() as tx:
138
- t = tx.bucket(clean_bucket_name).schema(SCHEMA).table(TABLE)
141
+ t = tx.bucket(test_bucket_name).schema(SCHEMA).table(TABLE)
139
142
  count = sum(len(rb) for rb in t.select([]))
140
143
  log.info("%s has %d rows: %s", t, count, t.stats)
141
144
 
@@ -154,6 +157,10 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
154
157
  r = random.Random(i)
155
158
  r.shuffle(row_group_indices)
156
159
 
160
+ pid = os.getpid()
161
+ tid = threading.get_native_id()
162
+ metrics_rows = []
163
+
157
164
  session = vastdb.connect(**(session_kwargs | {"endpoint": endpoint_url}))
158
165
  with session.transaction() as tx:
159
166
  t = tx.bucket(bucket_name).schema(SCHEMA).table(TABLE)
@@ -174,23 +181,37 @@ def run_query(session_kwargs, i, bucket_name, endpoint_url):
174
181
 
175
182
  for j, pred in enumerate(preds):
176
183
  log.info("%d) starting query #%d on %s", i, j, endpoint_url)
177
- t0 = time.time()
184
+
185
+ start = time.perf_counter()
178
186
  res = t.select(columns=cols, predicate=pred, config=config)
179
187
  rows = 0
180
188
  data = 0
181
189
  for rb in res:
182
190
  rows += len(rb)
183
191
  data += rb.nbytes
184
- dt = time.time() - t0
192
+ dt = time.perf_counter() - start
185
193
  log.info("%d) got query #%d batch %.3f[s], %.3f[GB] %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
186
194
 
187
- dt = time.time() - t0
195
+ finish = time.perf_counter()
196
+ dt = finish - start
188
197
  log.info("%d) finished query #%d %.3f[s], %.3f[GB], %.3f[MB/s], %.3f[Mrows]", i, j, dt, data / 1e9, data / 1e6 / dt, rows / 1e6)
189
198
 
199
+ metrics_rows.append(metrics.Row(
200
+ start=start, finish=finish, table_path=t.path, op="select",
201
+ nbytes=data, rows=rows, cols=len(cols),
202
+ pid=pid, tid=tid, sdk_version=sdk_version))
203
+
204
+
205
+ def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls, perf_metrics_db):
206
+ metrics_table = metrics.Table(perf_metrics_db, "query")
190
207
 
191
- def test_scan(test_bucket_name, session, num_workers, session_kwargs, tabular_endpoint_urls):
192
208
  log.info("starting %d workers, endpoints=%s", num_workers, tabular_endpoint_urls)
193
209
  with ProcessPoolExecutor(max_workers=num_workers) as executor:
194
- for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls)):
210
+ futures = [
195
211
  executor.submit(run_query, session_kwargs, i, test_bucket_name, url)
212
+ for i, url in zip(range(num_workers), itertools.cycle(tabular_endpoint_urls))
213
+ ]
214
+ for future in as_completed(futures):
215
+ metrics_table.insert(future.result())
216
+
196
217
  log.info("finished %d workers", num_workers)
@@ -0,0 +1,65 @@
1
+ """Configuration-related dataclasses."""
2
+
3
+ import logging
4
+ from dataclasses import dataclass, field
5
+ from typing import Callable, List, Optional
6
+
7
+ import backoff
8
+
9
+
10
+ @dataclass
11
+ class BackoffConfig:
12
+ """Retry configuration."""
13
+
14
+ wait_gen: Callable = field(default=backoff.expo)
15
+ max_value: Optional[float] = None # max duration for a single wait period
16
+ max_tries: int = 10
17
+ max_time: float = 60.0 # in seconds
18
+ backoff_log_level: int = logging.DEBUG
19
+
20
+
21
+ @dataclass
22
+ class QueryConfig:
23
+ """Query execution configiration."""
24
+
25
+ # allows server-side parallel processing by issuing multiple reads concurrently for a single RPC
26
+ num_sub_splits: int = 4
27
+
28
+ # used to split the table into disjoint subsets of rows, to be processed concurrently using multiple RPCs
29
+ # will be estimated from the table's row count, if not explicitly set
30
+ num_splits: Optional[int] = None
31
+
32
+ # each endpoint will be handled by a separate worker thread
33
+ # a single endpoint can be specified more than once to benefit from multithreaded execution
34
+ data_endpoints: Optional[List[str]] = None
35
+
36
+ # a subsplit fiber will finish after sending this number of rows back to the client
37
+ limit_rows_per_sub_split: int = 128 * 1024
38
+
39
+ # each fiber will read the following number of rowgroups coninuously before skipping
40
+ # in order to use semi-sorted projections this value must be 8 (this is the hard coded size of a row groups per row block).
41
+ num_row_groups_per_sub_split: int = 8
42
+
43
+ # can be disabled for benchmarking purposes
44
+ use_semi_sorted_projections: bool = True
45
+
46
+ # enforce using a specific semi-sorted projection (if enabled above)
47
+ semi_sorted_projection_name: Optional[str] = None
48
+
49
+ # used to estimate the number of splits, given the table rows' count
50
+ rows_per_split: int = 4000000
51
+
52
+ # used for worker threads' naming
53
+ query_id: str = ""
54
+
55
+ # non-negative integer, used for server-side prioritization of queued requests:
56
+ # - requests with lower values will be served before requests with higher values.
57
+ # - if unset, the request will be added to the queue's end.
58
+ queue_priority: Optional[int] = None
59
+
60
+
61
+ @dataclass
62
+ class ImportConfig:
63
+ """Import execution configiration."""
64
+
65
+ import_concurrency: int = 2
@@ -3,7 +3,6 @@ import xml.etree.ElementTree
3
3
  from dataclasses import dataclass
4
4
  from enum import Enum
5
5
 
6
- import pyarrow as pa
7
6
  import requests
8
7
 
9
8
 
@@ -170,11 +169,6 @@ class NotSupportedCommand(NotSupported):
170
169
  table: str
171
170
 
172
171
 
173
- @dataclass
174
- class NotSupportedType(NotSupported):
175
- field: pa.Field
176
-
177
-
178
172
  @dataclass
179
173
  class NotSupportedVersion(NotSupported):
180
174
  err_msg: str
@@ -0,0 +1,42 @@
1
+ """Version-dependent features."""
2
+
3
+ import logging
4
+
5
+ from .errors import NotSupportedVersion
6
+
7
+ log = logging.getLogger()
8
+
9
+
10
+ class Features:
11
+ """VAST database features - check if server is already support a feature."""
12
+
13
+ def __init__(self, vast_version):
14
+ """Save the server version."""
15
+ self.vast_version = vast_version
16
+
17
+ self.check_imports_table = self._check(
18
+ "Imported objects' table feature requires 5.2+ VAST release",
19
+ vast_version >= (5, 2))
20
+
21
+ self.check_return_row_ids = self._check(
22
+ "Returning row IDs requires 5.1+ VAST release",
23
+ vast_version >= (5, 1))
24
+
25
+ self.check_enforce_semisorted_projection = self._check(
26
+ "Semi-sorted projection enforcement requires 5.1+ VAST release",
27
+ vast_version >= (5, 1))
28
+
29
+ self.check_external_row_ids_allocation = self._check(
30
+ "External row IDs allocation requires 5.1+ VAST release",
31
+ vast_version >= (5, 1))
32
+
33
+ def _check(self, msg, supported):
34
+ log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
35
+ if not supported:
36
+ def fail():
37
+ raise NotSupportedVersion(msg, self.vast_version)
38
+ return fail
39
+
40
+ def noop():
41
+ pass
42
+ return noop
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional
10
10
 
11
11
  import pyarrow as pa
12
12
 
13
- from . import bucket, errors, schema, table, util
13
+ from . import bucket, errors, schema, table
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from .table import Table
@@ -86,7 +86,6 @@ class Schema:
86
86
  if use_external_row_ids_allocation:
87
87
  self.tx._rpc.features.check_external_row_ids_allocation()
88
88
 
89
- util.check_supported_types(columns)
90
89
  self.tx._rpc.api.create_table(self.bucket.name, self.name, table_name, columns, txid=self.tx.txid,
91
90
  use_external_row_ids_allocation=use_external_row_ids_allocation)
92
91
  log.info("Created table: %s", table_name)
@@ -7,51 +7,11 @@ For more details see:
7
7
  - [Tabular identity policy with the proper permissions](https://support.vastdata.com/s/article/UUID-14322b60-d6a2-89ac-3df0-3dfbb6974182)
8
8
  """
9
9
 
10
- import logging
11
10
  import os
12
- from typing import Optional
11
+ from typing import TYPE_CHECKING, Optional
13
12
 
14
- import boto3
15
-
16
- from . import _internal, errors, transaction
17
- from ._internal import BackoffConfig
18
-
19
- log = logging.getLogger()
20
-
21
-
22
- class Features:
23
- """VAST database features - check if server is already support a feature."""
24
-
25
- def __init__(self, vast_version):
26
- """Save the server version."""
27
- self.vast_version = vast_version
28
-
29
- self.check_imports_table = self._check(
30
- "Imported objects' table feature requires 5.2+ VAST release",
31
- vast_version >= (5, 2))
32
-
33
- self.check_return_row_ids = self._check(
34
- "Returning row IDs requires 5.1+ VAST release",
35
- vast_version >= (5, 1))
36
-
37
- self.check_enforce_semisorted_projection = self._check(
38
- "Semi-sorted projection enforcement requires 5.1+ VAST release",
39
- vast_version >= (5, 1))
40
-
41
- self.check_external_row_ids_allocation = self._check(
42
- "External row IDs allocation requires 5.1+ VAST release",
43
- vast_version >= (5, 1))
44
-
45
- def _check(self, msg, supported):
46
- log.debug("%s (current version is %s): supported=%s", msg, self.vast_version, supported)
47
- if not supported:
48
- def fail():
49
- raise errors.NotSupportedVersion(msg, self.vast_version)
50
- return fail
51
-
52
- def noop():
53
- pass
54
- return noop
13
+ if TYPE_CHECKING:
14
+ from .config import BackoffConfig
55
15
 
56
16
 
57
17
  class Session:
@@ -60,8 +20,13 @@ class Session:
60
20
  def __init__(self, access=None, secret=None, endpoint=None,
61
21
  *,
62
22
  ssl_verify=True,
63
- backoff_config: Optional[BackoffConfig] = None):
23
+ timeout=None,
24
+ backoff_config: Optional["BackoffConfig"] = None):
64
25
  """Connect to a VAST Database endpoint, using specified credentials."""
26
+ import boto3
27
+
28
+ from . import _internal, features
29
+
65
30
  if access is None:
66
31
  access = os.environ['AWS_ACCESS_KEY_ID']
67
32
  if secret is None:
@@ -74,8 +39,9 @@ class Session:
74
39
  access_key=access,
75
40
  secret_key=secret,
76
41
  ssl_verify=ssl_verify,
42
+ timeout=timeout,
77
43
  backoff_config=backoff_config)
78
- self.features = Features(self.api.vast_version)
44
+ self.features = features.Features(self.api.vast_version)
79
45
  self.s3 = boto3.client('s3',
80
46
  aws_access_key_id=access,
81
47
  aws_secret_access_key=secret,
@@ -93,4 +59,5 @@ class Session:
93
59
  with session.transaction() as tx:
94
60
  tx.bucket("bucket").create_schema("schema")
95
61
  """
62
+ from . import transaction
96
63
  return transaction.Transaction(self)