deeplake 3.9.8__tar.gz → 3.9.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (373) hide show
  1. {deeplake-3.9.8/deeplake.egg-info → deeplake-3.9.10}/PKG-INFO +64 -13
  2. {deeplake-3.9.8 → deeplake-3.9.10}/README.md +11 -10
  3. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/__init__.py +1 -1
  4. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/structured/dataframe.py +25 -10
  5. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/tests/test_ingestion.py +8 -1
  6. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/client.py +10 -16
  7. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/constants.py +1 -0
  8. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/dataset.py +9 -4
  9. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/indra_tensor_view.py +24 -12
  10. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/index_maintenance.py +40 -7
  11. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/tensor_meta.py +12 -0
  12. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tensor.py +180 -5
  13. deeplake-3.9.10/deeplake/core/tests/test_vdb_indexes.py +983 -0
  14. deeplake-3.9.10/deeplake/util/bugout_token.py +1 -0
  15. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/exceptions.py +6 -19
  16. {deeplake-3.9.8 → deeplake-3.9.10/deeplake.egg-info}/PKG-INFO +64 -13
  17. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake.egg-info/requires.txt +11 -11
  18. {deeplake-3.9.8 → deeplake-3.9.10}/setup.py +1 -1
  19. deeplake-3.9.8/deeplake/core/tests/test_vdb_indexes.py +0 -289
  20. deeplake-3.9.8/deeplake/util/bugout_token.py +0 -1
  21. {deeplake-3.9.8 → deeplake-3.9.10}/LICENSE +0 -0
  22. {deeplake-3.9.8 → deeplake-3.9.10}/MANIFEST.in +0 -0
  23. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/__init__.py +0 -0
  24. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/dataset.py +0 -0
  25. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/info.py +0 -0
  26. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/link.py +0 -0
  27. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/link_tiled.py +0 -0
  28. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/read.py +0 -0
  29. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/__init__.py +0 -0
  30. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_access_method.py +0 -0
  31. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_agreement.py +0 -0
  32. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_api.py +0 -0
  33. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_api_tiling.py +0 -0
  34. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_api_with_compression.py +0 -0
  35. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  36. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_connect_datasets.py +0 -0
  37. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_dataset.py +0 -0
  38. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_dicom.py +0 -0
  39. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_downsample.py +0 -0
  40. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_events.py +0 -0
  41. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_grayscale.py +0 -0
  42. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  43. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_info.py +0 -0
  44. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  45. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_json.py +0 -0
  46. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_link.py +0 -0
  47. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_link_tiled.py +0 -0
  48. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_linking.py +0 -0
  49. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_mesh.py +0 -0
  50. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_meta.py +0 -0
  51. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_nifti.py +0 -0
  52. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_none.py +0 -0
  53. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_partial_upload.py +0 -0
  54. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_pickle.py +0 -0
  55. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_point_cloud.py +0 -0
  56. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_polygons.py +0 -0
  57. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_pop.py +0 -0
  58. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_readonly.py +0 -0
  59. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_rechunk.py +0 -0
  60. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_reset.py +0 -0
  61. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_sample_info.py +0 -0
  62. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_tag.py +0 -0
  63. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_text.py +0 -0
  64. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_update_samples.py +0 -0
  65. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_video.py +0 -0
  66. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tests/test_views.py +0 -0
  67. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/api/tiled.py +0 -0
  68. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/__init__.py +0 -0
  69. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/structured/__init__.py +0 -0
  70. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/structured/base.py +0 -0
  71. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/tests/__init__.py +0 -0
  72. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/tests/test_coco_template.py +0 -0
  73. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/tests/test_kaggle.py +0 -0
  74. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/tests/test_yolo_template.py +0 -0
  75. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/__init__.py +0 -0
  76. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/base.py +0 -0
  77. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  78. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/coco/coco.py +0 -0
  79. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/coco/constants.py +0 -0
  80. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/coco/convert.py +0 -0
  81. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/coco/utils.py +0 -0
  82. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/image_classification.py +0 -0
  83. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/kaggle.py +0 -0
  84. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/util.py +0 -0
  85. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  86. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  87. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  88. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  89. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/__init__.py +0 -0
  90. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/auth/__init__.py +0 -0
  91. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/auth/activeloop.py +0 -0
  92. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/auth/auth_context.py +0 -0
  93. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/auth/azure.py +0 -0
  94. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/auth/test_auth_context.py +0 -0
  95. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/config.py +0 -0
  96. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/log.py +0 -0
  97. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/test_client.py +0 -0
  98. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/client/utils.py +0 -0
  99. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/compression.py +0 -0
  100. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/__init__.py +0 -0
  101. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/__init__.py +0 -0
  102. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/base_chunk.py +0 -0
  103. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  104. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  105. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  106. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  107. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/test_uncompressed.py +0 -0
  108. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  109. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/chunk_engine.py +0 -0
  110. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compression.py +0 -0
  111. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compute/__init__.py +0 -0
  112. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compute/process.py +0 -0
  113. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compute/provider.py +0 -0
  114. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compute/serial.py +0 -0
  115. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/compute/thread.py +0 -0
  116. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/__init__.py +0 -0
  117. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  118. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/indra_dataset_view.py +0 -0
  119. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/invalid_view.py +0 -0
  120. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/dataset/view_entry.py +0 -0
  121. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/distance_type.py +0 -0
  122. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/fast_forwarding.py +0 -0
  123. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/index/__init__.py +0 -0
  124. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/index/index.py +0 -0
  125. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/io.py +0 -0
  126. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/ipc.py +0 -0
  127. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/link_creds.py +0 -0
  128. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/linked_chunk_engine.py +0 -0
  129. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/linked_sample.py +0 -0
  130. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/linked_tiled_sample.py +0 -0
  131. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/lock.py +0 -0
  132. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/__init__.py +0 -0
  133. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/dataset_meta.py +0 -0
  134. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/__init__.py +0 -0
  135. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/base_encoder.py +0 -0
  136. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/byte_positions.py +0 -0
  137. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/chunk_id.py +0 -0
  138. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/creds.py +0 -0
  139. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/pad.py +0 -0
  140. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/sequence.py +0 -0
  141. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/shape.py +0 -0
  142. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  143. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/common.py +0 -0
  144. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  145. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  146. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  147. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  148. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  149. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/encode/tile.py +0 -0
  150. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/meta/meta.py +0 -0
  151. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/partial_reader.py +0 -0
  152. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/partial_sample.py +0 -0
  153. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/polygon.py +0 -0
  154. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/query/__init__.py +0 -0
  155. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/query/autocomplete.py +0 -0
  156. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/query/filter.py +0 -0
  157. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/query/query.py +0 -0
  158. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/sample.py +0 -0
  159. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/seed.py +0 -0
  160. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/serialize.py +0 -0
  161. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/__init__.py +0 -0
  162. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/azure.py +0 -0
  163. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  164. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/gcs.py +0 -0
  165. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/google_drive.py +0 -0
  166. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/indra.py +0 -0
  167. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/local.py +0 -0
  168. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/lru_cache.py +0 -0
  169. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/memory.py +0 -0
  170. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/provider.py +0 -0
  171. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/storage/s3.py +0 -0
  172. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tensor_link.py +0 -0
  173. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/test_serialize.py +0 -0
  174. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/__init__.py +0 -0
  175. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_compression.py +0 -0
  176. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_compute.py +0 -0
  177. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_dataset.py +0 -0
  178. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_indra_dataset.py +0 -0
  179. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_io.py +0 -0
  180. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_locking.py +0 -0
  181. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_query.py +0 -0
  182. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_readonly.py +0 -0
  183. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_seed.py +0 -0
  184. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tests/test_serialize.py +0 -0
  185. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/__init__.py +0 -0
  186. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/deserialize.py +0 -0
  187. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/optimizer.py +0 -0
  188. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/sample_tiles.py +0 -0
  189. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/serialize.py +0 -0
  190. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/test_optimizer.py +0 -0
  191. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/tiling/test_serialize.py +0 -0
  192. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/transform/__init__.py +0 -0
  193. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/transform/test_transform.py +0 -0
  194. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/transform/transform.py +0 -0
  195. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/transform/transform_dataset.py +0 -0
  196. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/transform/transform_tensor.py +0 -0
  197. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/__init__.py +0 -0
  198. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
  199. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
  200. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
  201. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
  202. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
  203. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
  204. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
  205. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
  206. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
  207. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
  208. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
  209. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  210. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  211. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  212. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  213. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  214. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  215. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  216. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  217. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  218. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  219. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  220. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  221. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  222. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  223. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  224. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  225. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  226. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  227. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  228. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  229. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  230. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  231. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  232. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  233. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  234. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  235. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/__init__.py +0 -0
  236. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  237. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/commit_diff.py +0 -0
  238. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/commit_node.py +0 -0
  239. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/dataset_diff.py +0 -0
  240. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/test_merge.py +0 -0
  241. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/core/version_control/test_version_control.py +0 -0
  242. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/__init__.py +0 -0
  243. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  244. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/dataloader.py +0 -0
  245. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/dummy_dataloader.py +0 -0
  246. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/libdeeplake_query.py +0 -0
  247. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/test_pytorch.py +0 -0
  248. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/test_query.py +0 -0
  249. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/test_tensorflow.py +0 -0
  250. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/enterprise/util.py +0 -0
  251. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/hooks.py +0 -0
  252. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/htype.py +0 -0
  253. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/__init__.py +0 -0
  254. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/huggingface/__init__.py +0 -0
  255. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/huggingface/huggingface.py +0 -0
  256. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/mmdet/__init__.py +0 -0
  257. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  258. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  259. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  260. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/pytorch/__init__.py +0 -0
  261. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/pytorch/common.py +0 -0
  262. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/pytorch/dataset.py +0 -0
  263. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/pytorch/pytorch.py +0 -0
  264. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  265. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/tf/__init__.py +0 -0
  266. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/tf/common.py +0 -0
  267. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  268. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  269. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/wandb/__init__.py +0 -0
  270. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/integrations/wandb/wandb.py +0 -0
  271. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/requirements/common.txt +0 -0
  272. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/requirements/docs.txt +0 -0
  273. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/requirements/plugins.txt +0 -0
  274. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/requirements/tests.txt +0 -0
  275. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/__init__.py +0 -0
  276. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/cache_fixtures.py +0 -0
  277. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/client_fixtures.py +0 -0
  278. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/common.py +0 -0
  279. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/dataset_fixtures.py +0 -0
  280. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/path_fixtures.py +0 -0
  281. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/tests/storage_fixtures.py +0 -0
  282. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/__init__.py +0 -0
  283. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/access_method.py +0 -0
  284. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/agreement.py +0 -0
  285. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/array_list.py +0 -0
  286. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/assert_byte_indexes.py +0 -0
  287. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/auto.py +0 -0
  288. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/bugout_reporter.py +0 -0
  289. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/cache_chain.py +0 -0
  290. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/casting.py +0 -0
  291. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/check_installation.py +0 -0
  292. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/check_latest_version.py +0 -0
  293. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/chunk_engine.py +0 -0
  294. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/class_label.py +0 -0
  295. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/compression.py +0 -0
  296. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/compute.py +0 -0
  297. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/connect_dataset.py +0 -0
  298. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/dataset.py +0 -0
  299. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/delete_entry.py +0 -0
  300. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/diff.py +0 -0
  301. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/downsample.py +0 -0
  302. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/empty_sample.py +0 -0
  303. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/encoder.py +0 -0
  304. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/exif.py +0 -0
  305. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/from_tfds.py +0 -0
  306. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/generate_id.py +0 -0
  307. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/hash.py +0 -0
  308. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/htype.py +0 -0
  309. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/image.py +0 -0
  310. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/invalid_view_op.py +0 -0
  311. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/iterable_ordered_dict.py +0 -0
  312. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/iteration_warning.py +0 -0
  313. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/join_chunks.py +0 -0
  314. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/json.py +0 -0
  315. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/keys.py +0 -0
  316. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/link.py +0 -0
  317. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/logging.py +0 -0
  318. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/merge.py +0 -0
  319. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/modified.py +0 -0
  320. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/notebook.py +0 -0
  321. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/__init__.py +0 -0
  322. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/mesh.py +0 -0
  323. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/mesh_parser.py +0 -0
  324. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/mesh_reader.py +0 -0
  325. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/object_base_3d.py +0 -0
  326. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_parser.py +0 -0
  327. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  328. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_parsers.py +0 -0
  329. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_reader.py +0 -0
  330. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  331. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/ply_readers.py +0 -0
  332. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/point_cloud.py +0 -0
  333. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/object_3d/read_3d_data.py +0 -0
  334. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/path.py +0 -0
  335. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/pretty_print.py +0 -0
  336. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/remove_cache.py +0 -0
  337. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/scheduling.py +0 -0
  338. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/shape_interval.py +0 -0
  339. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/shuffle.py +0 -0
  340. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/spinner.py +0 -0
  341. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/split.py +0 -0
  342. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/storage.py +0 -0
  343. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tag.py +0 -0
  344. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tensor_db.py +0 -0
  345. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/testing.py +0 -0
  346. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/__init__.py +0 -0
  347. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_auto.py +0 -0
  348. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_connect_dataset.py +0 -0
  349. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  350. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_keys.py +0 -0
  351. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_read.py +0 -0
  352. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_shape_interval.py +0 -0
  353. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_shuffle.py +0 -0
  354. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_split.py +0 -0
  355. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_tensor_db.py +0 -0
  356. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_token.py +0 -0
  357. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_version_control.py +0 -0
  358. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/tests/test_video.py +0 -0
  359. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/threading.py +0 -0
  360. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/token.py +0 -0
  361. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/transform.py +0 -0
  362. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/version_control.py +0 -0
  363. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/video.py +0 -0
  364. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/util/warnings.py +0 -0
  365. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/visualizer/__init__.py +0 -0
  366. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/visualizer/video_streaming.py +0 -0
  367. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake/visualizer/visualizer.py +0 -0
  368. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake.egg-info/SOURCES.txt +0 -0
  369. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake.egg-info/dependency_links.txt +0 -0
  370. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake.egg-info/not-zip-safe +0 -0
  371. {deeplake-3.9.8 → deeplake-3.9.10}/deeplake.egg-info/top_level.txt +0 -0
  372. {deeplake-3.9.8 → deeplake-3.9.10}/pyproject.toml +0 -0
  373. {deeplake-3.9.8 → deeplake-3.9.10}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.9.8
3
+ Version: 3.9.10
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -9,19 +9,69 @@ Project-URL: Documentation, https://docs.activeloop.ai/
9
9
  Project-URL: Source, https://github.com/activeloopai/deeplake
10
10
  Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
11
11
  Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: numpy
14
+ Requires-Dist: pillow~=10.2.0
15
+ Requires-Dist: boto3
16
+ Requires-Dist: click
17
+ Requires-Dist: pathos
18
+ Requires-Dist: humbug>=0.3.1
19
+ Requires-Dist: tqdm
20
+ Requires-Dist: lz4
21
+ Requires-Dist: pyjwt
22
+ Requires-Dist: aioboto3>=10.4.0; python_version >= "3.7" and sys_platform != "win32"
23
+ Requires-Dist: nest_asyncio; python_version >= "3.7" and sys_platform != "win32"
24
+ Requires-Dist: pydantic
25
+ Requires-Dist: libdeeplake==0.0.133
12
26
  Provides-Extra: audio
27
+ Requires-Dist: av>=8.1.0; (python_version >= "3.7" or sys_platform != "win32") and extra == "audio"
13
28
  Provides-Extra: video
29
+ Requires-Dist: av>=8.1.0; (python_version >= "3.7" or sys_platform != "win32") and extra == "video"
14
30
  Provides-Extra: av
31
+ Requires-Dist: av>=8.1.0; (python_version >= "3.7" or sys_platform != "win32") and extra == "av"
15
32
  Provides-Extra: gcp
33
+ Requires-Dist: google-cloud-storage~=1.42.0; extra == "gcp"
34
+ Requires-Dist: google-auth~=2.0.1; extra == "gcp"
35
+ Requires-Dist: google-auth-oauthlib~=0.4.5; extra == "gcp"
16
36
  Provides-Extra: azure
37
+ Requires-Dist: azure-cli; extra == "azure"
38
+ Requires-Dist: azure-identity; extra == "azure"
39
+ Requires-Dist: azure-storage-blob; extra == "azure"
17
40
  Provides-Extra: dicom
41
+ Requires-Dist: pydicom; extra == "dicom"
42
+ Requires-Dist: nibabel; extra == "dicom"
18
43
  Provides-Extra: medical
44
+ Requires-Dist: pydicom; extra == "medical"
45
+ Requires-Dist: nibabel; extra == "medical"
19
46
  Provides-Extra: visualizer
47
+ Requires-Dist: IPython; extra == "visualizer"
48
+ Requires-Dist: flask; extra == "visualizer"
20
49
  Provides-Extra: gdrive
21
- Provides-Extra: point_cloud
50
+ Requires-Dist: google-api-python-client~=2.31.0; extra == "gdrive"
51
+ Requires-Dist: oauth2client~=4.1.3; extra == "gdrive"
52
+ Requires-Dist: google-auth~=2.0.1; extra == "gdrive"
53
+ Requires-Dist: google-auth-oauthlib~=0.4.5; extra == "gdrive"
54
+ Provides-Extra: point-cloud
55
+ Requires-Dist: laspy; extra == "point-cloud"
22
56
  Provides-Extra: all
57
+ Requires-Dist: nibabel; extra == "all"
58
+ Requires-Dist: laspy; extra == "all"
59
+ Requires-Dist: google-api-python-client~=2.31.0; extra == "all"
60
+ Requires-Dist: flask; extra == "all"
61
+ Requires-Dist: av>=8.1.0; (python_version >= "3.7" or sys_platform != "win32") and extra == "all"
62
+ Requires-Dist: azure-identity; extra == "all"
63
+ Requires-Dist: google-cloud-storage~=1.42.0; extra == "all"
64
+ Requires-Dist: oauth2client~=4.1.3; extra == "all"
65
+ Requires-Dist: pydicom; extra == "all"
66
+ Requires-Dist: IPython; extra == "all"
67
+ Requires-Dist: google-auth-oauthlib~=0.4.5; extra == "all"
68
+ Requires-Dist: azure-storage-blob; extra == "all"
69
+ Requires-Dist: azure-cli; extra == "all"
70
+ Requires-Dist: google-auth~=2.0.1; extra == "all"
71
+ Requires-Dist: libdeeplake==0.0.133; extra == "all"
23
72
  Provides-Extra: enterprise
24
- License-File: LICENSE
73
+ Requires-Dist: libdeeplake==0.0.133; extra == "enterprise"
74
+ Requires-Dist: pyjwt; extra == "enterprise"
25
75
 
26
76
  <img src="https://static.scarf.sh/a.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82" /><p align="center">
27
77
  <img src="https://i.postimg.cc/rsjcWc3S/deeplake-logo.png" width="400"/>
@@ -61,7 +111,7 @@ Deep Lake is a Database for AI powered by a storage format optimized for deep-le
61
111
  1. Storing data and vectors while building LLM applications
62
112
  2. Managing datasets while training deep learning models
63
113
 
64
- Deep Lake simplifies the deployment of enterprise-grade LLM-based products by offering storage for all data types (embeddings, audio, text, videos, images, pdfs, annotations, etc.), querying and vector search, data streaming while training models at scale, data versioning and lineage, and integrations with popular tools such as LangChain, LlamaIndex, Weights & Biases, and many more. Deep Lake works with data of any size, it is serverless, and it enables you to store all of your data in your own cloud and in one place. Deep Lake is used by Intel, Bayer Radiology, Matterport, ZERO Systems, Red Cross, Yale, & Oxford.
114
+ Deep Lake simplifies the deployment of enterprise-grade LLM-based products by offering storage for all data types (embeddings, audio, text, videos, images, dicom, pdfs, annotations, [and more](https://docs.deeplake.ai/en/latest/Htypes.html)), querying and vector search, data streaming while training models at scale, data versioning and lineage, and integrations with popular tools such as LangChain, LlamaIndex, Weights & Biases, and many more. Deep Lake works with data of any size, it is serverless, and it enables you to store all of your data in your own cloud and in one place. Deep Lake is used by Intel, Bayer Radiology, Matterport, ZERO Systems, Red Cross, Yale, & Oxford.
65
115
 
66
116
  ### Deep Lake includes the following features:
67
117
 
@@ -120,19 +170,20 @@ pip3 install deeplake
120
170
 
121
171
  ### Vector Store Applications
122
172
  Using Deep Lake as a Vector Store for building LLM applications:
123
- ### - [Vector Store Quickstart](https://docs.activeloop.ai/quickstart)
124
- ### - [Vector Store Getting Started Guide](https://docs.activeloop.ai/getting-started/vector-store)
125
- ### - [Using Deep Lake with LangChain](https://docs.activeloop.ai/tutorials/vector-store/deep-lake-vector-store-in-langchain)
126
- ### - [Image Similarity Search with Deep Lake](https://docs.activeloop.ai/tutorials/vector-store/image-similarity-search)
173
+ ### - [Vector Store Quickstart](https://docs.activeloop.ai/examples/rag/quickstart)
174
+ ### - [Vector Store Tutorials](https://docs.activeloop.ai/examples/rag/tutorials)
175
+ ### - [LangChain Integration](https://docs.activeloop.ai/examples/rag/langchain-integration)
176
+ ### - [LlamaIndex Integration](https://docs.activeloop.ai/examples/rag/llamaindex-integration)
177
+ ### - [Image Similarity Search with Deep Lake](https://docs.activeloop.ai/examples/rag/tutorials/image-similarity-search)
127
178
 
128
179
 
129
180
  ### Deep Learning Applications
130
181
  Using Deep Lake for managing data while training Deep Learning models:
131
- ### - [Deep Learning Quickstart](https://docs.activeloop.ai/quickstart-dl)
132
- ### - [Deep Learning Getting Started Guide](https://docs.activeloop.ai/getting-started/deep-learning)
133
- ### - [Tutorials for Training Models](https://docs.activeloop.ai/tutorials/deep-learning/training-models)
134
- ### - [Tutorials for Creating Deep Learning Datasets](https://docs.activeloop.ai/tutorials/deep-learning/creating-datasets)
135
- ### - [Deep Learning Playbooks](https://docs.activeloop.ai/playbooks/evaluating-model-performance)
182
+ ### - [Deep Learning Quickstart](https://docs.activeloop.ai/examples/dl/quickstart)
183
+ ### - [Deep Learning Getting Started Guide](https://docs.activeloop.ai/examples/dl/guide)
184
+ ### - [Tutorials for Training Models](https://docs.activeloop.ai/examples/dl/tutorials/training-models)
185
+ ### - [Tutorials for Creating Deep Learning Datasets](https://docs.activeloop.ai/examples/dl/tutorials/creating-datasets)
186
+ ### - [Deep Learning Playbooks](https://docs.activeloop.ai/examples/dl/playbooks)
136
187
 
137
188
  ## ⚙️ Integrations
138
189
 
@@ -36,7 +36,7 @@ Deep Lake is a Database for AI powered by a storage format optimized for deep-le
36
36
  1. Storing data and vectors while building LLM applications
37
37
  2. Managing datasets while training deep learning models
38
38
 
39
- Deep Lake simplifies the deployment of enterprise-grade LLM-based products by offering storage for all data types (embeddings, audio, text, videos, images, pdfs, annotations, etc.), querying and vector search, data streaming while training models at scale, data versioning and lineage, and integrations with popular tools such as LangChain, LlamaIndex, Weights & Biases, and many more. Deep Lake works with data of any size, it is serverless, and it enables you to store all of your data in your own cloud and in one place. Deep Lake is used by Intel, Bayer Radiology, Matterport, ZERO Systems, Red Cross, Yale, & Oxford.
39
+ Deep Lake simplifies the deployment of enterprise-grade LLM-based products by offering storage for all data types (embeddings, audio, text, videos, images, dicom, pdfs, annotations, [and more](https://docs.deeplake.ai/en/latest/Htypes.html)), querying and vector search, data streaming while training models at scale, data versioning and lineage, and integrations with popular tools such as LangChain, LlamaIndex, Weights & Biases, and many more. Deep Lake works with data of any size, it is serverless, and it enables you to store all of your data in your own cloud and in one place. Deep Lake is used by Intel, Bayer Radiology, Matterport, ZERO Systems, Red Cross, Yale, & Oxford.
40
40
 
41
41
  ### Deep Lake includes the following features:
42
42
 
@@ -95,19 +95,20 @@ pip3 install deeplake
95
95
 
96
96
  ### Vector Store Applications
97
97
  Using Deep Lake as a Vector Store for building LLM applications:
98
- ### - [Vector Store Quickstart](https://docs.activeloop.ai/quickstart)
99
- ### - [Vector Store Getting Started Guide](https://docs.activeloop.ai/getting-started/vector-store)
100
- ### - [Using Deep Lake with LangChain](https://docs.activeloop.ai/tutorials/vector-store/deep-lake-vector-store-in-langchain)
101
- ### - [Image Similarity Search with Deep Lake](https://docs.activeloop.ai/tutorials/vector-store/image-similarity-search)
98
+ ### - [Vector Store Quickstart](https://docs.activeloop.ai/examples/rag/quickstart)
99
+ ### - [Vector Store Tutorials](https://docs.activeloop.ai/examples/rag/tutorials)
100
+ ### - [LangChain Integration](https://docs.activeloop.ai/examples/rag/langchain-integration)
101
+ ### - [LlamaIndex Integration](https://docs.activeloop.ai/examples/rag/llamaindex-integration)
102
+ ### - [Image Similarity Search with Deep Lake](https://docs.activeloop.ai/examples/rag/tutorials/image-similarity-search)
102
103
 
103
104
 
104
105
  ### Deep Learning Applications
105
106
  Using Deep Lake for managing data while training Deep Learning models:
106
- ### - [Deep Learning Quickstart](https://docs.activeloop.ai/quickstart-dl)
107
- ### - [Deep Learning Getting Started Guide](https://docs.activeloop.ai/getting-started/deep-learning)
108
- ### - [Tutorials for Training Models](https://docs.activeloop.ai/tutorials/deep-learning/training-models)
109
- ### - [Tutorials for Creating Deep Learning Datasets](https://docs.activeloop.ai/tutorials/deep-learning/creating-datasets)
110
- ### - [Deep Learning Playbooks](https://docs.activeloop.ai/playbooks/evaluating-model-performance)
107
+ ### - [Deep Learning Quickstart](https://docs.activeloop.ai/examples/dl/quickstart)
108
+ ### - [Deep Learning Getting Started Guide](https://docs.activeloop.ai/examples/dl/guide)
109
+ ### - [Tutorials for Training Models](https://docs.activeloop.ai/examples/dl/tutorials/training-models)
110
+ ### - [Tutorials for Creating Deep Learning Datasets](https://docs.activeloop.ai/examples/dl/tutorials/creating-datasets)
111
+ ### - [Deep Learning Playbooks](https://docs.activeloop.ai/examples/dl/playbooks)
111
112
 
112
113
  ## ⚙️ Integrations
113
114
 
@@ -89,7 +89,7 @@ __all__ = [
89
89
  ]
90
90
 
91
91
 
92
- __version__ = "3.9.8"
92
+ __version__ = "3.9.10"
93
93
  warn_if_update_required(__version__)
94
94
  __encoded_version__ = np.array(__version__)
95
95
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -10,8 +10,6 @@ from collections import defaultdict
10
10
  from typing import DefaultDict, List, Union, Optional, Dict
11
11
  from deeplake.core.sample import Sample
12
12
  from deeplake.core.linked_sample import LinkedSample
13
- import pathlib
14
-
15
13
 
16
14
  from deeplake.client.log import logger
17
15
 
@@ -30,14 +28,13 @@ class DataFrame(StructuredDataset):
30
28
  Raises:
31
29
  Exception: If source is not a pandas dataframe object.
32
30
  """
31
+
33
32
  import pandas as pd # type: ignore
34
33
 
35
34
  super().__init__(source)
36
35
  if not isinstance(self.source, pd.DataFrame):
37
36
  raise Exception("Source is not a pandas dataframe object.")
38
37
 
39
- self.source = self.source.replace({np.nan: None})
40
-
41
38
  self.creds = creds
42
39
  self.creds_key = creds_key
43
40
  self._initialize_params(column_params)
@@ -79,9 +76,14 @@ class DataFrame(StructuredDataset):
79
76
  )
80
77
  return most_frequent_image_extension
81
78
 
79
+ def _is_datetime(self, type: Union[type, np.dtype]):
80
+ return "datetime64[ns]" in str(type) or type == np.dtype(np.datetime64)
81
+
82
82
  def _parse_tensor_params(self, key: str, inspect_limit: int = 1000):
83
83
  """Parse the tensor parameters for a column. Required parameters that are not specified will be inferred by inspecting up to 'inspect_limit' rows in the data."""
84
84
 
85
+ import pandas as pd # type: ignore
86
+
85
87
  tensor_params: Dict = self.column_params[key]
86
88
 
87
89
  dtype = self.source[key].dtype
@@ -89,11 +91,12 @@ class DataFrame(StructuredDataset):
89
91
  if (
90
92
  "htype" not in tensor_params
91
93
  ): # Auto-set some typing parameters if htype is not specified
92
- if dtype == np.dtype("object"):
94
+ if dtype == np.dtype("object") or self._is_datetime(dtype):
95
+
96
+ column_data = self.source[key][0:inspect_limit]
97
+ column_data = column_data.where(pd.notnull(column_data), None).values
93
98
  types = [
94
- type(v)
95
- for v in self.source[key][0:inspect_limit].values
96
- if v is not None
99
+ type(v) for v in column_data if v is not None
97
100
  ] # Can be length 0 if all data is None
98
101
 
99
102
  if len(set(types)) > 1:
@@ -101,7 +104,7 @@ class DataFrame(StructuredDataset):
101
104
  f"Dataframe has different data types inside '{key}' column. Please make sure all data is given column is compatible with a single Deep Lake htype, or try specifying the htype manually."
102
105
  )
103
106
 
104
- if len(types) > 0 and types[0] == str:
107
+ if len(types) > 0 and (types[0] == str or self._is_datetime(types[0])):
105
108
  tensor_params.update(
106
109
  htype="text"
107
110
  ) # Use "text" htype for text data when the htype is not specified tensor_params
@@ -127,10 +130,22 @@ class DataFrame(StructuredDataset):
127
130
 
128
131
  def _get_extend_values(self, tensor_params: dict, key: str): # type: ignore
129
132
  """Method creates a list of values to be extended to the tensor, based on the tensor parameters and the data in the dataframe column"""
133
+
130
134
  import pandas as pd # type: ignore
131
135
 
132
136
  column_data = self.source[key]
133
- column_data = column_data.where(pd.notnull(column_data), None).values.tolist()
137
+
138
+ # Convert datetime arrays to strings. Other data can be uploaded as is.
139
+ if self._is_datetime(column_data.dtype):
140
+ column_data = (
141
+ column_data.where(pd.notnull(column_data), None)
142
+ .astype(str)
143
+ .values.tolist()
144
+ )
145
+ else:
146
+ column_data = column_data.where(
147
+ pd.notnull(column_data), None
148
+ ).values.tolist()
134
149
 
135
150
  extend_values: List[Optional[Union[Sample, LinkedSample, np.ndarray]]]
136
151
 
@@ -304,7 +304,7 @@ def test_dataframe_files(memory_ds: Dataset, dataframe_ingestion_data):
304
304
  assert ds[df_keys[2]][2].data()["text"][0] == df[df_keys[2]][2]
305
305
 
306
306
 
307
- def test_dataframe_array(memory_ds: Dataset):
307
+ def test_dataframe_mixed(memory_ds: Dataset):
308
308
  data = {
309
309
  "AA": ["Alice", "Bob", np.nan, None],
310
310
  "BB": [
@@ -322,8 +322,11 @@ def test_dataframe_array(memory_ds: Dataset):
322
322
  np.array([0, 56, 34]),
323
323
  ],
324
324
  "FF": [None, "Bob", "Charlie", "Dave"],
325
+ "GG": ["2024-01-01", None, "", "2024-04-01"],
325
326
  }
326
327
 
328
+ data["GG"] = pd.to_datetime(data["GG"])
329
+
327
330
  df = pd.DataFrame(data)
328
331
  df_keys = df.keys()
329
332
 
@@ -355,6 +358,10 @@ def test_dataframe_array(memory_ds: Dataset):
355
358
  assert ds[df_keys[4]][0].numpy().shape[0] == 0
356
359
  assert ds[df_keys[4]][1].numpy().shape[0] == 4
357
360
 
361
+ assert ds[df_keys[6]][0].text() in str(data["GG"][0]) # type: ignore
362
+ assert ds[df_keys[6]][2].text() == "NaT" # type: ignore
363
+ assert ds[df_keys[6]][1].text() == "NaT" # type: ignore
364
+
358
365
 
359
366
  def test_dataframe_array_bad(memory_ds: Dataset):
360
367
  data = {
@@ -1,19 +1,13 @@
1
- import os
2
-
3
1
  import deeplake
4
2
  import requests # type: ignore
5
- import textwrap
6
- from typing import Any, Optional, Dict, List, Union
3
+ from typing import Any, Optional, Dict
7
4
  from deeplake.util.exceptions import (
8
5
  AgreementNotAcceptedError,
9
6
  AuthorizationException,
10
- LoginException,
11
- InvalidPasswordException,
12
7
  ManagedCredentialsNotFoundError,
13
8
  NotLoggedInAgreementError,
14
9
  ResourceNotFoundException,
15
10
  InvalidTokenException,
16
- UserNotLoggedInException,
17
11
  TokenPermissionError,
18
12
  )
19
13
  from deeplake.client.utils import (
@@ -125,11 +119,6 @@ class DeepLakeBackendClient:
125
119
  headers["hub-cli-version"] = self.version
126
120
  headers = {**headers, **self.auth_context.get_auth_headers()}
127
121
 
128
- # clearer error than `ServerUnderMaintenence`
129
- if json is not None and "password" in json and json["password"] is None:
130
- # do NOT pass in the password here. `None` is explicitly typed.
131
- raise InvalidPasswordException("Password cannot be `None`.")
132
-
133
122
  status_code = None
134
123
  tries = 0
135
124
  while status_code is None or (status_code in retry_status_codes and tries < 3):
@@ -205,8 +194,11 @@ class DeepLakeBackendClient:
205
194
  ).json()
206
195
  except Exception as e:
207
196
  if isinstance(e, AuthorizationException):
208
- response_data = e.response.json()
209
- code = response_data.get("code")
197
+ code = -1
198
+ if e.response is not None:
199
+ response_data = e.response.json()
200
+ code = response_data.get("code")
201
+
210
202
  if code == 1:
211
203
  agreements = response_data["agreements"]
212
204
  agreements = [agreement["text"] for agreement in agreements]
@@ -215,11 +207,13 @@ class DeepLakeBackendClient:
215
207
  raise NotLoggedInAgreementError from e
216
208
  else:
217
209
  try:
218
- jwt.decode(self.token, options={"verify_signature": False})
210
+ jwt.decode(
211
+ self.get_token(), options={"verify_signature": False}
212
+ )
219
213
  except Exception:
220
214
  raise InvalidTokenException
221
215
 
222
- raise TokenPermissionError()
216
+ raise TokenPermissionError(e.original_message)
223
217
  raise
224
218
  full_url = response.get("path")
225
219
  repository = response.get("repository")
@@ -333,6 +333,7 @@ DEFAULT_VECTORSTORE_INDEX_PARAMS = {
333
333
  "additional_params": {
334
334
  "efConstruction": 600,
335
335
  "M": 32,
336
+ "partitions": 1,
336
337
  },
337
338
  }
338
339
  VECTORSTORE_EXTEND_BATCH_SIZE = 500
@@ -908,6 +908,7 @@ class Dataset:
908
908
  "point_cloud",
909
909
  "mesh",
910
910
  "nifti",
911
+ "segment_mask",
911
912
  ):
912
913
  self._create_sample_info_tensor(name)
913
914
  if create_shape_tensor and htype not in ("text", "json", "tag"):
@@ -3209,7 +3210,7 @@ class Dataset:
3209
3210
  ignore_errors: bool = False,
3210
3211
  progressbar: bool = False,
3211
3212
  ):
3212
- """Appends multiple samples (rows) to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
3213
+ """Appends multiple samples (rows) to multiple tensors at once. This method expects all tensors being updated to be of the same length.
3213
3214
 
3214
3215
  Args:
3215
3216
  samples (Dict[str, Any]): Dictionary with tensor names as keys and data as values. The values can be a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
@@ -3231,11 +3232,15 @@ class Dataset:
3231
3232
  >>> ds = deeplake.empty("../test/test_ds")
3232
3233
 
3233
3234
  >>> with ds:
3234
- >>> ds.create_tensor('data')
3235
- >>> ds.create_tensor('labels')
3235
+ >>> ds.create_tensor("data")
3236
+ >>> ds.create_tensor("labels", htype = "class_label")
3237
+ >>> ds.create_tensor("images", htype = "image", sample_compression = "jpeg")
3236
3238
 
3237
3239
  >>> # This operation will append 4 samples (rows) to the Deep Lake dataset
3238
- >>> ds.extend({"data": [1, 2, 3, 4], "labels":["table", "chair", "desk", "table"]})
3240
+ >>> ds.extend({"data": [1, 2, 3, 4],
3241
+ "labels":["table", "chair", "desk", "table"],
3242
+ "images": [deeplake.read("image1.jpg"), deeplake.read("image2.jpg"), deeplake.read("image3.jpg"), deeplake.read("image4.jpg")]
3243
+ })
3239
3244
 
3240
3245
  """
3241
3246
  extend = False
@@ -62,21 +62,33 @@ class IndraTensorView(tensor.Tensor):
62
62
 
63
63
  def text(self, fetch_chunks: bool = False):
64
64
  """Return text data. Only applicable for tensors with 'text' base htype."""
65
- bs = self.indra_tensor.bytes()
66
- if self.ndim == 1:
67
- return bs.decode()
68
- if isinstance(bs, bytes):
69
- return [bs.decode()]
70
- return list(b.decode() for b in bs)
65
+ try:
66
+ bs = self.indra_tensor.bytes()
67
+ if self.ndim == 1:
68
+ return bs.decode()
69
+ if isinstance(bs, bytes):
70
+ return [bs.decode()]
71
+ return list(b.decode() for b in bs)
72
+ except Exception as e:
73
+ bs = self.indra_tensor.numpy(aslist=True)
74
+ if self.ndim == 1:
75
+ return bs[0]
76
+ return list(b[0] for b in bs)
71
77
 
72
78
  def dict(self, fetch_chunks: bool = False):
73
79
  """Return json data. Only applicable for tensors with 'json' base htype."""
74
- bs = self.indra_tensor.bytes()
75
- if self.ndim == 1:
76
- return json.loads(bs.decode())
77
- if isinstance(bs, bytes):
78
- return [json.loads(bs.decode())]
79
- return list(json.loads(b.decode()) for b in self.indra_tensor.bytes())
80
+ try:
81
+ bs = self.indra_tensor.bytes()
82
+ if self.ndim == 1:
83
+ return json.loads(bs.decode())
84
+ if isinstance(bs, bytes):
85
+ return [json.loads(bs.decode())]
86
+ return list(json.loads(b.decode()) for b in self.indra_tensor.bytes())
87
+ except Exception as e:
88
+ bs = self.indra_tensor.numpy(aslist=True)
89
+ if self.ndim == 1:
90
+ return bs[0]
91
+ return list(b[0] for b in bs)
80
92
 
81
93
  @property
82
94
  def dtype(self):
@@ -62,6 +62,21 @@ def index_exists(dataset):
62
62
  return False
63
63
 
64
64
 
65
+ def index_partition_count(dataset):
66
+ emb_tensor = fetch_embedding_tensor(dataset)
67
+ if emb_tensor is not None:
68
+ vdb_indexes = emb_tensor.fetch_vdb_indexes()
69
+ if len(vdb_indexes) == 0:
70
+ return 1
71
+ else:
72
+ additional_params = vdb_indexes[0].get("additional_params", {})
73
+ if additional_params is None:
74
+ return 1
75
+ return additional_params.get("partitions", 1)
76
+ else:
77
+ return 1
78
+
79
+
65
80
  def index_used(exec_option):
66
81
  """Check if the index is used for the exec_option"""
67
82
  return exec_option in ("tensor_db", "compute_engine")
@@ -101,8 +116,17 @@ def check_index_params(self):
101
116
 
102
117
  existing_distance = existing_params.get("distance", "COS")
103
118
  if curr_distance == existing_distance:
104
- current_additional_params_dict = current_params.get("additional_params", None)
105
- existing_additional_params_dict = existing_params.get("additional_params", None)
119
+ current_additional_params_dict = current_params.get(
120
+ "additional_params", {}
121
+ ).copy()
122
+ existing_additional_params_dict = existing_params.get(
123
+ "additional_params", {}
124
+ ).copy()
125
+
126
+ # Remove the 'partitions' key from the copies of the dictionaries
127
+ current_additional_params_dict.pop("partitions", None)
128
+ existing_additional_params_dict.pop("partitions", None)
129
+
106
130
  if current_additional_params_dict == existing_additional_params_dict:
107
131
  return True
108
132
 
@@ -134,9 +158,9 @@ def get_index_metric(metric):
134
158
 
135
159
 
136
160
  def normalize_additional_params(params: dict) -> dict:
137
- mapping = {"efconstruction": "efConstruction", "m": "M"}
161
+ mapping = {"efconstruction": "efConstruction", "m": "M", "partitions": "partitions"}
138
162
 
139
- allowed_keys = ["efConstruction", "m"]
163
+ allowed_keys = ["efConstruction", "m", "partitions"]
140
164
 
141
165
  # New dictionary to store the result with desired key format
142
166
  result_dict = {}
@@ -180,7 +204,9 @@ def check_vdb_indexes(dataset):
180
204
  return False
181
205
 
182
206
 
183
- def _incr_maintenance_vdb_indexes(tensor, indexes, index_operation):
207
+ def _incr_maintenance_vdb_indexes(
208
+ tensor, indexes, index_operation, is_partitioned=False
209
+ ):
184
210
  try:
185
211
  is_embedding = tensor.htype == "embedding"
186
212
  has_vdb_indexes = hasattr(tensor.meta, "vdb_indexes")
@@ -194,6 +220,7 @@ def _incr_maintenance_vdb_indexes(tensor, indexes, index_operation):
194
220
  tensor.update_vdb_index(
195
221
  operation_kind=index_operation,
196
222
  row_ids=indexes,
223
+ is_partitioned=is_partitioned,
197
224
  )
198
225
  except Exception as e:
199
226
  raise Exception(f"An error occurred while regenerating VDB indexes: {e}")
@@ -247,7 +274,6 @@ def index_operation_dataset(self, dml_type, rowids):
247
274
 
248
275
  if index_operation_type == INDEX_OP_TYPE.NOOP:
249
276
  return
250
-
251
277
  if (
252
278
  index_operation_type == INDEX_OP_TYPE.CREATE_INDEX
253
279
  or index_operation_type == INDEX_OP_TYPE.REGENERATE_INDEX
@@ -272,6 +298,13 @@ def index_operation_dataset(self, dml_type, rowids):
272
298
  else:
273
299
  emb_tensor.create_vdb_index("hnsw_1", distance=distance)
274
300
  elif index_operation_type == INDEX_OP_TYPE.INCREMENTAL_INDEX:
275
- _incr_maintenance_vdb_indexes(emb_tensor, rowids, dml_type)
301
+ partition_count = index_partition_count(self)
302
+ print(f"Partition count: {partition_count}")
303
+ if partition_count > 1:
304
+ _incr_maintenance_vdb_indexes(
305
+ emb_tensor, rowids, dml_type, is_partitioned=True
306
+ )
307
+ else:
308
+ _incr_maintenance_vdb_indexes(emb_tensor, rowids, dml_type)
276
309
  else:
277
310
  raise Exception("Unknown index operation")
@@ -99,6 +99,18 @@ class TensorMeta(Meta):
99
99
  index_ids.append(index["id"])
100
100
  return index_ids
101
101
 
102
+ def update_vdb_partition(self, id: str, partition: int):
103
+ if not self.contains_vdb_index(id):
104
+ raise ValueError(f"Tensor meta has no vdb index with name '{id}'.")
105
+ for index in self.vdb_indexes:
106
+ if id == index["id"]:
107
+ additional_param = index["additional_params"]
108
+ if not isinstance(additional_param, dict):
109
+ raise ValueError("additional_params must be a dictionary")
110
+ additional_param["partitions"] = partition
111
+ self.is_dirty = True
112
+ return
113
+
102
114
  def add_vdb_index(self, id: str, type: str, distance: str, **kwargs):
103
115
  if self.contains_vdb_index(id):
104
116
  raise ValueError(f"Tensor meta already has a vdb index with name '{id}'.")