deeplake 3.8.0__tar.gz → 3.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (362) hide show
  1. {deeplake-3.8.0/deeplake.egg-info → deeplake-3.8.2}/PKG-INFO +1 -1
  2. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/__init__.py +1 -1
  3. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api.py +11 -2
  4. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api_with_compression.py +2 -2
  5. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_views.py +58 -0
  6. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/constants.py +2 -2
  7. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk_engine.py +28 -4
  8. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compression.py +5 -16
  9. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/dataset.py +26 -4
  10. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_dataset.py +9 -0
  11. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_tensor.py +7 -0
  12. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tensor.py +10 -14
  13. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_deeplake_indra_dataset.py +16 -0
  14. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform_dataset.py +19 -6
  15. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deep_memory.py +1 -1
  16. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deeplake_vectorstore.py +18 -14
  17. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +24 -0
  18. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/index.py +3 -19
  19. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/utils.py +2 -0
  20. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/dataloader.py +70 -19
  21. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/htype.py +1 -1
  22. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -0
  23. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/common.txt +1 -1
  24. deeplake-3.8.2/deeplake/util/bugout_token.py +1 -0
  25. {deeplake-3.8.0 → deeplake-3.8.2/deeplake.egg-info}/PKG-INFO +1 -1
  26. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/requires.txt +11 -11
  27. {deeplake-3.8.0 → deeplake-3.8.2}/setup.py +1 -1
  28. deeplake-3.8.0/deeplake/util/bugout_token.py +0 -1
  29. {deeplake-3.8.0 → deeplake-3.8.2}/LICENSE +0 -0
  30. {deeplake-3.8.0 → deeplake-3.8.2}/MANIFEST.in +0 -0
  31. {deeplake-3.8.0 → deeplake-3.8.2}/README.md +0 -0
  32. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/__init__.py +0 -0
  33. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/dataset.py +0 -0
  34. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/info.py +0 -0
  35. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/link.py +0 -0
  36. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/link_tiled.py +0 -0
  37. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/read.py +0 -0
  38. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/__init__.py +0 -0
  39. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_access_method.py +0 -0
  40. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_agreement.py +0 -0
  41. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api_tiling.py +0 -0
  42. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  43. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_connect_datasets.py +0 -0
  44. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_dataset.py +0 -0
  45. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_dicom.py +0 -0
  46. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_downsample.py +0 -0
  47. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_events.py +0 -0
  48. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_grayscale.py +0 -0
  49. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  50. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_info.py +0 -0
  51. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  52. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_json.py +0 -0
  53. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_link.py +0 -0
  54. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_link_tiled.py +0 -0
  55. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_linking.py +0 -0
  56. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_mesh.py +0 -0
  57. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_meta.py +0 -0
  58. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_nifti.py +0 -0
  59. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_none.py +0 -0
  60. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_partial_upload.py +0 -0
  61. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_pickle.py +0 -0
  62. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_point_cloud.py +0 -0
  63. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_polygons.py +0 -0
  64. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_pop.py +0 -0
  65. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_readonly.py +0 -0
  66. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_rechunk.py +0 -0
  67. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_reset.py +0 -0
  68. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_sample_info.py +0 -0
  69. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_text.py +0 -0
  70. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_update_samples.py +0 -0
  71. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_video.py +0 -0
  72. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tiled.py +0 -0
  73. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/__init__.py +0 -0
  74. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/__init__.py +0 -0
  75. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/base.py +0 -0
  76. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/dataframe.py +0 -0
  77. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/__init__.py +0 -0
  78. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_coco_template.py +0 -0
  79. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_ingestion.py +0 -0
  80. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_kaggle.py +0 -0
  81. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_yolo_template.py +0 -0
  82. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/__init__.py +0 -0
  83. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/base.py +0 -0
  84. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  85. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/coco.py +0 -0
  86. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/constants.py +0 -0
  87. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/convert.py +0 -0
  88. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/utils.py +0 -0
  89. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/image_classification.py +0 -0
  90. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/kaggle.py +0 -0
  91. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/util.py +0 -0
  92. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  93. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  94. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  95. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  96. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/__init__.py +0 -0
  97. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/auth.py +0 -0
  98. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/commands.py +0 -0
  99. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/test_cli.py +0 -0
  100. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/__init__.py +0 -0
  101. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/client.py +0 -0
  102. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/config.py +0 -0
  103. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/log.py +0 -0
  104. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/test_client.py +0 -0
  105. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/utils.py +0 -0
  106. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/compression.py +0 -0
  107. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/__init__.py +0 -0
  108. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/__init__.py +0 -0
  109. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/base_chunk.py +0 -0
  110. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  111. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  112. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  113. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  114. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_uncompressed.py +0 -0
  115. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  116. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/__init__.py +0 -0
  117. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/process.py +0 -0
  118. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/provider.py +0 -0
  119. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/ray.py +0 -0
  120. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/serial.py +0 -0
  121. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/thread.py +0 -0
  122. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/__init__.py +0 -0
  123. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  124. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/invalid_view.py +0 -0
  125. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/view_entry.py +0 -0
  126. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/distance_type.py +0 -0
  127. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/fast_forwarding.py +0 -0
  128. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/index/__init__.py +0 -0
  129. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/index/index.py +0 -0
  130. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/io.py +0 -0
  131. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/ipc.py +0 -0
  132. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/link_creds.py +0 -0
  133. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_chunk_engine.py +0 -0
  134. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_sample.py +0 -0
  135. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_tiled_sample.py +0 -0
  136. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/lock.py +0 -0
  137. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/__init__.py +0 -0
  138. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/dataset_meta.py +0 -0
  139. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/__init__.py +0 -0
  140. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/base_encoder.py +0 -0
  141. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/byte_positions.py +0 -0
  142. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/chunk_id.py +0 -0
  143. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/creds.py +0 -0
  144. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/pad.py +0 -0
  145. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/sequence.py +0 -0
  146. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/shape.py +0 -0
  147. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  148. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/common.py +0 -0
  149. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  150. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  151. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  152. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  153. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  154. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tile.py +0 -0
  155. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/meta.py +0 -0
  156. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/tensor_meta.py +0 -0
  157. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/partial_reader.py +0 -0
  158. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/partial_sample.py +0 -0
  159. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/polygon.py +0 -0
  160. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/__init__.py +0 -0
  161. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/autocomplete.py +0 -0
  162. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/filter.py +0 -0
  163. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/query.py +0 -0
  164. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/sample.py +0 -0
  165. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/seed.py +0 -0
  166. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/serialize.py +0 -0
  167. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/__init__.py +0 -0
  168. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/azure.py +0 -0
  169. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  170. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/gcs.py +0 -0
  171. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/google_drive.py +0 -0
  172. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/local.py +0 -0
  173. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/lru_cache.py +0 -0
  174. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/memory.py +0 -0
  175. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/provider.py +0 -0
  176. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/s3.py +0 -0
  177. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tensor_link.py +0 -0
  178. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/test_serialize.py +0 -0
  179. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/__init__.py +0 -0
  180. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_compression.py +0 -0
  181. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_compute.py +0 -0
  182. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_io.py +0 -0
  183. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_locking.py +0 -0
  184. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_readonly.py +0 -0
  185. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_seed.py +0 -0
  186. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_serialize.py +0 -0
  187. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_vdb_indexes.py +0 -0
  188. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/__init__.py +0 -0
  189. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/deserialize.py +0 -0
  190. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/optimizer.py +0 -0
  191. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/sample_tiles.py +0 -0
  192. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/serialize.py +0 -0
  193. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/test_optimizer.py +0 -0
  194. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/test_serialize.py +0 -0
  195. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/__init__.py +0 -0
  196. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/test_transform.py +0 -0
  197. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform.py +0 -0
  198. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform_tensor.py +0 -0
  199. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/__init__.py +0 -0
  200. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deepmemory_vectorstore.py +0 -0
  201. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deepmemory.py +0 -0
  202. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  203. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  204. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  205. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  206. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  207. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  208. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  209. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  210. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  211. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  212. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  213. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  214. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  215. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  216. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  217. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  218. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  219. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  220. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  221. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  222. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  223. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  224. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  225. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  226. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vectorstore_factory.py +0 -0
  227. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/__init__.py +0 -0
  228. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  229. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_diff.py +0 -0
  230. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_node.py +0 -0
  231. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/dataset_diff.py +0 -0
  232. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/test_merge.py +0 -0
  233. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/test_version_control.py +0 -0
  234. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/__init__.py +0 -0
  235. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  236. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/dummy_dataloader.py +0 -0
  237. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/libdeeplake_query.py +0 -0
  238. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_pytorch.py +0 -0
  239. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_query.py +0 -0
  240. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_tensorflow.py +0 -0
  241. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/util.py +0 -0
  242. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/hooks.py +0 -0
  243. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/__init__.py +0 -0
  244. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/huggingface/__init__.py +0 -0
  245. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/huggingface/huggingface.py +0 -0
  246. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/__init__.py +0 -0
  247. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  248. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  249. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  250. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/__init__.py +0 -0
  251. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/common.py +0 -0
  252. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/dataset.py +0 -0
  253. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/pytorch.py +0 -0
  254. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/__init__.py +0 -0
  255. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/common.py +0 -0
  256. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  257. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  258. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/wandb/__init__.py +0 -0
  259. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/wandb/wandb.py +0 -0
  260. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/docs.txt +0 -0
  261. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/plugins.txt +0 -0
  262. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/tests.txt +0 -0
  263. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/__init__.py +0 -0
  264. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/cache_fixtures.py +0 -0
  265. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/client_fixtures.py +0 -0
  266. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/common.py +0 -0
  267. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/dataset_fixtures.py +0 -0
  268. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/path_fixtures.py +0 -0
  269. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/storage_fixtures.py +0 -0
  270. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/__init__.py +0 -0
  271. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/access_method.py +0 -0
  272. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/agreement.py +0 -0
  273. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/array_list.py +0 -0
  274. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/assert_byte_indexes.py +0 -0
  275. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/auto.py +0 -0
  276. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/bugout_reporter.py +0 -0
  277. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/cache_chain.py +0 -0
  278. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/casting.py +0 -0
  279. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/check_installation.py +0 -0
  280. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/check_latest_version.py +0 -0
  281. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/chunk_engine.py +0 -0
  282. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/class_label.py +0 -0
  283. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/compression.py +0 -0
  284. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/compute.py +0 -0
  285. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/connect_dataset.py +0 -0
  286. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/dataset.py +0 -0
  287. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/delete_entry.py +0 -0
  288. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/diff.py +0 -0
  289. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/downsample.py +0 -0
  290. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/empty_sample.py +0 -0
  291. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/encoder.py +0 -0
  292. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/exceptions.py +0 -0
  293. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/exif.py +0 -0
  294. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/from_tfds.py +0 -0
  295. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/generate_id.py +0 -0
  296. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/hash.py +0 -0
  297. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/htype.py +0 -0
  298. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/image.py +0 -0
  299. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/invalid_view_op.py +0 -0
  300. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/iterable_ordered_dict.py +0 -0
  301. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/iteration_warning.py +0 -0
  302. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/join_chunks.py +0 -0
  303. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/json.py +0 -0
  304. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/keys.py +0 -0
  305. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/link.py +0 -0
  306. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/logging.py +0 -0
  307. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/merge.py +0 -0
  308. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/modified.py +0 -0
  309. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/notebook.py +0 -0
  310. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/__init__.py +0 -0
  311. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh.py +0 -0
  312. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_parser.py +0 -0
  313. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_reader.py +0 -0
  314. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/object_base_3d.py +0 -0
  315. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser.py +0 -0
  316. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  317. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parsers.py +0 -0
  318. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader.py +0 -0
  319. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  320. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_readers.py +0 -0
  321. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/point_cloud.py +0 -0
  322. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/read_3d_data.py +0 -0
  323. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/path.py +0 -0
  324. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/pretty_print.py +0 -0
  325. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/remove_cache.py +0 -0
  326. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/scheduling.py +0 -0
  327. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/shape_interval.py +0 -0
  328. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/shuffle.py +0 -0
  329. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/spinner.py +0 -0
  330. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/split.py +0 -0
  331. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/storage.py +0 -0
  332. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tag.py +0 -0
  333. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tensor_db.py +0 -0
  334. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/testing.py +0 -0
  335. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/__init__.py +0 -0
  336. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_auto.py +0 -0
  337. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_connect_dataset.py +0 -0
  338. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  339. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_keys.py +0 -0
  340. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_read.py +0 -0
  341. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_shape_interval.py +0 -0
  342. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_shuffle.py +0 -0
  343. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_split.py +0 -0
  344. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_tensor_db.py +0 -0
  345. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_token.py +0 -0
  346. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_version_control.py +0 -0
  347. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/threading.py +0 -0
  348. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/token.py +0 -0
  349. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/transform.py +0 -0
  350. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/version_control.py +0 -0
  351. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/video.py +0 -0
  352. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/warnings.py +0 -0
  353. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/__init__.py +0 -0
  354. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/video_streaming.py +0 -0
  355. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/visualizer.py +0 -0
  356. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/SOURCES.txt +0 -0
  357. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/dependency_links.txt +0 -0
  358. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/entry_points.txt +0 -0
  359. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/not-zip-safe +0 -0
  360. {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/top_level.txt +0 -0
  361. {deeplake-3.8.0 → deeplake-3.8.2}/pyproject.toml +0 -0
  362. {deeplake-3.8.0 → deeplake-3.8.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.8.0
3
+ Version: 3.8.2
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -87,7 +87,7 @@ __all__ = [
87
87
  ]
88
88
 
89
89
 
90
- __version__ = "3.8.0"
90
+ __version__ = "3.8.2"
91
91
  warn_if_update_required(__version__)
92
92
  __encoded_version__ = np.array(__version__)
93
93
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -2680,12 +2680,12 @@ def test_tensor_dtype_bug(local_path):
2680
2680
  ds.abc.append(deeplake.link(f"{data_path}/standard.nii.gz"))
2681
2681
 
2682
2682
  assert ds.abc[0].numpy().shape == (4, 5, 7)
2683
- assert ds.abc.dtype == np.dtype("<U1")
2683
+ assert ds.abc.dtype == np.dtype("uint8")
2684
2684
 
2685
2685
  ds2 = ds.copy(f"{local_path}_2", overwrite=True)
2686
2686
 
2687
2687
  assert ds2.abc[0].numpy().shape == (4, 5, 7)
2688
- assert ds2.abc.dtype == np.dtype("<U1")
2688
+ assert ds2.abc.dtype == np.dtype("uint8")
2689
2689
 
2690
2690
 
2691
2691
  def test_iterate_with_groups(memory_ds):
@@ -3043,3 +3043,12 @@ def test_change_htype_fail(local_ds_generator):
3043
3043
  ds.create_tensor("images3", htype="image", sample_compression="jpg")
3044
3044
  with pytest.raises(UnsupportedCompressionError):
3045
3045
  ds.images3.htype = "embedding"
3046
+
3047
+
3048
+ def test_append_non_uint8_to_image(local_ds):
3049
+ with local_ds as ds:
3050
+ ds.create_tensor("images", htype="image", sample_compression="png")
3051
+ ds.images.append(np.zeros((40, 40, 1), dtype=np.int16))
3052
+ ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
3053
+
3054
+ assert ds.images.dtype.name == "int16"
@@ -43,7 +43,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
43
43
  tiling_threshold=1 * MB,
44
44
  )
45
45
 
46
- assert images.meta.dtype == "uint8"
46
+ assert images.meta.dtype == None
47
47
  assert images.meta.sample_compression == "png"
48
48
 
49
49
  _populate_compressed_samples(images, cat_path, flower_path)
@@ -73,7 +73,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
73
73
  def test_iterate_compressed_samples(local_ds, cat_path, flower_path):
74
74
  images = local_ds.create_tensor(TENSOR_KEY, htype="image", sample_compression="png")
75
75
 
76
- assert images.meta.dtype == "uint8"
76
+ assert images.meta.dtype == None
77
77
  assert images.meta.sample_compression == "png"
78
78
 
79
79
  _populate_compressed_samples(images, cat_path, flower_path)
@@ -175,3 +175,61 @@ def test_save_view_ignore_errors(local_ds):
175
175
  assert view.images.shape == (8, 30, 20, 3)
176
176
 
177
177
  np.testing.assert_array_equal(view.labels.numpy(), np.array([[0]] * 8))
178
+
179
+
180
+ @pytest.mark.parametrize("optimize_first_view", [True, False])
181
+ @pytest.mark.parametrize("optimize_second_view", [True, False])
182
+ def test_save_view_of_view(
183
+ local_ds_generator, optimize_first_view, optimize_second_view
184
+ ):
185
+ with local_ds_generator() as ds:
186
+ ds.create_tensor("abc")
187
+ ds.abc.extend(list(range(100)))
188
+
189
+ ds.commit()
190
+
191
+ ds[:20].save_view(id="first_20", optimize=optimize_first_view)
192
+
193
+ view = ds.load_view("first_20")
194
+
195
+ view[:10].save_view(id="first_10", optimize=optimize_second_view)
196
+ view[10:].save_view(id="second_10", optimize=optimize_second_view)
197
+
198
+ first_10, second_10 = ds.load_view("first_10"), ds.load_view("second_10")
199
+ np.testing.assert_array_equal(first_10.abc.numpy(), ds[:10].abc.numpy())
200
+ np.testing.assert_array_equal(second_10.abc.numpy(), ds[10:20].abc.numpy())
201
+
202
+ with local_ds_generator() as ds:
203
+ first_10, second_10 = ds.load_view("first_10"), ds.load_view("second_10")
204
+ np.testing.assert_array_equal(first_10.abc.numpy(), ds[:10].abc.numpy())
205
+ np.testing.assert_array_equal(second_10.abc.numpy(), ds[10:20].abc.numpy())
206
+
207
+
208
+ @pytest.mark.parametrize("optimize_first_view", [True, False])
209
+ @pytest.mark.parametrize("optimize_second_view", [True, False])
210
+ @pytest.mark.parametrize("optimize_third_view", [True, False])
211
+ def test_save_view_of_view_of_view(
212
+ local_ds_generator, optimize_first_view, optimize_second_view, optimize_third_view
213
+ ):
214
+ with local_ds_generator() as ds:
215
+ ds.create_tensor("abc")
216
+ ds.abc.extend(list(range(100)))
217
+
218
+ ds.commit()
219
+
220
+ ds[:20].save_view(id="first_20", optimize=optimize_first_view)
221
+
222
+ view = ds.load_view("first_20")
223
+
224
+ view[:10].save_view(id="first_10", optimize=optimize_second_view)
225
+
226
+ first_10 = ds.load_view("first_10")
227
+
228
+ first_10[:5].save_view(id="first_5", optimize=optimize_third_view)
229
+
230
+ first_5 = ds.load_view("first_5")
231
+ np.testing.assert_array_equal(first_5.abc.numpy(), ds[:5].abc.numpy())
232
+
233
+ with local_ds_generator() as ds:
234
+ first_5 = ds.load_view("first_5")
235
+ np.testing.assert_array_equal(first_5.abc.numpy(), ds[:5].abc.numpy())
@@ -324,7 +324,7 @@ DEFAULT_VECTORSTORE_INDEX_PARAMS = {
324
324
  "threshold": -1,
325
325
  "distance_metric": DEFAULT_VECTORSTORE_DISTANCE_METRIC,
326
326
  "additional_params": {
327
- "efConstruction": 200,
328
- "M": 16,
327
+ "efConstruction": 600,
328
+ "M": 32,
329
329
  },
330
330
  }
@@ -15,7 +15,7 @@ from typing import (
15
15
  )
16
16
  from deeplake.api.info import Info
17
17
  from deeplake.core.link_creds import LinkCreds
18
- from deeplake.core.linked_sample import LinkedSample
18
+ from deeplake.core.linked_sample import LinkedSample, read_linked_sample
19
19
  from deeplake.core.meta.encode.base_encoder import LAST_SEEN_INDEX_COLUMN
20
20
  from deeplake.core.serialize import HEADER_SIZE_BYTES, text_to_bytes
21
21
  from deeplake.core.tensor_link import (
@@ -692,9 +692,33 @@ class ChunkEngine:
692
692
  if tensor_meta.htype is None and not all_empty:
693
693
  tensor_meta.set_htype(get_htype(samples))
694
694
  if tensor_meta.dtype is None and not all_empty:
695
- tensor_meta.set_dtype(
696
- get_dtype(next(filter(lambda x: x is not None, samples)))
697
- ) # first non empty sample
695
+ if tensor_meta.is_link:
696
+ try:
697
+ # download one sample to get dtype
698
+ sample = next(filter(lambda x: x is not None, samples))
699
+ assert isinstance(
700
+ sample, LinkedSample
701
+ ), "Sample must be LinkedSample"
702
+ dtype = np.dtype(
703
+ read_linked_sample(
704
+ sample.path, sample.creds_key, self.link_creds, True
705
+ )._typestr
706
+ )
707
+ except:
708
+ # assume uint8 if download fails
709
+ dtype = np.dtype("uint8")
710
+ else:
711
+ non_empty_samples = list(filter(lambda x: x is not None, samples))
712
+ for sample in non_empty_samples:
713
+ try:
714
+ dtype = get_dtype(sample)
715
+ break
716
+ except:
717
+ pass
718
+ else:
719
+ if not ignore_errors:
720
+ raise ValueError("Could not determine dtype of samples")
721
+ tensor_meta.set_dtype(dtype)
698
722
  if self._convert_to_list(samples):
699
723
  samples = list(samples)
700
724
  if self._is_temp_label_tensor:
@@ -25,11 +25,12 @@ from pathlib import Path
25
25
  from PIL import Image # type: ignore
26
26
  from io import BytesIO
27
27
 
28
+ import lz4.frame # type: ignore
29
+ import lz4.block # type: ignore
28
30
  import mmap
29
31
  import struct
30
32
  import sys
31
33
  import re
32
- import numcodecs.lz4 # type: ignore
33
34
  from numpy.core.fromnumeric import compress # type: ignore
34
35
  import math
35
36
  from pathlib import Path
@@ -42,12 +43,6 @@ try:
42
43
  except ImportError:
43
44
  _PYAV_INSTALLED = False
44
45
 
45
- try:
46
- import lz4.frame # type: ignore
47
-
48
- _LZ4_INSTALLED = True
49
- except ImportError:
50
- _LZ4_INSTALLED = False
51
46
 
52
47
  try:
53
48
  import nibabel as nib # type: ignore
@@ -157,7 +152,7 @@ def compress_bytes(
157
152
  if not buffer:
158
153
  return b""
159
154
  if compression == "lz4":
160
- return numcodecs.lz4.compress(buffer)
155
+ return lz4.block.compress(buffer)
161
156
  else:
162
157
  raise SampleCompressionError(
163
158
  (len(buffer),), compression, f"Not a byte compression: {compression}"
@@ -173,15 +168,9 @@ def decompress_bytes(
173
168
  # weird edge case of lz4 + empty string
174
169
  if buffer == b"\x00\x00\x00\x00\x00":
175
170
  return b""
176
- if (
177
- buffer[:4] == b'\x04"M\x18'
178
- ): # python-lz4 magic number (backward compatiblity)
179
- if not _LZ4_INSTALLED:
180
- raise ModuleNotFoundError(
181
- "Module lz4 not found. Install using `pip install lz4`."
182
- )
171
+ if buffer[:4] == b'\x04"M\x18': # python-lz4 magic number
183
172
  return lz4.frame.decompress(buffer)
184
- return numcodecs.lz4.decompress(buffer)
173
+ return lz4.block.decompress(buffer)
185
174
  else:
186
175
  raise SampleDecompressionError()
187
176
 
@@ -3361,7 +3361,7 @@ class Dataset:
3361
3361
  message: Optional[str] = None,
3362
3362
  copy: bool = False,
3363
3363
  ):
3364
- if self.has_head_changes:
3364
+ if self.has_head_changes and not self.is_optimized:
3365
3365
  raise DatasetViewSavingError(
3366
3366
  "The dataset's HEAD node has uncommitted changes. Please create a commit on"
3367
3367
  " the dataset object [ds.commit(<insert optional message>)] prior to saving the view."
@@ -3386,6 +3386,8 @@ class Dataset:
3386
3386
  if tql_query:
3387
3387
  info["tql_query"] = tql_query
3388
3388
  info["source-dataset-index"] = getattr(self, "_source_ds_idx", None)
3389
+ if not (query or tql_query):
3390
+ info["source-dataset-index"] = self.index.to_json()
3389
3391
  return info
3390
3392
 
3391
3393
  def _lock_queries_json(self):
@@ -3505,10 +3507,29 @@ class Dataset:
3505
3507
  """Saves this view under ".queries" sub directory of same storage."""
3506
3508
  info = self._get_view_info(id, message, copy)
3507
3509
  hash = info["id"]
3510
+ # creating sub-view of optimized view
3511
+ if self.is_optimized:
3512
+ view_entry = self._view_entry
3513
+ ds = view_entry._src_ds.no_view_dataset
3514
+
3515
+ if copy:
3516
+ view_info = view_entry.info
3517
+ info["source-dataset"] = ds.path
3518
+ info["source-dataset-version"] = view_info["source-dataset-version"]
3519
+ if "source-dataset-index" in view_info:
3520
+ original_idx = Index.from_json(view_info["source-dataset-index"])
3521
+ combined_idx = original_idx[self.index]
3522
+ info["source-dataset-index"] = combined_idx.to_json()
3523
+ else:
3524
+ info["source-dataset-version"] = (
3525
+ info["source-dataset-version"] or FIRST_COMMIT_ID
3526
+ )
3527
+ else:
3528
+ ds = self
3508
3529
  path = f".queries/{hash}"
3509
- vds = self._sub_ds(path, empty=True, verbose=False)
3530
+ vds = ds._sub_ds(path, empty=True, verbose=False)
3510
3531
  self._write_vds(vds, info, copy, tensors, num_workers, scheduler, ignore_errors)
3511
- self._append_to_queries_json(info)
3532
+ ds._append_to_queries_json(info)
3512
3533
  return vds
3513
3534
 
3514
3535
  def _save_view_in_path(
@@ -3683,7 +3704,8 @@ class Dataset:
3683
3704
  raise NotImplementedError(
3684
3705
  "Saving views inplace is not supported for in-memory datasets."
3685
3706
  )
3686
- if self.read_only and not base._locked_out:
3707
+
3708
+ if self.read_only and not base._locked_out and not self.is_optimized:
3687
3709
  if isinstance(self, deeplake.core.dataset.DeepLakeCloudDataset):
3688
3710
  try:
3689
3711
  with self._temp_write_access():
@@ -284,6 +284,15 @@ class DeepLakeQueryDataset(Dataset):
284
284
  def index(self):
285
285
  return self._index
286
286
 
287
+ @property
288
+ def sample_indices(self):
289
+ for t in self.tensors.values():
290
+ try:
291
+ return t.indra_tensor.indexes
292
+ except RuntimeError:
293
+ pass
294
+ return range(self.num_samples)
295
+
287
296
  def _tensors(
288
297
  self, include_hidden: bool = True, include_disabled=True
289
298
  ) -> Dict[str, Tensor]:
@@ -137,6 +137,13 @@ class DeepLakeQueryTensor(tensor.Tensor):
137
137
  def chunk_engine(self):
138
138
  raise NotImplementedError("Virtual tensor does not have chunk engine.")
139
139
 
140
+ @property
141
+ def sample_indices(self):
142
+ try:
143
+ return self.indra_tensor.indexes
144
+ except RuntimeError:
145
+ return range(self.num_samples)
146
+
140
147
  @property
141
148
  def shape(self):
142
149
  if (
@@ -1485,20 +1485,16 @@ class Tensor:
1485
1485
 
1486
1486
  def _regenerate_vdb_indexes(self):
1487
1487
  try:
1488
- is_embedding = self.htype == "embedding"
1489
- has_vdb_indexes = hasattr(self.meta, "vdb_indexes")
1490
- try:
1491
- vdb_index_ids_present = len(self.meta.vdb_indexes) > 0
1492
- except AttributeError:
1493
- vdb_index_ids_present = False
1494
-
1495
- if is_embedding and has_vdb_indexes and vdb_index_ids_present:
1496
- for vdb_index in self.meta.vdb_indexes:
1497
- id = vdb_index["id"]
1498
- distance = vdb_index["distance"]
1499
- self.delete_vdb_index(id)
1500
- # Recreate it back.
1501
- self.create_vdb_index(id, distance)
1488
+ vdb_indexes = self.get_vdb_indexes()
1489
+
1490
+ for vdb_index in vdb_indexes:
1491
+ self.delete_vdb_index(vdb_index["id"])
1492
+ # Recreate it back.
1493
+ self.create_vdb_index(
1494
+ vdb_index["id"],
1495
+ vdb_index["distance"],
1496
+ additional_params=vdb_index.get("additional_params", None),
1497
+ )
1502
1498
  except Exception as e:
1503
1499
  raise Exception(f"An error occurred while regenerating VDB indexes: {e}")
1504
1500
 
@@ -423,3 +423,19 @@ def test_virtual_tensors(local_auth_ds_generator):
423
423
  assert deeplake_indra_ds.score[100 - i].numpy() == [
424
424
  np.sqrt(2.0 / (i + 1) / (i + 1))
425
425
  ]
426
+
427
+ assert list(deeplake_indra_ds.sample_indices) == list(range(100))
428
+ deeplake_indra_ds = deeplake_ds.query(
429
+ "SELECT *, l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
430
+ )
431
+ assert list(deeplake_indra_ds.sample_indices) == list(reversed(range(100)))
432
+ assert list(deeplake_indra_ds.embeddings.sample_indices) == list(
433
+ reversed(range(100))
434
+ )
435
+ assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
436
+
437
+ deeplake_indra_ds = deeplake_ds.query(
438
+ "SELECT l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
439
+ )
440
+ assert list(deeplake_indra_ds.sample_indices) == list(range(100))
441
+ assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
@@ -161,7 +161,7 @@ class TransformDataset:
161
161
  # First sample in tensor
162
162
  # Flush to set meta attributes
163
163
  if dtype is None:
164
- self.flush()
164
+ self.flush(clear_on_fail=False)
165
165
  return
166
166
  sizeof_item = self._calculate_sample_size(item, dtype, htype)
167
167
  except:
@@ -201,22 +201,29 @@ class TransformDataset:
201
201
  updated_tensors[full_name] = len(items)
202
202
  tensor.items.clear()
203
203
 
204
- def _rollback(self, updated_tensors):
204
+ def _rollback(self, updated_tensors, no_dtype_tensors):
205
205
  for t in updated_tensors:
206
206
  chunk_engine = self.all_chunk_engines[t]
207
207
  num_samples = updated_tensors[t]
208
208
  for _ in range(num_samples):
209
209
  chunk_engine.pop(link_callback=chunk_engine._transform_pop_callback)
210
210
 
211
+ if t in no_dtype_tensors:
212
+ meta = chunk_engine.tensor_meta
213
+ meta.dtype = None
214
+ meta.typestr = None
215
+ meta.is_dirty = True
216
+
211
217
  def _clear(self):
212
218
  for tensor in self.data.values():
213
219
  tensor.items.clear()
214
220
  self.cache_used = 0
215
221
 
216
- def flush(self):
222
+ def flush(self, clear_on_fail=True):
217
223
  all_chunk_engines = self.all_chunk_engines
218
224
  label_temp_tensors = self.label_temp_tensors
219
225
  updated_tensors = {}
226
+ no_dtype_tensors = []
220
227
  try:
221
228
  for name, tensor in self.data.items():
222
229
  if not tensor.is_group:
@@ -225,6 +232,11 @@ class TransformDataset:
225
232
  chunk_engine = all_chunk_engines[name]
226
233
  callback = chunk_engine._transform_callback
227
234
 
235
+ meta = chunk_engine.tensor_meta
236
+ if meta.length == 0 and meta.dtype is None:
237
+ # for rolling back dtype change
238
+ no_dtype_tensors.append(name)
239
+
228
240
  if tensor.numpy_only:
229
241
  self._flush_numpy_tensor_to_chunk_engine(
230
242
  name, tensor, chunk_engine, callback, updated_tensors
@@ -234,9 +246,10 @@ class TransformDataset:
234
246
  name, tensor, chunk_engine, callback, updated_tensors
235
247
  )
236
248
  self.start_input_idx = None
249
+ self._clear()
237
250
  except Exception as e:
238
- self._rollback(updated_tensors)
251
+ self._rollback(updated_tensors, no_dtype_tensors)
252
+ if clear_on_fail:
253
+ self._clear()
239
254
  e = e.__cause__ if isinstance(e, SampleAppendError) else e # type: ignore
240
255
  raise SampleAppendError(name) from e
241
- finally:
242
- self._clear()
@@ -393,7 +393,7 @@ class DeepMemory:
393
393
  queries_data = {
394
394
  "text": queries,
395
395
  "metadata": [
396
- {"relvence": relevance_per_doc} for relevance_per_doc in relevance
396
+ {"relevance": relevance_per_doc} for relevance_per_doc in relevance
397
397
  ],
398
398
  "embedding": query_embs,
399
399
  "id": [uuid.uuid4().hex for _ in range(len(queries))],
@@ -330,13 +330,7 @@ class VectorStore:
330
330
  )
331
331
 
332
332
  assert id_ is not None
333
- utils.check_length_of_each_tensor(processed_tensors)
334
-
335
- # In Case prefilled dataset which is already having index defined
336
- # regenerate the index post ingestion.
337
- index_regeneration = False
338
- if len(self.dataset) > 0 and index.check_vdb_indexes(self.dataset):
339
- index_regeneration = True
333
+ data_length = utils.check_length_of_each_tensor(processed_tensors)
340
334
 
341
335
  dataset_utils.extend_or_ingest_dataset(
342
336
  processed_tensors=processed_tensors,
@@ -348,13 +342,7 @@ class VectorStore:
348
342
  rate_limiter=rate_limiter,
349
343
  )
350
344
 
351
- if utils.index_used(self.exec_option):
352
- index.index_cache_cleanup(self.dataset)
353
- self.distance_metric_index = index.validate_and_create_vector_index(
354
- dataset=self.dataset,
355
- index_params=self.index_params,
356
- regenerate_index=index_regeneration,
357
- )
345
+ self._update_index(regenerate_index=data_length > 0)
358
346
 
359
347
  try_flushing(self.dataset)
360
348
 
@@ -612,7 +600,11 @@ class VectorStore:
612
600
  return True
613
601
 
614
602
  dataset_utils.delete_and_without_commit(self.dataset, row_ids)
603
+
604
+ self._update_index(regenerate_index=len(row_ids) > 0 if row_ids else False)
605
+
615
606
  try_flushing(self.dataset)
607
+
616
608
  return True
617
609
 
618
610
  def update_embedding(
@@ -718,6 +710,9 @@ class VectorStore:
718
710
  )
719
711
 
720
712
  self.dataset[row_ids].update(embedding_tensor_data)
713
+
714
+ self._update_index(regenerate_index=len(row_ids) > 0 if row_ids else False)
715
+
721
716
  try_flushing(self.dataset)
722
717
 
723
718
  @staticmethod
@@ -784,5 +779,14 @@ class VectorStore:
784
779
  """Length of the dataset"""
785
780
  return len(self.dataset)
786
781
 
782
+ def _update_index(self, regenerate_index=False):
783
+ if utils.index_used(self.exec_option):
784
+ index.index_cache_cleanup(self.dataset)
785
+ self.distance_metric_index = index.validate_and_create_vector_index(
786
+ dataset=self.dataset,
787
+ index_params=self.index_params,
788
+ regenerate_index=regenerate_index,
789
+ )
790
+
787
791
 
788
792
  DeepLakeVectorStore = VectorStore
@@ -514,6 +514,7 @@ def test_search_basic(local_path, hub_cloud_dev_token):
514
514
  assert len(result) == 4
515
515
 
516
516
 
517
+ @pytest.mark.slow
517
518
  @requires_libdeeplake
518
519
  def test_index_basic(local_path, hub_cloud_dev_token):
519
520
  # Start by testing behavior without an index
@@ -545,6 +546,29 @@ def test_index_basic(local_path, hub_cloud_dev_token):
545
546
  == METRIC_TO_INDEX_METRIC[DEFAULT_VECTORSTORE_DISTANCE_METRIC]
546
547
  )
547
548
 
549
+ # Test index with sample updates
550
+ pre_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
551
+ vector_store.add(
552
+ embedding=[embeddings[0]], text=[texts[0]], metadata=[metadatas[0]]
553
+ )
554
+ post_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
555
+
556
+ assert pre_update_index == post_update_index
557
+
558
+ # Test index with sample deletion
559
+ pre_delete_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
560
+ vector_store.delete(row_ids=[len(vector_store) - 1])
561
+ post_delete_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
562
+
563
+ assert pre_delete_index == post_delete_index
564
+
565
+ # Test index with sample updating
566
+ pre_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
567
+ vector_store.update_embedding(row_ids=[0], embedding_function=embedding_fn)
568
+ post_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
569
+
570
+ assert pre_update_index == post_update_index
571
+
548
572
  # Check that distance metric throws a warning when there is an index
549
573
  with pytest.warns(None):
550
574
  vector_store.search(embedding=query_embedding, distance_metric="l1")
@@ -116,25 +116,6 @@ def validate_and_create_vector_index(dataset, index_params, regenerate_index=Fal
116
116
 
117
117
  tensors = dataset.tensors
118
118
 
119
- # TODO: BRING BACK WHEN IT IS IN USE
120
-
121
- # index_regen = False
122
- # Check if regenerate_index is true.
123
- # if regenerate_index:
124
- # for _, tensor in tensors.items():
125
- # is_embedding = utils.is_embedding_tensor(tensor)
126
- # has_vdb_indexes = hasattr(tensor.meta, "vdb_indexes")
127
- # try:
128
- # vdb_index_ids_present = len(tensor.get_vdb_indexes()) > 0
129
- # except AttributeError:
130
- # vdb_index_ids_present = False
131
-
132
- # if is_embedding and has_vdb_indexes and vdb_index_ids_present:
133
- # tensor._regenerate_vdb_indexes()
134
- # index_regen = True
135
- # if index_regen:
136
- # return
137
-
138
119
  # Check all tensors from the dataset.
139
120
  for _, tensor in tensors.items():
140
121
  is_embedding = utils.is_embedding_tensor(tensor)
@@ -159,6 +140,9 @@ def validate_and_create_vector_index(dataset, index_params, regenerate_index=Fal
159
140
  except ValueError as e:
160
141
  raise e
161
142
  elif len(vdb_indexes) > 0:
143
+ if regenerate_index:
144
+ tensor._regenerate_vdb_indexes()
145
+
162
146
  return vdb_indexes[0]["distance"]
163
147
 
164
148
  return None
@@ -158,6 +158,8 @@ def check_length_of_each_tensor(tensors):
158
158
  f"All of the tensors should have equal length. Currently tensors have different length: {tensor_lengths}"
159
159
  )
160
160
 
161
+ return tensor_length
162
+
161
163
 
162
164
  def create_tensor_to_length_str(tensors):
163
165
  tensor_lengths = "\n"