deeplake 3.6.18__tar.gz → 3.6.20__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. {deeplake-3.6.18/deeplake.egg-info → deeplake-3.6.20}/PKG-INFO +11 -7
  2. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/__init__.py +4 -1
  3. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api.py +14 -0
  4. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/constants.py +11 -0
  5. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/dataset.py +55 -10
  6. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_query_dataset.py +1 -0
  7. deeplake-3.6.20/deeplake/core/seed.py +25 -0
  8. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/azure.py +4 -1
  9. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/lru_cache.py +2 -0
  10. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_deeplake_indra_dataset.py +36 -0
  11. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/test_transform.py +4 -1
  12. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform.py +4 -2
  13. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/deeplake_vectorstore.py +2 -7
  14. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +23 -1
  15. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +2 -0
  16. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +111 -18
  17. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +64 -0
  18. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +2 -0
  19. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/utils.py +1 -1
  20. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/test_version_control.py +89 -0
  21. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/convert_to_libdeeplake.py +14 -3
  22. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/dataloader.py +22 -2
  23. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_pytorch.py +14 -0
  24. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_query.py +28 -0
  25. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/common.py +1 -1
  26. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -4
  27. deeplake-3.6.20/deeplake/util/bugout_token.py +1 -0
  28. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/exceptions.py +18 -1
  29. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/scheduling.py +4 -0
  30. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/shuffle.py +5 -0
  31. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/spinner.py +37 -29
  32. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_shuffle.py +2 -1
  33. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/transform.py +7 -2
  34. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/version_control.py +111 -0
  35. {deeplake-3.6.18 → deeplake-3.6.20/deeplake.egg-info}/PKG-INFO +11 -7
  36. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/SOURCES.txt +1 -0
  37. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/entry_points.txt +1 -0
  38. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/requires.txt +20 -20
  39. {deeplake-3.6.18 → deeplake-3.6.20}/setup.py +1 -1
  40. deeplake-3.6.18/deeplake/util/bugout_token.py +0 -1
  41. {deeplake-3.6.18 → deeplake-3.6.20}/LICENSE +0 -0
  42. {deeplake-3.6.18 → deeplake-3.6.20}/MANIFEST.in +0 -0
  43. {deeplake-3.6.18 → deeplake-3.6.20}/README.md +0 -0
  44. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/__init__.py +0 -0
  45. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/dataset.py +0 -0
  46. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/info.py +0 -0
  47. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/link.py +0 -0
  48. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/link_tiled.py +0 -0
  49. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/read.py +0 -0
  50. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/__init__.py +0 -0
  51. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_access_method.py +0 -0
  52. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_agreement.py +0 -0
  53. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api_tiling.py +0 -0
  54. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api_with_compression.py +0 -0
  55. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  56. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_connect_datasets.py +0 -0
  57. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_dataset.py +0 -0
  58. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_dicom.py +0 -0
  59. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_downsample.py +0 -0
  60. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_events.py +0 -0
  61. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_grayscale.py +0 -0
  62. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  63. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_info.py +0 -0
  64. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  65. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_json.py +0 -0
  66. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_link.py +0 -0
  67. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_link_tiled.py +0 -0
  68. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_linking.py +0 -0
  69. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_mesh.py +0 -0
  70. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_meta.py +0 -0
  71. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_nifti.py +0 -0
  72. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_none.py +0 -0
  73. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_partial_upload.py +0 -0
  74. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_pickle.py +0 -0
  75. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_point_cloud.py +0 -0
  76. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_polygons.py +0 -0
  77. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_pop.py +0 -0
  78. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_readonly.py +0 -0
  79. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_rechunk.py +0 -0
  80. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_reset.py +0 -0
  81. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_sample_info.py +0 -0
  82. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_text.py +0 -0
  83. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_update_samples.py +0 -0
  84. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_video.py +0 -0
  85. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_views.py +0 -0
  86. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tiled.py +0 -0
  87. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/__init__.py +0 -0
  88. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/__init__.py +0 -0
  89. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/base.py +0 -0
  90. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/dataframe.py +0 -0
  91. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/__init__.py +0 -0
  92. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_coco_template.py +0 -0
  93. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_ingestion.py +0 -0
  94. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_kaggle.py +0 -0
  95. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_yolo_template.py +0 -0
  96. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/__init__.py +0 -0
  97. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/base.py +0 -0
  98. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  99. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/coco.py +0 -0
  100. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/constants.py +0 -0
  101. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/convert.py +0 -0
  102. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/utils.py +0 -0
  103. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/image_classification.py +0 -0
  104. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/kaggle.py +0 -0
  105. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/util.py +0 -0
  106. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  107. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  108. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  109. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  110. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/__init__.py +0 -0
  111. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/auth.py +0 -0
  112. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/commands.py +0 -0
  113. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/test_cli.py +0 -0
  114. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/__init__.py +0 -0
  115. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/client.py +0 -0
  116. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/config.py +0 -0
  117. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/log.py +0 -0
  118. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/test_client.py +0 -0
  119. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/utils.py +0 -0
  120. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/compression.py +0 -0
  121. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/__init__.py +0 -0
  122. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/__init__.py +0 -0
  123. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/base_chunk.py +0 -0
  124. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  125. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  126. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  127. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  128. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_uncompressed.py +0 -0
  129. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  130. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk_engine.py +0 -0
  131. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compression.py +0 -0
  132. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/__init__.py +0 -0
  133. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/process.py +0 -0
  134. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/provider.py +0 -0
  135. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/ray.py +0 -0
  136. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/serial.py +0 -0
  137. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/thread.py +0 -0
  138. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/__init__.py +0 -0
  139. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  140. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
  141. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/invalid_view.py +0 -0
  142. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/view_entry.py +0 -0
  143. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/fast_forwarding.py +0 -0
  144. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/index/__init__.py +0 -0
  145. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/index/index.py +0 -0
  146. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/io.py +0 -0
  147. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/ipc.py +0 -0
  148. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/link_creds.py +0 -0
  149. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_chunk_engine.py +0 -0
  150. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_sample.py +0 -0
  151. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_tiled_sample.py +0 -0
  152. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/lock.py +0 -0
  153. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/__init__.py +0 -0
  154. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/dataset_meta.py +0 -0
  155. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/__init__.py +0 -0
  156. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/base_encoder.py +0 -0
  157. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/byte_positions.py +0 -0
  158. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/chunk_id.py +0 -0
  159. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/creds.py +0 -0
  160. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/pad.py +0 -0
  161. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/sequence.py +0 -0
  162. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/shape.py +0 -0
  163. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  164. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/common.py +0 -0
  165. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  166. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  167. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  168. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  169. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  170. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tile.py +0 -0
  171. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/meta.py +0 -0
  172. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/tensor_meta.py +0 -0
  173. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/partial_reader.py +0 -0
  174. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/partial_sample.py +0 -0
  175. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/polygon.py +0 -0
  176. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/__init__.py +0 -0
  177. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/autocomplete.py +0 -0
  178. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/filter.py +0 -0
  179. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/query.py +0 -0
  180. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/sample.py +0 -0
  181. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/serialize.py +0 -0
  182. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/__init__.py +0 -0
  183. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  184. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/gcs.py +0 -0
  185. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/google_drive.py +0 -0
  186. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/local.py +0 -0
  187. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/memory.py +0 -0
  188. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/provider.py +0 -0
  189. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/s3.py +0 -0
  190. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tensor.py +0 -0
  191. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tensor_link.py +0 -0
  192. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/test_serialize.py +0 -0
  193. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/__init__.py +0 -0
  194. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_compression.py +0 -0
  195. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_compute.py +0 -0
  196. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_io.py +0 -0
  197. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_locking.py +0 -0
  198. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_readonly.py +0 -0
  199. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_serialize.py +0 -0
  200. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/__init__.py +0 -0
  201. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/deserialize.py +0 -0
  202. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/optimizer.py +0 -0
  203. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/sample_tiles.py +0 -0
  204. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/serialize.py +0 -0
  205. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/test_optimizer.py +0 -0
  206. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/test_serialize.py +0 -0
  207. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/__init__.py +0 -0
  208. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform_dataset.py +0 -0
  209. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform_tensor.py +0 -0
  210. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/__init__.py +0 -0
  211. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  212. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  213. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  214. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  215. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  216. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  217. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  218. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  219. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  220. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  221. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  222. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  223. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  224. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  225. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  226. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  227. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  228. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  229. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  230. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  231. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/__init__.py +0 -0
  232. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  233. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_diff.py +0 -0
  234. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_node.py +0 -0
  235. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/dataset_diff.py +0 -0
  236. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/test_merge.py +0 -0
  237. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/__init__.py +0 -0
  238. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/dummy_dataloader.py +0 -0
  239. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/libdeeplake_query.py +0 -0
  240. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_tensorflow.py +0 -0
  241. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/util.py +0 -0
  242. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/hooks.py +0 -0
  243. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/htype.py +0 -0
  244. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/__init__.py +0 -0
  245. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/huggingface/__init__.py +0 -0
  246. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/huggingface/huggingface.py +0 -0
  247. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/__init__.py +0 -0
  248. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  249. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  250. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  251. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/__init__.py +0 -0
  252. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/dataset.py +0 -0
  253. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/pytorch.py +0 -0
  254. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/__init__.py +0 -0
  255. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/common.py +0 -0
  256. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  257. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  258. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/wandb/__init__.py +0 -0
  259. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/wandb/wandb.py +0 -0
  260. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/common.txt +0 -0
  261. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/docs.txt +0 -0
  262. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/plugins.txt +0 -0
  263. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/tests.txt +0 -0
  264. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/__init__.py +0 -0
  265. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/cache_fixtures.py +0 -0
  266. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/client_fixtures.py +0 -0
  267. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/common.py +0 -0
  268. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/dataset_fixtures.py +0 -0
  269. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/path_fixtures.py +0 -0
  270. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/storage_fixtures.py +0 -0
  271. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/__init__.py +0 -0
  272. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/access_method.py +0 -0
  273. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/agreement.py +0 -0
  274. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/array_list.py +0 -0
  275. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/assert_byte_indexes.py +0 -0
  276. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/auto.py +0 -0
  277. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/bugout_reporter.py +0 -0
  278. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/cache_chain.py +0 -0
  279. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/casting.py +0 -0
  280. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/check_installation.py +0 -0
  281. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/check_latest_version.py +0 -0
  282. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/chunk_engine.py +0 -0
  283. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/class_label.py +0 -0
  284. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/compression.py +0 -0
  285. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/compute.py +0 -0
  286. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/connect_dataset.py +0 -0
  287. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/dataset.py +0 -0
  288. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/delete_entry.py +0 -0
  289. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/diff.py +0 -0
  290. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/downsample.py +0 -0
  291. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/empty_sample.py +0 -0
  292. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/encoder.py +0 -0
  293. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/exif.py +0 -0
  294. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/from_tfds.py +0 -0
  295. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/generate_id.py +0 -0
  296. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/hash.py +0 -0
  297. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/htype.py +0 -0
  298. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/image.py +0 -0
  299. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/invalid_view_op.py +0 -0
  300. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/iterable_ordered_dict.py +0 -0
  301. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/iteration_warning.py +0 -0
  302. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/join_chunks.py +0 -0
  303. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/json.py +0 -0
  304. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/keys.py +0 -0
  305. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/link.py +0 -0
  306. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/logging.py +0 -0
  307. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/merge.py +0 -0
  308. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/modified.py +0 -0
  309. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/notebook.py +0 -0
  310. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/__init__.py +0 -0
  311. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh.py +0 -0
  312. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh_parser.py +0 -0
  313. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh_reader.py +0 -0
  314. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/object_base_3d.py +0 -0
  315. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parser.py +0 -0
  316. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  317. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parsers.py +0 -0
  318. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_reader.py +0 -0
  319. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  320. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_readers.py +0 -0
  321. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/point_cloud.py +0 -0
  322. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/read_3d_data.py +0 -0
  323. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/path.py +0 -0
  324. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/pretty_print.py +0 -0
  325. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/remove_cache.py +0 -0
  326. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/shape_interval.py +0 -0
  327. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/split.py +0 -0
  328. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/storage.py +0 -0
  329. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tag.py +0 -0
  330. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tensor_db.py +0 -0
  331. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/testing.py +0 -0
  332. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/__init__.py +0 -0
  333. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_auto.py +0 -0
  334. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_connect_dataset.py +0 -0
  335. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  336. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_keys.py +0 -0
  337. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_read.py +0 -0
  338. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_shape_interval.py +0 -0
  339. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_split.py +0 -0
  340. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_tensor_db.py +0 -0
  341. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_token.py +0 -0
  342. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_version_control.py +0 -0
  343. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/threading.py +0 -0
  344. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/token.py +0 -0
  345. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/video.py +0 -0
  346. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/warnings.py +0 -0
  347. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/__init__.py +0 -0
  348. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/video_streaming.py +0 -0
  349. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/visualizer.py +0 -0
  350. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/dependency_links.txt +0 -0
  351. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/not-zip-safe +0 -0
  352. {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/top_level.txt +0 -0
  353. {deeplake-3.6.18 → deeplake-3.6.20}/setup.cfg +0 -0
@@ -1,26 +1,28 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.6.18
3
+ Version: 3.6.20
4
4
  Summary: Activeloop Deep Lake
5
+ Home-page: UNKNOWN
5
6
  Author: activeloop.ai
6
7
  Author-email: support@activeloop.ai
7
8
  License: MPL-2.0
8
9
  Project-URL: Documentation, https://docs.activeloop.ai/
9
10
  Project-URL: Source, https://github.com/activeloopai/deeplake
11
+ Platform: UNKNOWN
10
12
  Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
11
13
  Description-Content-Type: text/markdown
14
+ Provides-Extra: all
12
15
  Provides-Extra: audio
13
- Provides-Extra: video
14
16
  Provides-Extra: av
15
- Provides-Extra: gcp
16
17
  Provides-Extra: azure
17
18
  Provides-Extra: dicom
18
- Provides-Extra: medical
19
- Provides-Extra: visualizer
19
+ Provides-Extra: enterprise
20
+ Provides-Extra: gcp
20
21
  Provides-Extra: gdrive
22
+ Provides-Extra: medical
21
23
  Provides-Extra: point_cloud
22
- Provides-Extra: all
23
- Provides-Extra: enterprise
24
+ Provides-Extra: video
25
+ Provides-Extra: visualizer
24
26
  License-File: LICENSE
25
27
 
26
28
  <img src="https://static.scarf.sh/a.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82" /><p align="center">
@@ -288,3 +290,5 @@ If you use Deep Lake in your research, please cite Activeloop using:
288
290
  ## Acknowledgment
289
291
 
290
292
  This technology was inspired by our research work at Princeton University. We would like to thank William Silversmith @SeungLab for his awesome [cloud-volume](https://github.com/seung-lab/cloud-volume) tool.
293
+
294
+
@@ -24,6 +24,7 @@ from .api.tiled import tiled
24
24
  from .core.dataset import Dataset
25
25
  from .core.transform import compute, compose
26
26
  from .core.tensor import Tensor
27
+ from .core.seed import DeeplakeRandom
27
28
  from .util.bugout_reporter import deeplake_reporter
28
29
  from .compression import SUPPORTED_COMPRESSIONS
29
30
  from .htype import HTYPE_CONFIGURATIONS
@@ -50,6 +51,7 @@ ingest_dataframe = api_dataset.ingest_dataframe
50
51
  ingest_huggingface = huggingface.ingest_huggingface
51
52
  dataset = api_dataset.init # type: ignore
52
53
  tensor = Tensor
54
+ random = DeeplakeRandom()
53
55
 
54
56
  __all__ = [
55
57
  "tensor",
@@ -76,10 +78,11 @@ __all__ = [
76
78
  "delete",
77
79
  "copy",
78
80
  "rename",
81
+ "random",
79
82
  ]
80
83
 
81
84
 
82
- __version__ = "3.6.18"
85
+ __version__ = "3.6.20"
83
86
  warn_if_update_required(__version__)
84
87
  __encoded_version__ = np.array(__version__)
85
88
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -40,6 +40,7 @@ from deeplake.util.exceptions import (
40
40
  InvalidDatasetNameException,
41
41
  UnsupportedParameterException,
42
42
  DynamicTensorNumpyError,
43
+ SampleExtendError,
43
44
  )
44
45
  from deeplake.util.path import convert_string_to_pathlib_if_needed, verify_dataset_name
45
46
  from deeplake.util.testing import assert_array_equal
@@ -2766,3 +2767,16 @@ def test_non_local_org_id():
2766
2767
  def test_azure_bad_path():
2767
2768
  with pytest.raises(ValueError):
2768
2769
  ds = deeplake.empty("az://storage_account")
2770
+
2771
+
2772
+ def test_dataset_extend_error_suggestion(local_ds):
2773
+ with local_ds as ds:
2774
+ ds.create_tensor("abc")
2775
+
2776
+ with pytest.raises(SampleExtendError) as e:
2777
+ ds.extend({"abc": [1, 2, 3, 4, "abcd", 5]})
2778
+
2779
+ assert (
2780
+ "If you wish to skip the samples that cause errors,"
2781
+ " please specify `ignore_errors=True`."
2782
+ ) in str(e)
@@ -246,3 +246,14 @@ DEFAULT_VECTORSTORE_TENSORS = [
246
246
  ]
247
247
 
248
248
  VIEW_SUMMARY_SAFE_LIMIT = 10000
249
+
250
+ # openai constants
251
+ OPENAI_TOKEN_SIZE = 2 * B
252
+ OPENAI_ADA_MAX_TOKENS_PER_MINUTE = 1000000
253
+ MAX_BYTES_PER_MINUTE = (
254
+ 0.9 * OPENAI_TOKEN_SIZE * OPENAI_ADA_MAX_TOKENS_PER_MINUTE
255
+ ) # 0.9 is a safety factor
256
+ TARGET_BYTE_SIZE = 10000
257
+
258
+ # Maximum default message length for saving query views
259
+ QUERY_MESSAGE_MAX_SIZE = 1000
@@ -24,6 +24,7 @@ from deeplake.util.version_control import (
24
24
  integrity_check,
25
25
  save_commit_info,
26
26
  rebuild_version_info,
27
+ _squash_main,
27
28
  )
28
29
  from deeplake.util.invalid_view_op import invalid_view_op
29
30
  from deeplake.util.spinner import spinner
@@ -44,6 +45,7 @@ from deeplake.constants import (
44
45
  SAMPLE_INFO_TENSOR_MAX_CHUNK_SIZE,
45
46
  DEFAULT_READONLY,
46
47
  ENV_HUB_DEV_USERNAME,
48
+ QUERY_MESSAGE_MAX_SIZE,
47
49
  )
48
50
  from deeplake.core.fast_forwarding import ffw_dataset_meta
49
51
  from deeplake.core.index import Index
@@ -104,6 +106,8 @@ from deeplake.util.exceptions import (
104
106
  CheckoutError,
105
107
  DatasetCorruptError,
106
108
  BadRequestException,
109
+ SampleAppendError,
110
+ SampleExtendError,
107
111
  )
108
112
  from deeplake.util.keys import (
109
113
  dataset_exists,
@@ -245,6 +249,7 @@ class Dataset:
245
249
  d["_lock_timeout"] = lock_timeout
246
250
  d["_temp_tensors"] = []
247
251
  d["_vc_info_updated"] = True
252
+ d["_query_string"] = None
248
253
  dct = self.__dict__
249
254
  dct.update(d)
250
255
 
@@ -1722,7 +1727,7 @@ class Dataset:
1722
1727
  Raises:
1723
1728
  CommitError: If ``branch`` could not be found.
1724
1729
  ReadOnlyModeError: If branch deletion is attempted in read-only mode.
1725
- Exception: If you have have the given branch currently checked out.
1730
+ Exception: If you have the given branch currently checked out.
1726
1731
 
1727
1732
  Examples:
1728
1733
 
@@ -1767,6 +1772,38 @@ class Dataset:
1767
1772
  self._set_read_only(read_only, err=True)
1768
1773
  self.storage.autoflush = self._initial_autoflush.pop()
1769
1774
 
1775
+ @invalid_view_op
1776
+ def _squash_main(self) -> None:
1777
+ """
1778
+ DEPRECATED: This method is deprecated and will be removed in a future release.
1779
+
1780
+ Squashes all commits in current branch into one commit.
1781
+ NOTE: This cannot be run if there are any branches besides ``main``
1782
+
1783
+ Raises:
1784
+ ReadOnlyModeError: If branch deletion is attempted in read-only mode.
1785
+ VersionControlError: If the branch cannot be squashed.
1786
+ """
1787
+ if self._is_filtered_view:
1788
+ raise Exception(
1789
+ "Cannot perform version control operations on a filtered dataset view."
1790
+ )
1791
+ read_only = self._read_only
1792
+ if read_only:
1793
+ raise ReadOnlyModeError()
1794
+
1795
+ try_flushing(self)
1796
+
1797
+ self._initial_autoflush.append(self.storage.autoflush)
1798
+ self.storage.autoflush = False
1799
+ try:
1800
+ self._unlock()
1801
+ _squash_main(self)
1802
+ finally:
1803
+ self._set_read_only(read_only, err=True)
1804
+ self.libdeeplake_dataset = None
1805
+ self.storage.autoflush = self._initial_autoflush.pop()
1806
+
1770
1807
  def log(self):
1771
1808
  """Displays the details of all the past commits."""
1772
1809
 
@@ -2257,7 +2294,15 @@ class Dataset:
2257
2294
 
2258
2295
  from deeplake.enterprise import query
2259
2296
 
2260
- return query(self, query_string)
2297
+ result = query(self, query_string)
2298
+
2299
+ if len(query_string) > QUERY_MESSAGE_MAX_SIZE:
2300
+ message = query_string[: QUERY_MESSAGE_MAX_SIZE - 3] + "..."
2301
+ else:
2302
+ message = query_string
2303
+ result._query_string = message
2304
+
2305
+ return result
2261
2306
 
2262
2307
  def sample_by(
2263
2308
  self,
@@ -3037,6 +3082,7 @@ class Dataset:
3037
3082
  TensorDoesNotExistError: If tensor in ``samples`` does not exist.
3038
3083
  ValueError: If all tensors being updated are not of the same length.
3039
3084
  NotImplementedError: If an error occurs while writing tiles.
3085
+ SampleExtendError: If the extend failed while appending a sample.
3040
3086
  Exception: Error while attempting to rollback appends.
3041
3087
  """
3042
3088
  extend = False
@@ -3079,6 +3125,8 @@ class Dataset:
3079
3125
  if ignore_errors:
3080
3126
  continue
3081
3127
  else:
3128
+ if isinstance(e, SampleAppendError):
3129
+ raise SampleExtendError(str(e)) from e.__cause__
3082
3130
  raise e
3083
3131
 
3084
3132
  @invalid_view_op
@@ -3479,7 +3527,7 @@ class Dataset:
3479
3527
  return self._save_view(
3480
3528
  path,
3481
3529
  id,
3482
- message,
3530
+ message or self._query_string,
3483
3531
  optimize,
3484
3532
  tensors,
3485
3533
  num_workers,
@@ -4463,8 +4511,6 @@ class Dataset:
4463
4511
  Raises:
4464
4512
  IndexError: If the index is out of range.
4465
4513
  """
4466
- self._initial_autoflush.append(self.storage.autoflush)
4467
- self.storage.autoflush = False
4468
4514
  max_len = max((t.num_samples for t in self.tensors.values()), default=0)
4469
4515
  if max_len == 0:
4470
4516
  raise IndexError("Can't pop from empty dataset.")
@@ -4479,11 +4525,10 @@ class Dataset:
4479
4525
  f"Index {index} is out of range. The longest tensor has {max_len} samples."
4480
4526
  )
4481
4527
 
4482
- for tensor in self.tensors.values():
4483
- if tensor.num_samples > index:
4484
- tensor.pop(index)
4485
-
4486
- self.storage.autoflush = self._initial_autoflush.pop()
4528
+ with self:
4529
+ for tensor in self.tensors.values():
4530
+ if tensor.num_samples > index:
4531
+ tensor.pop(index)
4487
4532
 
4488
4533
  @property
4489
4534
  def is_view(self) -> bool:
@@ -331,5 +331,6 @@ class DeepLakeQueryDataset(Dataset):
331
331
  def random_split(self, lengths: Sequence[Union[int, float]]):
332
332
  if math.isclose(sum(lengths), 1) and sum(lengths) <= 1:
333
333
  lengths = calculate_absolute_lengths(lengths, len(self))
334
+
334
335
  vs = self.indra_ds.random_split(lengths)
335
336
  return [DeepLakeQueryDataset(self.deeplake_ds, v) for v in vs]
@@ -0,0 +1,25 @@
1
+ import numpy as np
2
+ from typing import Optional
3
+
4
+ class DeeplakeRandom(object):
5
+ def __new__(cls):
6
+ if not hasattr(cls, 'instance'):
7
+ cls.instance = super(DeeplakeRandom, cls).__new__(cls)
8
+ cls.instance.internal_seed = None
9
+ cls.instance.indra_api = None
10
+ return cls.instance
11
+
12
+
13
+ def seed(self, seed: Optional[int] = None):
14
+ if isinstance(seed, Optional[int]):
15
+ self.internal_seed = seed
16
+ if self.indra_api is None:
17
+ from deeplake.enterprise.convert_to_libdeeplake import import_indra_api_silent
18
+ self.indra_api = import_indra_api_silent()
19
+ if self.indra_api is not None:
20
+ self.indra_api.set_seed(self.internal_seed)
21
+ else:
22
+ raise TypeError(f"provided seed type `{type(seed)}` is increect seed must be an integer")
23
+
24
+ def get_seed(self) -> Optional[int]:
25
+ return self.internal_seed
@@ -150,7 +150,10 @@ class AzureProvider(StorageProvider):
150
150
  blobs = [
151
151
  posixpath.join(self.root_folder, key) for key in self._all_keys(prefix)
152
152
  ]
153
- self.container_client.delete_blobs(*blobs)
153
+ # delete_blobs can only delete 256 blobs at a time
154
+ batches = [blobs[i : i + 256] for i in range(0, len(blobs), 256)]
155
+ for batch in batches:
156
+ self.container_client.delete_blobs(*batch)
154
157
 
155
158
  def get_sas_token(self):
156
159
  self._check_update_creds()
@@ -451,6 +451,7 @@ class LRUCache(StorageProvider):
451
451
  "next_storage": self.next_storage,
452
452
  "cache_storage": self.cache_storage,
453
453
  "cache_size": self.cache_size,
454
+ "use_async": self.use_async,
454
455
  }
455
456
 
456
457
  def __setstate__(self, state: Dict[str, Any]):
@@ -469,6 +470,7 @@ class LRUCache(StorageProvider):
469
470
  self.next_storage = state["next_storage"]
470
471
  self.cache_storage = state["cache_storage"]
471
472
  self.cache_size = state["cache_size"]
473
+ self.use_async = state["use_async"]
472
474
  self.lru_sizes = OrderedDict()
473
475
  self.dirty_keys = OrderedDict()
474
476
  self.cache_used = 0
@@ -293,6 +293,41 @@ def test_query_tensors_polygon_htype_consistency(local_auth_ds_generator):
293
293
  assert np.all(i == j)
294
294
 
295
295
 
296
+ @requires_libdeeplake
297
+ def test_random_split_with_seed(local_auth_ds_generator):
298
+ deeplake_ds = local_auth_ds_generator()
299
+ from deeplake.core.seed import DeeplakeRandom
300
+
301
+ with deeplake_ds:
302
+ deeplake_ds.create_tensor("label", htype="generic", dtype=np.int32)
303
+ for i in range(1000):
304
+ deeplake_ds.label.append(int(i % 100))
305
+
306
+ deeplake_indra_ds = deeplake_ds.query("SELECT * GROUP BY label")
307
+
308
+ initial_state = np.random.get_state()
309
+ DeeplakeRandom().seed(100)
310
+ split1 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
311
+ assert len(split1) == 3
312
+ assert len(split1[0]) == 20
313
+
314
+ DeeplakeRandom().seed(101)
315
+ split2 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
316
+ assert len(split2) == 3
317
+ assert len(split2[0]) == 20
318
+
319
+ DeeplakeRandom().seed(100)
320
+ split3 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
321
+ assert len(split3) == 3
322
+ assert len(split3[0]) == 20
323
+
324
+ for i in range(len(split1)):
325
+ assert np.all(split1[i].label.numpy() == split3[i].label.numpy())
326
+ assert not np.all(split1[i].label.numpy() == split2[i].label.numpy())
327
+
328
+ np.random.set_state(initial_state)
329
+
330
+
296
331
  @requires_libdeeplake
297
332
  def test_random_split(local_auth_ds_generator):
298
333
  deeplake_ds = local_auth_ds_generator()
@@ -306,6 +341,7 @@ def test_random_split(local_auth_ds_generator):
306
341
  split = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
307
342
  assert len(split) == 3
308
343
  assert len(split[0]) == 20
344
+
309
345
  l = split[0].dataloader().pytorch()
310
346
  for b in l:
311
347
  pass
@@ -1428,7 +1428,10 @@ def test_ds_append_errors(
1428
1428
  err_msg += re.escape(
1429
1429
  " Last checkpoint: 10 samples processed. You can slice the input to resume from this point."
1430
1430
  )
1431
- err_msg += re.escape(" See traceback for more details.")
1431
+ err_msg += re.escape(
1432
+ " See traceback for more details."
1433
+ " If you wish to skip the samples that cause errors, please specify `ignore_errors=True`."
1434
+ )
1432
1435
 
1433
1436
  samples.insert(17, bad_sample)
1434
1437
 
@@ -33,6 +33,7 @@ from deeplake.util.exceptions import (
33
33
  HubComposeEmptyListError,
34
34
  HubComposeIncompatibleFunction,
35
35
  TransformError,
36
+ SampleAppendError,
36
37
  )
37
38
  from deeplake.hooks import dataset_written, dataset_read
38
39
  from deeplake.util.version_control import auto_checkout
@@ -305,9 +306,9 @@ class Pipeline:
305
306
  target_ds.reset(force=True)
306
307
  target_ds._send_compute_progress(**progress_args, status="failed")
307
308
  close_states(compute_provider, pbar, pqueue)
308
- index, sample = None, None
309
+ index, sample, suggest = None, None, False
309
310
  if isinstance(e, TransformError):
310
- index, sample = e.index, e.sample
311
+ index, sample, suggest = e.index, e.sample, e.suggest
311
312
  if checkpointing_enabled and isinstance(index, int):
312
313
  index = samples_processed + index
313
314
  e = e.__cause__ # type: ignore
@@ -317,6 +318,7 @@ class Pipeline:
317
318
  index=index,
318
319
  sample=sample,
319
320
  samples_processed=samples_processed,
321
+ suggest=suggest,
320
322
  ) from e
321
323
  finally:
322
324
  reload_and_rechunk(
@@ -301,7 +301,7 @@ class VectorStore:
301
301
 
302
302
  def search(
303
303
  self,
304
- embedding_data=None,
304
+ embedding_data: Union[str, List[str]] = None,
305
305
  embedding_function: Optional[Callable] = None,
306
306
  embedding: Optional[Union[List[float], np.ndarray]] = None,
307
307
  k: int = 4,
@@ -344,7 +344,7 @@ class VectorStore:
344
344
 
345
345
  Args:
346
346
  embedding (Union[np.ndarray, List[float]], optional): Embedding representation for performing the search. Defaults to None. The ``embedding_data`` and ``embedding`` cannot both be specified.
347
- embedding_data: Data against which the search will be performed by embedding it using the `embedding_function`. Defaults to None. The `embedding_data` and `embedding` cannot both be specified.
347
+ embedding_data (List[str]): Data against which the search will be performed by embedding it using the `embedding_function`. Defaults to None. The `embedding_data` and `embedding` cannot both be specified.
348
348
  embedding_function (Optional[Callable], optional): function for converting `embedding_data` into embedding. Only valid if `embedding_data` is specified
349
349
  k (int): Number of elements to return after running query. Defaults to 4.
350
350
  distance_metric (str): Type of distance metric to use for sorting the data. Avaliable options are: ``"L1", "L2", "COS", "MAX"``. Defaults to ``"COS"``.
@@ -424,11 +424,6 @@ class VectorStore:
424
424
  embedding_data,
425
425
  embedding_function=embedding_function or self.embedding_function,
426
426
  )
427
- if isinstance(query_emb, np.ndarray):
428
- assert (
429
- query_emb.ndim == 1 or query_emb.shape[0] == 1
430
- ), "Query embedding must be 1-dimensional. Please consider using another embedding function for converting query string to embedding."
431
-
432
427
  return vector_search.search(
433
428
  query=query,
434
429
  logger=logger,
@@ -433,6 +433,24 @@ def test_search_basic(local_path, hub_cloud_dev_token):
433
433
  assert isinstance(data.text[0].data()["value"], str)
434
434
  assert data.embedding[0].numpy().size > 0
435
435
 
436
+ data = vector_store.search(
437
+ embedding_function=embedding_fn3,
438
+ embedding_data="dummy",
439
+ return_view=True,
440
+ k=2,
441
+ )
442
+ assert len(data) == 2
443
+ assert isinstance(data.text[0].data()["value"], str)
444
+ assert data.embedding[0].numpy().size > 0
445
+
446
+ with pytest.raises(NotImplementedError):
447
+ data = vector_store.search(
448
+ embedding_function=embedding_fn3,
449
+ embedding_data=["dummy", "dummy2"],
450
+ return_view=True,
451
+ k=2,
452
+ )
453
+
436
454
  data = vector_store.search(
437
455
  filter={"metadata": {"abcdefh": 1}},
438
456
  embedding=None,
@@ -552,6 +570,8 @@ def test_search_managed(hub_cloud_dev_token):
552
570
  exec_option="tensor_db",
553
571
  )
554
572
 
573
+ assert "vectordb/" in vector_store.dataset.base_storage.path
574
+
555
575
  assert len(data_ce["score"]) == len(data_db["score"])
556
576
  assert all(
557
577
  [
@@ -1661,9 +1681,11 @@ def test_multiple_embeddings(local_path, capsys):
1661
1681
  embedding_data=[texts, texts],
1662
1682
  embedding_tensor=["embedding_1", "embedding_2"],
1663
1683
  )
1684
+
1664
1685
  vector_store.add(
1665
1686
  text=texts, embedding_1=(embedding_fn, texts), embedding_2=(embedding_fn, texts)
1666
1687
  )
1688
+
1667
1689
  vector_store.add(
1668
1690
  text=texts,
1669
1691
  embedding_function=embedding_fn,
@@ -1736,7 +1758,7 @@ def test_query_dim(local_path):
1736
1758
  )
1737
1759
 
1738
1760
  vector_store.add(text=texts, embedding=embeddings)
1739
- with pytest.raises(AssertionError):
1761
+ with pytest.raises(NotImplementedError):
1740
1762
  vector_store.search([texts[0], texts[0]], embedding_fn3, k=1)
1741
1763
 
1742
1764
  vector_store.search([texts[0]], embedding_fn4, k=1)
@@ -9,4 +9,6 @@ from deeplake.core.vectorstore.vector_search.dataset.dataset import (
9
9
  extend_or_ingest_dataset,
10
10
  convert_id_to_row_id,
11
11
  search_row_ids,
12
+ extend,
13
+ chunk_by_bytes,
12
14
  )