deeplake 3.6.16__tar.gz → 3.6.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (352) hide show
  1. {deeplake-3.6.16/deeplake.egg-info → deeplake-3.6.18}/PKG-INFO +7 -11
  2. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/__init__.py +1 -1
  3. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/dataset.py +4 -0
  4. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_api.py +0 -7
  5. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/dataset.py +7 -1
  6. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/deeplake_vectorstore.py +2 -1
  7. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/dataloader.py +34 -4
  8. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/test_pytorch.py +13 -0
  9. deeplake-3.6.18/deeplake/util/bugout_token.py +1 -0
  10. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/storage.py +11 -1
  11. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/version_control.py +2 -2
  12. {deeplake-3.6.16 → deeplake-3.6.18/deeplake.egg-info}/PKG-INFO +7 -11
  13. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/entry_points.txt +0 -1
  14. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/requires.txt +20 -20
  15. {deeplake-3.6.16 → deeplake-3.6.18}/setup.py +1 -1
  16. deeplake-3.6.16/deeplake/util/bugout_token.py +0 -1
  17. {deeplake-3.6.16 → deeplake-3.6.18}/LICENSE +0 -0
  18. {deeplake-3.6.16 → deeplake-3.6.18}/MANIFEST.in +0 -0
  19. {deeplake-3.6.16 → deeplake-3.6.18}/README.md +0 -0
  20. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/__init__.py +0 -0
  21. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/info.py +0 -0
  22. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/link.py +0 -0
  23. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/link_tiled.py +0 -0
  24. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/read.py +0 -0
  25. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/__init__.py +0 -0
  26. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_access_method.py +0 -0
  27. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_agreement.py +0 -0
  28. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_api_tiling.py +0 -0
  29. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_api_with_compression.py +0 -0
  30. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  31. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_connect_datasets.py +0 -0
  32. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_dataset.py +0 -0
  33. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_dicom.py +0 -0
  34. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_downsample.py +0 -0
  35. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_events.py +0 -0
  36. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_grayscale.py +0 -0
  37. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  38. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_info.py +0 -0
  39. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  40. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_json.py +0 -0
  41. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_link.py +0 -0
  42. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_link_tiled.py +0 -0
  43. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_linking.py +0 -0
  44. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_mesh.py +0 -0
  45. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_meta.py +0 -0
  46. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_nifti.py +0 -0
  47. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_none.py +0 -0
  48. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_partial_upload.py +0 -0
  49. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_pickle.py +0 -0
  50. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_point_cloud.py +0 -0
  51. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_polygons.py +0 -0
  52. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_pop.py +0 -0
  53. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_readonly.py +0 -0
  54. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_rechunk.py +0 -0
  55. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_reset.py +0 -0
  56. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_sample_info.py +0 -0
  57. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_text.py +0 -0
  58. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_update_samples.py +0 -0
  59. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_video.py +0 -0
  60. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tests/test_views.py +0 -0
  61. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/api/tiled.py +0 -0
  62. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/__init__.py +0 -0
  63. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/structured/__init__.py +0 -0
  64. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/structured/base.py +0 -0
  65. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/structured/dataframe.py +0 -0
  66. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/tests/__init__.py +0 -0
  67. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/tests/test_coco_template.py +0 -0
  68. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/tests/test_ingestion.py +0 -0
  69. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/tests/test_kaggle.py +0 -0
  70. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/tests/test_yolo_template.py +0 -0
  71. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/__init__.py +0 -0
  72. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/base.py +0 -0
  73. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  74. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/coco/coco.py +0 -0
  75. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/coco/constants.py +0 -0
  76. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/coco/convert.py +0 -0
  77. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/coco/utils.py +0 -0
  78. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/image_classification.py +0 -0
  79. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/kaggle.py +0 -0
  80. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/util.py +0 -0
  81. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  82. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  83. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  84. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  85. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/cli/__init__.py +0 -0
  86. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/cli/auth.py +0 -0
  87. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/cli/commands.py +0 -0
  88. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/cli/test_cli.py +0 -0
  89. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/__init__.py +0 -0
  90. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/client.py +0 -0
  91. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/config.py +0 -0
  92. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/log.py +0 -0
  93. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/test_client.py +0 -0
  94. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/client/utils.py +0 -0
  95. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/compression.py +0 -0
  96. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/constants.py +0 -0
  97. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/__init__.py +0 -0
  98. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/__init__.py +0 -0
  99. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/base_chunk.py +0 -0
  100. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  101. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  102. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  103. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  104. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/test_uncompressed.py +0 -0
  105. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  106. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/chunk_engine.py +0 -0
  107. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compression.py +0 -0
  108. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/__init__.py +0 -0
  109. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/process.py +0 -0
  110. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/provider.py +0 -0
  111. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/ray.py +0 -0
  112. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/serial.py +0 -0
  113. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/compute/thread.py +0 -0
  114. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/__init__.py +0 -0
  115. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  116. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/deeplake_query_dataset.py +0 -0
  117. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
  118. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/invalid_view.py +0 -0
  119. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/dataset/view_entry.py +0 -0
  120. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/fast_forwarding.py +0 -0
  121. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/index/__init__.py +0 -0
  122. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/index/index.py +0 -0
  123. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/io.py +0 -0
  124. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/ipc.py +0 -0
  125. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/link_creds.py +0 -0
  126. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/linked_chunk_engine.py +0 -0
  127. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/linked_sample.py +0 -0
  128. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/linked_tiled_sample.py +0 -0
  129. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/lock.py +0 -0
  130. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/__init__.py +0 -0
  131. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/dataset_meta.py +0 -0
  132. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/__init__.py +0 -0
  133. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/base_encoder.py +0 -0
  134. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/byte_positions.py +0 -0
  135. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/chunk_id.py +0 -0
  136. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/creds.py +0 -0
  137. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/pad.py +0 -0
  138. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/sequence.py +0 -0
  139. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/shape.py +0 -0
  140. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  141. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/common.py +0 -0
  142. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  143. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  144. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  145. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  146. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  147. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/encode/tile.py +0 -0
  148. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/meta.py +0 -0
  149. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/meta/tensor_meta.py +0 -0
  150. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/partial_reader.py +0 -0
  151. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/partial_sample.py +0 -0
  152. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/polygon.py +0 -0
  153. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/query/__init__.py +0 -0
  154. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/query/autocomplete.py +0 -0
  155. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/query/filter.py +0 -0
  156. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/query/query.py +0 -0
  157. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/sample.py +0 -0
  158. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/serialize.py +0 -0
  159. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/__init__.py +0 -0
  160. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/azure.py +0 -0
  161. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  162. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/gcs.py +0 -0
  163. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/google_drive.py +0 -0
  164. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/local.py +0 -0
  165. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/lru_cache.py +0 -0
  166. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/memory.py +0 -0
  167. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/provider.py +0 -0
  168. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/storage/s3.py +0 -0
  169. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tensor.py +0 -0
  170. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tensor_link.py +0 -0
  171. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/test_serialize.py +0 -0
  172. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/__init__.py +0 -0
  173. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_compression.py +0 -0
  174. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_compute.py +0 -0
  175. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_deeplake_indra_dataset.py +0 -0
  176. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_io.py +0 -0
  177. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_locking.py +0 -0
  178. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_readonly.py +0 -0
  179. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tests/test_serialize.py +0 -0
  180. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/__init__.py +0 -0
  181. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/deserialize.py +0 -0
  182. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/optimizer.py +0 -0
  183. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/sample_tiles.py +0 -0
  184. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/serialize.py +0 -0
  185. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/test_optimizer.py +0 -0
  186. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/tiling/test_serialize.py +0 -0
  187. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/transform/__init__.py +0 -0
  188. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/transform/test_transform.py +0 -0
  189. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/transform/transform.py +0 -0
  190. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/transform/transform_dataset.py +0 -0
  191. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/transform/transform_tensor.py +0 -0
  192. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/__init__.py +0 -0
  193. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  194. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  195. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  196. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  197. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  198. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  199. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  200. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  201. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  202. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  203. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  204. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  205. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  206. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  207. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  208. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  209. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  210. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  211. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  212. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  213. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  214. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  215. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  216. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  217. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  218. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  219. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/__init__.py +0 -0
  220. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  221. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/commit_diff.py +0 -0
  222. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/commit_node.py +0 -0
  223. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/dataset_diff.py +0 -0
  224. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/test_merge.py +0 -0
  225. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/core/version_control/test_version_control.py +0 -0
  226. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/__init__.py +0 -0
  227. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  228. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/dummy_dataloader.py +0 -0
  229. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/libdeeplake_query.py +0 -0
  230. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/test_query.py +0 -0
  231. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/test_tensorflow.py +0 -0
  232. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/enterprise/util.py +0 -0
  233. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/hooks.py +0 -0
  234. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/htype.py +0 -0
  235. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/__init__.py +0 -0
  236. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/huggingface/__init__.py +0 -0
  237. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/huggingface/huggingface.py +0 -0
  238. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/mmdet/__init__.py +0 -0
  239. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  240. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  241. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  242. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/pytorch/__init__.py +0 -0
  243. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/pytorch/common.py +0 -0
  244. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/pytorch/dataset.py +0 -0
  245. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/pytorch/pytorch.py +0 -0
  246. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  247. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/tf/__init__.py +0 -0
  248. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/tf/common.py +0 -0
  249. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  250. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  251. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/wandb/__init__.py +0 -0
  252. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/integrations/wandb/wandb.py +0 -0
  253. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/requirements/common.txt +0 -0
  254. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/requirements/docs.txt +0 -0
  255. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/requirements/plugins.txt +0 -0
  256. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/requirements/tests.txt +0 -0
  257. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/__init__.py +0 -0
  258. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/cache_fixtures.py +0 -0
  259. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/client_fixtures.py +0 -0
  260. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/common.py +0 -0
  261. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/dataset_fixtures.py +0 -0
  262. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/path_fixtures.py +0 -0
  263. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/tests/storage_fixtures.py +0 -0
  264. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/__init__.py +0 -0
  265. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/access_method.py +0 -0
  266. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/agreement.py +0 -0
  267. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/array_list.py +0 -0
  268. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/assert_byte_indexes.py +0 -0
  269. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/auto.py +0 -0
  270. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/bugout_reporter.py +0 -0
  271. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/cache_chain.py +0 -0
  272. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/casting.py +0 -0
  273. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/check_installation.py +0 -0
  274. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/check_latest_version.py +0 -0
  275. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/chunk_engine.py +0 -0
  276. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/class_label.py +0 -0
  277. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/compression.py +0 -0
  278. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/compute.py +0 -0
  279. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/connect_dataset.py +0 -0
  280. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/dataset.py +0 -0
  281. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/delete_entry.py +0 -0
  282. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/diff.py +0 -0
  283. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/downsample.py +0 -0
  284. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/empty_sample.py +0 -0
  285. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/encoder.py +0 -0
  286. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/exceptions.py +0 -0
  287. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/exif.py +0 -0
  288. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/from_tfds.py +0 -0
  289. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/generate_id.py +0 -0
  290. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/hash.py +0 -0
  291. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/htype.py +0 -0
  292. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/image.py +0 -0
  293. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/invalid_view_op.py +0 -0
  294. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/iterable_ordered_dict.py +0 -0
  295. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/iteration_warning.py +0 -0
  296. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/join_chunks.py +0 -0
  297. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/json.py +0 -0
  298. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/keys.py +0 -0
  299. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/link.py +0 -0
  300. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/logging.py +0 -0
  301. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/merge.py +0 -0
  302. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/modified.py +0 -0
  303. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/notebook.py +0 -0
  304. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/__init__.py +0 -0
  305. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/mesh.py +0 -0
  306. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/mesh_parser.py +0 -0
  307. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/mesh_reader.py +0 -0
  308. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/object_base_3d.py +0 -0
  309. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_parser.py +0 -0
  310. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  311. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_parsers.py +0 -0
  312. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_reader.py +0 -0
  313. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  314. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/ply_readers.py +0 -0
  315. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/point_cloud.py +0 -0
  316. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/object_3d/read_3d_data.py +0 -0
  317. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/path.py +0 -0
  318. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/pretty_print.py +0 -0
  319. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/remove_cache.py +0 -0
  320. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/scheduling.py +0 -0
  321. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/shape_interval.py +0 -0
  322. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/shuffle.py +0 -0
  323. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/spinner.py +0 -0
  324. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/split.py +0 -0
  325. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tag.py +0 -0
  326. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tensor_db.py +0 -0
  327. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/testing.py +0 -0
  328. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/__init__.py +0 -0
  329. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_auto.py +0 -0
  330. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_connect_dataset.py +0 -0
  331. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  332. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_keys.py +0 -0
  333. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_read.py +0 -0
  334. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_shape_interval.py +0 -0
  335. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_shuffle.py +0 -0
  336. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_split.py +0 -0
  337. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_tensor_db.py +0 -0
  338. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_token.py +0 -0
  339. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/tests/test_version_control.py +0 -0
  340. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/threading.py +0 -0
  341. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/token.py +0 -0
  342. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/transform.py +0 -0
  343. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/video.py +0 -0
  344. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/util/warnings.py +0 -0
  345. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/visualizer/__init__.py +0 -0
  346. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/visualizer/video_streaming.py +0 -0
  347. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake/visualizer/visualizer.py +0 -0
  348. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/SOURCES.txt +0 -0
  349. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/dependency_links.txt +0 -0
  350. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/not-zip-safe +0 -0
  351. {deeplake-3.6.16 → deeplake-3.6.18}/deeplake.egg-info/top_level.txt +0 -0
  352. {deeplake-3.6.16 → deeplake-3.6.18}/setup.cfg +0 -0
@@ -1,28 +1,26 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.6.16
3
+ Version: 3.6.18
4
4
  Summary: Activeloop Deep Lake
5
- Home-page: UNKNOWN
6
5
  Author: activeloop.ai
7
6
  Author-email: support@activeloop.ai
8
7
  License: MPL-2.0
9
8
  Project-URL: Documentation, https://docs.activeloop.ai/
10
9
  Project-URL: Source, https://github.com/activeloopai/deeplake
11
- Platform: UNKNOWN
12
10
  Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
13
11
  Description-Content-Type: text/markdown
14
- Provides-Extra: all
15
12
  Provides-Extra: audio
13
+ Provides-Extra: video
16
14
  Provides-Extra: av
15
+ Provides-Extra: gcp
17
16
  Provides-Extra: azure
18
17
  Provides-Extra: dicom
19
- Provides-Extra: enterprise
20
- Provides-Extra: gcp
21
- Provides-Extra: gdrive
22
18
  Provides-Extra: medical
23
- Provides-Extra: point_cloud
24
- Provides-Extra: video
25
19
  Provides-Extra: visualizer
20
+ Provides-Extra: gdrive
21
+ Provides-Extra: point_cloud
22
+ Provides-Extra: all
23
+ Provides-Extra: enterprise
26
24
  License-File: LICENSE
27
25
 
28
26
  <img src="https://static.scarf.sh/a.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82" /><p align="center">
@@ -290,5 +288,3 @@ If you use Deep Lake in your research, please cite Activeloop using:
290
288
  ## Acknowledgment
291
289
 
292
290
  This technology was inspired by our research work at Princeton University. We would like to thank William Silversmith @SeungLab for his awesome [cloud-volume](https://github.com/seung-lab/cloud-volume) tool.
293
-
294
-
@@ -79,7 +79,7 @@ __all__ = [
79
79
  ]
80
80
 
81
81
 
82
- __version__ = "3.6.16"
82
+ __version__ = "3.6.18"
83
83
  warn_if_update_required(__version__)
84
84
  __encoded_version__ = np.array(__version__)
85
85
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -839,6 +839,10 @@ class dataset:
839
839
  read_only=False,
840
840
  token=token,
841
841
  )
842
+ if len(base_storage) == 0:
843
+ raise DatasetHandlerError(
844
+ f"Path {path} is empty or does not exist. Cannot delete."
845
+ )
842
846
  base_storage.clear()
843
847
  remove_path_from_backend(path, token)
844
848
  if verbose:
@@ -1121,13 +1121,6 @@ def test_deepcopy_errors(path, hub_token):
1121
1121
  )
1122
1122
 
1123
1123
 
1124
- def test_cloud_delete_doesnt_exist(hub_cloud_path, hub_cloud_dev_token):
1125
- username = hub_cloud_path.split("/")[2]
1126
- # this dataset doesn't exist
1127
- new_path = f"hub://{username}/doesntexist123"
1128
- deeplake.delete(new_path, token=hub_cloud_dev_token, force=True)
1129
-
1130
-
1131
1124
  def test_invalid_tensor_name(memory_ds):
1132
1125
  with pytest.raises(InvalidTensorNameError):
1133
1126
  memory_ds.create_tensor("group/version_state")
@@ -3021,6 +3021,7 @@ class Dataset:
3021
3021
  skip_ok: bool = False,
3022
3022
  append_empty: bool = False,
3023
3023
  ignore_errors: bool = False,
3024
+ progressbar: bool = False,
3024
3025
  ):
3025
3026
  """Appends multiple rows of samples to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
3026
3027
 
@@ -3029,6 +3030,7 @@ class Dataset:
3029
3030
  skip_ok (bool): Skip tensors not in ``samples`` if set to True.
3030
3031
  append_empty (bool): Append empty samples to tensors not specified in ``sample`` if set to ``True``. If True, ``skip_ok`` is ignored.
3031
3032
  ignore_errors (bool): Skip samples that cause errors while extending, if set to ``True``.
3033
+ progressbar (bool): Displays a progress bar if set to ``True``.
3032
3034
 
3033
3035
  Raises:
3034
3036
  KeyError: If any tensor in the dataset is not a key in ``samples`` and ``skip_ok`` is ``False``.
@@ -3062,7 +3064,11 @@ class Dataset:
3062
3064
  samples, extend=True, skip_ok=skip_ok, append_empty=append_empty
3063
3065
  )
3064
3066
  with self:
3065
- for i in range(n):
3067
+ if progressbar:
3068
+ indices = tqdm(range(n))
3069
+ else:
3070
+ indices = range(n)
3071
+ for i in indices:
3066
3072
  try:
3067
3073
  self.append(
3068
3074
  {k: v[i] for k, v in samples.items()},
@@ -634,6 +634,7 @@ class VectorStore:
634
634
  def delete_by_path(
635
635
  path: Union[str, pathlib.Path],
636
636
  token: Optional[str] = None,
637
+ force: bool = False,
637
638
  ) -> None:
638
639
  """Deleted the Vector Store at the specified path.
639
640
 
@@ -651,7 +652,7 @@ class VectorStore:
651
652
  {},
652
653
  token=token,
653
654
  )
654
- deeplake.delete(path, large_ok=True, token=token)
655
+ deeplake.delete(path, large_ok=True, token=token, force=force)
655
656
 
656
657
  def tensors(self):
657
658
  """Returns the list of tensors present in the dataset"""
@@ -62,6 +62,7 @@ def indra_available() -> bool:
62
62
 
63
63
  def import_indra_loader():
64
64
  global INDRA_LOADER
65
+ global INDRA_API
65
66
  if INDRA_LOADER:
66
67
  return INDRA_LOADER
67
68
  if not importlib.util.find_spec("indra"):
@@ -70,11 +71,13 @@ def import_indra_loader():
70
71
  from indra import api # type: ignore
71
72
  from indra.pytorch.loader import Loader # type:ignore
72
73
 
74
+ INDRA_API = api
73
75
  INDRA_LOADER = Loader
74
76
  return Loader
75
77
  except Exception as e:
76
78
  if not deeplake.constants.RETURN_DUMMY_DATA_FOR_DATALOADER:
77
79
  raise_indra_installation_error(e)
80
+ INDRA_API = None
78
81
  INDRA_LOADER = None
79
82
 
80
83
 
@@ -214,10 +217,13 @@ class DeepLakeDataLoader(DataLoader):
214
217
  return get_collate_fn(self._collate, self._mode)
215
218
 
216
219
  def __len__(self):
217
- round_fn = math.floor if self._drop_last else math.ceil
218
- return round_fn(
219
- len(self._orig_dataset) / ((self.batch_size) * self._world_size)
220
+ len_ds = (
221
+ len(self._orig_dataset[self._tensors])
222
+ if self._tensors is not None
223
+ else len(self._orig_dataset)
220
224
  )
225
+ round_fn = math.floor if self._drop_last else math.ceil
226
+ return round_fn(len_ds / ((self.batch_size) * self._world_size))
221
227
 
222
228
  def batch(self, batch_size: int, drop_last: bool = False):
223
229
  """Returns a batched :class:`DeepLakeDataLoader` object.
@@ -584,6 +590,24 @@ class DeepLakeDataLoader(DataLoader):
584
590
  all_vars["_dataloader"] = None
585
591
  return self.__class__(**all_vars)
586
592
 
593
+ def _get_suboptimal_thread_count(self) -> Optional[int]:
594
+ assert self._distributed
595
+
596
+ if self._num_threads is None and self._mode == "pytorch":
597
+ import torch
598
+
599
+ num_devices = (
600
+ torch.cuda.device_count() if torch.cuda.is_available() else None
601
+ )
602
+
603
+ num_suboptimal_threads = (
604
+ int(INDRA_API.num_available_threads() / num_devices)
605
+ if INDRA_API is not None and num_devices is not None
606
+ else None
607
+ )
608
+ return num_suboptimal_threads
609
+ return self._num_threads
610
+
587
611
  def __iter__(self):
588
612
  if self._dataloader is None:
589
613
  dataset = self._orig_dataset
@@ -641,10 +665,16 @@ class DeepLakeDataLoader(DataLoader):
641
665
  indra_dataset = dataset_to_libdeeplake(dataset)
642
666
  else:
643
667
  indra_dataset = self._indra_dataset
668
+
669
+ num_threads = (
670
+ self._get_suboptimal_thread_count()
671
+ if self._distributed
672
+ else self._num_threads
673
+ )
644
674
  self._dataloader = INDRA_LOADER(
645
675
  indra_dataset,
646
676
  batch_size=self._batch_size,
647
- num_threads=self._num_threads,
677
+ num_threads=num_threads,
648
678
  shuffle=self._shuffle,
649
679
  num_workers=self._num_workers,
650
680
  collate_fn=collate_fn,
@@ -153,6 +153,19 @@ def test_pytorch_transform(hub_cloud_ds):
153
153
  np.testing.assert_array_equal(actual_image2, expected_image2)
154
154
 
155
155
 
156
+ @requires_libdeeplake
157
+ def test_inequal_tensors_dataloader_length(local_auth_ds):
158
+ with local_auth_ds as ds:
159
+ ds.create_tensor("images")
160
+ ds.create_tensor("label")
161
+ ds.images.extend(([i * np.ones((i + 1, i + 1)) for i in range(16)]))
162
+
163
+ ld = local_auth_ds.dataloader().batch(1).pytorch()
164
+ assert len(ld) == 0
165
+ ld1 = local_auth_ds.dataloader().batch(2).pytorch(tensors=["images"])
166
+ assert len(ld1) == 8
167
+
168
+
156
169
  @requires_torch
157
170
  @requires_libdeeplake
158
171
  def test_pytorch_transform_dict(hub_cloud_ds):
@@ -0,0 +1 @@
1
+ BUGOUT_TOKEN = '7358369f-00a8-469e-af74-a5eb512b183a'
@@ -57,9 +57,18 @@ def storage_provider_from_path(
57
57
  path, read_only, db_engine=db_engine, token=token, creds=creds
58
58
  )
59
59
  else:
60
- if isinstance(creds, str):
60
+ if isinstance(creds, str) and not path.startswith("s3://"):
61
61
  creds = {}
62
62
  if path.startswith("s3://"):
63
+ creds_used = "PLATFORM"
64
+ if creds == "ENV":
65
+ creds_used = "ENV"
66
+ elif isinstance(creds, dict) and set(creds.keys()) == {"profile_name"}:
67
+ creds_used = "ENV"
68
+ elif isinstance(creds, dict) and bool(creds):
69
+ creds_used = "DICT"
70
+ if isinstance(creds, str):
71
+ creds = {}
63
72
  key = creds.get("aws_access_key_id")
64
73
  secret = creds.get("aws_secret_access_key")
65
74
  session_token = creds.get("aws_session_token")
@@ -76,6 +85,7 @@ def storage_provider_from_path(
76
85
  profile_name=profile,
77
86
  token=token,
78
87
  )
88
+ storage.creds_used = creds_used
79
89
  elif (
80
90
  path.startswith("gcp://")
81
91
  or path.startswith("gcs://")
@@ -369,7 +369,7 @@ def delete_branch(
369
369
 
370
370
 
371
371
  def _delete_branch_and_commits(
372
- branch_name: str, all_branch_commits: list[str], dataset, storage
372
+ branch_name: str, all_branch_commits: List[str], dataset, storage
373
373
  ) -> None:
374
374
  """
375
375
  Physically deletes the given branch and list of commits from the version_control_info.json and versions directories.
@@ -401,7 +401,7 @@ def _delete_branch_and_commits(
401
401
  ).encode("utf-8")
402
402
 
403
403
 
404
- def _find_branch_commits(branch_name: str, version_state: dict) -> list[str]:
404
+ def _find_branch_commits(branch_name: str, version_state: dict) -> List[str]:
405
405
  """
406
406
  Returns a list of all commits used by the given branch
407
407
  """
@@ -1,28 +1,26 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.6.16
3
+ Version: 3.6.18
4
4
  Summary: Activeloop Deep Lake
5
- Home-page: UNKNOWN
6
5
  Author: activeloop.ai
7
6
  Author-email: support@activeloop.ai
8
7
  License: MPL-2.0
9
8
  Project-URL: Documentation, https://docs.activeloop.ai/
10
9
  Project-URL: Source, https://github.com/activeloopai/deeplake
11
- Platform: UNKNOWN
12
10
  Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
13
11
  Description-Content-Type: text/markdown
14
- Provides-Extra: all
15
12
  Provides-Extra: audio
13
+ Provides-Extra: video
16
14
  Provides-Extra: av
15
+ Provides-Extra: gcp
17
16
  Provides-Extra: azure
18
17
  Provides-Extra: dicom
19
- Provides-Extra: enterprise
20
- Provides-Extra: gcp
21
- Provides-Extra: gdrive
22
18
  Provides-Extra: medical
23
- Provides-Extra: point_cloud
24
- Provides-Extra: video
25
19
  Provides-Extra: visualizer
20
+ Provides-Extra: gdrive
21
+ Provides-Extra: point_cloud
22
+ Provides-Extra: all
23
+ Provides-Extra: enterprise
26
24
  License-File: LICENSE
27
25
 
28
26
  <img src="https://static.scarf.sh/a.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82" /><p align="center">
@@ -290,5 +288,3 @@ If you use Deep Lake in your research, please cite Activeloop using:
290
288
  ## Acknowledgment
291
289
 
292
290
  This technology was inspired by our research work at Princeton University. We would like to thank William Silversmith @SeungLab for his awesome [cloud-volume](https://github.com/seung-lab/cloud-volume) tool.
293
-
294
-
@@ -1,3 +1,2 @@
1
1
  [console_scripts]
2
2
  activeloop = deeplake.cli.commands:cli
3
-
@@ -1,32 +1,32 @@
1
+ numpy
2
+ pillow
1
3
  boto3
2
4
  click
5
+ pathos
3
6
  humbug>=0.3.1
7
+ tqdm
4
8
  numcodecs
5
- numpy
6
- pathos
7
- pillow
8
9
  pyjwt
9
- tqdm
10
10
 
11
11
  [:python_version >= "3.7" and sys_platform != "win32"]
12
12
  aioboto3>=10.4.0
13
13
  nest_asyncio
14
14
 
15
15
  [all]
16
+ google-api-python-client~=2.31.0
17
+ oauth2client~=4.1.3
18
+ pydicom
16
19
  IPython
20
+ google-cloud-storage~=1.42.0
21
+ google-auth~=2.0.1
17
22
  azure-cli
18
- azure-identity
19
- azure-storage-blob
20
23
  flask
21
- google-api-python-client~=2.31.0
22
- google-auth-oauthlib~=0.4.5
23
- google-auth~=2.0.1
24
- google-cloud-storage~=1.42.0
25
24
  laspy
26
- libdeeplake==0.0.64
27
25
  nibabel
28
- oauth2client~=4.1.3
29
- pydicom
26
+ azure-storage-blob
27
+ azure-identity
28
+ google-auth-oauthlib~=0.4.5
29
+ libdeeplake==0.0.68
30
30
 
31
31
  [all:python_version >= "3.7" or sys_platform != "win32"]
32
32
  av>=8.1.0
@@ -47,27 +47,27 @@ azure-identity
47
47
  azure-storage-blob
48
48
 
49
49
  [dicom]
50
- nibabel
51
50
  pydicom
51
+ nibabel
52
52
 
53
53
  [enterprise]
54
- libdeeplake==0.0.64
54
+ libdeeplake==0.0.68
55
55
  pyjwt
56
56
 
57
57
  [gcp]
58
- google-auth-oauthlib~=0.4.5
59
- google-auth~=2.0.1
60
58
  google-cloud-storage~=1.42.0
59
+ google-auth~=2.0.1
60
+ google-auth-oauthlib~=0.4.5
61
61
 
62
62
  [gdrive]
63
63
  google-api-python-client~=2.31.0
64
- google-auth-oauthlib~=0.4.5
65
- google-auth~=2.0.1
66
64
  oauth2client~=4.1.3
65
+ google-auth~=2.0.1
66
+ google-auth-oauthlib~=0.4.5
67
67
 
68
68
  [medical]
69
- nibabel
70
69
  pydicom
70
+ nibabel
71
71
 
72
72
  [point_cloud]
73
73
  laspy
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
70
70
  extras_require["all"] = [req_map[r] for r in all_extras]
71
71
 
72
72
  if libdeeplake_availabe():
73
- libdeeplake = "libdeeplake==0.0.64"
73
+ libdeeplake = "libdeeplake==0.0.68"
74
74
  extras_require["enterprise"] = [libdeeplake, "pyjwt"]
75
75
  extras_require["all"].append(libdeeplake)
76
76
 
@@ -1 +0,0 @@
1
- BUGOUT_TOKEN = "7c122c87-f50b-4684-8ed5-9465bff67873"
File without changes
File without changes
File without changes