deeplake 3.9.7__tar.gz → 3.9.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (373) hide show
  1. {deeplake-3.9.7/deeplake.egg-info → deeplake-3.9.8}/PKG-INFO +1 -1
  2. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/__init__.py +1 -1
  3. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/dataset.py +103 -53
  4. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_api.py +56 -49
  5. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_connect_datasets.py +27 -0
  6. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/dataset.py +50 -33
  7. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/deeplake_cloud_dataset.py +10 -6
  8. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/view_entry.py +1 -1
  9. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_compute.py +0 -2
  10. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/transform/test_transform.py +0 -2
  11. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/transform/transform.py +4 -4
  12. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/requirements/plugins.txt +0 -1
  13. deeplake-3.9.8/deeplake/util/bugout_token.py +1 -0
  14. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/check_installation.py +0 -4
  15. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/compute.py +0 -10
  16. {deeplake-3.9.7 → deeplake-3.9.8/deeplake.egg-info}/PKG-INFO +1 -1
  17. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake.egg-info/SOURCES.txt +0 -1
  18. deeplake-3.9.7/deeplake/core/compute/ray.py +0 -24
  19. deeplake-3.9.7/deeplake/util/bugout_token.py +0 -1
  20. {deeplake-3.9.7 → deeplake-3.9.8}/LICENSE +0 -0
  21. {deeplake-3.9.7 → deeplake-3.9.8}/MANIFEST.in +0 -0
  22. {deeplake-3.9.7 → deeplake-3.9.8}/README.md +0 -0
  23. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/__init__.py +0 -0
  24. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/info.py +0 -0
  25. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/link.py +0 -0
  26. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/link_tiled.py +0 -0
  27. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/read.py +0 -0
  28. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/__init__.py +0 -0
  29. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_access_method.py +0 -0
  30. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_agreement.py +0 -0
  31. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_api_tiling.py +0 -0
  32. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_api_with_compression.py +0 -0
  33. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  34. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_dataset.py +0 -0
  35. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_dicom.py +0 -0
  36. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_downsample.py +0 -0
  37. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_events.py +0 -0
  38. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_grayscale.py +0 -0
  39. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  40. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_info.py +0 -0
  41. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  42. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_json.py +0 -0
  43. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_link.py +0 -0
  44. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_link_tiled.py +0 -0
  45. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_linking.py +0 -0
  46. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_mesh.py +0 -0
  47. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_meta.py +0 -0
  48. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_nifti.py +0 -0
  49. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_none.py +0 -0
  50. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_partial_upload.py +0 -0
  51. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_pickle.py +0 -0
  52. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_point_cloud.py +0 -0
  53. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_polygons.py +0 -0
  54. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_pop.py +0 -0
  55. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_readonly.py +0 -0
  56. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_rechunk.py +0 -0
  57. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_reset.py +0 -0
  58. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_sample_info.py +0 -0
  59. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_tag.py +0 -0
  60. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_text.py +0 -0
  61. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_update_samples.py +0 -0
  62. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_video.py +0 -0
  63. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tests/test_views.py +0 -0
  64. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/api/tiled.py +0 -0
  65. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/__init__.py +0 -0
  66. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/structured/__init__.py +0 -0
  67. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/structured/base.py +0 -0
  68. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/structured/dataframe.py +0 -0
  69. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/tests/__init__.py +0 -0
  70. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/tests/test_coco_template.py +0 -0
  71. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/tests/test_ingestion.py +0 -0
  72. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/tests/test_kaggle.py +0 -0
  73. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/tests/test_yolo_template.py +0 -0
  74. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/__init__.py +0 -0
  75. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/base.py +0 -0
  76. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  77. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/coco/coco.py +0 -0
  78. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/coco/constants.py +0 -0
  79. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/coco/convert.py +0 -0
  80. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/coco/utils.py +0 -0
  81. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/image_classification.py +0 -0
  82. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/kaggle.py +0 -0
  83. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/util.py +0 -0
  84. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  85. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  86. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  87. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  88. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/__init__.py +0 -0
  89. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/auth/__init__.py +0 -0
  90. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/auth/activeloop.py +0 -0
  91. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/auth/auth_context.py +0 -0
  92. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/auth/azure.py +0 -0
  93. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/auth/test_auth_context.py +0 -0
  94. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/client.py +0 -0
  95. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/config.py +0 -0
  96. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/log.py +0 -0
  97. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/test_client.py +0 -0
  98. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/client/utils.py +0 -0
  99. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/compression.py +0 -0
  100. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/constants.py +0 -0
  101. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/__init__.py +0 -0
  102. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/__init__.py +0 -0
  103. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/base_chunk.py +0 -0
  104. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  105. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  106. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  107. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  108. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/test_uncompressed.py +0 -0
  109. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  110. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/chunk_engine.py +0 -0
  111. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compression.py +0 -0
  112. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compute/__init__.py +0 -0
  113. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compute/process.py +0 -0
  114. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compute/provider.py +0 -0
  115. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compute/serial.py +0 -0
  116. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/compute/thread.py +0 -0
  117. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/__init__.py +0 -0
  118. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/indra_dataset_view.py +0 -0
  119. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/indra_tensor_view.py +0 -0
  120. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/dataset/invalid_view.py +0 -0
  121. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/distance_type.py +0 -0
  122. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/fast_forwarding.py +0 -0
  123. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/index/__init__.py +0 -0
  124. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/index/index.py +0 -0
  125. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/index_maintenance.py +0 -0
  126. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/io.py +0 -0
  127. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/ipc.py +0 -0
  128. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/link_creds.py +0 -0
  129. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/linked_chunk_engine.py +0 -0
  130. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/linked_sample.py +0 -0
  131. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/linked_tiled_sample.py +0 -0
  132. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/lock.py +0 -0
  133. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/__init__.py +0 -0
  134. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/dataset_meta.py +0 -0
  135. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/__init__.py +0 -0
  136. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/base_encoder.py +0 -0
  137. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/byte_positions.py +0 -0
  138. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/chunk_id.py +0 -0
  139. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/creds.py +0 -0
  140. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/pad.py +0 -0
  141. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/sequence.py +0 -0
  142. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/shape.py +0 -0
  143. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  144. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/common.py +0 -0
  145. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  146. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  147. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  148. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  149. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  150. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/encode/tile.py +0 -0
  151. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/meta.py +0 -0
  152. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/meta/tensor_meta.py +0 -0
  153. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/partial_reader.py +0 -0
  154. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/partial_sample.py +0 -0
  155. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/polygon.py +0 -0
  156. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/query/__init__.py +0 -0
  157. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/query/autocomplete.py +0 -0
  158. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/query/filter.py +0 -0
  159. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/query/query.py +0 -0
  160. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/sample.py +0 -0
  161. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/seed.py +0 -0
  162. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/serialize.py +0 -0
  163. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/__init__.py +0 -0
  164. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/azure.py +0 -0
  165. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  166. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/gcs.py +0 -0
  167. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/google_drive.py +0 -0
  168. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/indra.py +0 -0
  169. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/local.py +0 -0
  170. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/lru_cache.py +0 -0
  171. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/memory.py +0 -0
  172. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/provider.py +0 -0
  173. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/storage/s3.py +0 -0
  174. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tensor.py +0 -0
  175. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tensor_link.py +0 -0
  176. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/test_serialize.py +0 -0
  177. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/__init__.py +0 -0
  178. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_compression.py +0 -0
  179. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_dataset.py +0 -0
  180. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_indra_dataset.py +0 -0
  181. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_io.py +0 -0
  182. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_locking.py +0 -0
  183. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_query.py +0 -0
  184. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_readonly.py +0 -0
  185. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_seed.py +0 -0
  186. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_serialize.py +0 -0
  187. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tests/test_vdb_indexes.py +0 -0
  188. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/__init__.py +0 -0
  189. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/deserialize.py +0 -0
  190. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/optimizer.py +0 -0
  191. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/sample_tiles.py +0 -0
  192. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/serialize.py +0 -0
  193. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/test_optimizer.py +0 -0
  194. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/tiling/test_serialize.py +0 -0
  195. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/transform/__init__.py +0 -0
  196. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/transform/transform_dataset.py +0 -0
  197. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/transform/transform_tensor.py +0 -0
  198. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/__init__.py +0 -0
  199. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
  200. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
  201. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
  202. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
  203. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
  204. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
  205. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
  206. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
  207. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
  208. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
  209. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
  210. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  211. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  212. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  213. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  214. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  215. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  216. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  217. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  218. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  219. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  220. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  221. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  222. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  223. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  224. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  225. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  226. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  227. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  228. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  229. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  230. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  231. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  232. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  233. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  234. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  235. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  236. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/__init__.py +0 -0
  237. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  238. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/commit_diff.py +0 -0
  239. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/commit_node.py +0 -0
  240. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/dataset_diff.py +0 -0
  241. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/test_merge.py +0 -0
  242. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/core/version_control/test_version_control.py +0 -0
  243. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/__init__.py +0 -0
  244. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  245. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/dataloader.py +0 -0
  246. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/dummy_dataloader.py +0 -0
  247. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/libdeeplake_query.py +0 -0
  248. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/test_pytorch.py +0 -0
  249. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/test_query.py +0 -0
  250. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/test_tensorflow.py +0 -0
  251. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/enterprise/util.py +0 -0
  252. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/hooks.py +0 -0
  253. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/htype.py +0 -0
  254. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/__init__.py +0 -0
  255. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/huggingface/__init__.py +0 -0
  256. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/huggingface/huggingface.py +0 -0
  257. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/mmdet/__init__.py +0 -0
  258. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  259. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  260. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  261. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/pytorch/__init__.py +0 -0
  262. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/pytorch/common.py +0 -0
  263. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/pytorch/dataset.py +0 -0
  264. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/pytorch/pytorch.py +0 -0
  265. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  266. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/tf/__init__.py +0 -0
  267. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/tf/common.py +0 -0
  268. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  269. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  270. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/wandb/__init__.py +0 -0
  271. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/integrations/wandb/wandb.py +0 -0
  272. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/requirements/common.txt +0 -0
  273. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/requirements/docs.txt +0 -0
  274. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/requirements/tests.txt +0 -0
  275. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/__init__.py +0 -0
  276. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/cache_fixtures.py +0 -0
  277. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/client_fixtures.py +0 -0
  278. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/common.py +0 -0
  279. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/dataset_fixtures.py +0 -0
  280. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/path_fixtures.py +0 -0
  281. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/tests/storage_fixtures.py +0 -0
  282. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/__init__.py +0 -0
  283. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/access_method.py +0 -0
  284. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/agreement.py +0 -0
  285. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/array_list.py +0 -0
  286. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/assert_byte_indexes.py +0 -0
  287. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/auto.py +0 -0
  288. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/bugout_reporter.py +0 -0
  289. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/cache_chain.py +0 -0
  290. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/casting.py +0 -0
  291. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/check_latest_version.py +0 -0
  292. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/chunk_engine.py +0 -0
  293. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/class_label.py +0 -0
  294. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/compression.py +0 -0
  295. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/connect_dataset.py +0 -0
  296. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/dataset.py +0 -0
  297. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/delete_entry.py +0 -0
  298. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/diff.py +0 -0
  299. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/downsample.py +0 -0
  300. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/empty_sample.py +0 -0
  301. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/encoder.py +0 -0
  302. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/exceptions.py +0 -0
  303. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/exif.py +0 -0
  304. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/from_tfds.py +0 -0
  305. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/generate_id.py +0 -0
  306. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/hash.py +0 -0
  307. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/htype.py +0 -0
  308. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/image.py +0 -0
  309. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/invalid_view_op.py +0 -0
  310. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/iterable_ordered_dict.py +0 -0
  311. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/iteration_warning.py +0 -0
  312. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/join_chunks.py +0 -0
  313. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/json.py +0 -0
  314. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/keys.py +0 -0
  315. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/link.py +0 -0
  316. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/logging.py +0 -0
  317. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/merge.py +0 -0
  318. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/modified.py +0 -0
  319. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/notebook.py +0 -0
  320. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/__init__.py +0 -0
  321. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/mesh.py +0 -0
  322. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/mesh_parser.py +0 -0
  323. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/mesh_reader.py +0 -0
  324. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/object_base_3d.py +0 -0
  325. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_parser.py +0 -0
  326. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  327. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_parsers.py +0 -0
  328. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_reader.py +0 -0
  329. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  330. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/ply_readers.py +0 -0
  331. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/point_cloud.py +0 -0
  332. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/object_3d/read_3d_data.py +0 -0
  333. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/path.py +0 -0
  334. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/pretty_print.py +0 -0
  335. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/remove_cache.py +0 -0
  336. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/scheduling.py +0 -0
  337. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/shape_interval.py +0 -0
  338. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/shuffle.py +0 -0
  339. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/spinner.py +0 -0
  340. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/split.py +0 -0
  341. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/storage.py +0 -0
  342. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tag.py +0 -0
  343. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tensor_db.py +0 -0
  344. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/testing.py +0 -0
  345. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/__init__.py +0 -0
  346. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_auto.py +0 -0
  347. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_connect_dataset.py +0 -0
  348. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  349. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_keys.py +0 -0
  350. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_read.py +0 -0
  351. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_shape_interval.py +0 -0
  352. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_shuffle.py +0 -0
  353. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_split.py +0 -0
  354. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_tensor_db.py +0 -0
  355. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_token.py +0 -0
  356. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_version_control.py +0 -0
  357. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/tests/test_video.py +0 -0
  358. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/threading.py +0 -0
  359. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/token.py +0 -0
  360. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/transform.py +0 -0
  361. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/version_control.py +0 -0
  362. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/video.py +0 -0
  363. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/util/warnings.py +0 -0
  364. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/visualizer/__init__.py +0 -0
  365. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/visualizer/video_streaming.py +0 -0
  366. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake/visualizer/visualizer.py +0 -0
  367. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake.egg-info/dependency_links.txt +0 -0
  368. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake.egg-info/not-zip-safe +0 -0
  369. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake.egg-info/requires.txt +7 -7
  370. {deeplake-3.9.7 → deeplake-3.9.8}/deeplake.egg-info/top_level.txt +0 -0
  371. {deeplake-3.9.7 → deeplake-3.9.8}/pyproject.toml +0 -0
  372. {deeplake-3.9.7 → deeplake-3.9.8}/setup.cfg +0 -0
  373. {deeplake-3.9.7 → deeplake-3.9.8}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.9.7
3
+ Version: 3.9.8
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -89,7 +89,7 @@ __all__ = [
89
89
  ]
90
90
 
91
91
 
92
- __version__ = "3.9.7"
92
+ __version__ = "3.9.8"
93
93
  warn_if_update_required(__version__)
94
94
  __encoded_version__ = np.array(__version__)
95
95
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -11,6 +11,7 @@ from deeplake.auto.unstructured.kaggle import download_kaggle_dataset
11
11
  from deeplake.auto.unstructured.image_classification import ImageClassification
12
12
  from deeplake.auto.unstructured.coco.coco import CocoDataset
13
13
  from deeplake.auto.unstructured.yolo.yolo import YoloDataset
14
+ from deeplake.client.client import DeepLakeBackendClient
14
15
  from deeplake.client.log import logger
15
16
  from deeplake.core.dataset import Dataset, dataset_factory
16
17
  from deeplake.core.dataset.indra_dataset_view import IndraDatasetView
@@ -70,6 +71,7 @@ from deeplake.util.exceptions import (
70
71
  ReadOnlyModeError,
71
72
  LockedException,
72
73
  BadRequestException,
74
+ RenameError,
73
75
  )
74
76
  from deeplake.util.storage import (
75
77
  get_storage_and_cache_chain,
@@ -91,6 +93,30 @@ def _check_indra_and_read_only_flags(indra: bool, read_only: Optional[bool]):
91
93
  )
92
94
 
93
95
 
96
+ def _fetch_creds_from_key(
97
+ creds: Union[dict, str], org_id: Optional[str], token: Optional[str]
98
+ ) -> Optional[str]:
99
+ if not isinstance(creds, dict):
100
+ return None
101
+
102
+ if "creds_key" in creds:
103
+ if len(creds) != 1:
104
+ raise ValueError("Other creds values are not allowed with creds_key")
105
+
106
+ client = DeepLakeBackendClient(token)
107
+ creds_key = creds["creds_key"]
108
+
109
+ if not org_id:
110
+ raise ValueError("Please specify org_id when using creds_key")
111
+
112
+ creds.update(client.get_managed_creds(org_id, creds_key))
113
+ del creds["creds_key"]
114
+
115
+ return creds_key
116
+
117
+ return None
118
+
119
+
94
120
  class dataset:
95
121
  @staticmethod
96
122
  @spinner
@@ -119,28 +145,28 @@ class dataset:
119
145
 
120
146
  Examples:
121
147
 
122
- >>> ds = deeplake.dataset("hub://username/dataset")
123
- >>> ds = deeplake.dataset("s3://mybucket/my_dataset")
124
- >>> ds = deeplake.dataset("./datasets/my_dataset", overwrite=True)
148
+ >>> ds = deeplake.dataset("hub://org_id/dataset") # Initialize dataset managed by Deep Lake.
149
+ >>> ds = deeplake.dataset("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Initialize dataset stored in your cloud using your own credentials.
150
+ >>> ds = deeplake.dataset("./datasets/my_dataset", overwrite=True) # Overwrite the dataset currently at the path
125
151
 
126
152
  Loading to a specfic version:
127
153
 
128
- >>> ds = deeplake.dataset("hub://username/dataset@new_branch")
129
- >>> ds = deeplake.dataset("hub://username/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
154
+ >>> ds = deeplake.dataset("hub://org_id/dataset@new_branch")
155
+ >>> ds = deeplake.dataset("hub://org_id/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
130
156
 
131
157
  >>> my_commit_id = "3e49cded62b6b335c74ff07e97f8451a37aca7b2"
132
- >>> ds = deeplake.dataset(f"hub://username/dataset@{my_commit_id}")
158
+ >>> ds = deeplake.dataset(f"hub://org_id/dataset@{my_commit_id}")
133
159
 
134
160
  Args:
135
161
  path (str, pathlib.Path): - The full path to the dataset. Can be:
136
- - a Deep Lake cloud path of the form ``hub://username/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
162
+ - a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
137
163
  - an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
138
164
  - a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
139
165
  - a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
140
166
  - Loading to a specific version:
141
167
 
142
168
  - You can also specify a ``commit_id`` or ``branch`` to load the dataset to that version directly by using the ``@`` symbol.
143
- - The path will then be of the form ``hub://username/dataset@{branch}`` or ``hub://username/dataset@{commit_id}``.
169
+ - The path will then be of the form ``hub://org_id/dataset@{branch}`` or ``hub://org_id/dataset@{commit_id}``.
144
170
  - See examples above.
145
171
  runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub:// paths.
146
172
  read_only (bool, optional): Opens dataset in read only mode if this is passed as ``True``. Defaults to ``False``.
@@ -152,9 +178,10 @@ class dataset:
152
178
  creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
153
179
  - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
154
180
  - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
181
+ - To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
155
182
  - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
156
183
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
157
- org_id (str, Optional): Organization id to be used for enabling high-performance features. Only applicable for local datasets.
184
+ org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
158
185
  verbose (bool): If ``True``, logs will be printed. Defaults to ``True``.
159
186
  access_method (str): The access method to use for the dataset. Can be:
160
187
 
@@ -223,12 +250,11 @@ class dataset:
223
250
  path, address = process_dataset_path(path)
224
251
  verify_dataset_name(path)
225
252
 
226
- if org_id is not None and get_path_type(path) != "local":
227
- raise ValueError("org_id parameter can only be used with local datasets")
228
-
229
253
  if creds is None:
230
254
  creds = {}
231
255
 
256
+ dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
257
+
232
258
  db_engine = parse_runtime_parameters(path, runtime)["tensor_db"]
233
259
 
234
260
  try:
@@ -285,6 +311,11 @@ class dataset:
285
311
  "index_params": index_params,
286
312
  }
287
313
 
314
+ if dataset_creds_key:
315
+ dataset_kwargs["dataset_creds_key"] = dataset_creds_key
316
+ dataset_kwargs["dataset_creds_key_org_id"] = org_id
317
+ dataset_kwargs["dataset_creds_key_token"] = token
318
+
288
319
  if access_method == "stream":
289
320
  dataset_kwargs.update(
290
321
  {
@@ -339,6 +370,7 @@ class dataset:
339
370
  path: Union[str, pathlib.Path],
340
371
  creds: Optional[Union[Dict, str]] = None,
341
372
  token: Optional[str] = None,
373
+ org_id: Optional[str] = None,
342
374
  ) -> bool:
343
375
  """Checks if a dataset exists at the given ``path``.
344
376
 
@@ -347,8 +379,10 @@ class dataset:
347
379
  creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
348
380
  - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
349
381
  - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
382
+ - To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
350
383
  - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
351
384
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
385
+ org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
352
386
 
353
387
  Returns:
354
388
  A boolean confirming whether the dataset exists or not at the given path.
@@ -365,6 +399,9 @@ class dataset:
365
399
 
366
400
  if creds is None:
367
401
  creds = {}
402
+
403
+ _fetch_creds_from_key(creds, org_id, token)
404
+
368
405
  try:
369
406
  storage, cache_chain = get_storage_and_cache_chain(
370
407
  path=path,
@@ -397,6 +434,13 @@ class dataset:
397
434
  ) -> Dataset:
398
435
  """Creates an empty Deep Lake dataset.
399
436
 
437
+ Examples:
438
+
439
+ >>> ds = deeplake.empty("hub://org_id/dataset") # Create dataset in the default storage for your organization.
440
+ >>> ds = deeplake.empty("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Create dataset stored in your cloud using your own credentials.
441
+ >>> ds = deeplake.empty("s3://mybucket/my_dataset", creds = {"creds_key": "managed_creds_key"}, org_id = "my_org_id") # Create dataset stored in your cloud using Deep Lake managed credentials.
442
+ >>> ds = deeplake.empty("./datasets/my_dataset", overwrite=True) # Overwrite the dataset currently at the path
443
+
400
444
  Args:
401
445
  path (str, pathlib.Path): - The full path to the dataset. It can be:
402
446
  - a Deep Lake cloud path of the form ``hub://org_id/dataset_name``. Requires registration with Deep Lake.
@@ -411,9 +455,10 @@ class dataset:
411
455
  creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
412
456
  - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
413
457
  - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
458
+ - To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
414
459
  - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
415
460
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
416
- org_id (str, Optional): Organization id to be used for enabling high-performance features. Only applicable for local datasets.
461
+ org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
417
462
  verbose (bool): If True, logs will be printed. Defaults to True.
418
463
  lock_timeout (int): Number of seconds to wait before throwing a LockException. If None, wait indefinitely
419
464
  lock_enabled (bool): If true, the dataset manages a write lock. NOTE: Only set to False if you are managing concurrent access externally.
@@ -434,8 +479,6 @@ class dataset:
434
479
  """
435
480
  path, address = process_dataset_path(path)
436
481
 
437
- if org_id is not None and get_path_type(path) != "local":
438
- raise ValueError("org_id parameter can only be used with local datasets")
439
482
  db_engine = parse_runtime_parameters(path, runtime)["tensor_db"]
440
483
 
441
484
  if address:
@@ -448,6 +491,8 @@ class dataset:
448
491
  if creds is None:
449
492
  creds = {}
450
493
 
494
+ dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
495
+
451
496
  try:
452
497
  storage, cache_chain = get_storage_and_cache_chain(
453
498
  path=path,
@@ -506,6 +551,9 @@ class dataset:
506
551
  "lock_timeout": lock_timeout,
507
552
  "lock_enabled": lock_enabled,
508
553
  "index_params": index_params,
554
+ "dataset_creds_key": dataset_creds_key,
555
+ "dataset_creds_key_org_id": org_id,
556
+ "dataset_creds_key_token": token,
509
557
  }
510
558
  ret = dataset._load(dataset_kwargs, create=True)
511
559
  return ret
@@ -530,32 +578,32 @@ class dataset:
530
578
  lock_enabled: Optional[bool] = True,
531
579
  index_params: Optional[Dict[str, Union[int, str]]] = None,
532
580
  ) -> Dataset:
533
- """Loads an existing Deep Lake dataset
581
+ """Loads an existing Deep Lake dataset.
534
582
 
535
583
  Examples:
536
584
 
537
- >>> ds = deeplake.load("hub://username/dataset")
538
- >>> ds = deeplake.load("s3://mybucket/my_dataset")
539
- >>> ds = deeplake.load("./datasets/my_dataset", overwrite=True)
585
+ >>> ds = deeplake.load("hub://org_id/dataset") # Load dataset managed by Deep Lake.
586
+ >>> ds = deeplake.load("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Load dataset stored in your cloud using your own credentials.
587
+ >>> ds = deeplake.load("s3://mybucket/my_dataset", creds = {"creds_key": "managed_creds_key"}, org_id = "my_org_id") # Load dataset stored in your cloud using Deep Lake managed credentials.
540
588
 
541
589
  Loading to a specfic version:
542
590
 
543
- >>> ds = deeplake.load("hub://username/dataset@new_branch")
544
- >>> ds = deeplake.load("hub://username/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
591
+ >>> ds = deeplake.load("hub://org_id/dataset@new_branch")
592
+ >>> ds = deeplake.load("hub://org_id/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
545
593
 
546
594
  >>> my_commit_id = "3e49cded62b6b335c74ff07e97f8451a37aca7b2"
547
- >>> ds = deeplake.load(f"hub://username/dataset@{my_commit_id}")
595
+ >>> ds = deeplake.load(f"hub://org_id/dataset@{my_commit_id}")
548
596
 
549
597
  Args:
550
598
  path (str, pathlib.Path): - The full path to the dataset. Can be:
551
- - a Deep Lake cloud path of the form ``hub://username/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
599
+ - a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
552
600
  - an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
553
601
  - a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
554
602
  - a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
555
603
  - Loading to a specific version:
556
604
 
557
605
  - You can also specify a ``commit_id`` or ``branch`` to load the dataset to that version directly by using the ``@`` symbol.
558
- - The path will then be of the form ``hub://username/dataset@{branch}`` or ``hub://username/dataset@{commit_id}``.
606
+ - The path will then be of the form ``hub://org_id/dataset@{branch}`` or ``hub://org_id/dataset@{commit_id}``.
559
607
  - See examples above.
560
608
  read_only (bool, optional): Opens dataset in read only mode if this is passed as ``True``. Defaults to ``False``.
561
609
  Datasets stored on Deep Lake cloud that your account does not have write access to will automatically open in read mode.
@@ -564,6 +612,7 @@ class dataset:
564
612
  creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
565
613
  - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
566
614
  - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
615
+ - To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
567
616
  - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
568
617
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
569
618
  org_id (str, Optional): Organization id to be used for enabling high-performance features. Only applicable for local datasets.
@@ -632,8 +681,7 @@ class dataset:
632
681
  if creds is None:
633
682
  creds = {}
634
683
 
635
- if org_id is not None and get_path_type(path) != "local":
636
- raise ValueError("org_id parameter can only be used with local datasets")
684
+ dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
637
685
 
638
686
  try:
639
687
  storage, cache_chain = get_storage_and_cache_chain(
@@ -681,6 +729,11 @@ class dataset:
681
729
  "index_params": index_params,
682
730
  }
683
731
 
732
+ if dataset_creds_key:
733
+ dataset_kwargs["dataset_creds_key"] = dataset_creds_key
734
+ dataset_kwargs["dataset_creds_key_org_id"] = org_id
735
+ dataset_kwargs["dataset_creds_key_token"] = token
736
+
684
737
  if access_method == "stream":
685
738
  dataset_kwargs.update(
686
739
  {
@@ -803,45 +856,37 @@ class dataset:
803
856
 
804
857
  @staticmethod
805
858
  def rename(
806
- old_path: Union[str, pathlib.Path],
807
- new_path: Union[str, pathlib.Path],
808
- creds: Optional[Union[dict, str]] = None,
859
+ path: str,
860
+ new_name: str,
809
861
  token: Optional[str] = None,
810
862
  ) -> Dataset:
811
- """Renames dataset at ``old_path`` to ``new_path``.
863
+ """Renames managed dataset at ``path`` to ``new_name``.
812
864
 
813
865
  Examples:
814
-
815
- >>> deeplake.rename("hub://username/image_ds", "hub://username/new_ds")
816
- >>> deeplake.rename("s3://mybucket/my_ds", "s3://mybucket/renamed_ds")
866
+ >>> deeplake.rename("hub://username/image_ds", "new_ds")
867
+ >>> deeplake.rename("hub://org_id/image_ds", "new_ds") ## New dataset path is `hub://org_id/new_ds`
817
868
 
818
869
  Args:
819
- old_path (str, pathlib.Path): The path to the dataset to be renamed.
820
- new_path (str, pathlib.Path): Path to the dataset after renaming.
821
- creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
822
- - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
823
- - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
824
- - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
825
- token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
870
+ path (str): The path to the dataset to be renamed.
871
+ new_name (str): New name of the dataset.
872
+ token (str, optional): Activeloop token for accessing the dataset
826
873
 
827
874
  Returns:
828
875
  Dataset: The renamed Dataset.
829
876
 
830
877
  Raises:
831
- DatasetHandlerError: If a Dataset does not exist at the given path or if new path is to a different directory.
878
+ RenameError: If a Dataset does not exist at the given path or if new path is not a managed dataset.
832
879
  """
833
- old_path = convert_pathlib_to_string_if_needed(old_path)
834
- new_path = convert_pathlib_to_string_if_needed(new_path)
835
880
 
836
- if creds is None:
837
- creds = {}
881
+ if not path.startswith("hub://"):
882
+ raise RenameError("Rename is not available for non-managed datasets")
838
883
 
839
- feature_report_path(old_path, "rename", {}, token=token)
884
+ feature_report_path(path, "rename", {}, token=token)
840
885
 
841
- deeplake.deepcopy(old_path, new_path, verbose=False, token=token, creds=creds)
842
- deeplake.delete(old_path, token=token, creds=creds)
886
+ ds = deeplake.load(path, verbose=False, token=token)
887
+ ds.rename(new_name)
843
888
 
844
- return deeplake.load(new_path, verbose=False, token=token, creds=creds)
889
+ return ds
845
890
 
846
891
  @staticmethod
847
892
  @spinner
@@ -852,6 +897,7 @@ class dataset:
852
897
  creds: Optional[Union[dict, str]] = None,
853
898
  token: Optional[str] = None,
854
899
  verbose: bool = False,
900
+ org_id: Optional[str] = None,
855
901
  ) -> None:
856
902
  """Deletes a dataset at a given path.
857
903
 
@@ -863,9 +909,11 @@ class dataset:
863
909
  creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
864
910
  - If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
865
911
  - It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
912
+ - To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
866
913
  - If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
867
914
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
868
915
  verbose (bool): If True, logs will be printed. Defaults to True.
916
+ org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
869
917
 
870
918
  Raises:
871
919
  DatasetHandlerError: If a Dataset does not exist at the given path and ``force = False``.
@@ -886,6 +934,8 @@ class dataset:
886
934
  if creds is None:
887
935
  creds = {}
888
936
 
937
+ _fetch_creds_from_key(creds, org_id, token)
938
+
889
939
  feature_report_path(
890
940
  path, "delete", {"Force": force, "Large_OK": large_ok}, token=token
891
941
  )
@@ -1136,7 +1186,7 @@ class dataset:
1136
1186
  dest_creds (dict, optional): creds required to create / overwrite datasets at ``dest``.
1137
1187
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
1138
1188
  num_workers (int): The number of workers to use for copying. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
1139
- scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
1189
+ scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', and 'processed'.
1140
1190
  Defaults to 'threaded'.
1141
1191
  progressbar (bool): Displays a progress bar if True (default).
1142
1192
  **kwargs (dict): Additional keyword arguments
@@ -1222,7 +1272,7 @@ class dataset:
1222
1272
  dest_creds (dict, optional): creds required to create / overwrite datasets at ``dest``.
1223
1273
  token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
1224
1274
  num_workers (int): The number of workers to use for copying. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
1225
- scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
1275
+ scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', and 'processed'.
1226
1276
  Defaults to 'threaded'.
1227
1277
  progressbar (bool): Displays a progress bar if True (default).
1228
1278
  public (bool): Defines if the dataset will have public access. Applicable only if Deep Lake cloud storage is used and a new Dataset is being created. Defaults to ``False``.
@@ -1943,7 +1993,7 @@ class dataset:
1943
1993
  tag (str): Kaggle dataset tag. Example: ``"coloradokb/dandelionimages"`` points to https://www.kaggle.com/coloradokb/dandelionimages
1944
1994
  src (str, pathlib.Path): Local path to where the raw kaggle dataset will be downlaoded to.
1945
1995
  dest (str, pathlib.Path): - The full path to the dataset. Can be:
1946
- - a Deep Lake cloud path of the form ``hub://username/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
1996
+ - a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
1947
1997
  - an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
1948
1998
  - a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
1949
1999
  - a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
@@ -2056,7 +2106,7 @@ class dataset:
2056
2106
  src (pd.DataFrame): The pandas dataframe to be converted.
2057
2107
  dest (str, pathlib.Path):
2058
2108
  - A Dataset or The full path to the dataset. Can be:
2059
- - a Deep Lake cloud path of the form ``hub://username/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
2109
+ - a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
2060
2110
  - an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
2061
2111
  - a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
2062
2112
  - a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
@@ -1,5 +1,7 @@
1
1
  import os
2
2
  import sys
3
+ import time
4
+
3
5
  import numpy as np
4
6
  import pathlib
5
7
  import pytest
@@ -924,63 +926,28 @@ def test_invalid_token():
924
926
  )
925
927
 
926
928
 
927
- @pytest.mark.parametrize(
928
- ("ds_generator", "path", "hub_token"),
929
- [
930
- ("local_ds_generator", "local_path", "hub_cloud_dev_token"),
931
- pytest.param(
932
- "s3_ds_generator", "s3_path", "hub_cloud_dev_token", marks=pytest.mark.slow
933
- ),
934
- pytest.param(
935
- "gcs_ds_generator",
936
- "gcs_path",
937
- "hub_cloud_dev_token",
938
- marks=pytest.mark.slow,
939
- ),
940
- pytest.param(
941
- "azure_ds_generator",
942
- "azure_path",
943
- "hub_cloud_dev_token",
944
- marks=pytest.mark.slow,
945
- ),
946
- pytest.param(
947
- "hub_cloud_ds_generator",
948
- "hub_cloud_path",
949
- "hub_cloud_dev_token",
950
- marks=pytest.mark.slow,
951
- ),
952
- ],
953
- indirect=True,
954
- )
955
- @pytest.mark.parametrize("convert_to_pathlib", [True, False])
956
- def test_dataset_rename(ds_generator, path, hub_token, convert_to_pathlib):
957
- ds = ds_generator()
929
+ @pytest.mark.slow
930
+ def test_dataset_rename(hub_cloud_ds_generator, hub_cloud_path, hub_cloud_dev_token):
931
+ ds = hub_cloud_ds_generator()
958
932
  ds.create_tensor("abc")
959
933
  ds.abc.append([1, 2, 3, 4])
934
+ original_path = ds.path
960
935
 
961
- new_path = "_".join([path, "renamed"])
962
-
963
- ds.path = convert_string_to_pathlib_if_needed(ds.path, convert_to_pathlib)
964
- new_path = convert_string_to_pathlib_if_needed(new_path, convert_to_pathlib)
936
+ new_name = f"renamed-{time.time()}"
937
+ new_path = original_path.rsplit("/", 1)[0] + "/" + new_name
965
938
 
966
939
  with pytest.raises(RenameError):
967
- ds.rename("wrongfolder/new_ds")
940
+ ds.rename("hub://wrong_org/test")
968
941
 
969
- if str(ds.path).startswith("hub://"):
970
- with pytest.raises(BadRequestException):
971
- ds.rename(ds.path)
972
- else:
973
- with pytest.raises(PathNotEmptyException):
974
- ds.rename(ds.path)
942
+ ds.rename(new_name)
975
943
 
976
- ds = deeplake.rename(ds.path, new_path, token=hub_token)
977
- assert ds.path == str(new_path)
944
+ assert ds.path.endswith("/" + new_name)
978
945
  assert_array_equal(ds.abc.numpy(), np.array([[1, 2, 3, 4]]))
979
946
 
980
- ds = deeplake.load(new_path, token=hub_token)
947
+ ds = deeplake.load(new_path, token=hub_cloud_dev_token)
981
948
  assert_array_equal(ds.abc.numpy(), np.array([[1, 2, 3, 4]]))
982
949
 
983
- deeplake.delete(new_path, token=hub_token)
950
+ deeplake.delete(new_path, token=hub_cloud_dev_token)
984
951
 
985
952
 
986
953
  @pytest.mark.parametrize(
@@ -2807,13 +2774,13 @@ def test_slice_shape_interval(memory_ds):
2807
2774
 
2808
2775
 
2809
2776
  def test_non_local_org_id():
2810
- with pytest.raises(ValueError):
2777
+ with pytest.raises(InvalidTokenException):
2811
2778
  ds = deeplake.dataset("hub://test/test_dataset", org_id="test")
2812
2779
 
2813
- with pytest.raises(ValueError):
2780
+ with pytest.raises(InvalidTokenException):
2814
2781
  ds = deeplake.empty("hub://test/test_dataset", org_id="test")
2815
2782
 
2816
- with pytest.raises(ValueError):
2783
+ with pytest.raises(InvalidTokenException):
2817
2784
  ds = deeplake.load("hub://test/test_dataset", org_id="test")
2818
2785
 
2819
2786
  with pytest.raises(ValueError):
@@ -3030,3 +2997,43 @@ def test_append_non_uint8_to_image(local_ds):
3030
2997
  ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
3031
2998
 
3032
2999
  assert ds.images.dtype.name == "int16"
3000
+
3001
+
3002
+ @pytest.mark.slow
3003
+ def test_create_and_load_with_managed_credentials(
3004
+ hub_cloud_path: str, hub_cloud_dev_token
3005
+ ):
3006
+ old_environ = dict(os.environ)
3007
+ os.environ.pop("AWS_ACCESS_KEY_ID", None)
3008
+ os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
3009
+ os.environ.pop("AWS_SESSION_TOKEN", None)
3010
+
3011
+ try:
3012
+ dir_name = hub_cloud_path.rsplit("/", 1)[1]
3013
+ ds = deeplake.empty(
3014
+ f"s3://deeplake-tests/{dir_name}",
3015
+ creds={"creds_key": "aws_creds"},
3016
+ org_id="testingacc2",
3017
+ token=hub_cloud_dev_token,
3018
+ )
3019
+ ds.create_tensor("id", htype="text")
3020
+
3021
+ assert ds.path == f"s3://deeplake-tests/{dir_name}"
3022
+
3023
+ ds = deeplake.load(
3024
+ f"s3://deeplake-tests/{dir_name}",
3025
+ creds={"creds_key": "aws_creds"},
3026
+ org_id="testingacc2",
3027
+ token=hub_cloud_dev_token,
3028
+ )
3029
+ assert "id" in ds.tensors
3030
+
3031
+ deeplake.delete(
3032
+ f"s3://deeplake-tests/{dir_name}",
3033
+ creds={"creds_key": "aws_creds"},
3034
+ org_id="testingacc2",
3035
+ token=hub_cloud_dev_token,
3036
+ )
3037
+ finally:
3038
+ os.environ.clear()
3039
+ os.environ.update(old_environ)
@@ -1,3 +1,5 @@
1
+ import os
2
+
1
3
  import pytest
2
4
 
3
5
  import deeplake
@@ -87,3 +89,28 @@ def test_connect_user_not_in_org(s3_ds_generator, hub_cloud_dev_token):
87
89
  token=hub_cloud_dev_token,
88
90
  )
89
91
  assert "organization id" in str(e)
92
+
93
+
94
+ # @pytest.mark.slow
95
+ def test_connect_from_managed_credentials(hub_cloud_path: str, hub_cloud_dev_token):
96
+ old_environ = dict(os.environ)
97
+ os.environ.pop("AWS_ACCESS_KEY_ID", None)
98
+ os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
99
+ os.environ.pop("AWS_SESSION_TOKEN", None)
100
+
101
+ try:
102
+ dir_name = hub_cloud_path.rsplit("/", 1)[1]
103
+ ds = deeplake.empty(
104
+ f"s3://deeplake-tests/{dir_name}",
105
+ creds={"creds_key": "aws_creds"},
106
+ org_id="testingacc2",
107
+ token=hub_cloud_dev_token,
108
+ )
109
+ ds.create_tensor("id", htype="text")
110
+
111
+ ds.connect()
112
+ assert ds.path == f"hub://testingacc2/{dir_name}"
113
+
114
+ finally:
115
+ os.environ.clear()
116
+ os.environ.update(old_environ)