deeplake 3.9.6__tar.gz → 3.9.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (373) hide show
  1. {deeplake-3.9.6/deeplake.egg-info → deeplake-3.9.7}/PKG-INFO +1 -1
  2. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/__init__.py +1 -1
  3. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/dataset.py +8 -7
  4. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/dataset.py +90 -67
  5. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tensor.py +2 -4
  6. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform.py +73 -40
  7. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/convert_to_libdeeplake.py +1 -0
  8. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/dataloader.py +5 -3
  9. deeplake-3.9.7/deeplake/util/bugout_token.py +1 -0
  10. {deeplake-3.9.6 → deeplake-3.9.7/deeplake.egg-info}/PKG-INFO +1 -1
  11. deeplake-3.9.6/deeplake/util/bugout_token.py +0 -1
  12. {deeplake-3.9.6 → deeplake-3.9.7}/LICENSE +0 -0
  13. {deeplake-3.9.6 → deeplake-3.9.7}/MANIFEST.in +0 -0
  14. {deeplake-3.9.6 → deeplake-3.9.7}/README.md +0 -0
  15. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/__init__.py +0 -0
  16. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/info.py +0 -0
  17. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/link.py +0 -0
  18. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/link_tiled.py +0 -0
  19. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/read.py +0 -0
  20. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/__init__.py +0 -0
  21. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_access_method.py +0 -0
  22. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_agreement.py +0 -0
  23. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api.py +0 -0
  24. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api_tiling.py +0 -0
  25. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api_with_compression.py +0 -0
  26. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  27. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_connect_datasets.py +0 -0
  28. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_dataset.py +0 -0
  29. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_dicom.py +0 -0
  30. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_downsample.py +0 -0
  31. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_events.py +0 -0
  32. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_grayscale.py +0 -0
  33. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  34. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_info.py +0 -0
  35. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  36. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_json.py +0 -0
  37. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_link.py +0 -0
  38. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_link_tiled.py +0 -0
  39. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_linking.py +0 -0
  40. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_mesh.py +0 -0
  41. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_meta.py +0 -0
  42. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_nifti.py +0 -0
  43. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_none.py +0 -0
  44. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_partial_upload.py +0 -0
  45. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_pickle.py +0 -0
  46. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_point_cloud.py +0 -0
  47. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_polygons.py +0 -0
  48. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_pop.py +0 -0
  49. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_readonly.py +0 -0
  50. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_rechunk.py +0 -0
  51. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_reset.py +0 -0
  52. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_sample_info.py +0 -0
  53. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_tag.py +0 -0
  54. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_text.py +0 -0
  55. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_update_samples.py +0 -0
  56. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_video.py +0 -0
  57. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_views.py +0 -0
  58. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tiled.py +0 -0
  59. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/__init__.py +0 -0
  60. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/__init__.py +0 -0
  61. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/base.py +0 -0
  62. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/dataframe.py +0 -0
  63. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/__init__.py +0 -0
  64. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_coco_template.py +0 -0
  65. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_ingestion.py +0 -0
  66. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_kaggle.py +0 -0
  67. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_yolo_template.py +0 -0
  68. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/__init__.py +0 -0
  69. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/base.py +0 -0
  70. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  71. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/coco.py +0 -0
  72. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/constants.py +0 -0
  73. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/convert.py +0 -0
  74. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/utils.py +0 -0
  75. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/image_classification.py +0 -0
  76. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/kaggle.py +0 -0
  77. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/util.py +0 -0
  78. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  79. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  80. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  81. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  82. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/__init__.py +0 -0
  83. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/__init__.py +0 -0
  84. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/activeloop.py +0 -0
  85. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/auth_context.py +0 -0
  86. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/azure.py +0 -0
  87. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/test_auth_context.py +0 -0
  88. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/client.py +0 -0
  89. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/config.py +0 -0
  90. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/log.py +0 -0
  91. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/test_client.py +0 -0
  92. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/utils.py +0 -0
  93. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/compression.py +0 -0
  94. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/constants.py +0 -0
  95. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/__init__.py +0 -0
  96. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/__init__.py +0 -0
  97. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/base_chunk.py +0 -0
  98. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  99. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  100. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  101. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  102. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_uncompressed.py +0 -0
  103. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  104. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk_engine.py +0 -0
  105. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compression.py +0 -0
  106. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/__init__.py +0 -0
  107. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/process.py +0 -0
  108. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/provider.py +0 -0
  109. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/ray.py +0 -0
  110. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/serial.py +0 -0
  111. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/thread.py +0 -0
  112. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/__init__.py +0 -0
  113. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  114. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/indra_dataset_view.py +0 -0
  115. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/indra_tensor_view.py +0 -0
  116. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/invalid_view.py +0 -0
  117. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/view_entry.py +0 -0
  118. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/distance_type.py +0 -0
  119. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/fast_forwarding.py +0 -0
  120. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index/__init__.py +0 -0
  121. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index/index.py +0 -0
  122. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index_maintenance.py +0 -0
  123. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/io.py +0 -0
  124. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/ipc.py +0 -0
  125. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/link_creds.py +0 -0
  126. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_chunk_engine.py +0 -0
  127. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_sample.py +0 -0
  128. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_tiled_sample.py +0 -0
  129. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/lock.py +0 -0
  130. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/__init__.py +0 -0
  131. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/dataset_meta.py +0 -0
  132. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/__init__.py +0 -0
  133. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/base_encoder.py +0 -0
  134. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/byte_positions.py +0 -0
  135. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/chunk_id.py +0 -0
  136. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/creds.py +0 -0
  137. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/pad.py +0 -0
  138. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/sequence.py +0 -0
  139. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/shape.py +0 -0
  140. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  141. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/common.py +0 -0
  142. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  143. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  144. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  145. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  146. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  147. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tile.py +0 -0
  148. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/meta.py +0 -0
  149. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/tensor_meta.py +0 -0
  150. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/partial_reader.py +0 -0
  151. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/partial_sample.py +0 -0
  152. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/polygon.py +0 -0
  153. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/__init__.py +0 -0
  154. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/autocomplete.py +0 -0
  155. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/filter.py +0 -0
  156. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/query.py +0 -0
  157. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/sample.py +0 -0
  158. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/seed.py +0 -0
  159. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/serialize.py +0 -0
  160. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/__init__.py +0 -0
  161. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/azure.py +0 -0
  162. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  163. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/gcs.py +0 -0
  164. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/google_drive.py +0 -0
  165. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/indra.py +0 -0
  166. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/local.py +0 -0
  167. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/lru_cache.py +0 -0
  168. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/memory.py +0 -0
  169. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/provider.py +0 -0
  170. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/s3.py +0 -0
  171. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tensor_link.py +0 -0
  172. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/test_serialize.py +0 -0
  173. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/__init__.py +0 -0
  174. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_compression.py +0 -0
  175. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_compute.py +0 -0
  176. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_dataset.py +0 -0
  177. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_indra_dataset.py +0 -0
  178. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_io.py +0 -0
  179. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_locking.py +0 -0
  180. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_query.py +0 -0
  181. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_readonly.py +0 -0
  182. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_seed.py +0 -0
  183. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_serialize.py +0 -0
  184. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_vdb_indexes.py +0 -0
  185. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/__init__.py +0 -0
  186. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/deserialize.py +0 -0
  187. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/optimizer.py +0 -0
  188. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/sample_tiles.py +0 -0
  189. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/serialize.py +0 -0
  190. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/test_optimizer.py +0 -0
  191. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/test_serialize.py +0 -0
  192. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/__init__.py +0 -0
  193. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/test_transform.py +0 -0
  194. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform_dataset.py +0 -0
  195. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform_tensor.py +0 -0
  196. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/__init__.py +0 -0
  197. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
  198. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
  199. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
  200. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
  201. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
  202. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
  203. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
  204. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
  205. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
  206. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
  207. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
  208. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  209. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  210. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  211. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  212. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  213. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  214. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  215. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  216. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  217. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  218. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  219. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  220. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  221. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  222. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  223. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  224. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  225. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  226. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  227. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  228. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  229. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  230. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  231. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  232. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  233. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  234. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/__init__.py +0 -0
  235. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  236. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_diff.py +0 -0
  237. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_node.py +0 -0
  238. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/dataset_diff.py +0 -0
  239. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/test_merge.py +0 -0
  240. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/test_version_control.py +0 -0
  241. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/__init__.py +0 -0
  242. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/dummy_dataloader.py +0 -0
  243. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/libdeeplake_query.py +0 -0
  244. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_pytorch.py +0 -0
  245. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_query.py +0 -0
  246. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_tensorflow.py +0 -0
  247. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/util.py +0 -0
  248. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/hooks.py +0 -0
  249. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/htype.py +0 -0
  250. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/__init__.py +0 -0
  251. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/huggingface/__init__.py +0 -0
  252. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/huggingface/huggingface.py +0 -0
  253. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/__init__.py +0 -0
  254. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  255. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  256. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  257. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/__init__.py +0 -0
  258. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/common.py +0 -0
  259. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/dataset.py +0 -0
  260. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/pytorch.py +0 -0
  261. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  262. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/__init__.py +0 -0
  263. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/common.py +0 -0
  264. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  265. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  266. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/wandb/__init__.py +0 -0
  267. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/wandb/wandb.py +0 -0
  268. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/common.txt +0 -0
  269. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/docs.txt +0 -0
  270. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/plugins.txt +0 -0
  271. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/tests.txt +0 -0
  272. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/__init__.py +0 -0
  273. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/cache_fixtures.py +0 -0
  274. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/client_fixtures.py +0 -0
  275. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/common.py +0 -0
  276. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/dataset_fixtures.py +0 -0
  277. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/path_fixtures.py +0 -0
  278. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/storage_fixtures.py +0 -0
  279. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/__init__.py +0 -0
  280. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/access_method.py +0 -0
  281. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/agreement.py +0 -0
  282. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/array_list.py +0 -0
  283. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/assert_byte_indexes.py +0 -0
  284. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/auto.py +0 -0
  285. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/bugout_reporter.py +0 -0
  286. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/cache_chain.py +0 -0
  287. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/casting.py +0 -0
  288. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/check_installation.py +0 -0
  289. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/check_latest_version.py +0 -0
  290. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/chunk_engine.py +0 -0
  291. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/class_label.py +0 -0
  292. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/compression.py +0 -0
  293. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/compute.py +0 -0
  294. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/connect_dataset.py +0 -0
  295. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/dataset.py +0 -0
  296. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/delete_entry.py +0 -0
  297. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/diff.py +0 -0
  298. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/downsample.py +0 -0
  299. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/empty_sample.py +0 -0
  300. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/encoder.py +0 -0
  301. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/exceptions.py +0 -0
  302. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/exif.py +0 -0
  303. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/from_tfds.py +0 -0
  304. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/generate_id.py +0 -0
  305. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/hash.py +0 -0
  306. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/htype.py +0 -0
  307. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/image.py +0 -0
  308. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/invalid_view_op.py +0 -0
  309. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/iterable_ordered_dict.py +0 -0
  310. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/iteration_warning.py +0 -0
  311. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/join_chunks.py +0 -0
  312. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/json.py +0 -0
  313. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/keys.py +0 -0
  314. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/link.py +0 -0
  315. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/logging.py +0 -0
  316. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/merge.py +0 -0
  317. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/modified.py +0 -0
  318. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/notebook.py +0 -0
  319. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/__init__.py +0 -0
  320. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh.py +0 -0
  321. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_parser.py +0 -0
  322. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_reader.py +0 -0
  323. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/object_base_3d.py +0 -0
  324. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser.py +0 -0
  325. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  326. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parsers.py +0 -0
  327. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader.py +0 -0
  328. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  329. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_readers.py +0 -0
  330. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/point_cloud.py +0 -0
  331. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/read_3d_data.py +0 -0
  332. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/path.py +0 -0
  333. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/pretty_print.py +0 -0
  334. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/remove_cache.py +0 -0
  335. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/scheduling.py +0 -0
  336. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/shape_interval.py +0 -0
  337. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/shuffle.py +0 -0
  338. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/spinner.py +0 -0
  339. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/split.py +0 -0
  340. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/storage.py +0 -0
  341. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tag.py +0 -0
  342. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tensor_db.py +0 -0
  343. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/testing.py +0 -0
  344. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/__init__.py +0 -0
  345. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_auto.py +0 -0
  346. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_connect_dataset.py +0 -0
  347. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  348. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_keys.py +0 -0
  349. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_read.py +0 -0
  350. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_shape_interval.py +0 -0
  351. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_shuffle.py +0 -0
  352. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_split.py +0 -0
  353. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_tensor_db.py +0 -0
  354. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_token.py +0 -0
  355. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_version_control.py +0 -0
  356. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_video.py +0 -0
  357. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/threading.py +0 -0
  358. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/token.py +0 -0
  359. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/transform.py +0 -0
  360. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/version_control.py +0 -0
  361. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/video.py +0 -0
  362. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/warnings.py +0 -0
  363. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/__init__.py +0 -0
  364. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/video_streaming.py +0 -0
  365. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/visualizer.py +0 -0
  366. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/SOURCES.txt +0 -0
  367. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/dependency_links.txt +0 -0
  368. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/not-zip-safe +0 -0
  369. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/requires.txt +7 -7
  370. {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/top_level.txt +0 -0
  371. {deeplake-3.9.6 → deeplake-3.9.7}/pyproject.toml +0 -0
  372. {deeplake-3.9.6 → deeplake-3.9.7}/setup.cfg +0 -0
  373. {deeplake-3.9.6 → deeplake-3.9.7}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.9.6
3
+ Version: 3.9.7
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -89,7 +89,7 @@ __all__ = [
89
89
  ]
90
90
 
91
91
 
92
- __version__ = "3.9.6"
92
+ __version__ = "3.9.7"
93
93
  warn_if_update_required(__version__)
94
94
  __encoded_version__ = np.array(__version__)
95
95
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -395,7 +395,7 @@ class dataset:
395
395
  verbose: bool = True,
396
396
  index_params: Optional[Dict[str, Union[int, str]]] = None,
397
397
  ) -> Dataset:
398
- """Creates an empty dataset
398
+ """Creates an empty Deep Lake dataset.
399
399
 
400
400
  Args:
401
401
  path (str, pathlib.Path): - The full path to the dataset. It can be:
@@ -530,7 +530,7 @@ class dataset:
530
530
  lock_enabled: Optional[bool] = True,
531
531
  index_params: Optional[Dict[str, Union[int, str]]] = None,
532
532
  ) -> Dataset:
533
- """Loads an existing dataset
533
+ """Loads an existing Deep Lake dataset
534
534
 
535
535
  Examples:
536
536
 
@@ -1120,7 +1120,8 @@ class dataset:
1120
1120
  progressbar=True,
1121
1121
  **kwargs,
1122
1122
  ):
1123
- """Copies dataset at ``src`` to ``dest``. Version control history is not included.
1123
+ """Copies dataset at ``src`` to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
1124
+ For fast copying, we recommend using ``deepake.deepcopy()`` instead.
1124
1125
 
1125
1126
  Args:
1126
1127
  src (str, Dataset, pathlib.Path): The Dataset or the path to the dataset to be copied.
@@ -1206,7 +1207,7 @@ class dataset:
1206
1207
  verbose: bool = True,
1207
1208
  **kwargs,
1208
1209
  ):
1209
- """Copies dataset at ``src`` to ``dest`` including version control history.
1210
+ """Copies dataset at ``src`` to ``dest`` including version control history. This is the fastest method for copying datasets.
1210
1211
 
1211
1212
  Args:
1212
1213
  src (str, pathlib.Path, Dataset): The Dataset or the path to the dataset to be copied.
@@ -1452,9 +1453,9 @@ class dataset:
1452
1453
 
1453
1454
  Examples:
1454
1455
  >>> # Connect an s3 dataset
1455
- >>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key", token="my_activeloop_token")
1456
+ >>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key")
1456
1457
  >>> # or
1457
- >>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key", token="my_activeloop_token")
1458
+ >>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key")
1458
1459
 
1459
1460
  Args:
1460
1461
  src_path (str): Cloud path to the source dataset. Can be:
@@ -1653,9 +1654,9 @@ class dataset:
1653
1654
  >>> "path/to/data/directory",
1654
1655
  >>> dest="hub://org_id/dataset",
1655
1656
  >>> allow_no_annotation=True,
1656
- >>> token="my_activeloop_token",
1657
1657
  >>> num_workers=4,
1658
1658
  >>> )
1659
+
1659
1660
  >>> # Ingest data from your cloud into another Deep Lake dataset in your cloud, and connect that dataset to the Deep Lake backend.
1660
1661
  >>> ds = deeplake.ingest_yolo(
1661
1662
  >>> "s3://bucket/data_directory",
@@ -372,7 +372,7 @@ class Dataset:
372
372
  return self._client
373
373
 
374
374
  def __len__(self, warn: bool = True):
375
- """Returns the length of the smallest tensor."""
375
+ """Returns the length (number of rows) of the shortest tensor in the dataset."""
376
376
  tensor_lengths = [len(tensor) for tensor in self.tensors.values()]
377
377
  pad_tensors = self._pad_tensors
378
378
  if (
@@ -390,7 +390,7 @@ class Dataset:
390
390
 
391
391
  @property
392
392
  def max_len(self):
393
- """Return the maximum length of the tensor."""
393
+ """Returns the length (number of rows) of the longest tensor in the dataset."""
394
394
  return (
395
395
  max([len(tensor) for tensor in self.tensors.values()])
396
396
  if self.tensors
@@ -399,7 +399,7 @@ class Dataset:
399
399
 
400
400
  @property
401
401
  def min_len(self):
402
- """Return the minimum length of the tensor."""
402
+ """Returns the length (number of rows) of the shortest tensor in the dataset."""
403
403
  return (
404
404
  min([len(tensor) for tensor in self.tensors.values()])
405
405
  if self.tensors
@@ -668,32 +668,34 @@ class Dataset:
668
668
  tiling_threshold: Optional[int] = None,
669
669
  **kwargs,
670
670
  ):
671
- """Creates a new tensor in the dataset.
671
+ """Creates a new tensor in the Deep Lake dataset. Specifying the tensor's htype is highly recommended for complex data such as images, video, dicom, text, json, etc.
672
+ Specifying htype is not necessary for simple numeric date such as arrays and scalars.
672
673
 
673
674
  Examples:
674
- >>> # create dataset
675
- >>> ds = deeplake.dataset("path/to/dataset")
676
-
677
- >>> # create tensors
678
- >>> ds.create_tensor("images", htype="image", sample_compression="jpg")
679
- >>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
680
- >>> ds.create_tensor("data")
681
- >>> ds.create_tensor("point_clouds", htype="point_cloud")
675
+ >>> # Create dataset
676
+ >>> ds = deeplake.empty("path/to/dataset")
682
677
 
683
- >>> # append data
684
- >>> ds.images.append(np.ones((400, 400, 3), dtype='uint8'))
685
- >>> ds.videos.append(deeplake.read("videos/sample_video.mp4"))
686
- >>> ds.data.append(np.zeros((100, 100, 2)))
678
+ >>> # Create tensors
679
+ >>> with ds:
680
+ >>> ds.create_tensor("images", htype="image", sample_compression="jpg")
681
+ >>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
682
+ >>> ds.create_tensor("data")
683
+ >>> ds.create_tensor("point_clouds", htype="point_cloud")
684
+ >>>
685
+ >>> # Append data
686
+ >>> ds.images.append(np.ones((400, 400, 3), dtype='uint8'))
687
+ >>> ds.videos.append(deeplake.read("videos/sample_video.mp4"))
688
+ >>> ds.data.append(np.zeros((100, 100, 2)))
687
689
 
688
690
  Args:
689
691
  name (str): The name of the tensor to be created.
690
692
  htype (str):
691
- - The class of data for the tensor.
692
- - The defaults for other parameters are determined in terms of this value.
693
+ - The class of data for the tensor. Specifying ``htype`` is highly recommended for complex data such as images, video, dicom, text, json, etc. Specifying htype is not necessary for simple numeric date such as arrays and scalars.
694
+ - The defaults for other parameters are determined in terms of this value, thus improving error checking and visualization.
693
695
  - For example, ``htype="image"`` would have ``dtype`` default to ``uint8``.
694
696
  - These defaults can be overridden by explicitly passing any of the other parameters to this function.
695
697
  - May also modify the defaults for other parameters.
696
- dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``.
698
+ dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``. Specifying ``dtype`` is typically not necessary unless you are dealing with mixed numerical data in a single tensor (column).
697
699
  sample_compression (str): All samples will be compressed in the provided format. If ``None``, samples are uncompressed. For ``link[]`` tensors, ``sample_compression`` is used only for optimizing dataset views.
698
700
  chunk_compression (str): All chunks will be compressed in the provided format. If ``None``, chunks are uncompressed. For ``link[]`` tensors, ``chunk_compression`` is used only for optimizing dataset views.
699
701
  hidden (bool): If ``True``, the tensor will be hidden from ds.tensors but can still be accessed via ``ds[tensor_name]``.
@@ -1178,7 +1180,7 @@ class Dataset:
1178
1180
  def create_tensor_like(
1179
1181
  self, name: str, source: "Tensor", unlink: bool = False
1180
1182
  ) -> "Tensor":
1181
- """Copies the ``source`` tensor's meta information and creates a new tensor with it. No samples are copied, only the meta/info for the tensor is.
1183
+ """Creates a tensor with the same properties as ``source``. No samples or data copied, other than the tensor meta/info.
1182
1184
 
1183
1185
  Examples:
1184
1186
  >>> ds.create_tensor_like("cats", ds["images"])
@@ -2093,13 +2095,16 @@ class Dataset:
2093
2095
  *args,
2094
2096
  **kwargs,
2095
2097
  ):
2096
- """Converts the dataset into a pytorch Dataloader.
2098
+ """Creates a PyTorch Dataloader from the Deep Lake dataset. During iteration, the data from all tensors will be streamed on-the-fly from the storage location.
2099
+ Understanding the parameters below is critical for achieving fast streaming for your use-case
2097
2100
 
2098
2101
  Args:
2099
2102
  *args: Additional args to be passed to torch_dataset
2100
2103
  **kwargs: Additional kwargs to be passed to torch_dataset
2101
2104
  transform (Callable, Optional): Transformation function to be applied to each sample.
2102
- tensors (List, Optional): Optionally provide a list of tensor names in the ordering that your training script expects. For example, if you have a dataset that has "image" and "label" tensors, if ``tensors=["image", "label"]``, your training script should expect each batch will be provided as a tuple of (image, label).
2105
+ tensors (List, Optional): List of tensors to load. If ``None``, all tensors are loaded. Defaults to ``None``.
2106
+ For datasets with many tensors, its extremely important to stream only the data that is needed for training the model, in order to avoid bottlenecks associated with streaming unused data.
2107
+ For example, if you have a dataset that has ``image``, ``label``, and ``metadata`` tensors, if ``tensors=["image", "label"]``, the Data Loader will only stream the ``image`` and ``label`` tensors.
2103
2108
  num_workers (int): The number of workers to use for fetching data in parallel.
2104
2109
  batch_size (int): Number of samples per batch to load. Default value is 1.
2105
2110
  drop_last (bool): Set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
@@ -2564,10 +2569,10 @@ class Dataset:
2564
2569
 
2565
2570
  @spinner
2566
2571
  def flush(self):
2567
- """Necessary operation after writes if caches are being used.
2568
- Writes all the dirty data from the cache layers (if any) to the underlying storage.
2569
- Here dirty data corresponds to data that has been changed/assigned and but hasn't yet been sent to the
2570
- underlying storage.
2572
+ """
2573
+ Writes all the data that has been changed/assigned from the cache layers (if any) to the underlying storage.
2574
+
2575
+ NOTE: The high-level APIs flush the cache automatically and users generally do not have explicitly run the ``flush`` command.
2571
2576
  """
2572
2577
  self._flush_vc_info()
2573
2578
  self.storage.flush()
@@ -2582,11 +2587,13 @@ class Dataset:
2582
2587
 
2583
2588
  def clear_cache(self):
2584
2589
  """
2585
- - Flushes (see :func:`Dataset.flush`) the contents of the cache layers (if any) and then deletes contents of all the layers of it.
2586
- - This doesn't delete data from the actual storage.
2587
- - This is useful if you have multiple datasets with memory caches open, taking up too much RAM.
2588
- - Also useful when local cache is no longer needed for certain datasets and is taking up storage space.
2590
+ Flushes (see :func:`Dataset.flush`) the contents of the cache layers (if any) and then deletes contents of all the layers of it.
2591
+ This doesn't delete data from the actual storage.
2592
+ This is useful if you have multiple datasets with memory caches open, taking up too much RAM, or when local cache is no longer needed for certain datasets and is taking up storage space.
2593
+
2594
+ NOTE: The high-level APIs clear the cache automatically and users generally do not have explicitly run the ``clear_cache`` command.
2589
2595
  """
2596
+
2590
2597
  if hasattr(self.storage, "clear_cache"):
2591
2598
  self.storage.clear_cache()
2592
2599
 
@@ -2628,7 +2635,7 @@ class Dataset:
2628
2635
 
2629
2636
  @invalid_view_op
2630
2637
  def delete(self, large_ok=False):
2631
- """Deletes the entire dataset from the cache layers (if any) and the underlying storage.
2638
+ """Deletes the entire dataset from the underlying storage and cache layers (if any).
2632
2639
  This is an **IRREVERSIBLE** operation. Data once deleted can not be recovered.
2633
2640
 
2634
2641
  Args:
@@ -2672,10 +2679,10 @@ class Dataset:
2672
2679
  self.storage.clear()
2673
2680
 
2674
2681
  def summary(self, force: bool = False):
2675
- """Prints a summary of the dataset.
2682
+ """Prints a summary of the dataset, including the tensor names and their lengths, shapes, htypes, dtypes, compressions, and other relevant information.
2676
2683
 
2677
2684
  Args:
2678
- force (bool): Dataset views with more than 10000 samples might take a long time to summarize. If `force=True`,
2685
+ force (bool): Dataset views with more than 10000 samples might take several seconds of minutes to summarize. If `force=True`,
2679
2686
  the summary will be printed regardless. An error will be raised otherwise.
2680
2687
 
2681
2688
  Raises:
@@ -2976,7 +2983,8 @@ class Dataset:
2976
2983
  return self[fullname]
2977
2984
 
2978
2985
  def create_group(self, name: str, exist_ok=False) -> "Dataset":
2979
- """Creates a tensor group. Intermediate groups in the path are also created.
2986
+ """Creates a tensor group, which is a collection of tensors that can be reference together.
2987
+ Groups are recommended for use-cases with many tensors in order to organize and reference data more easily.
2980
2988
 
2981
2989
  Args:
2982
2990
  name: The name of the group to create.
@@ -2992,11 +3000,17 @@ class Dataset:
2992
3000
  Examples:
2993
3001
 
2994
3002
  >>> ds.create_group("images")
2995
- >>> ds['images'].create_tensor("cats")
3003
+ >>> ds.images.create_tensor("left_camera", htype = "image", "sample_compression" = "jpeg")
3004
+ >>> ds.images.create_tensor("center_camera", htype = "image", "sample_compression" = "jpeg")
3005
+ >>> ds.images.create_tensor("right_camera", htype = "image", "sample_compression" = "jpeg")
2996
3006
 
2997
- >>> ds.create_groups("images/jpg/cats")
2998
- >>> ds["images"].create_tensor("png")
2999
- >>> ds["images/jpg"].create_group("dogs")
3007
+ >>> # Data from a tensors in groups is referenced using:
3008
+ >>> ds.images.right_camera[0].numpy()
3009
+ >>> # OR
3010
+ >>> ds["images/right_camera"].numpy()
3011
+
3012
+ >>> # "/" Notation can also be used to create groups with ``create_tensor``
3013
+ >>> ds.create_tensor("images/right_camera", htype = "image", "sample_compression" = "jpeg")
3000
3014
  """
3001
3015
 
3002
3016
  deeplake_reporter.feature_report(
@@ -3104,15 +3118,6 @@ class Dataset:
3104
3118
  Exception: Error while attempting to rollback appends.
3105
3119
  SampleAppendingError: Error that occurs when someone tries to append a tensor value directly to the dataset without specifying tensor name.
3106
3120
 
3107
- Examples:
3108
-
3109
- >>> ds = deeplake.empty("../test/test_ds")
3110
- >>> ds.create_tensor('data')
3111
- Tensor(key='data')
3112
- >>> ds.create_tensor('labels')
3113
- Tensor(key='labels')
3114
- >>> ds.append({"data": [1, 2, 3, 4], "labels":[0, 1, 2, 3]})
3115
-
3116
3121
  """
3117
3122
  tensors = self.tensors
3118
3123
  if isinstance(sample, Dataset):
@@ -3205,10 +3210,10 @@ class Dataset:
3205
3210
  ignore_errors: bool = False,
3206
3211
  progressbar: bool = False,
3207
3212
  ):
3208
- """Appends multiple rows of samples to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
3213
+ """Appends multiple samples (rows) to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
3209
3214
 
3210
3215
  Args:
3211
- samples (Dict[str, Any]): Dictionary with tensor names as keys and samples as values.
3216
+ samples (Dict[str, Any]): Dictionary with tensor names as keys and data as values. The values can be a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
3212
3217
  skip_ok (bool): Skip tensors not in ``samples`` if set to True.
3213
3218
  append_empty (bool): Append empty samples to tensors not specified in ``sample`` if set to ``True``. If True, ``skip_ok`` is ignored.
3214
3219
  ignore_errors (bool): Skip samples that cause errors while extending, if set to ``True``.
@@ -3221,6 +3226,18 @@ class Dataset:
3221
3226
  NotImplementedError: If an error occurs while writing tiles.
3222
3227
  SampleExtendError: If the extend failed while appending a sample.
3223
3228
  Exception: Error while attempting to rollback appends.
3229
+
3230
+ Examples:
3231
+
3232
+ >>> ds = deeplake.empty("../test/test_ds")
3233
+
3234
+ >>> with ds:
3235
+ >>> ds.create_tensor('data')
3236
+ >>> ds.create_tensor('labels')
3237
+
3238
+ >>> # This operation will append 4 samples (rows) to the Deep Lake dataset
3239
+ >>> ds.extend({"data": [1, 2, 3, 4], "labels":["table", "chair", "desk", "table"]})
3240
+
3224
3241
  """
3225
3242
  extend = False
3226
3243
  if isinstance(samples, Dataset):
@@ -3283,7 +3300,7 @@ class Dataset:
3283
3300
  skip_ok: bool = False,
3284
3301
  append_empty: bool = False,
3285
3302
  ):
3286
- """Append samples to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
3303
+ """Append a single sample (row) to multiple tensors at once.
3287
3304
 
3288
3305
  Args:
3289
3306
  sample (dict): Dictionary with tensor names as keys and samples as values.
@@ -3301,11 +3318,13 @@ class Dataset:
3301
3318
  Examples:
3302
3319
 
3303
3320
  >>> ds = deeplake.empty("../test/test_ds")
3304
- >>> ds.create_tensor('data')
3305
- Tensor(key='data')
3306
- >>> ds.create_tensor('labels')
3307
- Tensor(key='labels')
3308
- >>> ds.append({"data": [1, 2, 3, 4], "labels":[0, 1, 2, 3]})
3321
+
3322
+ >>> with ds:
3323
+ >>> ds.create_tensor('data')
3324
+ >>> ds.create_tensor('labels')
3325
+
3326
+ >>> # This operation will append 1 sample (row) to the Deep Lake dataset
3327
+ >>> ds.append({"data": 1, "labels": "table"})
3309
3328
 
3310
3329
  """
3311
3330
  new_row_ids = [self.__len__(warn=False)]
@@ -4300,12 +4319,12 @@ class Dataset:
4300
4319
  progressbar=True,
4301
4320
  public: bool = False,
4302
4321
  ):
4303
- """Copies this dataset or dataset view to ``dest``. Version control history is not included.
4322
+ """Copies this dataset or dataset view to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
4304
4323
 
4305
4324
  Args:
4306
4325
  dest (str, pathlib.Path): Destination dataset or path to copy to. If a Dataset instance is provided, it is expected to be empty.
4307
4326
  tensors (List[str], optional): Names of tensors (and groups) to be copied. If not specified all tensors are copied.
4308
- runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub:// paths.
4327
+ runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub://... paths.
4309
4328
  overwrite (bool): If ``True`` and a dataset exists at `destination`, it will be overwritten. Defaults to False.
4310
4329
  creds (dict, Optional): creds required to create / overwrite datasets at `dest`.
4311
4330
  token (str, Optional): token used to for fetching credentials to `dest`.
@@ -4398,21 +4417,25 @@ class Dataset:
4398
4417
  ds_name: Optional[str] = None,
4399
4418
  token: Optional[str] = None,
4400
4419
  ):
4401
- """Connect a Deep Lake cloud dataset through a deeplake path.
4420
+ """Connect a Deep Lake dataset stored in your cloud to the Deep Lake App.
4402
4421
 
4403
4422
  Examples:
4404
- >>> # create/load an s3 dataset
4405
- >>> s3_ds = deeplake.dataset("s3://bucket/dataset")
4406
- >>> ds = s3_ds.connect(dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key", token="my_activeloop_token)
4407
- >>> # or
4408
- >>> ds = s3_ds.connect(org_id="my_org", creds_key="my_managed_credentials_key", token="my_activeloop_token")
4423
+ >>> # Load an s3 dataset
4424
+ >>> s3_ds = deeplake.load("s3://bucket/dataset")
4425
+ >>>
4426
+ >>> # Specify the org_id and managed creds_key, and the dataset name is inferred from the currently storage location path
4427
+ >>> ds = s3_ds.connect(org_id="my_org_id", creds_key="managed_creds_key")
4428
+ >>>
4429
+ >>> # Or if you want to explicitly specify the full path, which contains the org_id
4430
+ >>> ds = s3_ds.connect(dest_path="hub://my_org/dataset", creds_key="managed_creds_key")
4431
+
4409
4432
 
4410
4433
  Args:
4411
4434
  creds_key (str): The managed credentials to be used for accessing the source path.
4412
- dest_path (str, optional): The full path to where the connected Deep Lake dataset will reside. Can be:
4413
- a Deep Lake path like ``hub://organization/dataset``
4414
- org_id (str, optional): The organization to where the connected Deep Lake dataset will be added.
4415
- ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or ``src_path`` if not provided.
4435
+ dest_path (str, optional): The full path to which the connected Deep Lake dataset will reside. Can be:
4436
+ a Deep Lake path like ``hub://<org_id>/<dataset_name>``
4437
+ org_id (str, optional): The organization to which the connected Deep Lake dataset will be added. The dataset name will be the same as parent folder for the dataset in the cloud storage location.
4438
+ ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or storage location path if not provided.
4416
4439
  token (str, optional): Activeloop token used to fetch the managed credentials.
4417
4440
 
4418
4441
  Raises:
@@ -326,8 +326,7 @@ class Tensor:
326
326
  progressbar: bool = False,
327
327
  ignore_errors: bool = False,
328
328
  ):
329
- """Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence, a single batched numpy array,
330
- or a sequence of :func:`deeplake.read` outputs, which can be used to load files. See examples down below.
329
+ """Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
331
330
 
332
331
  Example:
333
332
  Numpy input:
@@ -429,8 +428,7 @@ class Tensor:
429
428
 
430
429
  @invalid_view_op
431
430
  def append(self, sample: InputSample):
432
- """Appends a single sample to the end of the tensor. Can be an array, scalar value, or the return value from :func:`deeplake.read`,
433
- which can be used to load files. See examples down below.
431
+ """Appends a single sample (row) to the end of the tensor.
434
432
 
435
433
  Examples:
436
434
  Numpy input:
@@ -34,19 +34,18 @@ from deeplake.util.exceptions import (
34
34
  HubComposeEmptyListError,
35
35
  HubComposeIncompatibleFunction,
36
36
  TransformError,
37
- SampleAppendError,
38
37
  )
39
38
  from deeplake.hooks import dataset_written, dataset_read
40
39
  from deeplake.util.version_control import auto_checkout
41
40
  from deeplake.util.class_label import sync_labels
42
41
  from deeplake.constants import DEFAULT_TRANSFORM_SAMPLE_CACHE_SIZE
43
42
 
44
- import posixpath
45
-
46
43
 
47
44
  class ComputeFunction:
48
45
  def __init__(self, func, args, kwargs, name: Optional[str] = None):
49
- """Creates a ComputeFunction object that can be evaluated using .eval or used as a part of a Pipeline."""
46
+ """Creates a ComputeFunction object that can be evaluated using ``.eval()`` or used as a part of a Pipeline.
47
+ Compute Functions are evaluated in parallel, where the input data iterable is devided between the workers for processing.
48
+ """
50
49
  self.func = func
51
50
  self.args = args
52
51
  self.kwargs = kwargs
@@ -68,37 +67,58 @@ class ComputeFunction:
68
67
  ignore_errors: bool = False,
69
68
  **kwargs,
70
69
  ):
71
- """Evaluates the ComputeFunction on data_in to produce an output dataset ds_out.
70
+ """
71
+ Evaluates the ComputeFunction on ``data_in`` to produce an output dataset ``ds_out``. The purpose of compute functions is to process the input data in parallel,
72
+ which is useful when rapidly ingesting data to a Deep Lake dataset. Compute Functions can also be executed in-place, where it modifies the input dataset (see ``ds_out`` parameters below) instead of writing to a new dataset.
72
73
 
73
74
  Args:
74
- data_in: Input passed to the transform to generate output dataset. Should support \__getitem__ and \__len__. Can be a Deep Lake dataset.
75
- ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, data_in will be overwritten if it is a Deep Lake dataset, otherwise error will be raised.
76
- It should have all keys being generated in output already present as tensors. It's initial state should be either:-
75
+ data_in: Input passed to the transform to generate output dataset. Should support ``__getitem__`` and ``__len__`` operations. Can be a Deep Lake dataset.
76
+ ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, the ComputeFunction will operate in-place, which means that data will be written to tensors in ``data_in`` .
77
+ All tensors modified in the ComputeFunction should already be defined in ``ds_out``. It's initial state should be either:
77
78
  - Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
78
79
  - All tensors are populated and have same length. In this case new samples are appended to the dataset.
79
80
  num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
80
- scheduler (str): The scheduler to be used to compute the transformation. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
81
- Defaults to 'threaded'.
81
+ scheduler (str): The scheduler to be used to compute the transformation. Supported values include: ``serial``, ``threaded``, and ``processed``.
82
+ Defaults to ``threaded``.
82
83
  progressbar (bool): Displays a progress bar if True (default).
83
- skip_ok (bool): If True, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
84
+ skip_ok (bool): If ``True``, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
84
85
  This is especially useful for inplace transformations in which certain tensors are not modified. Defaults to False.
85
- check_lengths (bool): If True, checks whether ds_out has tensors of same lengths initially.
86
- pad_data_in (bool): NOTE: This is only applicable if data_in is a Deep Lake dataset. If True, pads tensors of data_in to match the length of the largest tensor in data_in.
86
+ check_lengths (bool): If ``True``, checks whether ds_out has tensors of same lengths initially.
87
+ pad_data_in (bool): NOTE: This is only applicable if ``data_in`` is a Deep Lake dataset. If True, pads tensors of data_in to match the length of the largest tensor in data_in.
87
88
  Defaults to False.
88
89
  read_only_ok (bool): If ``True`` and output dataset is same as input dataset, the read-only check is skipped. This can be used to read data in parallel without making changes to underlying dataset.
89
90
  Defaults to False.
90
91
  cache_size (int): Cache size to be used by transform per worker.
91
- checkpoint_interval (int): If > 0, the transform will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
92
- If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if num_workers > 0. Defaults to 0.
92
+ checkpoint_interval (int): If > 0, the ComputeFunction will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
93
+ If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if ``num_workers`` > 0. Defaults to 0.
93
94
  ignore_errors (bool): If ``True``, input samples that causes transform to fail will be skipped and the errors will be ignored **if possible**.
94
95
  **kwargs: Additional arguments.
95
96
 
96
97
  Raises:
97
- InvalidInputDataError: If data_in passed to transform is invalid. It should support \__getitem__ and \__len__ operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as data_in will also raise this.
98
+ InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as data_in will also raise this.
98
99
  InvalidOutputDatasetError: If all the tensors of ds_out passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ds_out will also raise this.
99
100
  TensorMismatchError: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
100
- UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
101
+ UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: ``serial``, ``threaded``, and ``processed``.
102
+ TransformError: All other exceptions raised if there are problems while running the ComputeFunction.
101
103
  ValueError: If ``num_workers`` > 0 and ``checkpoint_interval`` is not a multiple of ``num_workers`` or if ``checkpoint_interval`` > 0 and ds_out is None.
104
+ AllSamplesSkippedError: If all samples are skipped during execution of the Pipeline.
105
+
106
+ Example:
107
+ # Suppose we have a list of dictionaries that we want to upload in parallel to a Deep Lake dataset.
108
+ data_in = [{"label": "cat", "score": 0.9}, {"label": "dog", "score": 0.8}]
109
+
110
+ # First, we define a function that takes a single element of the `data_in` list and uploads it to a Deep Lake dataset
111
+ @deeplake.compute
112
+ def data_upload(data_dict_in, sample_out, score_multiplier):
113
+ label = data_dict_in["label"]
114
+ score = data_dict_in["score"] * score_multiplier
115
+ sample_out.append({"label": label, "score": score})
116
+
117
+ # To evaluate the function, static parameters (if any) are specified to the function input, and the input data iterable and output dataset are specified to `.eval(...)`
118
+ data_upload(score_multiplier).eval(data_in, ds_out, scheduler="threaded", num_workers=4)
119
+
120
+ Note:
121
+ ``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.
102
122
  """
103
123
 
104
124
  pipeline = Pipeline([self])
@@ -124,7 +144,7 @@ class ComputeFunction:
124
144
 
125
145
  class Pipeline:
126
146
  def __init__(self, functions: List[ComputeFunction]):
127
- """Takes a list of functions decorated using :func:`deeplake.compute` and creates a pipeline that can be evaluated using .eval"""
147
+ """Takes a list of functions decorated using :func:`deeplake.compute` and creates a pipeline that can be evaluated using ``.eval()``"""
128
148
  self.functions = functions
129
149
 
130
150
  def __len__(self):
@@ -147,16 +167,18 @@ class Pipeline:
147
167
  verbose: bool = True,
148
168
  **kwargs,
149
169
  ):
150
- """Evaluates the pipeline on ``data_in`` to produce an output dataset ``ds_out``.
170
+ """
171
+ Evaluates the Pipeline of ComputeFunctions on ``data_in`` to produce an output dataset ``ds_out``. The purpose of compute functions is to process the input data in parallel,
172
+ which is useful when rapidly ingesting data to a Deep Lake dataset. Pipelines can also be executed in-place, where it modifies the input dataset (see ``ds_out`` parameters below) instead of writing to a new dataset.
151
173
 
152
174
  Args:
153
- data_in: Input passed to the transform to generate output dataset. Should support \__getitem__ and \__len__. Can be a Deep Lake dataset.
154
- ds_out (Dataset, optional): - The dataset object to which the transform will get written. If this is not provided, ``data_in`` will be overwritten if it is a Deep Lake dataset, otherwise error will be raised.
155
- - It should have all keys being generated in output already present as tensors. It's initial state should be either:
156
- - **Empty**, i.e., all tensors have no samples. In this case all samples are added to the dataset.
157
- - **All tensors are populated and have same length.** In this case new samples are appended to the dataset.
175
+ data_in: Input passed to the transform to generate output dataset. Should support ``__getitem__`` and ``__len__`` operations. Can be a Deep Lake dataset.
176
+ ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, the ComputeFunction will operate in-place, which means that data will be written to tensors in ``data_in`` .
177
+ All tensors modified in the ComputeFunction should already be defined in ``ds_out``. It's initial state should be either:
178
+ - Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
179
+ - All tensors are populated and have same length. In this case new samples are appended to the dataset.
158
180
  num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
159
- scheduler (str): The scheduler to be used to compute the transformation. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
181
+ scheduler (str): The scheduler to be used to compute the transformation. Supported values include: ``serial``, ``threaded``, and ``processed``.
160
182
  Defaults to 'threaded'.
161
183
  progressbar (bool): Displays a progress bar if ``True`` (default).
162
184
  skip_ok (bool): If ``True``, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
@@ -167,38 +189,49 @@ class Pipeline:
167
189
  read_only_ok (bool): If ``True`` and output dataset is same as input dataset, the read-only check is skipped.
168
190
  Defaults to False.
169
191
  cache_size (int): Cache size to be used by transform per worker.
170
- checkpoint_interval (int): If > 0, the transform will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
171
- If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if num_workers > 0. Defaults to 0.
192
+ checkpoint_interval (int): If > 0, the ComputeFunction will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
193
+ If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if ``num_workers`` > 0. Defaults to 0.
172
194
  ignore_errors (bool): If ``True``, input samples that causes transform to fail will be skipped and the errors will be ignored **if possible**.
173
195
  verbose (bool): If ``True``, prints additional information about the transform.
174
196
  **kwargs: Additional arguments.
175
197
 
176
198
  Raises:
177
- InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support \__getitem__ and \__len__ operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``data_in`` will also raise this.
199
+ InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than ``threaded`` with deeplake dataset having base storage as memory as ``data_in`` will also raise this.
178
200
  InvalidOutputDatasetError: If all the tensors of ``ds_out`` passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``ds_out`` will also raise this.
179
201
  TensorMismatchError: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
180
- UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
202
+ UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: ``serial``, ``threaded``, and ``processed``.
181
203
  TransformError: All other exceptions raised if there are problems while running the pipeline.
182
204
  ValueError: If ``num_workers`` > 0 and ``checkpoint_interval`` is not a multiple of ``num_workers`` or if ``checkpoint_interval`` > 0 and ds_out is None.
183
-
205
+ AllSamplesSkippedError: If all samples are skipped during execution of the Pipeline.
206
+ ModuleNotInstalledException: If the module ``ray`` is not installed and the scheduler is set to ``ray``.
184
207
 
185
208
  # noqa: DAR401
186
209
 
187
- Example::
210
+ Example:
211
+
212
+ # Suppose we have a series of operations that we want to perform in parallel on images using reusable pipelines.
213
+ # We use the pipeline to ingest the transfomed data from one dataset to another dataset.
214
+
215
+ # First, we define the ComputeFunctions that will be used in the pipeline
216
+ @deeplake.compute
217
+ def flip_vertical(sample_in, sample_out):
218
+ sample_out.append({'labels': sample_in.labels.numpy(),
219
+ 'images': np.flip(sample_in.images.numpy(), axis = 0)})
188
220
 
189
221
  @deeplake.compute
190
- def my_fn(sample_in: Any, samples_out, my_arg0, my_arg1=0):
191
- samples_out.my_tensor.append(my_arg0 * my_arg1)
222
+ def resize(sample_in, sample_out, new_size):
223
+ sample_out.append({"labels": sample_in.labels.numpy(),
224
+ "images": np.array(Image.fromarray(sample_in.images.numpy()).resize(new_size))})
192
225
 
193
- # This transform can be used using the eval method in one of these 2 ways:-
226
+ # Append the label and image to the output sample
227
+ sample_out.labels.append(sample_in.labels.numpy())
228
+ sample_out.images.append(np.array(Image.fromarray(sample_in.images.numpy()).resize(new_size)))
194
229
 
195
- # Directly evaluating the method
196
- # here arg0 and arg1 correspond to the 3rd and 4th argument in my_fn
197
- my_fn(arg0, arg1).eval(data_in, ds_out, scheduler="threaded", num_workers=5)
230
+ # We can define the pipeline using:
231
+ pipeline = deeplake.compose([flip_vertical(), resize(new_size = (64,64))])
198
232
 
199
- # As a part of a Transform pipeline containing other functions
200
- pipeline = deeplake.compose([my_fn(a, b), another_function(x=2)])
201
- pipeline.eval(data_in, ds_out, scheduler="processed", num_workers=2)
233
+ # Finally, we can evaluate the pipeline using:
234
+ pipeline.eval(ds_in, ds_out, num_workers = 4)
202
235
 
203
236
  Note:
204
237
  ``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.
@@ -149,6 +149,7 @@ def _get_indra_ds_from_s3_provider(
149
149
  read_only=provider.read_only,
150
150
  token=token,
151
151
  origin_path=provider.root,
152
+ profile_name=provider.profile_name,
152
153
  aws_access_key_id=provider.aws_access_key_id,
153
154
  aws_secret_access_key=provider.aws_secret_access_key,
154
155
  aws_session_token=provider.aws_session_token,