deeplake 3.2.19__tar.gz → 3.2.21__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (317) hide show
  1. {deeplake-3.2.19 → deeplake-3.2.21}/PKG-INFO +1 -1
  2. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/__init__.py +1 -1
  3. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/dataset.py +1 -1
  4. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_link.py +19 -0
  5. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk_engine.py +11 -1
  6. deeplake-3.2.21/deeplake/core/compute/provider.py +85 -0
  7. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/serial.py +12 -4
  8. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/dataset.py +9 -3
  9. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/deeplake_cloud_dataset.py +6 -2
  10. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/link_creds.py +15 -4
  11. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_chunk_engine.py +12 -19
  12. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/filter.py +2 -2
  13. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tensor.py +9 -4
  14. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_compute.py +1 -1
  15. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/test_transform.py +50 -0
  16. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform.py +107 -52
  17. deeplake-3.2.21/deeplake/util/bugout_token.py +1 -0
  18. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/exceptions.py +12 -7
  19. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/iterable_ordered_dict.py +1 -2
  20. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/link.py +3 -1
  21. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/transform.py +73 -5
  22. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/PKG-INFO +1 -1
  23. deeplake-3.2.19/deeplake/core/compute/provider.py +0 -78
  24. deeplake-3.2.19/deeplake/util/bugout_token.py +0 -1
  25. {deeplake-3.2.19 → deeplake-3.2.21}/MANIFEST.in +0 -0
  26. {deeplake-3.2.19 → deeplake-3.2.21}/README.md +0 -0
  27. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/__init__.py +0 -0
  28. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/info.py +0 -0
  29. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/link.py +0 -0
  30. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/link_tiled.py +0 -0
  31. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/read.py +0 -0
  32. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/__init__.py +0 -0
  33. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_access_method.py +0 -0
  34. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_agreement.py +0 -0
  35. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api.py +0 -0
  36. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api_tiling.py +0 -0
  37. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api_with_compression.py +0 -0
  38. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  39. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_connect_datasets.py +0 -0
  40. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_dataset.py +0 -0
  41. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_dicom.py +0 -0
  42. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_downsample.py +0 -0
  43. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_events.py +0 -0
  44. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_grayscale.py +0 -0
  45. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  46. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_info.py +0 -0
  47. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  48. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_json.py +0 -0
  49. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_link_tiled.py +0 -0
  50. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_linking.py +0 -0
  51. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_mesh.py +0 -0
  52. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_meta.py +0 -0
  53. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_nifti.py +0 -0
  54. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_none.py +0 -0
  55. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_partial_upload.py +0 -0
  56. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_pickle.py +0 -0
  57. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_point_cloud.py +0 -0
  58. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_polygons.py +0 -0
  59. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_pop.py +0 -0
  60. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_readonly.py +0 -0
  61. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_rechunk.py +0 -0
  62. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_reset.py +0 -0
  63. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_sample_info.py +0 -0
  64. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_text.py +0 -0
  65. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_update_samples.py +0 -0
  66. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_video.py +0 -0
  67. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_views.py +0 -0
  68. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tiled.py +0 -0
  69. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/__init__.py +0 -0
  70. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/__init__.py +0 -0
  71. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/base.py +0 -0
  72. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/dataframe.py +0 -0
  73. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/__init__.py +0 -0
  74. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_coco_template.py +0 -0
  75. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_ingestion.py +0 -0
  76. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_kaggle.py +0 -0
  77. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_yolo_template.py +0 -0
  78. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/__init__.py +0 -0
  79. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/base.py +0 -0
  80. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  81. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/coco.py +0 -0
  82. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/constants.py +0 -0
  83. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/convert.py +0 -0
  84. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/utils.py +0 -0
  85. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/image_classification.py +0 -0
  86. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/kaggle.py +0 -0
  87. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/util.py +0 -0
  88. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  89. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  90. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  91. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  92. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/__init__.py +0 -0
  93. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/auth.py +0 -0
  94. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/commands.py +0 -0
  95. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/test_cli.py +0 -0
  96. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/__init__.py +0 -0
  97. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/client.py +0 -0
  98. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/config.py +0 -0
  99. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/log.py +0 -0
  100. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/test_client.py +0 -0
  101. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/utils.py +0 -0
  102. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/compression.py +0 -0
  103. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/constants.py +0 -0
  104. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/__init__.py +0 -0
  105. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/__init__.py +0 -0
  106. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/base_chunk.py +0 -0
  107. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  108. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  109. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  110. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  111. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_uncompressed.py +0 -0
  112. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  113. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compression.py +0 -0
  114. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/__init__.py +0 -0
  115. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/process.py +0 -0
  116. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/ray.py +0 -0
  117. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/thread.py +0 -0
  118. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/__init__.py +0 -0
  119. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/invalid_view.py +0 -0
  120. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/view_entry.py +0 -0
  121. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/fast_forwarding.py +0 -0
  122. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/index/__init__.py +0 -0
  123. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/index/index.py +0 -0
  124. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/io.py +0 -0
  125. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/ipc.py +0 -0
  126. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_sample.py +0 -0
  127. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_tiled_sample.py +0 -0
  128. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/lock.py +0 -0
  129. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/__init__.py +0 -0
  130. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/dataset_meta.py +0 -0
  131. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/__init__.py +0 -0
  132. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/base_encoder.py +0 -0
  133. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/byte_positions.py +0 -0
  134. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/chunk_id.py +0 -0
  135. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/creds.py +0 -0
  136. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/pad.py +0 -0
  137. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/sequence.py +0 -0
  138. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/shape.py +0 -0
  139. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  140. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/common.py +0 -0
  141. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  142. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  143. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  144. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  145. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  146. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tile.py +0 -0
  147. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/meta.py +0 -0
  148. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/tensor_meta.py +0 -0
  149. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/partial_reader.py +0 -0
  150. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/partial_sample.py +0 -0
  151. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/polygon.py +0 -0
  152. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/__init__.py +0 -0
  153. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/autocomplete.py +0 -0
  154. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/query.py +0 -0
  155. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/sample.py +0 -0
  156. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/serialize.py +0 -0
  157. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/__init__.py +0 -0
  158. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  159. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/gcs.py +0 -0
  160. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/google_drive.py +0 -0
  161. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/local.py +0 -0
  162. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/lru_cache.py +0 -0
  163. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/memory.py +0 -0
  164. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/provider.py +0 -0
  165. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/s3.py +0 -0
  166. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tensor_link.py +0 -0
  167. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/test_serialize.py +0 -0
  168. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/__init__.py +0 -0
  169. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_compression.py +0 -0
  170. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_io.py +0 -0
  171. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_locking.py +0 -0
  172. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_readonly.py +0 -0
  173. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_serialize.py +0 -0
  174. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/__init__.py +0 -0
  175. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/deserialize.py +0 -0
  176. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/optimizer.py +0 -0
  177. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/sample_tiles.py +0 -0
  178. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/serialize.py +0 -0
  179. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/test_optimizer.py +0 -0
  180. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/test_serialize.py +0 -0
  181. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/__init__.py +0 -0
  182. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform_dataset.py +0 -0
  183. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform_tensor.py +0 -0
  184. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/__init__.py +0 -0
  185. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  186. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_diff.py +0 -0
  187. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_node.py +0 -0
  188. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/dataset_diff.py +0 -0
  189. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/test_merge.py +0 -0
  190. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/test_version_control.py +0 -0
  191. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/__init__.py +0 -0
  192. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  193. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/dataloader.py +0 -0
  194. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/dummy_dataloader.py +0 -0
  195. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/libdeeplake_query.py +0 -0
  196. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_pytorch.py +0 -0
  197. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_query.py +0 -0
  198. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_tensorflow.py +0 -0
  199. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/util.py +0 -0
  200. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/hooks.py +0 -0
  201. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/htype.py +0 -0
  202. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/__init__.py +0 -0
  203. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/huggingface/__init__.py +0 -0
  204. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/huggingface/huggingface.py +0 -0
  205. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/__init__.py +0 -0
  206. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  207. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  208. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  209. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/__init__.py +0 -0
  210. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/common.py +0 -0
  211. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/dataset.py +0 -0
  212. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/pytorch.py +0 -0
  213. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  214. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/__init__.py +0 -0
  215. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/common.py +0 -0
  216. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  217. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  218. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/wandb/__init__.py +0 -0
  219. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/wandb/wandb.py +0 -0
  220. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/common.txt +0 -0
  221. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/docs.txt +0 -0
  222. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/plugins.txt +0 -0
  223. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/tests.txt +0 -0
  224. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/__init__.py +0 -0
  225. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/cache_fixtures.py +0 -0
  226. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/client_fixtures.py +0 -0
  227. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/common.py +0 -0
  228. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/dataset_fixtures.py +0 -0
  229. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/path_fixtures.py +0 -0
  230. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/storage_fixtures.py +0 -0
  231. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/__init__.py +0 -0
  232. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/access_method.py +0 -0
  233. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/agreement.py +0 -0
  234. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/array_list.py +0 -0
  235. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/assert_byte_indexes.py +0 -0
  236. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/auto.py +0 -0
  237. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/bugout_reporter.py +0 -0
  238. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/cache_chain.py +0 -0
  239. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/casting.py +0 -0
  240. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/check_installation.py +0 -0
  241. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/check_latest_version.py +0 -0
  242. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/chunk_engine.py +0 -0
  243. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/class_label.py +0 -0
  244. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/compression.py +0 -0
  245. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/compute.py +0 -0
  246. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/connect_dataset.py +0 -0
  247. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/dataset.py +0 -0
  248. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/delete_entry.py +0 -0
  249. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/diff.py +0 -0
  250. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/downsample.py +0 -0
  251. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/empty_sample.py +0 -0
  252. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/encoder.py +0 -0
  253. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/exif.py +0 -0
  254. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/from_tfds.py +0 -0
  255. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/generate_id.py +0 -0
  256. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/hash.py +0 -0
  257. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/htype.py +0 -0
  258. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/image.py +0 -0
  259. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/invalid_view_op.py +0 -0
  260. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/iteration_warning.py +0 -0
  261. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/join_chunks.py +0 -0
  262. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/json.py +0 -0
  263. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/keys.py +0 -0
  264. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/logging.py +0 -0
  265. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/merge.py +0 -0
  266. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/modified.py +0 -0
  267. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/notebook.py +0 -0
  268. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/__init__.py +0 -0
  269. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh.py +0 -0
  270. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh_parser.py +0 -0
  271. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh_reader.py +0 -0
  272. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/object_base_3d.py +0 -0
  273. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parser.py +0 -0
  274. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  275. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parsers.py +0 -0
  276. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_reader.py +0 -0
  277. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  278. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_readers.py +0 -0
  279. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/point_cloud.py +0 -0
  280. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/read_3d_data.py +0 -0
  281. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/path.py +0 -0
  282. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/pretty_print.py +0 -0
  283. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/remove_cache.py +0 -0
  284. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/scheduling.py +0 -0
  285. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/shape_interval.py +0 -0
  286. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/shuffle.py +0 -0
  287. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/spinner.py +0 -0
  288. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/split.py +0 -0
  289. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/storage.py +0 -0
  290. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tag.py +0 -0
  291. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/testing.py +0 -0
  292. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/__init__.py +0 -0
  293. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_auto.py +0 -0
  294. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_connect_dataset.py +0 -0
  295. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  296. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_read.py +0 -0
  297. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_shape_interval.py +0 -0
  298. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_shuffle.py +0 -0
  299. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_split.py +0 -0
  300. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_token.py +0 -0
  301. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_version_control.py +0 -0
  302. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/threading.py +0 -0
  303. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/token.py +0 -0
  304. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/version_control.py +0 -0
  305. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/video.py +0 -0
  306. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/warnings.py +0 -0
  307. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/__init__.py +0 -0
  308. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/video_streaming.py +0 -0
  309. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/visualizer.py +0 -0
  310. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/SOURCES.txt +0 -0
  311. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/dependency_links.txt +0 -0
  312. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/entry_points.txt +0 -0
  313. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/not-zip-safe +0 -0
  314. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/requires.txt +6 -6
  315. {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/top_level.txt +0 -0
  316. {deeplake-3.2.19 → deeplake-3.2.21}/setup.cfg +0 -0
  317. {deeplake-3.2.19 → deeplake-3.2.21}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.2.19
3
+ Version: 3.2.21
4
4
  Summary: Activeloop Deep Lake
5
5
  Home-page: UNKNOWN
6
6
  Author: activeloop.ai
@@ -79,7 +79,7 @@ __all__ = [
79
79
  ]
80
80
 
81
81
 
82
- __version__ = "3.2.19"
82
+ __version__ = "3.2.21"
83
83
  warn_if_update_required(__version__)
84
84
  __encoded_version__ = np.array(__version__)
85
85
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -1175,7 +1175,7 @@ class dataset:
1175
1175
  compute_provider = get_compute_provider(scheduler, num_workers)
1176
1176
  try:
1177
1177
  if progressbar:
1178
- compute_provider.map_with_progressbar(
1178
+ compute_provider.map_with_progress_bar(
1179
1179
  copy_func_with_progress_bar,
1180
1180
  keys,
1181
1181
  len_keys,
@@ -644,3 +644,22 @@ def test_rgb_gray(local_ds, cat_path, hopper_gray_path):
644
644
  assert len(ds.abc[1].shape) == 3
645
645
  assert len(ds.abc[0].numpy().shape) == 3
646
646
  assert len(ds.abc[1].numpy().shape) == 3
647
+
648
+
649
+ def test_creds(hub_cloud_ds_generator, cat_path):
650
+ creds_key = "ENV"
651
+ ds = hub_cloud_ds_generator()
652
+ ds.add_creds_key(creds_key)
653
+ ds.populate_creds(creds_key, from_environment=True)
654
+ with ds:
655
+ tensor = ds.create_tensor("abc", "link[image]", sample_compression="jpeg")
656
+ tensor.append(deeplake.link(cat_path, creds_key))
657
+
658
+ assert tensor[0].creds_key() == creds_key
659
+ ds.add_creds_key("my_s3_creds", True)
660
+ assert ds.get_managed_creds_keys() == ["my_s3_creds"]
661
+ assert set(ds.get_creds_keys()) == {"my_s3_creds", "ENV"}
662
+ ds.update_creds_key("my_s3_creds", managed=True)
663
+ ds = hub_cloud_ds_generator()
664
+ assert ds.get_managed_creds_keys() == ["my_s3_creds"]
665
+ assert set(ds.get_creds_keys()) == {"my_s3_creds", "ENV"}
@@ -1399,6 +1399,7 @@ class ChunkEngine:
1399
1399
  del self.cache[from_chunk.key] # type: ignore
1400
1400
  except KeyError:
1401
1401
  pass
1402
+ self.cache[to_chunk.key] = to_chunk # type: ignore
1402
1403
  return True
1403
1404
 
1404
1405
  def _is_tiled(self, row: int) -> bool:
@@ -1436,7 +1437,11 @@ class ChunkEngine:
1436
1437
  if next_chunk_size + chunk.num_data_bytes < next_chunk.min_chunk_size:
1437
1438
  if next_chunk_commit_id != self.commit_id:
1438
1439
  next_chunk = self.copy_chunk_to_new_commit(next_chunk, next_chunk_name)
1439
- # merge with next chunk
1440
+ chunk_id = chunk.id
1441
+ chunk_name = ChunkIdEncoder.name_from_id(chunk_id)
1442
+ chunk_commit_id, tkey = self.get_chunk_commit(chunk_name)
1443
+ if chunk_commit_id != self.commit_id:
1444
+ chunk = self.copy_chunk_to_new_commit(chunk, chunk_name)
1440
1445
  return self._merge_chunks(
1441
1446
  from_chunk=next_chunk,
1442
1447
  from_chunk_row=next_chunk_row,
@@ -1462,6 +1467,11 @@ class ChunkEngine:
1462
1467
  if prev_chunk_size + chunk.num_data_bytes < prev_chunk.min_chunk_size:
1463
1468
  if prev_chunk_commit_id != self.commit_id:
1464
1469
  prev_chunk = self.copy_chunk_to_new_commit(prev_chunk, prev_chunk_name)
1470
+ chunk_id = chunk.id
1471
+ chunk_name = ChunkIdEncoder.name_from_id(chunk_id)
1472
+ chunk_commit_id, tkey = self.get_chunk_commit(chunk_name)
1473
+ if chunk_commit_id != self.commit_id:
1474
+ chunk = self.copy_chunk_to_new_commit(chunk, chunk_name)
1465
1475
  # merge with previous chunk
1466
1476
  return self._merge_chunks(
1467
1477
  from_chunk=chunk,
@@ -0,0 +1,85 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import Optional
3
+ import threading
4
+ import warnings
5
+ from tqdm.std import tqdm # type: ignore
6
+ from tqdm import TqdmWarning # type: ignore
7
+
8
+
9
+ def get_progress_bar(total_length, desc):
10
+ warnings.simplefilter("ignore", TqdmWarning)
11
+
12
+ return tqdm(
13
+ total=total_length,
14
+ desc=desc,
15
+ bar_format="{desc}: {percentage:.0f}%|{bar}| {n:.0f}/{total_fmt} [{elapsed}<{remaining}",
16
+ )
17
+
18
+
19
+ def get_progress_thread(progress_bar, progress_queue):
20
+ def update_pg(bar, queue):
21
+ while True:
22
+ r = queue.get()
23
+ if r is not None:
24
+ bar.update(r)
25
+ else:
26
+ break
27
+
28
+ progress_thread = threading.Thread(
29
+ target=update_pg, args=(progress_bar, progress_queue)
30
+ )
31
+ progress_thread.start()
32
+ return progress_thread
33
+
34
+
35
+ class ComputeProvider(ABC):
36
+ """An abstract base class for implementing a compute provider."""
37
+
38
+ def __init__(self, workers):
39
+ self.workers = workers
40
+
41
+ def map_with_progress_bar(
42
+ self,
43
+ func,
44
+ iterable,
45
+ total_length: int,
46
+ desc: Optional[str] = None,
47
+ pbar=None,
48
+ pqueue=None,
49
+ ):
50
+ progress_bar = pbar or get_progress_bar(total_length, desc)
51
+ progress_queue = pqueue or self.create_queue()
52
+
53
+ def sub_func(*args, **kwargs):
54
+ def pg_callback(value: int):
55
+ progress_queue.put(value) # type: ignore[trust]
56
+
57
+ return func(pg_callback, *args, **kwargs)
58
+
59
+ progress_thread = get_progress_thread(progress_bar, progress_queue)
60
+
61
+ try:
62
+ result = self.map(sub_func, iterable)
63
+ finally:
64
+ progress_queue.put(None) # type: ignore[trust]
65
+ if pqueue is None and hasattr(progress_queue, "close"):
66
+ progress_queue.close()
67
+ progress_thread.join()
68
+ if pbar is None:
69
+ progress_bar.close()
70
+
71
+ return result
72
+
73
+ @abstractmethod
74
+ def create_queue(self):
75
+ """Creates queue for specific provider"""
76
+
77
+ @abstractmethod
78
+ def map(self, func, iterable):
79
+ """Applies 'func' to each element in 'iterable', collecting the results
80
+ in a list that is returned.
81
+ """
82
+
83
+ @abstractmethod
84
+ def close(self):
85
+ """Closes the provider."""
@@ -1,4 +1,4 @@
1
- from deeplake.core.compute.provider import ComputeProvider
1
+ from deeplake.core.compute.provider import ComputeProvider, get_progress_bar
2
2
 
3
3
 
4
4
  class SerialProvider(ComputeProvider):
@@ -8,8 +8,16 @@ class SerialProvider(ComputeProvider):
8
8
  def map(self, func, iterable):
9
9
  return list(map(func, iterable))
10
10
 
11
- def map_with_progressbar(self, func, iterable, total_length: int, desc=None):
12
- progress_bar = self.get_progressbar(total_length, desc)
11
+ def map_with_progress_bar(
12
+ self,
13
+ func,
14
+ iterable,
15
+ total_length: int,
16
+ desc=None,
17
+ pbar=None,
18
+ pqueue=None,
19
+ ):
20
+ progress_bar = pbar or get_progress_bar(total_length, desc)
13
21
 
14
22
  def sub_func(*args, **kwargs):
15
23
  def pg_callback(value: int):
@@ -22,7 +30,7 @@ class SerialProvider(ComputeProvider):
22
30
  return result
23
31
 
24
32
  def create_queue(self):
25
- raise NotImplementedError("no queues in serial provider")
33
+ return None
26
34
 
27
35
  def close(self):
28
36
  return
@@ -3537,7 +3537,7 @@ class Dataset:
3537
3537
 
3538
3538
  @invalid_view_op
3539
3539
  @spinner
3540
- def reset(self):
3540
+ def reset(self, force: bool = False):
3541
3541
  """Resets the uncommitted changes present in the branch.
3542
3542
 
3543
3543
  Note:
@@ -3547,7 +3547,7 @@ class Dataset:
3547
3547
  if version_state["commit_node"].children:
3548
3548
  print("You are not at the head node of the branch, cannot reset.")
3549
3549
  return
3550
- if not self.has_head_changes:
3550
+ if not self.has_head_changes and not force:
3551
3551
  print("There are no uncommitted changes on this branch.")
3552
3552
  return
3553
3553
 
@@ -3716,7 +3716,13 @@ class Dataset:
3716
3716
 
3717
3717
  def get_creds_keys(self) -> List[str]:
3718
3718
  """Returns the list of creds keys added to the dataset. These are used to fetch external data in linked tensors"""
3719
- return self.link_creds.creds_keys
3719
+ return list(self.link_creds.creds_keys)
3720
+
3721
+ def get_managed_creds_keys(self) -> List[str]:
3722
+ """Returns the list of creds keys added to the dataset that are managed by Activeloop platform. These are used to fetch external data in linked tensors."""
3723
+ raise ValueError(
3724
+ "Managed creds are not supported for datasets that are not connected to activeloop platform."
3725
+ )
3720
3726
 
3721
3727
  def visualize(
3722
3728
  self, width: Union[int, str, None] = None, height: Union[int, str, None] = None
@@ -1,5 +1,5 @@
1
1
  import posixpath
2
- from typing import Any, Dict, Optional, Union
2
+ from typing import Any, Dict, List, Optional, Union
3
3
  from deeplake.client.utils import get_user_name
4
4
  from deeplake.constants import HUB_CLOUD_DEV_USERNAME
5
5
  from deeplake.core.dataset import Dataset
@@ -336,7 +336,7 @@ class DeepLakeCloudDataset(Dataset):
336
336
  if management_changed:
337
337
  managed_info = (managed, key_index)
338
338
  if original_key_is_managed and managed is not False:
339
- self.link_creds.populate_single_manged_creds(creds_key)
339
+ self.link_creds.populate_single_managed_creds(creds_key)
340
340
  except Exception:
341
341
  if replaced_index is not None:
342
342
  # revert the change
@@ -383,3 +383,7 @@ class DeepLakeCloudDataset(Dataset):
383
383
  raise InvalidSourcePathError(
384
384
  f"The dataset being connected is already accessible via Deep Lake path {self.path}"
385
385
  )
386
+
387
+ def get_managed_creds_keys(self) -> List[str]:
388
+ """Returns the list of creds keys added to the dataset that are managed by Activeloop platform. These are used to fetch external data in linked tensors."""
389
+ return list(self.link_creds.managed_creds_keys)
@@ -5,7 +5,11 @@ from deeplake.constants import ALL_CLOUD_PREFIXES
5
5
  from deeplake.core.storage.deeplake_memory_object import DeepLakeMemoryObject
6
6
  from deeplake.core.storage import StorageProvider, storage_factory
7
7
  from deeplake.core.storage.s3 import S3Provider
8
- from deeplake.util.exceptions import MissingCredsError, MissingManagedCredsError
8
+ from deeplake.util.exceptions import (
9
+ ManagedCredentialsNotFoundError,
10
+ MissingCredsError,
11
+ MissingManagedCredsError,
12
+ )
9
13
  from deeplake.util.token import expires_in_to_expires_at, is_expired_token
10
14
  from deeplake.client.log import logger
11
15
 
@@ -168,7 +172,7 @@ class LinkCreds(DeepLakeMemoryObject):
168
172
  obj.creds_mapping = {k: i + 1 for i, k in enumerate(obj.creds_keys)}
169
173
  obj.managed_creds_keys = set(d["managed_creds_keys"])
170
174
  obj.used_creds_keys = set(d["used_creds_keys"])
171
- if "ENV" in obj.used_creds_keys:
175
+ if "ENV" in obj.used_creds_keys and "ENV" not in obj.creds_keys:
172
176
  obj.creds_keys = ["ENV"] + obj.creds_keys
173
177
  obj.creds_mapping["ENV"] = 0
174
178
  obj.is_dirty = False
@@ -224,9 +228,16 @@ class LinkCreds(DeepLakeMemoryObject):
224
228
  assert self.client is not None
225
229
  assert self.org_id is not None
226
230
  for creds_key in self.managed_creds_keys:
227
- self.populate_single_manged_creds(creds_key, verbose=verbose)
231
+ try:
232
+ self.populate_single_managed_creds(creds_key, verbose=verbose)
233
+ except ManagedCredentialsNotFoundError:
234
+ logger.warning(
235
+ f"Credentials '{creds_key}' not found in Activeloop platform. "
236
+ f"Please make sure the credentials are added to the platform and reload the dataset. "
237
+ f"Alternatively, use ds.update_creds_key(key_name, managed=False) to disable the managed credentials.)"
238
+ )
228
239
 
229
- def populate_single_manged_creds(self, creds_key: str, verbose: bool = True):
240
+ def populate_single_managed_creds(self, creds_key: str, verbose: bool = True):
230
241
  assert self.client is not None
231
242
  assert self.org_id is not None
232
243
  creds = self.fetch_managed_creds(creds_key, verbose=verbose)
@@ -96,9 +96,7 @@ class LinkedChunkEngine(ChunkEngine):
96
96
  def linked_sample(
97
97
  self, global_sample_index: int
98
98
  ) -> Union[LinkedSample, LinkedTiledSample]:
99
- creds_encoder = self.creds_encoder
100
- sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
101
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
99
+ sample_creds_key = self.creds_key(global_sample_index)
102
100
  if self._is_tiled_sample(global_sample_index):
103
101
  path_array: np.ndarray = (
104
102
  super()
@@ -114,11 +112,15 @@ class LinkedChunkEngine(ChunkEngine):
114
112
  sample_path = self.get_path(global_sample_index, fetch_chunks=True)
115
113
  return LinkedSample(sample_path, sample_creds_key)
116
114
 
115
+ def creds_key(self, global_sample_index: int):
116
+ sample_creds_encoded = self.creds_encoder.get_encoded_creds_key(
117
+ global_sample_index
118
+ )
119
+ return self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
120
+
117
121
  def get_video_url(self, global_sample_index):
118
- creds_encoder = self.creds_encoder
119
122
  sample_path = self.get_path(global_sample_index)
120
- sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
121
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
123
+ sample_creds_key = self.creds_key(global_sample_index)
122
124
  storage = None
123
125
  if sample_path.startswith(("gcs://", "gcp://", "s3://")):
124
126
  provider_type = "s3" if sample_path.startswith("s3://") else "gcs"
@@ -166,10 +168,7 @@ class LinkedChunkEngine(ChunkEngine):
166
168
  .path_array
167
169
  )
168
170
 
169
- sample_creds_encoded = self.creds_encoder.get_encoded_creds_key(
170
- global_sample_index
171
- )
172
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
171
+ sample_creds_key = self.creds_key(global_sample_index)
173
172
  tiled_arrays = [
174
173
  read_linked_sample(path, sample_creds_key, self.link_creds, False).array
175
174
  for path in iter(path_array.flat)
@@ -195,10 +194,7 @@ class LinkedChunkEngine(ChunkEngine):
195
194
  )
196
195
  required_tile_paths = ordered_tile_paths[tiles_index]
197
196
 
198
- sample_creds_encoded = self.creds_encoder.get_encoded_creds_key(
199
- global_sample_index
200
- )
201
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
197
+ sample_creds_key = self.creds_key(global_sample_index)
202
198
 
203
199
  tiles = np.vectorize(
204
200
  lambda path: read_linked_sample(
@@ -230,8 +226,7 @@ class LinkedChunkEngine(ChunkEngine):
230
226
  sample_path = self.get_path(global_sample_index, fetch_chunks)
231
227
  if not sample_path:
232
228
  return None
233
- sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
234
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
229
+ sample_creds_key = self.creds_key(global_sample_index)
235
230
  return read_linked_sample(sample_path, sample_creds_key, self.link_creds, False)
236
231
 
237
232
  @property
@@ -329,11 +324,9 @@ class LinkedChunkEngine(ChunkEngine):
329
324
  local_sample_index, cast=cast, copy=copy, decompress=decompress
330
325
  )[0]
331
326
 
332
- creds_encoder = self.creds_encoder
333
327
  if not sample_path:
334
328
  return self.get_empty_sample()
335
- sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
336
- sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
329
+ sample_creds_key = self.creds_key(global_sample_index)
337
330
  read_sample = read_linked_sample(
338
331
  sample_path, sample_creds_key, self.link_creds, False
339
332
  )
@@ -219,7 +219,7 @@ def filter_with_compute(
219
219
 
220
220
  try:
221
221
  if progressbar:
222
- result = compute.map_with_progressbar(pg_filter_slice, idx, total_length=len(dataset)) # type: ignore
222
+ result = compute.map_with_progress_bar(pg_filter_slice, idx, total_length=len(dataset)) # type: ignore
223
223
  else:
224
224
  result = compute.map(filter_slice, idx) # type: ignore
225
225
  index_map = [k for x in result for k in x] # unfold the result map
@@ -446,7 +446,7 @@ def query_inplace(
446
446
  ]
447
447
 
448
448
  if progressbar:
449
- result = compute.map_with_progressbar(pg_subquery, subdatasets, total_length=num_samples) # type: ignore
449
+ result = compute.map_with_progress_bar(pg_subquery, subdatasets, total_length=num_samples) # type: ignore
450
450
  else:
451
451
  result = compute.map(subquery, subdatasets) # type: ignore
452
452
 
@@ -406,7 +406,6 @@ class Tensor:
406
406
  def clear(self):
407
407
  """Deletes all samples from the tensor"""
408
408
  self.chunk_engine.clear()
409
- sample_id_key = get_sample_id_tensor_key(self.key)
410
409
  try:
411
410
  for t in self._all_tensor_links():
412
411
  t.chunk_engine.clear()
@@ -1039,9 +1038,6 @@ class Tensor:
1039
1038
  else:
1040
1039
  val = cast_to_type(val, tensor.dtype)
1041
1040
  tensor[global_sample_index] = val
1042
- # if self.meta.is_link and not has_shape_tensor:
1043
- # func = get_link_transform("update_shape")
1044
- # func(new_sample, link_creds=self.link_creds, tensor_meta=self.meta)
1045
1041
 
1046
1042
  @invalid_view_op
1047
1043
  def pop(self, index: Optional[int] = None):
@@ -1316,6 +1312,15 @@ class Tensor:
1316
1312
  assert isinstance(self.chunk_engine, LinkedChunkEngine)
1317
1313
  return self.chunk_engine.path(self.index, fetch_chunks=fetch_chunks)
1318
1314
 
1315
+ def creds_key(self):
1316
+ """Return path data. Only applicable for linked tensors"""
1317
+ if not self.is_link:
1318
+ raise Exception(f"Only supported for linked tensors.")
1319
+ if self.index.values[0].subscriptable() or len(self.index.values) > 1:
1320
+ raise ValueError("_linked_sample can be used only on exatcly 1 sample.")
1321
+ assert isinstance(self.chunk_engine, LinkedChunkEngine)
1322
+ return self.chunk_engine.creds_key(self.index.values[0].value)
1323
+
1319
1324
  def invalidate_libdeeplake_dataset(self):
1320
1325
  """Invalidates the libdeeplake dataset object."""
1321
1326
  self.dataset.libdeeplake_dataset = None
@@ -15,7 +15,7 @@ def test_compute_with_progress_bar(scheduler):
15
15
 
16
16
  compute = get_compute_provider(scheduler=scheduler, num_workers=2)
17
17
  try:
18
- r = compute.map_with_progressbar(f, range(1000), 1000)
18
+ r = compute.map_with_progress_bar(f, range(1000), 1000)
19
19
 
20
20
  assert r is not None
21
21
  assert len(r) == 1000
@@ -1280,6 +1280,56 @@ def test_none_rechunk_post_transform(local_ds):
1280
1280
  assert num_chunks == 2
1281
1281
 
1282
1282
 
1283
+ @pytest.mark.parametrize("scheduler", ["serial", "threaded", "processed"])
1284
+ def test_transform_checkpointing(local_ds, scheduler):
1285
+ @deeplake.compute
1286
+ def upload(i, ds):
1287
+ if i == 45:
1288
+ raise Exception("test")
1289
+ ds.abc.append(i)
1290
+
1291
+ @deeplake.compute
1292
+ def double(data_in, ds):
1293
+ ds.abc.append(data_in.abc * 2)
1294
+
1295
+ data_in = list(range(100))
1296
+
1297
+ with local_ds as ds:
1298
+ ds.create_tensor("abc")
1299
+
1300
+ # not divisible by num_workers
1301
+ with pytest.raises(ValueError):
1302
+ upload().eval(
1303
+ data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=51
1304
+ )
1305
+
1306
+ # greater than len(data_in)
1307
+ with pytest.raises(ValueError):
1308
+ upload().eval(
1309
+ data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=102
1310
+ )
1311
+
1312
+ # less than 10% of data_in, shows warning
1313
+ with pytest.warns(UserWarning, match="10%"):
1314
+ with pytest.raises(TransformError):
1315
+ upload().eval(
1316
+ data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=8
1317
+ )
1318
+
1319
+ assert len(ds.abc) == 40
1320
+ assert ds.abc.numpy(aslist=True) == list(range(40))
1321
+ with pytest.raises(ValueError):
1322
+ double().eval(ds, num_workers=2, scheduler=scheduler, checkpoint_interval=10)
1323
+
1324
+ # fix input data
1325
+ data_in[45] = 0
1326
+
1327
+ upload().eval(
1328
+ data_in[40:], ds, num_workers=2, scheduler=scheduler, checkpoint_interval=10
1329
+ )
1330
+ assert ds.abc.numpy(aslist=True) == data_in
1331
+
1332
+
1283
1333
  def create_test_ds(path):
1284
1334
  ds = deeplake.empty(path, overwrite=True)
1285
1335
  ds.create_tensor("images", htype="image", sample_compression="jpg")