deeplake 3.8.1__tar.gz → 3.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (362) hide show
  1. {deeplake-3.8.1/deeplake.egg-info → deeplake-3.8.2}/PKG-INFO +1 -1
  2. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/__init__.py +1 -1
  3. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api.py +11 -2
  4. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api_with_compression.py +2 -2
  5. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk_engine.py +28 -4
  6. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compression.py +1 -1
  7. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_deeplake_indra_dataset.py +4 -4
  8. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform_dataset.py +19 -6
  9. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/dataloader.py +70 -19
  10. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/htype.py +1 -1
  11. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -0
  12. deeplake-3.8.2/deeplake/util/bugout_token.py +1 -0
  13. {deeplake-3.8.1 → deeplake-3.8.2/deeplake.egg-info}/PKG-INFO +1 -1
  14. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/requires.txt +11 -11
  15. {deeplake-3.8.1 → deeplake-3.8.2}/setup.py +1 -1
  16. deeplake-3.8.1/deeplake/util/bugout_token.py +0 -1
  17. {deeplake-3.8.1 → deeplake-3.8.2}/LICENSE +0 -0
  18. {deeplake-3.8.1 → deeplake-3.8.2}/MANIFEST.in +0 -0
  19. {deeplake-3.8.1 → deeplake-3.8.2}/README.md +0 -0
  20. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/__init__.py +0 -0
  21. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/dataset.py +0 -0
  22. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/info.py +0 -0
  23. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/link.py +0 -0
  24. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/link_tiled.py +0 -0
  25. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/read.py +0 -0
  26. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/__init__.py +0 -0
  27. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_access_method.py +0 -0
  28. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_agreement.py +0 -0
  29. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api_tiling.py +0 -0
  30. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  31. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_connect_datasets.py +0 -0
  32. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_dataset.py +0 -0
  33. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_dicom.py +0 -0
  34. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_downsample.py +0 -0
  35. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_events.py +0 -0
  36. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_grayscale.py +0 -0
  37. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  38. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_info.py +0 -0
  39. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  40. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_json.py +0 -0
  41. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_link.py +0 -0
  42. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_link_tiled.py +0 -0
  43. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_linking.py +0 -0
  44. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_mesh.py +0 -0
  45. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_meta.py +0 -0
  46. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_nifti.py +0 -0
  47. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_none.py +0 -0
  48. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_partial_upload.py +0 -0
  49. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_pickle.py +0 -0
  50. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_point_cloud.py +0 -0
  51. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_polygons.py +0 -0
  52. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_pop.py +0 -0
  53. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_readonly.py +0 -0
  54. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_rechunk.py +0 -0
  55. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_reset.py +0 -0
  56. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_sample_info.py +0 -0
  57. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_text.py +0 -0
  58. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_update_samples.py +0 -0
  59. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_video.py +0 -0
  60. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_views.py +0 -0
  61. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tiled.py +0 -0
  62. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/__init__.py +0 -0
  63. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/__init__.py +0 -0
  64. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/base.py +0 -0
  65. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/dataframe.py +0 -0
  66. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/__init__.py +0 -0
  67. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_coco_template.py +0 -0
  68. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_ingestion.py +0 -0
  69. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_kaggle.py +0 -0
  70. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_yolo_template.py +0 -0
  71. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/__init__.py +0 -0
  72. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/base.py +0 -0
  73. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  74. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/coco.py +0 -0
  75. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/constants.py +0 -0
  76. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/convert.py +0 -0
  77. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/utils.py +0 -0
  78. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/image_classification.py +0 -0
  79. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/kaggle.py +0 -0
  80. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/util.py +0 -0
  81. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  82. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  83. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  84. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  85. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/__init__.py +0 -0
  86. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/auth.py +0 -0
  87. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/commands.py +0 -0
  88. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/test_cli.py +0 -0
  89. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/__init__.py +0 -0
  90. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/client.py +0 -0
  91. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/config.py +0 -0
  92. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/log.py +0 -0
  93. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/test_client.py +0 -0
  94. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/utils.py +0 -0
  95. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/compression.py +0 -0
  96. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/constants.py +0 -0
  97. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/__init__.py +0 -0
  98. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/__init__.py +0 -0
  99. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/base_chunk.py +0 -0
  100. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  101. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  102. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  103. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  104. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_uncompressed.py +0 -0
  105. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  106. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/__init__.py +0 -0
  107. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/process.py +0 -0
  108. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/provider.py +0 -0
  109. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/ray.py +0 -0
  110. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/serial.py +0 -0
  111. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/thread.py +0 -0
  112. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/__init__.py +0 -0
  113. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/dataset.py +0 -0
  114. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  115. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_dataset.py +0 -0
  116. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
  117. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/invalid_view.py +0 -0
  118. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/view_entry.py +0 -0
  119. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/distance_type.py +0 -0
  120. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/fast_forwarding.py +0 -0
  121. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/index/__init__.py +0 -0
  122. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/index/index.py +0 -0
  123. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/io.py +0 -0
  124. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/ipc.py +0 -0
  125. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/link_creds.py +0 -0
  126. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_chunk_engine.py +0 -0
  127. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_sample.py +0 -0
  128. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_tiled_sample.py +0 -0
  129. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/lock.py +0 -0
  130. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/__init__.py +0 -0
  131. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/dataset_meta.py +0 -0
  132. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/__init__.py +0 -0
  133. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/base_encoder.py +0 -0
  134. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/byte_positions.py +0 -0
  135. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/chunk_id.py +0 -0
  136. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/creds.py +0 -0
  137. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/pad.py +0 -0
  138. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/sequence.py +0 -0
  139. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/shape.py +0 -0
  140. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  141. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/common.py +0 -0
  142. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  143. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  144. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  145. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  146. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  147. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tile.py +0 -0
  148. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/meta.py +0 -0
  149. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/tensor_meta.py +0 -0
  150. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/partial_reader.py +0 -0
  151. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/partial_sample.py +0 -0
  152. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/polygon.py +0 -0
  153. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/__init__.py +0 -0
  154. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/autocomplete.py +0 -0
  155. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/filter.py +0 -0
  156. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/query.py +0 -0
  157. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/sample.py +0 -0
  158. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/seed.py +0 -0
  159. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/serialize.py +0 -0
  160. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/__init__.py +0 -0
  161. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/azure.py +0 -0
  162. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  163. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/gcs.py +0 -0
  164. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/google_drive.py +0 -0
  165. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/local.py +0 -0
  166. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/lru_cache.py +0 -0
  167. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/memory.py +0 -0
  168. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/provider.py +0 -0
  169. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/s3.py +0 -0
  170. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tensor.py +0 -0
  171. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tensor_link.py +0 -0
  172. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/test_serialize.py +0 -0
  173. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/__init__.py +0 -0
  174. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_compression.py +0 -0
  175. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_compute.py +0 -0
  176. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_io.py +0 -0
  177. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_locking.py +0 -0
  178. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_readonly.py +0 -0
  179. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_seed.py +0 -0
  180. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_serialize.py +0 -0
  181. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_vdb_indexes.py +0 -0
  182. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/__init__.py +0 -0
  183. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/deserialize.py +0 -0
  184. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/optimizer.py +0 -0
  185. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/sample_tiles.py +0 -0
  186. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/serialize.py +0 -0
  187. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/test_optimizer.py +0 -0
  188. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/test_serialize.py +0 -0
  189. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/__init__.py +0 -0
  190. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/test_transform.py +0 -0
  191. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform.py +0 -0
  192. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform_tensor.py +0 -0
  193. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/__init__.py +0 -0
  194. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deep_memory.py +0 -0
  195. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
  196. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deepmemory_vectorstore.py +0 -0
  197. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  198. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deepmemory.py +0 -0
  199. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  200. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  201. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  202. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  203. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  204. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  205. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  206. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  207. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/index.py +0 -0
  208. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  209. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  210. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  211. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  212. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  213. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  214. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  215. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  216. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  217. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  218. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  219. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  220. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  221. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  222. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  223. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  224. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  225. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vectorstore_factory.py +0 -0
  226. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/__init__.py +0 -0
  227. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  228. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_diff.py +0 -0
  229. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_node.py +0 -0
  230. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/dataset_diff.py +0 -0
  231. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/test_merge.py +0 -0
  232. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/test_version_control.py +0 -0
  233. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/__init__.py +0 -0
  234. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  235. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/dummy_dataloader.py +0 -0
  236. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/libdeeplake_query.py +0 -0
  237. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_pytorch.py +0 -0
  238. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_query.py +0 -0
  239. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_tensorflow.py +0 -0
  240. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/util.py +0 -0
  241. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/hooks.py +0 -0
  242. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/__init__.py +0 -0
  243. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/huggingface/__init__.py +0 -0
  244. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/huggingface/huggingface.py +0 -0
  245. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/__init__.py +0 -0
  246. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  247. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  248. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  249. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/__init__.py +0 -0
  250. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/common.py +0 -0
  251. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/dataset.py +0 -0
  252. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/pytorch.py +0 -0
  253. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/__init__.py +0 -0
  254. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/common.py +0 -0
  255. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  256. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  257. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/wandb/__init__.py +0 -0
  258. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/wandb/wandb.py +0 -0
  259. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/common.txt +0 -0
  260. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/docs.txt +0 -0
  261. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/plugins.txt +0 -0
  262. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/tests.txt +0 -0
  263. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/__init__.py +0 -0
  264. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/cache_fixtures.py +0 -0
  265. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/client_fixtures.py +0 -0
  266. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/common.py +0 -0
  267. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/dataset_fixtures.py +0 -0
  268. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/path_fixtures.py +0 -0
  269. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/storage_fixtures.py +0 -0
  270. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/__init__.py +0 -0
  271. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/access_method.py +0 -0
  272. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/agreement.py +0 -0
  273. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/array_list.py +0 -0
  274. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/assert_byte_indexes.py +0 -0
  275. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/auto.py +0 -0
  276. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/bugout_reporter.py +0 -0
  277. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/cache_chain.py +0 -0
  278. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/casting.py +0 -0
  279. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/check_installation.py +0 -0
  280. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/check_latest_version.py +0 -0
  281. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/chunk_engine.py +0 -0
  282. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/class_label.py +0 -0
  283. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/compression.py +0 -0
  284. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/compute.py +0 -0
  285. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/connect_dataset.py +0 -0
  286. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/dataset.py +0 -0
  287. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/delete_entry.py +0 -0
  288. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/diff.py +0 -0
  289. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/downsample.py +0 -0
  290. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/empty_sample.py +0 -0
  291. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/encoder.py +0 -0
  292. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/exceptions.py +0 -0
  293. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/exif.py +0 -0
  294. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/from_tfds.py +0 -0
  295. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/generate_id.py +0 -0
  296. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/hash.py +0 -0
  297. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/htype.py +0 -0
  298. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/image.py +0 -0
  299. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/invalid_view_op.py +0 -0
  300. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/iterable_ordered_dict.py +0 -0
  301. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/iteration_warning.py +0 -0
  302. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/join_chunks.py +0 -0
  303. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/json.py +0 -0
  304. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/keys.py +0 -0
  305. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/link.py +0 -0
  306. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/logging.py +0 -0
  307. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/merge.py +0 -0
  308. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/modified.py +0 -0
  309. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/notebook.py +0 -0
  310. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/__init__.py +0 -0
  311. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh.py +0 -0
  312. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_parser.py +0 -0
  313. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_reader.py +0 -0
  314. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/object_base_3d.py +0 -0
  315. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser.py +0 -0
  316. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  317. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parsers.py +0 -0
  318. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader.py +0 -0
  319. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  320. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_readers.py +0 -0
  321. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/point_cloud.py +0 -0
  322. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/read_3d_data.py +0 -0
  323. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/path.py +0 -0
  324. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/pretty_print.py +0 -0
  325. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/remove_cache.py +0 -0
  326. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/scheduling.py +0 -0
  327. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/shape_interval.py +0 -0
  328. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/shuffle.py +0 -0
  329. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/spinner.py +0 -0
  330. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/split.py +0 -0
  331. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/storage.py +0 -0
  332. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tag.py +0 -0
  333. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tensor_db.py +0 -0
  334. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/testing.py +0 -0
  335. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/__init__.py +0 -0
  336. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_auto.py +0 -0
  337. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_connect_dataset.py +0 -0
  338. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  339. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_keys.py +0 -0
  340. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_read.py +0 -0
  341. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_shape_interval.py +0 -0
  342. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_shuffle.py +0 -0
  343. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_split.py +0 -0
  344. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_tensor_db.py +0 -0
  345. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_token.py +0 -0
  346. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_version_control.py +0 -0
  347. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/threading.py +0 -0
  348. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/token.py +0 -0
  349. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/transform.py +0 -0
  350. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/version_control.py +0 -0
  351. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/video.py +0 -0
  352. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/warnings.py +0 -0
  353. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/__init__.py +0 -0
  354. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/video_streaming.py +0 -0
  355. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/visualizer.py +0 -0
  356. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/SOURCES.txt +0 -0
  357. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/dependency_links.txt +0 -0
  358. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/entry_points.txt +0 -0
  359. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/not-zip-safe +0 -0
  360. {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/top_level.txt +0 -0
  361. {deeplake-3.8.1 → deeplake-3.8.2}/pyproject.toml +0 -0
  362. {deeplake-3.8.1 → deeplake-3.8.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.8.1
3
+ Version: 3.8.2
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -87,7 +87,7 @@ __all__ = [
87
87
  ]
88
88
 
89
89
 
90
- __version__ = "3.8.1"
90
+ __version__ = "3.8.2"
91
91
  warn_if_update_required(__version__)
92
92
  __encoded_version__ = np.array(__version__)
93
93
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -2680,12 +2680,12 @@ def test_tensor_dtype_bug(local_path):
2680
2680
  ds.abc.append(deeplake.link(f"{data_path}/standard.nii.gz"))
2681
2681
 
2682
2682
  assert ds.abc[0].numpy().shape == (4, 5, 7)
2683
- assert ds.abc.dtype == np.dtype("<U1")
2683
+ assert ds.abc.dtype == np.dtype("uint8")
2684
2684
 
2685
2685
  ds2 = ds.copy(f"{local_path}_2", overwrite=True)
2686
2686
 
2687
2687
  assert ds2.abc[0].numpy().shape == (4, 5, 7)
2688
- assert ds2.abc.dtype == np.dtype("<U1")
2688
+ assert ds2.abc.dtype == np.dtype("uint8")
2689
2689
 
2690
2690
 
2691
2691
  def test_iterate_with_groups(memory_ds):
@@ -3043,3 +3043,12 @@ def test_change_htype_fail(local_ds_generator):
3043
3043
  ds.create_tensor("images3", htype="image", sample_compression="jpg")
3044
3044
  with pytest.raises(UnsupportedCompressionError):
3045
3045
  ds.images3.htype = "embedding"
3046
+
3047
+
3048
+ def test_append_non_uint8_to_image(local_ds):
3049
+ with local_ds as ds:
3050
+ ds.create_tensor("images", htype="image", sample_compression="png")
3051
+ ds.images.append(np.zeros((40, 40, 1), dtype=np.int16))
3052
+ ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
3053
+
3054
+ assert ds.images.dtype.name == "int16"
@@ -43,7 +43,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
43
43
  tiling_threshold=1 * MB,
44
44
  )
45
45
 
46
- assert images.meta.dtype == "uint8"
46
+ assert images.meta.dtype == None
47
47
  assert images.meta.sample_compression == "png"
48
48
 
49
49
  _populate_compressed_samples(images, cat_path, flower_path)
@@ -73,7 +73,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
73
73
  def test_iterate_compressed_samples(local_ds, cat_path, flower_path):
74
74
  images = local_ds.create_tensor(TENSOR_KEY, htype="image", sample_compression="png")
75
75
 
76
- assert images.meta.dtype == "uint8"
76
+ assert images.meta.dtype == None
77
77
  assert images.meta.sample_compression == "png"
78
78
 
79
79
  _populate_compressed_samples(images, cat_path, flower_path)
@@ -15,7 +15,7 @@ from typing import (
15
15
  )
16
16
  from deeplake.api.info import Info
17
17
  from deeplake.core.link_creds import LinkCreds
18
- from deeplake.core.linked_sample import LinkedSample
18
+ from deeplake.core.linked_sample import LinkedSample, read_linked_sample
19
19
  from deeplake.core.meta.encode.base_encoder import LAST_SEEN_INDEX_COLUMN
20
20
  from deeplake.core.serialize import HEADER_SIZE_BYTES, text_to_bytes
21
21
  from deeplake.core.tensor_link import (
@@ -692,9 +692,33 @@ class ChunkEngine:
692
692
  if tensor_meta.htype is None and not all_empty:
693
693
  tensor_meta.set_htype(get_htype(samples))
694
694
  if tensor_meta.dtype is None and not all_empty:
695
- tensor_meta.set_dtype(
696
- get_dtype(next(filter(lambda x: x is not None, samples)))
697
- ) # first non empty sample
695
+ if tensor_meta.is_link:
696
+ try:
697
+ # download one sample to get dtype
698
+ sample = next(filter(lambda x: x is not None, samples))
699
+ assert isinstance(
700
+ sample, LinkedSample
701
+ ), "Sample must be LinkedSample"
702
+ dtype = np.dtype(
703
+ read_linked_sample(
704
+ sample.path, sample.creds_key, self.link_creds, True
705
+ )._typestr
706
+ )
707
+ except:
708
+ # assume uint8 if download fails
709
+ dtype = np.dtype("uint8")
710
+ else:
711
+ non_empty_samples = list(filter(lambda x: x is not None, samples))
712
+ for sample in non_empty_samples:
713
+ try:
714
+ dtype = get_dtype(sample)
715
+ break
716
+ except:
717
+ pass
718
+ else:
719
+ if not ignore_errors:
720
+ raise ValueError("Could not determine dtype of samples")
721
+ tensor_meta.set_dtype(dtype)
698
722
  if self._convert_to_list(samples):
699
723
  samples = list(samples)
700
724
  if self._is_temp_label_tensor:
@@ -152,7 +152,7 @@ def compress_bytes(
152
152
  if not buffer:
153
153
  return b""
154
154
  if compression == "lz4":
155
- return lz4.frame.compress(buffer)
155
+ return lz4.block.compress(buffer)
156
156
  else:
157
157
  raise SampleCompressionError(
158
158
  (len(buffer),), compression, f"Not a byte compression: {compression}"
@@ -424,7 +424,7 @@ def test_virtual_tensors(local_auth_ds_generator):
424
424
  np.sqrt(2.0 / (i + 1) / (i + 1))
425
425
  ]
426
426
 
427
- assert deeplake_indra_ds.sample_indices == slice(0, 100, 1)
427
+ assert list(deeplake_indra_ds.sample_indices) == list(range(100))
428
428
  deeplake_indra_ds = deeplake_ds.query(
429
429
  "SELECT *, l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
430
430
  )
@@ -432,10 +432,10 @@ def test_virtual_tensors(local_auth_ds_generator):
432
432
  assert list(deeplake_indra_ds.embeddings.sample_indices) == list(
433
433
  reversed(range(100))
434
434
  )
435
- assert deeplake_indra_ds.score.sample_indices == slice(0, 100, 1)
435
+ assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
436
436
 
437
437
  deeplake_indra_ds = deeplake_ds.query(
438
438
  "SELECT l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
439
439
  )
440
- assert deeplake_indra_ds.sample_indices == slice(0, 100, 1)
441
- assert deeplake_indra_ds.score.sample_indices == slice(0, 100, 1)
440
+ assert list(deeplake_indra_ds.sample_indices) == list(range(100))
441
+ assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
@@ -161,7 +161,7 @@ class TransformDataset:
161
161
  # First sample in tensor
162
162
  # Flush to set meta attributes
163
163
  if dtype is None:
164
- self.flush()
164
+ self.flush(clear_on_fail=False)
165
165
  return
166
166
  sizeof_item = self._calculate_sample_size(item, dtype, htype)
167
167
  except:
@@ -201,22 +201,29 @@ class TransformDataset:
201
201
  updated_tensors[full_name] = len(items)
202
202
  tensor.items.clear()
203
203
 
204
- def _rollback(self, updated_tensors):
204
+ def _rollback(self, updated_tensors, no_dtype_tensors):
205
205
  for t in updated_tensors:
206
206
  chunk_engine = self.all_chunk_engines[t]
207
207
  num_samples = updated_tensors[t]
208
208
  for _ in range(num_samples):
209
209
  chunk_engine.pop(link_callback=chunk_engine._transform_pop_callback)
210
210
 
211
+ if t in no_dtype_tensors:
212
+ meta = chunk_engine.tensor_meta
213
+ meta.dtype = None
214
+ meta.typestr = None
215
+ meta.is_dirty = True
216
+
211
217
  def _clear(self):
212
218
  for tensor in self.data.values():
213
219
  tensor.items.clear()
214
220
  self.cache_used = 0
215
221
 
216
- def flush(self):
222
+ def flush(self, clear_on_fail=True):
217
223
  all_chunk_engines = self.all_chunk_engines
218
224
  label_temp_tensors = self.label_temp_tensors
219
225
  updated_tensors = {}
226
+ no_dtype_tensors = []
220
227
  try:
221
228
  for name, tensor in self.data.items():
222
229
  if not tensor.is_group:
@@ -225,6 +232,11 @@ class TransformDataset:
225
232
  chunk_engine = all_chunk_engines[name]
226
233
  callback = chunk_engine._transform_callback
227
234
 
235
+ meta = chunk_engine.tensor_meta
236
+ if meta.length == 0 and meta.dtype is None:
237
+ # for rolling back dtype change
238
+ no_dtype_tensors.append(name)
239
+
228
240
  if tensor.numpy_only:
229
241
  self._flush_numpy_tensor_to_chunk_engine(
230
242
  name, tensor, chunk_engine, callback, updated_tensors
@@ -234,9 +246,10 @@ class TransformDataset:
234
246
  name, tensor, chunk_engine, callback, updated_tensors
235
247
  )
236
248
  self.start_input_idx = None
249
+ self._clear()
237
250
  except Exception as e:
238
- self._rollback(updated_tensors)
251
+ self._rollback(updated_tensors, no_dtype_tensors)
252
+ if clear_on_fail:
253
+ self._clear()
239
254
  e = e.__cause__ if isinstance(e, SampleAppendError) else e # type: ignore
240
255
  raise SampleAppendError(name) from e
241
- finally:
242
- self._clear()
@@ -40,6 +40,7 @@ import warnings
40
40
  import math
41
41
 
42
42
  import itertools
43
+ import multiprocessing
43
44
 
44
45
  original_islice = itertools.islice
45
46
 
@@ -146,6 +147,7 @@ class DeepLakeDataLoader(DataLoader):
146
147
  self._IterableDataset_len_called = None
147
148
  self._iterator = None
148
149
  self._worker_init_fn = None
150
+ self.__multiprocessing_context = None
149
151
 
150
152
  @property
151
153
  def batch_size(self):
@@ -187,7 +189,46 @@ class DeepLakeDataLoader(DataLoader):
187
189
 
188
190
  @property # type: ignore
189
191
  def multiprocessing_context(self):
190
- return None
192
+ return self.__multiprocessing_context
193
+
194
+ @multiprocessing_context.setter
195
+ def multiprocessing_context(self, multiprocessing_context):
196
+ if multiprocessing_context is not None:
197
+ if self.num_workers > 0:
198
+ if isinstance(multiprocessing_context, str):
199
+ valid_start_methods = multiprocessing.get_all_start_methods()
200
+ if multiprocessing_context not in valid_start_methods:
201
+ raise ValueError(
202
+ (
203
+ "multiprocessing_context option "
204
+ "should specify a valid start method in {!r}, but got "
205
+ "multiprocessing_context={!r}"
206
+ ).format(valid_start_methods, multiprocessing_context)
207
+ )
208
+ multiprocessing_context = multiprocessing.get_context(
209
+ multiprocessing_context
210
+ )
211
+
212
+ if not isinstance(
213
+ multiprocessing_context, multiprocessing.context.BaseContext
214
+ ):
215
+ raise TypeError(
216
+ (
217
+ "multiprocessing_context option should be a valid context "
218
+ "object or a string specifying the start method, but got "
219
+ "multiprocessing_context={}"
220
+ ).format(multiprocessing_context)
221
+ )
222
+ else:
223
+ raise ValueError(
224
+ (
225
+ "multiprocessing_context can only be used with "
226
+ "multi-process loading (num_workers > 0), but got "
227
+ "num_workers={}"
228
+ ).format(self.num_workers)
229
+ )
230
+
231
+ self.__multiprocessing_context = multiprocessing_context
191
232
 
192
233
  @property
193
234
  def _dataset_kind(self):
@@ -424,9 +465,9 @@ class DeepLakeDataLoader(DataLoader):
424
465
 
425
466
  def close(self):
426
467
  """Shuts down the workers and releases the resources."""
427
- if self._iterator is not None:
468
+ if hasattr(self, "_iterator") and self._iterator is not None:
428
469
  self._iterator.close()
429
- if self._dataloader is not None:
470
+ if hasattr(self, "_dataloader") and self._dataloader is not None:
430
471
  self._dataloader = None
431
472
 
432
473
  def pytorch(
@@ -697,6 +738,30 @@ class DeepLakeDataLoader(DataLoader):
697
738
  "not equal after each initialization and iteration through the dataloader. If you intend to shuffle data while "
698
739
  "preserving the offset for resuming iteration at a predictable index and order, please set a random seed using deeplake.random()"
699
740
  )
741
+ from indra.pytorch.tensorinfo import TensorsInfo, LoaderMetaInfo # type:ignore
742
+
743
+ info = TensorsInfo(
744
+ raw_tensors=raw_tensors or [],
745
+ htype_dict=htype_dict or {},
746
+ ndim_dict=ndim_dict or {},
747
+ tensor_info_dict=tensor_info_dict or {},
748
+ pil_compressed_tensors=pil_compressed_tensors or [],
749
+ json_tensors=json_tensors or [],
750
+ list_tensors=list_tensors or [],
751
+ )
752
+
753
+ loader_meta = LoaderMetaInfo(
754
+ context=self.multiprocessing_context,
755
+ distributed=self._distributed,
756
+ upcast=self._mode == "pytorch", # upcast to handle unsupported dtypes,
757
+ return_index=self._return_index,
758
+ verbose=self._verbose,
759
+ ignore_errors=self._ignore_errors,
760
+ prefetch_factor=self._prefetch_factor,
761
+ offset=self._offset,
762
+ primary_tensor=self._primary_tensor_name,
763
+ worker_init_fn=self.worker_init_fn,
764
+ )
700
765
 
701
766
  return INDRA_LOADER( # type: ignore [misc]
702
767
  indra_dataset,
@@ -706,26 +771,12 @@ class DeepLakeDataLoader(DataLoader):
706
771
  num_workers=self._num_workers,
707
772
  collate_fn=self.collate_fn,
708
773
  transform_fn=self._transform,
709
- distributed=self._distributed,
710
- prefetch_factor=self._prefetch_factor,
711
774
  tensors=tensors,
712
775
  drop_last=self._drop_last,
713
- ignore_errors=self._ignore_errors,
714
- upcast=self._mode == "pytorch", # upcast to handle unsupported dtypes,
715
- return_index=self._return_index,
716
- primary_tensor=self._primary_tensor_name,
717
776
  buffer_size=self._buffer_size,
718
- raw_tensors=raw_tensors,
719
- pil_compressed_tensors=pil_compressed_tensors,
720
- json_tensors=json_tensors,
721
- list_tensors=list_tensors,
722
777
  persistent_workers=self._persistent_workers,
723
- htype_dict=htype_dict,
724
- ndim_dict=ndim_dict,
725
- tensor_info_dict=tensor_info_dict,
726
- verbose=self._verbose,
727
- offset=self._offset,
728
- worker_init_fn=self.worker_init_fn,
778
+ loader_meta=loader_meta,
779
+ info=info,
729
780
  )
730
781
 
731
782
  def __iter__(self):
@@ -50,7 +50,7 @@ UNSPECIFIED = "unspecified"
50
50
  HTYPE_CONFIGURATIONS: Dict[str, Dict] = {
51
51
  htype.DEFAULT: {"dtype": None},
52
52
  htype.IMAGE: {
53
- "dtype": "uint8",
53
+ "dtype": None,
54
54
  "intrinsics": None,
55
55
  "_info": ["intrinsics"],
56
56
  },
@@ -46,6 +46,11 @@ class ShuffleBuffer:
46
46
  )
47
47
  self.pbar_closed = False
48
48
 
49
+ def reset(self):
50
+ self.buffer = list()
51
+ self.buffer_used = 0
52
+ self.num_torch_tensors = 0
53
+
49
54
  def exchange(self, sample):
50
55
  """Shuffle with existing elements in a buffer and return value if buffer is full or if `None` is provided as argument.
51
56
 
@@ -0,0 +1 @@
1
+ BUGOUT_TOKEN = '120ee6dc-8a93-4c96-8032-edafd33d2dac'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.8.1
3
+ Version: 3.8.2
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -7,27 +7,27 @@ humbug>=0.3.1
7
7
  tqdm
8
8
  lz4
9
9
  pyjwt
10
- libdeeplake==0.0.83
10
+ libdeeplake==0.0.84
11
11
 
12
12
  [:python_version >= "3.7" and sys_platform != "win32"]
13
13
  aioboto3>=10.4.0
14
14
  nest_asyncio
15
15
 
16
16
  [all]
17
- google-cloud-storage~=1.42.0
17
+ google-auth-oauthlib~=0.4.5
18
+ azure-cli
19
+ azure-identity
20
+ IPython
18
21
  google-api-python-client~=2.31.0
22
+ flask
19
23
  azure-storage-blob
20
- azure-identity
24
+ oauth2client~=4.1.3
25
+ pydicom
21
26
  laspy
22
- google-auth-oauthlib~=0.4.5
27
+ google-cloud-storage~=1.42.0
23
28
  google-auth~=2.0.1
24
- azure-cli
25
29
  nibabel
26
- oauth2client~=4.1.3
27
- pydicom
28
- IPython
29
- flask
30
- libdeeplake==0.0.83
30
+ libdeeplake==0.0.84
31
31
 
32
32
  [all:python_version >= "3.7" or sys_platform != "win32"]
33
33
  av>=8.1.0
@@ -52,7 +52,7 @@ pydicom
52
52
  nibabel
53
53
 
54
54
  [enterprise]
55
- libdeeplake==0.0.83
55
+ libdeeplake==0.0.84
56
56
  pyjwt
57
57
 
58
58
  [gcp]
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
70
70
  extras_require["all"] = [req_map[r] for r in all_extras]
71
71
 
72
72
  if libdeeplake_available():
73
- libdeeplake = "libdeeplake==0.0.83"
73
+ libdeeplake = "libdeeplake==0.0.84"
74
74
  extras_require["enterprise"] = [libdeeplake, "pyjwt"]
75
75
  extras_require["all"].append(libdeeplake)
76
76
  install_requires.append(libdeeplake)
@@ -1 +0,0 @@
1
- BUGOUT_TOKEN = '718a213b-6509-495b-aadb-f9e63c4c7240'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes