deeplake 3.9.4__tar.gz → 3.9.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (373) hide show
  1. {deeplake-3.9.4/deeplake.egg-info → deeplake-3.9.6}/PKG-INFO +1 -1
  2. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/__init__.py +1 -1
  3. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compression.py +28 -3
  4. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/dataset.py +10 -2
  5. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/sample.py +10 -3
  6. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/gcs.py +3 -1
  7. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/s3.py +9 -8
  8. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +9 -9
  9. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/dataloader.py +44 -16
  10. deeplake-3.9.6/deeplake/util/bugout_token.py +1 -0
  11. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/storage.py +2 -0
  12. {deeplake-3.9.4 → deeplake-3.9.6/deeplake.egg-info}/PKG-INFO +1 -1
  13. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/requires.txt +9 -9
  14. {deeplake-3.9.4 → deeplake-3.9.6}/setup.py +1 -1
  15. deeplake-3.9.4/deeplake/util/bugout_token.py +0 -1
  16. {deeplake-3.9.4 → deeplake-3.9.6}/LICENSE +0 -0
  17. {deeplake-3.9.4 → deeplake-3.9.6}/MANIFEST.in +0 -0
  18. {deeplake-3.9.4 → deeplake-3.9.6}/README.md +0 -0
  19. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/__init__.py +0 -0
  20. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/dataset.py +0 -0
  21. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/info.py +0 -0
  22. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/link.py +0 -0
  23. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/link_tiled.py +0 -0
  24. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/read.py +0 -0
  25. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/__init__.py +0 -0
  26. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_access_method.py +0 -0
  27. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_agreement.py +0 -0
  28. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api.py +0 -0
  29. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api_tiling.py +0 -0
  30. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api_with_compression.py +0 -0
  31. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_chunk_sizes.py +0 -0
  32. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_connect_datasets.py +0 -0
  33. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_dataset.py +0 -0
  34. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_dicom.py +0 -0
  35. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_downsample.py +0 -0
  36. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_events.py +0 -0
  37. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_grayscale.py +0 -0
  38. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_hosted_datasets.py +0 -0
  39. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_info.py +0 -0
  40. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
  41. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_json.py +0 -0
  42. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_link.py +0 -0
  43. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_link_tiled.py +0 -0
  44. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_linking.py +0 -0
  45. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_mesh.py +0 -0
  46. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_meta.py +0 -0
  47. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_nifti.py +0 -0
  48. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_none.py +0 -0
  49. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_partial_upload.py +0 -0
  50. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_pickle.py +0 -0
  51. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_point_cloud.py +0 -0
  52. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_polygons.py +0 -0
  53. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_pop.py +0 -0
  54. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_readonly.py +0 -0
  55. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_rechunk.py +0 -0
  56. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_reset.py +0 -0
  57. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_sample_info.py +0 -0
  58. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_tag.py +0 -0
  59. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_text.py +0 -0
  60. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_update_samples.py +0 -0
  61. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_video.py +0 -0
  62. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_views.py +0 -0
  63. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tiled.py +0 -0
  64. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/__init__.py +0 -0
  65. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/__init__.py +0 -0
  66. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/base.py +0 -0
  67. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/dataframe.py +0 -0
  68. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/__init__.py +0 -0
  69. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_coco_template.py +0 -0
  70. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_ingestion.py +0 -0
  71. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_kaggle.py +0 -0
  72. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_yolo_template.py +0 -0
  73. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/__init__.py +0 -0
  74. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/base.py +0 -0
  75. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/__init__.py +0 -0
  76. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/coco.py +0 -0
  77. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/constants.py +0 -0
  78. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/convert.py +0 -0
  79. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/utils.py +0 -0
  80. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/image_classification.py +0 -0
  81. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/kaggle.py +0 -0
  82. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/util.py +0 -0
  83. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
  84. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/constants.py +0 -0
  85. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/utils.py +0 -0
  86. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
  87. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/__init__.py +0 -0
  88. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/__init__.py +0 -0
  89. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/activeloop.py +0 -0
  90. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/auth_context.py +0 -0
  91. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/azure.py +0 -0
  92. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/test_auth_context.py +0 -0
  93. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/client.py +0 -0
  94. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/config.py +0 -0
  95. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/log.py +0 -0
  96. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/test_client.py +0 -0
  97. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/utils.py +0 -0
  98. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/compression.py +0 -0
  99. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/constants.py +0 -0
  100. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/__init__.py +0 -0
  101. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/__init__.py +0 -0
  102. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/base_chunk.py +0 -0
  103. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
  104. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
  105. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
  106. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_sample_compressed.py +0 -0
  107. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_uncompressed.py +0 -0
  108. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
  109. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk_engine.py +0 -0
  110. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/__init__.py +0 -0
  111. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/process.py +0 -0
  112. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/provider.py +0 -0
  113. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/ray.py +0 -0
  114. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/serial.py +0 -0
  115. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/thread.py +0 -0
  116. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/__init__.py +0 -0
  117. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
  118. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/indra_dataset_view.py +0 -0
  119. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/indra_tensor_view.py +0 -0
  120. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/invalid_view.py +0 -0
  121. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/view_entry.py +0 -0
  122. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/distance_type.py +0 -0
  123. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/fast_forwarding.py +0 -0
  124. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index/__init__.py +0 -0
  125. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index/index.py +0 -0
  126. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index_maintenance.py +0 -0
  127. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/io.py +0 -0
  128. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/ipc.py +0 -0
  129. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/link_creds.py +0 -0
  130. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_chunk_engine.py +0 -0
  131. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_sample.py +0 -0
  132. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_tiled_sample.py +0 -0
  133. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/lock.py +0 -0
  134. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/__init__.py +0 -0
  135. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/dataset_meta.py +0 -0
  136. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/__init__.py +0 -0
  137. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/base_encoder.py +0 -0
  138. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/byte_positions.py +0 -0
  139. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/chunk_id.py +0 -0
  140. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/creds.py +0 -0
  141. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/pad.py +0 -0
  142. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/sequence.py +0 -0
  143. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/shape.py +0 -0
  144. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/__init__.py +0 -0
  145. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/common.py +0 -0
  146. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
  147. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
  148. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
  149. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
  150. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
  151. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tile.py +0 -0
  152. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/meta.py +0 -0
  153. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/tensor_meta.py +0 -0
  154. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/partial_reader.py +0 -0
  155. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/partial_sample.py +0 -0
  156. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/polygon.py +0 -0
  157. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/__init__.py +0 -0
  158. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/autocomplete.py +0 -0
  159. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/filter.py +0 -0
  160. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/query.py +0 -0
  161. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/seed.py +0 -0
  162. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/serialize.py +0 -0
  163. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/__init__.py +0 -0
  164. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/azure.py +0 -0
  165. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/deeplake_memory_object.py +0 -0
  166. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/google_drive.py +0 -0
  167. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/indra.py +0 -0
  168. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/local.py +0 -0
  169. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/lru_cache.py +0 -0
  170. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/memory.py +0 -0
  171. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/provider.py +0 -0
  172. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tensor.py +0 -0
  173. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tensor_link.py +0 -0
  174. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/test_serialize.py +0 -0
  175. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/__init__.py +0 -0
  176. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_compression.py +0 -0
  177. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_compute.py +0 -0
  178. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_dataset.py +0 -0
  179. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_indra_dataset.py +0 -0
  180. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_io.py +0 -0
  181. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_locking.py +0 -0
  182. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_query.py +0 -0
  183. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_readonly.py +0 -0
  184. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_seed.py +0 -0
  185. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_serialize.py +0 -0
  186. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_vdb_indexes.py +0 -0
  187. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/__init__.py +0 -0
  188. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/deserialize.py +0 -0
  189. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/optimizer.py +0 -0
  190. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/sample_tiles.py +0 -0
  191. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/serialize.py +0 -0
  192. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/test_optimizer.py +0 -0
  193. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/test_serialize.py +0 -0
  194. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/__init__.py +0 -0
  195. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/test_transform.py +0 -0
  196. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform.py +0 -0
  197. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform_dataset.py +0 -0
  198. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform_tensor.py +0 -0
  199. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/__init__.py +0 -0
  200. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
  201. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
  202. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
  203. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
  204. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
  205. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
  206. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
  207. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
  208. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
  209. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
  210. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
  211. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
  212. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
  213. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
  214. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
  215. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
  216. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
  217. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
  218. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
  219. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
  220. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
  221. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
  222. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
  223. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
  224. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
  225. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
  226. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
  227. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
  228. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
  229. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
  230. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
  231. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
  232. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
  233. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
  234. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
  235. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
  236. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/__init__.py +0 -0
  237. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_chunk_map.py +0 -0
  238. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_diff.py +0 -0
  239. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_node.py +0 -0
  240. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/dataset_diff.py +0 -0
  241. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/test_merge.py +0 -0
  242. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/test_version_control.py +0 -0
  243. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/__init__.py +0 -0
  244. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
  245. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/dummy_dataloader.py +0 -0
  246. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/libdeeplake_query.py +0 -0
  247. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_pytorch.py +0 -0
  248. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_query.py +0 -0
  249. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_tensorflow.py +0 -0
  250. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/util.py +0 -0
  251. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/hooks.py +0 -0
  252. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/htype.py +0 -0
  253. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/__init__.py +0 -0
  254. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/huggingface/__init__.py +0 -0
  255. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/huggingface/huggingface.py +0 -0
  256. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/__init__.py +0 -0
  257. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_.py +0 -0
  258. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
  259. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
  260. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/__init__.py +0 -0
  261. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/common.py +0 -0
  262. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/dataset.py +0 -0
  263. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/pytorch.py +0 -0
  264. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
  265. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/__init__.py +0 -0
  266. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/common.py +0 -0
  267. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
  268. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
  269. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/wandb/__init__.py +0 -0
  270. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/wandb/wandb.py +0 -0
  271. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/common.txt +0 -0
  272. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/docs.txt +0 -0
  273. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/plugins.txt +0 -0
  274. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/tests.txt +0 -0
  275. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/__init__.py +0 -0
  276. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/cache_fixtures.py +0 -0
  277. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/client_fixtures.py +0 -0
  278. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/common.py +0 -0
  279. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/dataset_fixtures.py +0 -0
  280. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/path_fixtures.py +0 -0
  281. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/storage_fixtures.py +0 -0
  282. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/__init__.py +0 -0
  283. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/access_method.py +0 -0
  284. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/agreement.py +0 -0
  285. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/array_list.py +0 -0
  286. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/assert_byte_indexes.py +0 -0
  287. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/auto.py +0 -0
  288. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/bugout_reporter.py +0 -0
  289. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/cache_chain.py +0 -0
  290. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/casting.py +0 -0
  291. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/check_installation.py +0 -0
  292. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/check_latest_version.py +0 -0
  293. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/chunk_engine.py +0 -0
  294. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/class_label.py +0 -0
  295. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/compression.py +0 -0
  296. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/compute.py +0 -0
  297. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/connect_dataset.py +0 -0
  298. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/dataset.py +0 -0
  299. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/delete_entry.py +0 -0
  300. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/diff.py +0 -0
  301. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/downsample.py +0 -0
  302. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/empty_sample.py +0 -0
  303. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/encoder.py +0 -0
  304. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/exceptions.py +0 -0
  305. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/exif.py +0 -0
  306. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/from_tfds.py +0 -0
  307. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/generate_id.py +0 -0
  308. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/hash.py +0 -0
  309. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/htype.py +0 -0
  310. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/image.py +0 -0
  311. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/invalid_view_op.py +0 -0
  312. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/iterable_ordered_dict.py +0 -0
  313. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/iteration_warning.py +0 -0
  314. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/join_chunks.py +0 -0
  315. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/json.py +0 -0
  316. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/keys.py +0 -0
  317. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/link.py +0 -0
  318. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/logging.py +0 -0
  319. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/merge.py +0 -0
  320. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/modified.py +0 -0
  321. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/notebook.py +0 -0
  322. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/__init__.py +0 -0
  323. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh.py +0 -0
  324. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_parser.py +0 -0
  325. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_reader.py +0 -0
  326. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/object_base_3d.py +0 -0
  327. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser.py +0 -0
  328. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser_base.py +0 -0
  329. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parsers.py +0 -0
  330. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader.py +0 -0
  331. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader_base.py +0 -0
  332. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_readers.py +0 -0
  333. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/point_cloud.py +0 -0
  334. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/read_3d_data.py +0 -0
  335. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/path.py +0 -0
  336. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/pretty_print.py +0 -0
  337. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/remove_cache.py +0 -0
  338. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/scheduling.py +0 -0
  339. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/shape_interval.py +0 -0
  340. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/shuffle.py +0 -0
  341. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/spinner.py +0 -0
  342. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/split.py +0 -0
  343. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tag.py +0 -0
  344. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tensor_db.py +0 -0
  345. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/testing.py +0 -0
  346. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/__init__.py +0 -0
  347. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_auto.py +0 -0
  348. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_connect_dataset.py +0 -0
  349. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
  350. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_keys.py +0 -0
  351. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_read.py +0 -0
  352. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_shape_interval.py +0 -0
  353. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_shuffle.py +0 -0
  354. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_split.py +0 -0
  355. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_tensor_db.py +0 -0
  356. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_token.py +0 -0
  357. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_version_control.py +0 -0
  358. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_video.py +0 -0
  359. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/threading.py +0 -0
  360. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/token.py +0 -0
  361. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/transform.py +0 -0
  362. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/version_control.py +0 -0
  363. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/video.py +0 -0
  364. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/warnings.py +0 -0
  365. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/__init__.py +0 -0
  366. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/video_streaming.py +0 -0
  367. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/visualizer.py +0 -0
  368. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/SOURCES.txt +0 -0
  369. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/dependency_links.txt +0 -0
  370. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/not-zip-safe +0 -0
  371. {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/top_level.txt +0 -0
  372. {deeplake-3.9.4 → deeplake-3.9.6}/pyproject.toml +0 -0
  373. {deeplake-3.9.4 → deeplake-3.9.6}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.9.4
3
+ Version: 3.9.6
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -89,7 +89,7 @@ __all__ = [
89
89
  ]
90
90
 
91
91
 
92
- __version__ = "3.9.4"
92
+ __version__ = "3.9.6"
93
93
  warn_if_update_required(__version__)
94
94
  __encoded_version__ = np.array(__version__)
95
95
  config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
@@ -43,6 +43,8 @@ else:
43
43
  _NATIVE_INT32 = ">i4"
44
44
  _NATIVE_FLOAT32 = ">f4"
45
45
 
46
+ HEADER_MAX_BYTES = 132
47
+
46
48
  DIMS_RE = re.compile(rb" ([0-9]+)x([0-9]+)")
47
49
  FPS_RE = re.compile(rb" ([0-9]+) fps,")
48
50
  DURATION_RE = re.compile(rb"Duration: ([0-9:.]+),")
@@ -489,11 +491,34 @@ def get_compression(header=None, path=None):
489
491
  ".nii",
490
492
  ".nii.gz",
491
493
  ]
492
- path = str(path).lower()
494
+ path = str(path).lower().partition("?")[0].partition("#")[0].partition(";")[0]
493
495
  for fmt in file_formats:
494
496
  if path.endswith(fmt):
495
497
  return fmt[1:]
496
498
  if header:
499
+ if (
500
+ header[4:12] == b"\x66\x74\x79\x70\x4D\x53\x4E\x56"
501
+ or header[4:12] == b"\x66\x74\x79\x70\x69\x73\x6F\x6D"
502
+ ):
503
+ return "mp4"
504
+ if header[0:4] == b"\x1A\x45\xDF\xA3":
505
+ return "mkv"
506
+ if (
507
+ header[0:2] == b"\xff\xfb"
508
+ or header[0:2] == b"\xff\xf3"
509
+ or header[0:2] == b"\xff\xf2"
510
+ ):
511
+ return "mp3"
512
+ if header[0:2] == b"\x66\x4c\x61\x43":
513
+ return "flac"
514
+ if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x57\x41\x56\x45":
515
+ return "wav"
516
+ if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x41\x56\x49\x20":
517
+ return "avi"
518
+ if header[128:132] == b"\x44\x49\x43\x4D":
519
+ return "dcm"
520
+ if header[0:4] == b"\x6e\x2b\x31\x00":
521
+ return "nii"
497
522
  if not Image.OPEN:
498
523
  Image.init()
499
524
  for fmt in Image.OPEN:
@@ -651,10 +676,10 @@ def read_meta_from_compressed_file(
651
676
  try:
652
677
  if compression is None:
653
678
  if hasattr(f, "read"):
654
- compression = get_compression(f.read(32), path)
679
+ compression = get_compression(f.read(HEADER_MAX_BYTES), path)
655
680
  f.seek(0)
656
681
  else:
657
- compression = get_compression(f[:32], path) # type: ignore
682
+ compression = get_compression(f[:HEADER_MAX_BYTES], path) # type: ignore
658
683
  if compression == "jpeg":
659
684
  try:
660
685
  shape, typestr = _read_jpeg_shape(f), "|u1"
@@ -391,12 +391,20 @@ class Dataset:
391
391
  @property
392
392
  def max_len(self):
393
393
  """Return the maximum length of the tensor."""
394
- return max([len(tensor) for tensor in self.tensors.values()])
394
+ return (
395
+ max([len(tensor) for tensor in self.tensors.values()])
396
+ if self.tensors
397
+ else 0
398
+ )
395
399
 
396
400
  @property
397
401
  def min_len(self):
398
402
  """Return the minimum length of the tensor."""
399
- return min([len(tensor) for tensor in self.tensors.values()])
403
+ return (
404
+ min([len(tensor) for tensor in self.tensors.values()])
405
+ if self.tensors
406
+ else 0
407
+ )
400
408
 
401
409
  def __getstate__(self) -> Dict[str, Any]:
402
410
  """Returns a dict that can be pickled and used to restore this dataset.
@@ -10,6 +10,7 @@ from deeplake.core.compression import (
10
10
  _read_audio_meta,
11
11
  _read_3d_data_meta,
12
12
  _open_nifti,
13
+ HEADER_MAX_BYTES,
13
14
  )
14
15
  from deeplake.compression import (
15
16
  get_compression_type,
@@ -104,7 +105,9 @@ class Sample:
104
105
  self._compression = get_compression(path=self.path)
105
106
  compressed_bytes = self._read_from_path()
106
107
  if self._compression is None:
107
- self._compression = get_compression(header=compressed_bytes[:32])
108
+ self._compression = get_compression(
109
+ header=compressed_bytes[:HEADER_MAX_BYTES]
110
+ )
108
111
  self._shape, self._typestr = verify_compressed_file(compressed_bytes, self._compression) # type: ignore
109
112
 
110
113
  if array is not None:
@@ -318,7 +321,9 @@ class Sample:
318
321
  self._compression = get_compression(path=self.path)
319
322
  compressed_bytes = self._read_from_path()
320
323
  if self._compression is None:
321
- self._compression = get_compression(header=compressed_bytes[:32])
324
+ self._compression = get_compression(
325
+ header=compressed_bytes[:HEADER_MAX_BYTES]
326
+ )
322
327
  if self._compression == compression:
323
328
  if self._shape is None:
324
329
  _, self._shape, self._typestr = read_meta_from_compressed_file(
@@ -328,7 +333,9 @@ class Sample:
328
333
  compressed_bytes = self._recompress(compressed_bytes, compression)
329
334
  elif self._buffer is not None:
330
335
  if self._compression is None:
331
- self._compression = get_compression(header=self._buffer[:32])
336
+ self._compression = get_compression(
337
+ header=self._buffer[:HEADER_MAX_BYTES]
338
+ )
332
339
  if self._compression == compression:
333
340
  compressed_bytes = self._buffer
334
341
  else:
@@ -528,5 +528,7 @@ class GCSProvider(StorageProvider):
528
528
 
529
529
  def get_creds(self):
530
530
  d = self.scoped_credentials.get_token_info()
531
- d["expiration"] = self.expiration or ""
531
+ d["expiration"] = (
532
+ self.expiration if hasattr(self, "expiration") and self.expiration else ""
533
+ )
532
534
  return d
@@ -6,7 +6,7 @@ import boto3
6
6
  import botocore # type: ignore
7
7
  import posixpath
8
8
  import ssl
9
- from typing import Dict, Optional, Tuple, Type
9
+ from typing import Dict, Optional, Tuple, Type, Any
10
10
  from datetime import datetime, timezone
11
11
  from botocore.session import ComponentLocator
12
12
  from deeplake.client.client import DeepLakeBackendClient
@@ -96,6 +96,7 @@ class S3Provider(StorageProvider):
96
96
  aws_region: Optional[str] = None,
97
97
  profile_name: Optional[str] = None,
98
98
  token: Optional[str] = None,
99
+ config: Optional[Any] = None,
99
100
  **kwargs,
100
101
  ):
101
102
  """Initializes the S3Provider
@@ -118,6 +119,7 @@ class S3Provider(StorageProvider):
118
119
  profile_name (str, optional): Specifies the AWS profile name to use.
119
120
  token (str, optional): Activeloop token, used for fetching credentials for Deep Lake datasets (if this is underlying storage for Deep Lake dataset).
120
121
  This is optional, tokens are normally autogenerated.
122
+ config (Any, Optional): s3 client configuration provided by the user. Defaults to None.
121
123
  **kwargs: Additional arguments to pass to the S3 client. Includes: ``expiration``.
122
124
  """
123
125
  self.root = root
@@ -132,7 +134,7 @@ class S3Provider(StorageProvider):
132
134
  self.tag: Optional[str] = None
133
135
  self.token: Optional[str] = token
134
136
  self.loaded_creds_from_environment = False
135
- self.client_config = deeplake.config["s3"]
137
+ self.client_config = config
136
138
  self.start_time = time.time()
137
139
  self.profile_name = profile_name
138
140
  self._initialize_s3_parameters()
@@ -234,17 +236,16 @@ class S3Provider(StorageProvider):
234
236
  def _get_bytes(
235
237
  self, path, start_byte: Optional[int] = None, end_byte: Optional[int] = None
236
238
  ):
239
+ range_kwarg = {}
237
240
  if start_byte is not None and end_byte is not None:
238
241
  if start_byte == end_byte:
239
242
  return b""
240
- range = f"bytes={start_byte}-{end_byte - 1}"
243
+ range_kwarg["Range"] = f"bytes={start_byte}-{end_byte - 1}"
241
244
  elif start_byte is not None:
242
- range = f"bytes={start_byte}-"
245
+ range_kwarg["Range"] = f"bytes={start_byte}-"
243
246
  elif end_byte is not None:
244
- range = f"bytes=0-{end_byte - 1}"
245
- else:
246
- range = ""
247
- resp = self.client.get_object(Bucket=self.bucket, Key=path, Range=range)
247
+ range_kwarg["Range"] = f"bytes=0-{end_byte - 1}"
248
+ resp = self.client.get_object(Bucket=self.bucket, Key=path, **range_kwarg)
248
249
  return resp["Body"].read()
249
250
 
250
251
  def get_bytes(
@@ -195,15 +195,15 @@ def test_deepmemory_evaluate(
195
195
  "recall@100": 0.9,
196
196
  }
197
197
 
198
- # TODO: add this back when the issue with the backend is resolved
199
- # assert recall["with model"] == {
200
- # "recall@1": 0.8,
201
- # "recall@3": 0.8,
202
- # "recall@5": 0.9,
203
- # "recall@10": 0.9,
204
- # "recall@50": 0.9,
205
- # "recall@100": 0.9,
206
- # }
198
+ assert recall["with model"] == {
199
+ "recall@1": 0.9,
200
+ "recall@3": 0.9,
201
+ "recall@5": 0.9,
202
+ "recall@10": 0.9,
203
+ "recall@50": 0.9,
204
+ "recall@100": 0.9,
205
+ }
206
+
207
207
  sleep(15)
208
208
  queries_dataset = VectorStore(
209
209
  path=query_path,
@@ -105,6 +105,7 @@ class DeepLakeDataLoader(DataLoader):
105
105
  _return_index=None,
106
106
  _primary_tensor_name=None,
107
107
  _buffer_size=None,
108
+ _orig_dataset=None,
108
109
  _decode_method=None,
109
110
  _persistent_workers=None,
110
111
  _dataloader=None,
@@ -116,6 +117,7 @@ class DeepLakeDataLoader(DataLoader):
116
117
  ):
117
118
  import_indra_loader()
118
119
  self.dataset = dataset
120
+ self._orig_dataset = _orig_dataset or dataset
119
121
  self._batch_size = _batch_size
120
122
  self._shuffle = _shuffle
121
123
  self._num_threads = _num_threads
@@ -235,7 +237,7 @@ class DeepLakeDataLoader(DataLoader):
235
237
  @property
236
238
  def sampler(self):
237
239
  return (
238
- DistributedSampler(self.dataset)
240
+ DistributedSampler(self._orig_dataset)
239
241
  if self._distributed
240
242
  else _InfiniteConstantSampler()
241
243
  )
@@ -277,9 +279,9 @@ class DeepLakeDataLoader(DataLoader):
277
279
 
278
280
  def __len__(self):
279
281
  len_ds = (
280
- len(self.dataset[self._tensors])
282
+ len(self._orig_dataset[self._tensors])
281
283
  if self._tensors is not None
282
- else len(self.dataset)
284
+ else len(self._orig_dataset)
283
285
  )
284
286
  round_fn = math.floor if self._drop_last else math.ceil
285
287
  return round_fn(len_ds / ((self.batch_size) * self._world_size))
@@ -344,10 +346,12 @@ class DeepLakeDataLoader(DataLoader):
344
346
  all_vars["_shuffle"] = shuffle
345
347
  all_vars["_buffer_size"] = buffer_size
346
348
  if shuffle:
347
- schedule = create_fetching_schedule(self.dataset, self._primary_tensor_name)
349
+ schedule = create_fetching_schedule(
350
+ self._orig_dataset, self._primary_tensor_name
351
+ )
348
352
  if schedule is not None:
349
- ds = self.dataset.no_view_dataset # type: ignore
350
- all_vars["dataset"] = ds[schedule]
353
+ ds = self._orig_dataset # type: ignore
354
+ all_vars["_orig_dataset"] = ds[schedule]
351
355
  all_vars["_dataloader"] = None
352
356
  return self.__class__(**all_vars)
353
357
 
@@ -374,12 +378,12 @@ class DeepLakeDataLoader(DataLoader):
374
378
  all_vars = self.__dict__.copy()
375
379
  if isinstance(transform, dict):
376
380
  tensors = [k for k in transform.keys() if k != "index"]
377
- tensors = map_tensor_keys(self.dataset, tensors)
381
+ tensors = map_tensor_keys(self._orig_dataset, tensors)
378
382
  if self._tensors:
379
383
  raise ValueError(
380
384
  f"Tensors have already been specified in the .{self._mode} method."
381
385
  )
382
- all_vars["_tensors"] = map_tensor_keys(self.dataset, tensors)
386
+ all_vars["_tensors"] = map_tensor_keys(self._orig_dataset, tensors)
383
387
  transform = PytorchTransformFunction(transform_dict=transform)
384
388
  else:
385
389
  if kwargs:
@@ -410,7 +414,7 @@ class DeepLakeDataLoader(DataLoader):
410
414
  >>> query_ds_train = ds_train.dataloader().query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
411
415
  """
412
416
  all_vars = self.__dict__.copy()
413
- all_vars["dataset"] = query(self.dataset, query_string)
417
+ all_vars["_orig_dataset"] = query(self._orig_dataset, query_string)
414
418
  all_vars["_dataloader"] = None
415
419
  return self.__class__(**all_vars)
416
420
 
@@ -455,8 +459,8 @@ class DeepLakeDataLoader(DataLoader):
455
459
 
456
460
  """
457
461
  all_vars = self.__dict__.copy()
458
- all_vars["dataset"] = sample_by(
459
- self.dataset, weights, replace=replace, size=size
462
+ all_vars["_orig_dataset"] = sample_by(
463
+ self._orig_dataset, weights, replace=replace, size=size
460
464
  )
461
465
  all_vars["_dataloader"] = None
462
466
  return self.__class__(**all_vars)
@@ -539,7 +543,7 @@ class DeepLakeDataLoader(DataLoader):
539
543
  all_vars = self.__dict__.copy()
540
544
  all_vars["_num_workers"] = num_workers
541
545
  all_vars["_collate"] = collate_fn
542
- validate_tensors(tensors, self.dataset, all_vars)
546
+ validate_tensors(tensors, self._orig_dataset, all_vars)
543
547
  all_vars["_decode_method"] = decode_method
544
548
  all_vars["_num_threads"] = num_threads
545
549
  all_vars["_prefetch_factor"] = prefetch_factor
@@ -614,7 +618,7 @@ class DeepLakeDataLoader(DataLoader):
614
618
  all_vars = self.__dict__.copy()
615
619
  all_vars["_num_workers"] = num_workers
616
620
  all_vars["_collate"] = collate_fn
617
- validate_tensors(tensors, self.dataset, all_vars)
621
+ validate_tensors(tensors, self._orig_dataset, all_vars)
618
622
  all_vars["_decode_method"] = decode_method
619
623
  all_vars["_num_threads"] = num_threads
620
624
  all_vars["_prefetch_factor"] = prefetch_factor
@@ -659,7 +663,7 @@ class DeepLakeDataLoader(DataLoader):
659
663
  handle_mode(self._mode, mode)
660
664
  all_vars = self.__dict__.copy()
661
665
  all_vars["_num_workers"] = num_workers
662
- validate_tensors(tensors, self.dataset, all_vars)
666
+ validate_tensors(tensors, self._orig_dataset, all_vars)
663
667
  all_vars["_decode_method"] = decode_method
664
668
  all_vars["_tensors"] = self._tensors or tensors
665
669
  all_vars["_num_threads"] = num_threads
@@ -801,9 +805,30 @@ class DeepLakeDataLoader(DataLoader):
801
805
  info=info,
802
806
  )
803
807
 
808
+ def _fill_sample_info_tensors(
809
+ self,
810
+ dataset,
811
+ sample_info_tensors,
812
+ json_tensors,
813
+ list_tensors,
814
+ ):
815
+ for tensor_name in sample_info_tensors:
816
+ tensor = dataset._get_tensor_from_root(tensor_name)
817
+ if len(tensor) == 0:
818
+ raise EmptyTensorError(
819
+ f" the dataset has an empty tensor {tensor_name}, pytorch dataloader can't be created."
820
+ f" Please either populate the tensor or pass tensors argument to .pytorch that excludes this"
821
+ f" tensor."
822
+ )
823
+ meta = tensor.meta
824
+ if meta.htype == "json":
825
+ json_tensors.append(tensor_name)
826
+ elif meta.htype in ["list", "tag"]:
827
+ list_tensors.append(tensor_name)
828
+
804
829
  def __iter__(self):
805
830
  if self._dataloader is None:
806
- dataset = self.dataset
831
+ dataset = self._orig_dataset
807
832
  tensors = self._tensors or map_tensor_keys(dataset, None)
808
833
 
809
834
  jpeg_png_compressed_tensors, json_tensors, list_tensors = check_tensors(
@@ -825,6 +850,9 @@ class DeepLakeDataLoader(DataLoader):
825
850
  sample_info_tensors, tensor_info_tensors = find_additional_tensors_and_info(
826
851
  dataset, data_tensors
827
852
  )
853
+ self._fill_sample_info_tensors(
854
+ dataset, sample_info_tensors, json_tensors, list_tensors
855
+ )
828
856
  tensors.extend(sample_info_tensors)
829
857
  htype_dict, ndim_dict, tensor_info_dict = get_htype_ndim_tensor_info_dicts(
830
858
  dataset, data_tensors, tensor_info_tensors
@@ -855,7 +883,7 @@ class DeepLakeDataLoader(DataLoader):
855
883
  tensor_info_dict=tensor_info_dict,
856
884
  )
857
885
 
858
- dataset_read(self.dataset)
886
+ dataset_read(self._orig_dataset)
859
887
 
860
888
  if self._iterator is not None:
861
889
  self._iterator = iter(self._dataloader)
@@ -0,0 +1 @@
1
+ BUGOUT_TOKEN = 'd9276895-9cfa-4805-9800-a09f06f79d7b'
@@ -84,6 +84,7 @@ def storage_provider_from_path(
84
84
  session_token = creds.get("aws_session_token")
85
85
  endpoint_url = creds.get("endpoint_url")
86
86
  region = creds.get("aws_region") or creds.get("region")
87
+ config = creds.get("config", None) or deeplake.config["s3"]
87
88
  profile = creds.get("profile_name")
88
89
  storage = S3Provider(
89
90
  path,
@@ -94,6 +95,7 @@ def storage_provider_from_path(
94
95
  region,
95
96
  profile_name=profile,
96
97
  token=token,
98
+ config=config,
97
99
  )
98
100
  storage.creds_used = creds_used
99
101
  else:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deeplake
3
- Version: 3.9.4
3
+ Version: 3.9.6
4
4
  Summary: Activeloop Deep Lake
5
5
  Author: activeloop.ai
6
6
  Author-email: support@activeloop.ai
@@ -8,27 +8,27 @@ tqdm
8
8
  lz4
9
9
  pyjwt
10
10
  pydantic
11
- libdeeplake==0.0.126
11
+ libdeeplake==0.0.129
12
12
 
13
13
  [:python_version >= "3.7" and sys_platform != "win32"]
14
14
  aioboto3>=10.4.0
15
15
  nest_asyncio
16
16
 
17
17
  [all]
18
- IPython
19
- google-cloud-storage~=1.42.0
20
- oauth2client~=4.1.3
21
- azure-identity
22
18
  laspy
23
- azure-storage-blob
24
19
  flask
25
20
  google-auth~=2.0.1
26
21
  nibabel
27
22
  google-auth-oauthlib~=0.4.5
28
23
  pydicom
29
- google-api-python-client~=2.31.0
24
+ IPython
25
+ oauth2client~=4.1.3
26
+ azure-storage-blob
30
27
  azure-cli
31
- libdeeplake==0.0.126
28
+ google-cloud-storage~=1.42.0
29
+ google-api-python-client~=2.31.0
30
+ azure-identity
31
+ libdeeplake==0.0.129
32
32
 
33
33
  [all:python_version >= "3.7" or sys_platform != "win32"]
34
34
  av>=8.1.0
@@ -53,7 +53,7 @@ pydicom
53
53
  nibabel
54
54
 
55
55
  [enterprise]
56
- libdeeplake==0.0.126
56
+ libdeeplake==0.0.129
57
57
  pyjwt
58
58
 
59
59
  [gcp]
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
70
70
  extras_require["all"] = [req_map[r] for r in all_extras]
71
71
 
72
72
  if libdeeplake_available():
73
- libdeeplake = "libdeeplake==0.0.126"
73
+ libdeeplake = "libdeeplake==0.0.129"
74
74
  extras_require["enterprise"] = [libdeeplake, "pyjwt"]
75
75
  extras_require["all"].append(libdeeplake)
76
76
  install_requires.append(libdeeplake)
@@ -1 +0,0 @@
1
- BUGOUT_TOKEN = '1ba3468c-36d3-44a0-8180-5bab73ecb13c'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes