deeplake 3.9.4__tar.gz → 3.9.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.4/deeplake.egg-info → deeplake-3.9.6}/PKG-INFO +1 -1
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/__init__.py +1 -1
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compression.py +28 -3
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/dataset.py +10 -2
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/sample.py +10 -3
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/gcs.py +3 -1
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/s3.py +9 -8
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +9 -9
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/dataloader.py +44 -16
- deeplake-3.9.6/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/storage.py +2 -0
- {deeplake-3.9.4 → deeplake-3.9.6/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/requires.txt +9 -9
- {deeplake-3.9.4 → deeplake-3.9.6}/setup.py +1 -1
- deeplake-3.9.4/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.4 → deeplake-3.9.6}/LICENSE +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/MANIFEST.in +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/README.md +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/info.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/link.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/read.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/client.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/config.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/log.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/compression.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/constants.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/io.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tensor.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/hooks.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/htype.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/image.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/json.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/link.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/path.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/split.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/token.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/video.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/pyproject.toml +0 -0
- {deeplake-3.9.4 → deeplake-3.9.6}/setup.cfg +0 -0
|
@@ -43,6 +43,8 @@ else:
|
|
|
43
43
|
_NATIVE_INT32 = ">i4"
|
|
44
44
|
_NATIVE_FLOAT32 = ">f4"
|
|
45
45
|
|
|
46
|
+
HEADER_MAX_BYTES = 132
|
|
47
|
+
|
|
46
48
|
DIMS_RE = re.compile(rb" ([0-9]+)x([0-9]+)")
|
|
47
49
|
FPS_RE = re.compile(rb" ([0-9]+) fps,")
|
|
48
50
|
DURATION_RE = re.compile(rb"Duration: ([0-9:.]+),")
|
|
@@ -489,11 +491,34 @@ def get_compression(header=None, path=None):
|
|
|
489
491
|
".nii",
|
|
490
492
|
".nii.gz",
|
|
491
493
|
]
|
|
492
|
-
path = str(path).lower()
|
|
494
|
+
path = str(path).lower().partition("?")[0].partition("#")[0].partition(";")[0]
|
|
493
495
|
for fmt in file_formats:
|
|
494
496
|
if path.endswith(fmt):
|
|
495
497
|
return fmt[1:]
|
|
496
498
|
if header:
|
|
499
|
+
if (
|
|
500
|
+
header[4:12] == b"\x66\x74\x79\x70\x4D\x53\x4E\x56"
|
|
501
|
+
or header[4:12] == b"\x66\x74\x79\x70\x69\x73\x6F\x6D"
|
|
502
|
+
):
|
|
503
|
+
return "mp4"
|
|
504
|
+
if header[0:4] == b"\x1A\x45\xDF\xA3":
|
|
505
|
+
return "mkv"
|
|
506
|
+
if (
|
|
507
|
+
header[0:2] == b"\xff\xfb"
|
|
508
|
+
or header[0:2] == b"\xff\xf3"
|
|
509
|
+
or header[0:2] == b"\xff\xf2"
|
|
510
|
+
):
|
|
511
|
+
return "mp3"
|
|
512
|
+
if header[0:2] == b"\x66\x4c\x61\x43":
|
|
513
|
+
return "flac"
|
|
514
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x57\x41\x56\x45":
|
|
515
|
+
return "wav"
|
|
516
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x41\x56\x49\x20":
|
|
517
|
+
return "avi"
|
|
518
|
+
if header[128:132] == b"\x44\x49\x43\x4D":
|
|
519
|
+
return "dcm"
|
|
520
|
+
if header[0:4] == b"\x6e\x2b\x31\x00":
|
|
521
|
+
return "nii"
|
|
497
522
|
if not Image.OPEN:
|
|
498
523
|
Image.init()
|
|
499
524
|
for fmt in Image.OPEN:
|
|
@@ -651,10 +676,10 @@ def read_meta_from_compressed_file(
|
|
|
651
676
|
try:
|
|
652
677
|
if compression is None:
|
|
653
678
|
if hasattr(f, "read"):
|
|
654
|
-
compression = get_compression(f.read(
|
|
679
|
+
compression = get_compression(f.read(HEADER_MAX_BYTES), path)
|
|
655
680
|
f.seek(0)
|
|
656
681
|
else:
|
|
657
|
-
compression = get_compression(f[:
|
|
682
|
+
compression = get_compression(f[:HEADER_MAX_BYTES], path) # type: ignore
|
|
658
683
|
if compression == "jpeg":
|
|
659
684
|
try:
|
|
660
685
|
shape, typestr = _read_jpeg_shape(f), "|u1"
|
|
@@ -391,12 +391,20 @@ class Dataset:
|
|
|
391
391
|
@property
|
|
392
392
|
def max_len(self):
|
|
393
393
|
"""Return the maximum length of the tensor."""
|
|
394
|
-
return
|
|
394
|
+
return (
|
|
395
|
+
max([len(tensor) for tensor in self.tensors.values()])
|
|
396
|
+
if self.tensors
|
|
397
|
+
else 0
|
|
398
|
+
)
|
|
395
399
|
|
|
396
400
|
@property
|
|
397
401
|
def min_len(self):
|
|
398
402
|
"""Return the minimum length of the tensor."""
|
|
399
|
-
return
|
|
403
|
+
return (
|
|
404
|
+
min([len(tensor) for tensor in self.tensors.values()])
|
|
405
|
+
if self.tensors
|
|
406
|
+
else 0
|
|
407
|
+
)
|
|
400
408
|
|
|
401
409
|
def __getstate__(self) -> Dict[str, Any]:
|
|
402
410
|
"""Returns a dict that can be pickled and used to restore this dataset.
|
|
@@ -10,6 +10,7 @@ from deeplake.core.compression import (
|
|
|
10
10
|
_read_audio_meta,
|
|
11
11
|
_read_3d_data_meta,
|
|
12
12
|
_open_nifti,
|
|
13
|
+
HEADER_MAX_BYTES,
|
|
13
14
|
)
|
|
14
15
|
from deeplake.compression import (
|
|
15
16
|
get_compression_type,
|
|
@@ -104,7 +105,9 @@ class Sample:
|
|
|
104
105
|
self._compression = get_compression(path=self.path)
|
|
105
106
|
compressed_bytes = self._read_from_path()
|
|
106
107
|
if self._compression is None:
|
|
107
|
-
self._compression = get_compression(
|
|
108
|
+
self._compression = get_compression(
|
|
109
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
110
|
+
)
|
|
108
111
|
self._shape, self._typestr = verify_compressed_file(compressed_bytes, self._compression) # type: ignore
|
|
109
112
|
|
|
110
113
|
if array is not None:
|
|
@@ -318,7 +321,9 @@ class Sample:
|
|
|
318
321
|
self._compression = get_compression(path=self.path)
|
|
319
322
|
compressed_bytes = self._read_from_path()
|
|
320
323
|
if self._compression is None:
|
|
321
|
-
self._compression = get_compression(
|
|
324
|
+
self._compression = get_compression(
|
|
325
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
326
|
+
)
|
|
322
327
|
if self._compression == compression:
|
|
323
328
|
if self._shape is None:
|
|
324
329
|
_, self._shape, self._typestr = read_meta_from_compressed_file(
|
|
@@ -328,7 +333,9 @@ class Sample:
|
|
|
328
333
|
compressed_bytes = self._recompress(compressed_bytes, compression)
|
|
329
334
|
elif self._buffer is not None:
|
|
330
335
|
if self._compression is None:
|
|
331
|
-
self._compression = get_compression(
|
|
336
|
+
self._compression = get_compression(
|
|
337
|
+
header=self._buffer[:HEADER_MAX_BYTES]
|
|
338
|
+
)
|
|
332
339
|
if self._compression == compression:
|
|
333
340
|
compressed_bytes = self._buffer
|
|
334
341
|
else:
|
|
@@ -528,5 +528,7 @@ class GCSProvider(StorageProvider):
|
|
|
528
528
|
|
|
529
529
|
def get_creds(self):
|
|
530
530
|
d = self.scoped_credentials.get_token_info()
|
|
531
|
-
d["expiration"] =
|
|
531
|
+
d["expiration"] = (
|
|
532
|
+
self.expiration if hasattr(self, "expiration") and self.expiration else ""
|
|
533
|
+
)
|
|
532
534
|
return d
|
|
@@ -6,7 +6,7 @@ import boto3
|
|
|
6
6
|
import botocore # type: ignore
|
|
7
7
|
import posixpath
|
|
8
8
|
import ssl
|
|
9
|
-
from typing import Dict, Optional, Tuple, Type
|
|
9
|
+
from typing import Dict, Optional, Tuple, Type, Any
|
|
10
10
|
from datetime import datetime, timezone
|
|
11
11
|
from botocore.session import ComponentLocator
|
|
12
12
|
from deeplake.client.client import DeepLakeBackendClient
|
|
@@ -96,6 +96,7 @@ class S3Provider(StorageProvider):
|
|
|
96
96
|
aws_region: Optional[str] = None,
|
|
97
97
|
profile_name: Optional[str] = None,
|
|
98
98
|
token: Optional[str] = None,
|
|
99
|
+
config: Optional[Any] = None,
|
|
99
100
|
**kwargs,
|
|
100
101
|
):
|
|
101
102
|
"""Initializes the S3Provider
|
|
@@ -118,6 +119,7 @@ class S3Provider(StorageProvider):
|
|
|
118
119
|
profile_name (str, optional): Specifies the AWS profile name to use.
|
|
119
120
|
token (str, optional): Activeloop token, used for fetching credentials for Deep Lake datasets (if this is underlying storage for Deep Lake dataset).
|
|
120
121
|
This is optional, tokens are normally autogenerated.
|
|
122
|
+
config (Any, Optional): s3 client configuration provided by the user. Defaults to None.
|
|
121
123
|
**kwargs: Additional arguments to pass to the S3 client. Includes: ``expiration``.
|
|
122
124
|
"""
|
|
123
125
|
self.root = root
|
|
@@ -132,7 +134,7 @@ class S3Provider(StorageProvider):
|
|
|
132
134
|
self.tag: Optional[str] = None
|
|
133
135
|
self.token: Optional[str] = token
|
|
134
136
|
self.loaded_creds_from_environment = False
|
|
135
|
-
self.client_config =
|
|
137
|
+
self.client_config = config
|
|
136
138
|
self.start_time = time.time()
|
|
137
139
|
self.profile_name = profile_name
|
|
138
140
|
self._initialize_s3_parameters()
|
|
@@ -234,17 +236,16 @@ class S3Provider(StorageProvider):
|
|
|
234
236
|
def _get_bytes(
|
|
235
237
|
self, path, start_byte: Optional[int] = None, end_byte: Optional[int] = None
|
|
236
238
|
):
|
|
239
|
+
range_kwarg = {}
|
|
237
240
|
if start_byte is not None and end_byte is not None:
|
|
238
241
|
if start_byte == end_byte:
|
|
239
242
|
return b""
|
|
240
|
-
|
|
243
|
+
range_kwarg["Range"] = f"bytes={start_byte}-{end_byte - 1}"
|
|
241
244
|
elif start_byte is not None:
|
|
242
|
-
|
|
245
|
+
range_kwarg["Range"] = f"bytes={start_byte}-"
|
|
243
246
|
elif end_byte is not None:
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
range = ""
|
|
247
|
-
resp = self.client.get_object(Bucket=self.bucket, Key=path, Range=range)
|
|
247
|
+
range_kwarg["Range"] = f"bytes=0-{end_byte - 1}"
|
|
248
|
+
resp = self.client.get_object(Bucket=self.bucket, Key=path, **range_kwarg)
|
|
248
249
|
return resp["Body"].read()
|
|
249
250
|
|
|
250
251
|
def get_bytes(
|
|
@@ -195,15 +195,15 @@ def test_deepmemory_evaluate(
|
|
|
195
195
|
"recall@100": 0.9,
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
198
|
+
assert recall["with model"] == {
|
|
199
|
+
"recall@1": 0.9,
|
|
200
|
+
"recall@3": 0.9,
|
|
201
|
+
"recall@5": 0.9,
|
|
202
|
+
"recall@10": 0.9,
|
|
203
|
+
"recall@50": 0.9,
|
|
204
|
+
"recall@100": 0.9,
|
|
205
|
+
}
|
|
206
|
+
|
|
207
207
|
sleep(15)
|
|
208
208
|
queries_dataset = VectorStore(
|
|
209
209
|
path=query_path,
|
|
@@ -105,6 +105,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
105
105
|
_return_index=None,
|
|
106
106
|
_primary_tensor_name=None,
|
|
107
107
|
_buffer_size=None,
|
|
108
|
+
_orig_dataset=None,
|
|
108
109
|
_decode_method=None,
|
|
109
110
|
_persistent_workers=None,
|
|
110
111
|
_dataloader=None,
|
|
@@ -116,6 +117,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
116
117
|
):
|
|
117
118
|
import_indra_loader()
|
|
118
119
|
self.dataset = dataset
|
|
120
|
+
self._orig_dataset = _orig_dataset or dataset
|
|
119
121
|
self._batch_size = _batch_size
|
|
120
122
|
self._shuffle = _shuffle
|
|
121
123
|
self._num_threads = _num_threads
|
|
@@ -235,7 +237,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
235
237
|
@property
|
|
236
238
|
def sampler(self):
|
|
237
239
|
return (
|
|
238
|
-
DistributedSampler(self.
|
|
240
|
+
DistributedSampler(self._orig_dataset)
|
|
239
241
|
if self._distributed
|
|
240
242
|
else _InfiniteConstantSampler()
|
|
241
243
|
)
|
|
@@ -277,9 +279,9 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
277
279
|
|
|
278
280
|
def __len__(self):
|
|
279
281
|
len_ds = (
|
|
280
|
-
len(self.
|
|
282
|
+
len(self._orig_dataset[self._tensors])
|
|
281
283
|
if self._tensors is not None
|
|
282
|
-
else len(self.
|
|
284
|
+
else len(self._orig_dataset)
|
|
283
285
|
)
|
|
284
286
|
round_fn = math.floor if self._drop_last else math.ceil
|
|
285
287
|
return round_fn(len_ds / ((self.batch_size) * self._world_size))
|
|
@@ -344,10 +346,12 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
344
346
|
all_vars["_shuffle"] = shuffle
|
|
345
347
|
all_vars["_buffer_size"] = buffer_size
|
|
346
348
|
if shuffle:
|
|
347
|
-
schedule = create_fetching_schedule(
|
|
349
|
+
schedule = create_fetching_schedule(
|
|
350
|
+
self._orig_dataset, self._primary_tensor_name
|
|
351
|
+
)
|
|
348
352
|
if schedule is not None:
|
|
349
|
-
ds = self.
|
|
350
|
-
all_vars["
|
|
353
|
+
ds = self._orig_dataset # type: ignore
|
|
354
|
+
all_vars["_orig_dataset"] = ds[schedule]
|
|
351
355
|
all_vars["_dataloader"] = None
|
|
352
356
|
return self.__class__(**all_vars)
|
|
353
357
|
|
|
@@ -374,12 +378,12 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
374
378
|
all_vars = self.__dict__.copy()
|
|
375
379
|
if isinstance(transform, dict):
|
|
376
380
|
tensors = [k for k in transform.keys() if k != "index"]
|
|
377
|
-
tensors = map_tensor_keys(self.
|
|
381
|
+
tensors = map_tensor_keys(self._orig_dataset, tensors)
|
|
378
382
|
if self._tensors:
|
|
379
383
|
raise ValueError(
|
|
380
384
|
f"Tensors have already been specified in the .{self._mode} method."
|
|
381
385
|
)
|
|
382
|
-
all_vars["_tensors"] = map_tensor_keys(self.
|
|
386
|
+
all_vars["_tensors"] = map_tensor_keys(self._orig_dataset, tensors)
|
|
383
387
|
transform = PytorchTransformFunction(transform_dict=transform)
|
|
384
388
|
else:
|
|
385
389
|
if kwargs:
|
|
@@ -410,7 +414,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
410
414
|
>>> query_ds_train = ds_train.dataloader().query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
|
|
411
415
|
"""
|
|
412
416
|
all_vars = self.__dict__.copy()
|
|
413
|
-
all_vars["
|
|
417
|
+
all_vars["_orig_dataset"] = query(self._orig_dataset, query_string)
|
|
414
418
|
all_vars["_dataloader"] = None
|
|
415
419
|
return self.__class__(**all_vars)
|
|
416
420
|
|
|
@@ -455,8 +459,8 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
455
459
|
|
|
456
460
|
"""
|
|
457
461
|
all_vars = self.__dict__.copy()
|
|
458
|
-
all_vars["
|
|
459
|
-
self.
|
|
462
|
+
all_vars["_orig_dataset"] = sample_by(
|
|
463
|
+
self._orig_dataset, weights, replace=replace, size=size
|
|
460
464
|
)
|
|
461
465
|
all_vars["_dataloader"] = None
|
|
462
466
|
return self.__class__(**all_vars)
|
|
@@ -539,7 +543,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
539
543
|
all_vars = self.__dict__.copy()
|
|
540
544
|
all_vars["_num_workers"] = num_workers
|
|
541
545
|
all_vars["_collate"] = collate_fn
|
|
542
|
-
validate_tensors(tensors, self.
|
|
546
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
543
547
|
all_vars["_decode_method"] = decode_method
|
|
544
548
|
all_vars["_num_threads"] = num_threads
|
|
545
549
|
all_vars["_prefetch_factor"] = prefetch_factor
|
|
@@ -614,7 +618,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
614
618
|
all_vars = self.__dict__.copy()
|
|
615
619
|
all_vars["_num_workers"] = num_workers
|
|
616
620
|
all_vars["_collate"] = collate_fn
|
|
617
|
-
validate_tensors(tensors, self.
|
|
621
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
618
622
|
all_vars["_decode_method"] = decode_method
|
|
619
623
|
all_vars["_num_threads"] = num_threads
|
|
620
624
|
all_vars["_prefetch_factor"] = prefetch_factor
|
|
@@ -659,7 +663,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
659
663
|
handle_mode(self._mode, mode)
|
|
660
664
|
all_vars = self.__dict__.copy()
|
|
661
665
|
all_vars["_num_workers"] = num_workers
|
|
662
|
-
validate_tensors(tensors, self.
|
|
666
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
663
667
|
all_vars["_decode_method"] = decode_method
|
|
664
668
|
all_vars["_tensors"] = self._tensors or tensors
|
|
665
669
|
all_vars["_num_threads"] = num_threads
|
|
@@ -801,9 +805,30 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
801
805
|
info=info,
|
|
802
806
|
)
|
|
803
807
|
|
|
808
|
+
def _fill_sample_info_tensors(
|
|
809
|
+
self,
|
|
810
|
+
dataset,
|
|
811
|
+
sample_info_tensors,
|
|
812
|
+
json_tensors,
|
|
813
|
+
list_tensors,
|
|
814
|
+
):
|
|
815
|
+
for tensor_name in sample_info_tensors:
|
|
816
|
+
tensor = dataset._get_tensor_from_root(tensor_name)
|
|
817
|
+
if len(tensor) == 0:
|
|
818
|
+
raise EmptyTensorError(
|
|
819
|
+
f" the dataset has an empty tensor {tensor_name}, pytorch dataloader can't be created."
|
|
820
|
+
f" Please either populate the tensor or pass tensors argument to .pytorch that excludes this"
|
|
821
|
+
f" tensor."
|
|
822
|
+
)
|
|
823
|
+
meta = tensor.meta
|
|
824
|
+
if meta.htype == "json":
|
|
825
|
+
json_tensors.append(tensor_name)
|
|
826
|
+
elif meta.htype in ["list", "tag"]:
|
|
827
|
+
list_tensors.append(tensor_name)
|
|
828
|
+
|
|
804
829
|
def __iter__(self):
|
|
805
830
|
if self._dataloader is None:
|
|
806
|
-
dataset = self.
|
|
831
|
+
dataset = self._orig_dataset
|
|
807
832
|
tensors = self._tensors or map_tensor_keys(dataset, None)
|
|
808
833
|
|
|
809
834
|
jpeg_png_compressed_tensors, json_tensors, list_tensors = check_tensors(
|
|
@@ -825,6 +850,9 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
825
850
|
sample_info_tensors, tensor_info_tensors = find_additional_tensors_and_info(
|
|
826
851
|
dataset, data_tensors
|
|
827
852
|
)
|
|
853
|
+
self._fill_sample_info_tensors(
|
|
854
|
+
dataset, sample_info_tensors, json_tensors, list_tensors
|
|
855
|
+
)
|
|
828
856
|
tensors.extend(sample_info_tensors)
|
|
829
857
|
htype_dict, ndim_dict, tensor_info_dict = get_htype_ndim_tensor_info_dicts(
|
|
830
858
|
dataset, data_tensors, tensor_info_tensors
|
|
@@ -855,7 +883,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
855
883
|
tensor_info_dict=tensor_info_dict,
|
|
856
884
|
)
|
|
857
885
|
|
|
858
|
-
dataset_read(self.
|
|
886
|
+
dataset_read(self._orig_dataset)
|
|
859
887
|
|
|
860
888
|
if self._iterator is not None:
|
|
861
889
|
self._iterator = iter(self._dataloader)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = 'd9276895-9cfa-4805-9800-a09f06f79d7b'
|
|
@@ -84,6 +84,7 @@ def storage_provider_from_path(
|
|
|
84
84
|
session_token = creds.get("aws_session_token")
|
|
85
85
|
endpoint_url = creds.get("endpoint_url")
|
|
86
86
|
region = creds.get("aws_region") or creds.get("region")
|
|
87
|
+
config = creds.get("config", None) or deeplake.config["s3"]
|
|
87
88
|
profile = creds.get("profile_name")
|
|
88
89
|
storage = S3Provider(
|
|
89
90
|
path,
|
|
@@ -94,6 +95,7 @@ def storage_provider_from_path(
|
|
|
94
95
|
region,
|
|
95
96
|
profile_name=profile,
|
|
96
97
|
token=token,
|
|
98
|
+
config=config,
|
|
97
99
|
)
|
|
98
100
|
storage.creds_used = creds_used
|
|
99
101
|
else:
|
|
@@ -8,27 +8,27 @@ tqdm
|
|
|
8
8
|
lz4
|
|
9
9
|
pyjwt
|
|
10
10
|
pydantic
|
|
11
|
-
libdeeplake==0.0.
|
|
11
|
+
libdeeplake==0.0.129
|
|
12
12
|
|
|
13
13
|
[:python_version >= "3.7" and sys_platform != "win32"]
|
|
14
14
|
aioboto3>=10.4.0
|
|
15
15
|
nest_asyncio
|
|
16
16
|
|
|
17
17
|
[all]
|
|
18
|
-
IPython
|
|
19
|
-
google-cloud-storage~=1.42.0
|
|
20
|
-
oauth2client~=4.1.3
|
|
21
|
-
azure-identity
|
|
22
18
|
laspy
|
|
23
|
-
azure-storage-blob
|
|
24
19
|
flask
|
|
25
20
|
google-auth~=2.0.1
|
|
26
21
|
nibabel
|
|
27
22
|
google-auth-oauthlib~=0.4.5
|
|
28
23
|
pydicom
|
|
29
|
-
|
|
24
|
+
IPython
|
|
25
|
+
oauth2client~=4.1.3
|
|
26
|
+
azure-storage-blob
|
|
30
27
|
azure-cli
|
|
31
|
-
|
|
28
|
+
google-cloud-storage~=1.42.0
|
|
29
|
+
google-api-python-client~=2.31.0
|
|
30
|
+
azure-identity
|
|
31
|
+
libdeeplake==0.0.129
|
|
32
32
|
|
|
33
33
|
[all:python_version >= "3.7" or sys_platform != "win32"]
|
|
34
34
|
av>=8.1.0
|
|
@@ -53,7 +53,7 @@ pydicom
|
|
|
53
53
|
nibabel
|
|
54
54
|
|
|
55
55
|
[enterprise]
|
|
56
|
-
libdeeplake==0.0.
|
|
56
|
+
libdeeplake==0.0.129
|
|
57
57
|
pyjwt
|
|
58
58
|
|
|
59
59
|
[gcp]
|
|
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
|
|
|
70
70
|
extras_require["all"] = [req_map[r] for r in all_extras]
|
|
71
71
|
|
|
72
72
|
if libdeeplake_available():
|
|
73
|
-
libdeeplake = "libdeeplake==0.0.
|
|
73
|
+
libdeeplake = "libdeeplake==0.0.129"
|
|
74
74
|
extras_require["enterprise"] = [libdeeplake, "pyjwt"]
|
|
75
75
|
extras_require["all"].append(libdeeplake)
|
|
76
76
|
install_requires.append(libdeeplake)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
BUGOUT_TOKEN = '1ba3468c-36d3-44a0-8180-5bab73ecb13c'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|