deeplake 3.9.5__tar.gz → 3.9.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.5/deeplake.egg-info → deeplake-3.9.6}/PKG-INFO +1 -1
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/__init__.py +1 -1
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compression.py +28 -3
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/sample.py +10 -3
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +9 -9
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/dataloader.py +20 -16
- deeplake-3.9.6/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.5 → deeplake-3.9.6/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake.egg-info/requires.txt +11 -11
- {deeplake-3.9.5 → deeplake-3.9.6}/setup.py +1 -1
- deeplake-3.9.5/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.5 → deeplake-3.9.6}/LICENSE +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/MANIFEST.in +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/README.md +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/read.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/client.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/config.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/log.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/io.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tensor.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/hooks.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/htype.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/image.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/json.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/path.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/split.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/storage.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/token.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/pyproject.toml +0 -0
- {deeplake-3.9.5 → deeplake-3.9.6}/setup.cfg +0 -0
|
@@ -43,6 +43,8 @@ else:
|
|
|
43
43
|
_NATIVE_INT32 = ">i4"
|
|
44
44
|
_NATIVE_FLOAT32 = ">f4"
|
|
45
45
|
|
|
46
|
+
HEADER_MAX_BYTES = 132
|
|
47
|
+
|
|
46
48
|
DIMS_RE = re.compile(rb" ([0-9]+)x([0-9]+)")
|
|
47
49
|
FPS_RE = re.compile(rb" ([0-9]+) fps,")
|
|
48
50
|
DURATION_RE = re.compile(rb"Duration: ([0-9:.]+),")
|
|
@@ -489,11 +491,34 @@ def get_compression(header=None, path=None):
|
|
|
489
491
|
".nii",
|
|
490
492
|
".nii.gz",
|
|
491
493
|
]
|
|
492
|
-
path = str(path).lower()
|
|
494
|
+
path = str(path).lower().partition("?")[0].partition("#")[0].partition(";")[0]
|
|
493
495
|
for fmt in file_formats:
|
|
494
496
|
if path.endswith(fmt):
|
|
495
497
|
return fmt[1:]
|
|
496
498
|
if header:
|
|
499
|
+
if (
|
|
500
|
+
header[4:12] == b"\x66\x74\x79\x70\x4D\x53\x4E\x56"
|
|
501
|
+
or header[4:12] == b"\x66\x74\x79\x70\x69\x73\x6F\x6D"
|
|
502
|
+
):
|
|
503
|
+
return "mp4"
|
|
504
|
+
if header[0:4] == b"\x1A\x45\xDF\xA3":
|
|
505
|
+
return "mkv"
|
|
506
|
+
if (
|
|
507
|
+
header[0:2] == b"\xff\xfb"
|
|
508
|
+
or header[0:2] == b"\xff\xf3"
|
|
509
|
+
or header[0:2] == b"\xff\xf2"
|
|
510
|
+
):
|
|
511
|
+
return "mp3"
|
|
512
|
+
if header[0:2] == b"\x66\x4c\x61\x43":
|
|
513
|
+
return "flac"
|
|
514
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x57\x41\x56\x45":
|
|
515
|
+
return "wav"
|
|
516
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x41\x56\x49\x20":
|
|
517
|
+
return "avi"
|
|
518
|
+
if header[128:132] == b"\x44\x49\x43\x4D":
|
|
519
|
+
return "dcm"
|
|
520
|
+
if header[0:4] == b"\x6e\x2b\x31\x00":
|
|
521
|
+
return "nii"
|
|
497
522
|
if not Image.OPEN:
|
|
498
523
|
Image.init()
|
|
499
524
|
for fmt in Image.OPEN:
|
|
@@ -651,10 +676,10 @@ def read_meta_from_compressed_file(
|
|
|
651
676
|
try:
|
|
652
677
|
if compression is None:
|
|
653
678
|
if hasattr(f, "read"):
|
|
654
|
-
compression = get_compression(f.read(
|
|
679
|
+
compression = get_compression(f.read(HEADER_MAX_BYTES), path)
|
|
655
680
|
f.seek(0)
|
|
656
681
|
else:
|
|
657
|
-
compression = get_compression(f[:
|
|
682
|
+
compression = get_compression(f[:HEADER_MAX_BYTES], path) # type: ignore
|
|
658
683
|
if compression == "jpeg":
|
|
659
684
|
try:
|
|
660
685
|
shape, typestr = _read_jpeg_shape(f), "|u1"
|
|
@@ -10,6 +10,7 @@ from deeplake.core.compression import (
|
|
|
10
10
|
_read_audio_meta,
|
|
11
11
|
_read_3d_data_meta,
|
|
12
12
|
_open_nifti,
|
|
13
|
+
HEADER_MAX_BYTES,
|
|
13
14
|
)
|
|
14
15
|
from deeplake.compression import (
|
|
15
16
|
get_compression_type,
|
|
@@ -104,7 +105,9 @@ class Sample:
|
|
|
104
105
|
self._compression = get_compression(path=self.path)
|
|
105
106
|
compressed_bytes = self._read_from_path()
|
|
106
107
|
if self._compression is None:
|
|
107
|
-
self._compression = get_compression(
|
|
108
|
+
self._compression = get_compression(
|
|
109
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
110
|
+
)
|
|
108
111
|
self._shape, self._typestr = verify_compressed_file(compressed_bytes, self._compression) # type: ignore
|
|
109
112
|
|
|
110
113
|
if array is not None:
|
|
@@ -318,7 +321,9 @@ class Sample:
|
|
|
318
321
|
self._compression = get_compression(path=self.path)
|
|
319
322
|
compressed_bytes = self._read_from_path()
|
|
320
323
|
if self._compression is None:
|
|
321
|
-
self._compression = get_compression(
|
|
324
|
+
self._compression = get_compression(
|
|
325
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
326
|
+
)
|
|
322
327
|
if self._compression == compression:
|
|
323
328
|
if self._shape is None:
|
|
324
329
|
_, self._shape, self._typestr = read_meta_from_compressed_file(
|
|
@@ -328,7 +333,9 @@ class Sample:
|
|
|
328
333
|
compressed_bytes = self._recompress(compressed_bytes, compression)
|
|
329
334
|
elif self._buffer is not None:
|
|
330
335
|
if self._compression is None:
|
|
331
|
-
self._compression = get_compression(
|
|
336
|
+
self._compression = get_compression(
|
|
337
|
+
header=self._buffer[:HEADER_MAX_BYTES]
|
|
338
|
+
)
|
|
332
339
|
if self._compression == compression:
|
|
333
340
|
compressed_bytes = self._buffer
|
|
334
341
|
else:
|
|
@@ -195,15 +195,15 @@ def test_deepmemory_evaluate(
|
|
|
195
195
|
"recall@100": 0.9,
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
198
|
+
assert recall["with model"] == {
|
|
199
|
+
"recall@1": 0.9,
|
|
200
|
+
"recall@3": 0.9,
|
|
201
|
+
"recall@5": 0.9,
|
|
202
|
+
"recall@10": 0.9,
|
|
203
|
+
"recall@50": 0.9,
|
|
204
|
+
"recall@100": 0.9,
|
|
205
|
+
}
|
|
206
|
+
|
|
207
207
|
sleep(15)
|
|
208
208
|
queries_dataset = VectorStore(
|
|
209
209
|
path=query_path,
|
|
@@ -105,6 +105,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
105
105
|
_return_index=None,
|
|
106
106
|
_primary_tensor_name=None,
|
|
107
107
|
_buffer_size=None,
|
|
108
|
+
_orig_dataset=None,
|
|
108
109
|
_decode_method=None,
|
|
109
110
|
_persistent_workers=None,
|
|
110
111
|
_dataloader=None,
|
|
@@ -116,6 +117,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
116
117
|
):
|
|
117
118
|
import_indra_loader()
|
|
118
119
|
self.dataset = dataset
|
|
120
|
+
self._orig_dataset = _orig_dataset or dataset
|
|
119
121
|
self._batch_size = _batch_size
|
|
120
122
|
self._shuffle = _shuffle
|
|
121
123
|
self._num_threads = _num_threads
|
|
@@ -235,7 +237,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
235
237
|
@property
|
|
236
238
|
def sampler(self):
|
|
237
239
|
return (
|
|
238
|
-
DistributedSampler(self.
|
|
240
|
+
DistributedSampler(self._orig_dataset)
|
|
239
241
|
if self._distributed
|
|
240
242
|
else _InfiniteConstantSampler()
|
|
241
243
|
)
|
|
@@ -277,9 +279,9 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
277
279
|
|
|
278
280
|
def __len__(self):
|
|
279
281
|
len_ds = (
|
|
280
|
-
len(self.
|
|
282
|
+
len(self._orig_dataset[self._tensors])
|
|
281
283
|
if self._tensors is not None
|
|
282
|
-
else len(self.
|
|
284
|
+
else len(self._orig_dataset)
|
|
283
285
|
)
|
|
284
286
|
round_fn = math.floor if self._drop_last else math.ceil
|
|
285
287
|
return round_fn(len_ds / ((self.batch_size) * self._world_size))
|
|
@@ -344,10 +346,12 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
344
346
|
all_vars["_shuffle"] = shuffle
|
|
345
347
|
all_vars["_buffer_size"] = buffer_size
|
|
346
348
|
if shuffle:
|
|
347
|
-
schedule = create_fetching_schedule(
|
|
349
|
+
schedule = create_fetching_schedule(
|
|
350
|
+
self._orig_dataset, self._primary_tensor_name
|
|
351
|
+
)
|
|
348
352
|
if schedule is not None:
|
|
349
|
-
ds = self.
|
|
350
|
-
all_vars["
|
|
353
|
+
ds = self._orig_dataset # type: ignore
|
|
354
|
+
all_vars["_orig_dataset"] = ds[schedule]
|
|
351
355
|
all_vars["_dataloader"] = None
|
|
352
356
|
return self.__class__(**all_vars)
|
|
353
357
|
|
|
@@ -374,12 +378,12 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
374
378
|
all_vars = self.__dict__.copy()
|
|
375
379
|
if isinstance(transform, dict):
|
|
376
380
|
tensors = [k for k in transform.keys() if k != "index"]
|
|
377
|
-
tensors = map_tensor_keys(self.
|
|
381
|
+
tensors = map_tensor_keys(self._orig_dataset, tensors)
|
|
378
382
|
if self._tensors:
|
|
379
383
|
raise ValueError(
|
|
380
384
|
f"Tensors have already been specified in the .{self._mode} method."
|
|
381
385
|
)
|
|
382
|
-
all_vars["_tensors"] = map_tensor_keys(self.
|
|
386
|
+
all_vars["_tensors"] = map_tensor_keys(self._orig_dataset, tensors)
|
|
383
387
|
transform = PytorchTransformFunction(transform_dict=transform)
|
|
384
388
|
else:
|
|
385
389
|
if kwargs:
|
|
@@ -410,7 +414,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
410
414
|
>>> query_ds_train = ds_train.dataloader().query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
|
|
411
415
|
"""
|
|
412
416
|
all_vars = self.__dict__.copy()
|
|
413
|
-
all_vars["
|
|
417
|
+
all_vars["_orig_dataset"] = query(self._orig_dataset, query_string)
|
|
414
418
|
all_vars["_dataloader"] = None
|
|
415
419
|
return self.__class__(**all_vars)
|
|
416
420
|
|
|
@@ -455,8 +459,8 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
455
459
|
|
|
456
460
|
"""
|
|
457
461
|
all_vars = self.__dict__.copy()
|
|
458
|
-
all_vars["
|
|
459
|
-
self.
|
|
462
|
+
all_vars["_orig_dataset"] = sample_by(
|
|
463
|
+
self._orig_dataset, weights, replace=replace, size=size
|
|
460
464
|
)
|
|
461
465
|
all_vars["_dataloader"] = None
|
|
462
466
|
return self.__class__(**all_vars)
|
|
@@ -539,7 +543,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
539
543
|
all_vars = self.__dict__.copy()
|
|
540
544
|
all_vars["_num_workers"] = num_workers
|
|
541
545
|
all_vars["_collate"] = collate_fn
|
|
542
|
-
validate_tensors(tensors, self.
|
|
546
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
543
547
|
all_vars["_decode_method"] = decode_method
|
|
544
548
|
all_vars["_num_threads"] = num_threads
|
|
545
549
|
all_vars["_prefetch_factor"] = prefetch_factor
|
|
@@ -614,7 +618,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
614
618
|
all_vars = self.__dict__.copy()
|
|
615
619
|
all_vars["_num_workers"] = num_workers
|
|
616
620
|
all_vars["_collate"] = collate_fn
|
|
617
|
-
validate_tensors(tensors, self.
|
|
621
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
618
622
|
all_vars["_decode_method"] = decode_method
|
|
619
623
|
all_vars["_num_threads"] = num_threads
|
|
620
624
|
all_vars["_prefetch_factor"] = prefetch_factor
|
|
@@ -659,7 +663,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
659
663
|
handle_mode(self._mode, mode)
|
|
660
664
|
all_vars = self.__dict__.copy()
|
|
661
665
|
all_vars["_num_workers"] = num_workers
|
|
662
|
-
validate_tensors(tensors, self.
|
|
666
|
+
validate_tensors(tensors, self._orig_dataset, all_vars)
|
|
663
667
|
all_vars["_decode_method"] = decode_method
|
|
664
668
|
all_vars["_tensors"] = self._tensors or tensors
|
|
665
669
|
all_vars["_num_threads"] = num_threads
|
|
@@ -824,7 +828,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
824
828
|
|
|
825
829
|
def __iter__(self):
|
|
826
830
|
if self._dataloader is None:
|
|
827
|
-
dataset = self.
|
|
831
|
+
dataset = self._orig_dataset
|
|
828
832
|
tensors = self._tensors or map_tensor_keys(dataset, None)
|
|
829
833
|
|
|
830
834
|
jpeg_png_compressed_tensors, json_tensors, list_tensors = check_tensors(
|
|
@@ -879,7 +883,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
879
883
|
tensor_info_dict=tensor_info_dict,
|
|
880
884
|
)
|
|
881
885
|
|
|
882
|
-
dataset_read(self.
|
|
886
|
+
dataset_read(self._orig_dataset)
|
|
883
887
|
|
|
884
888
|
if self._iterator is not None:
|
|
885
889
|
self._iterator = iter(self._dataloader)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = 'd9276895-9cfa-4805-9800-a09f06f79d7b'
|
|
@@ -8,27 +8,27 @@ tqdm
|
|
|
8
8
|
lz4
|
|
9
9
|
pyjwt
|
|
10
10
|
pydantic
|
|
11
|
-
libdeeplake==0.0.
|
|
11
|
+
libdeeplake==0.0.129
|
|
12
12
|
|
|
13
13
|
[:python_version >= "3.7" and sys_platform != "win32"]
|
|
14
14
|
aioboto3>=10.4.0
|
|
15
15
|
nest_asyncio
|
|
16
16
|
|
|
17
17
|
[all]
|
|
18
|
-
azure-identity
|
|
19
|
-
google-auth-oauthlib~=0.4.5
|
|
20
|
-
nibabel
|
|
21
|
-
IPython
|
|
22
18
|
laspy
|
|
23
|
-
|
|
24
|
-
pydicom
|
|
19
|
+
flask
|
|
25
20
|
google-auth~=2.0.1
|
|
21
|
+
nibabel
|
|
22
|
+
google-auth-oauthlib~=0.4.5
|
|
23
|
+
pydicom
|
|
24
|
+
IPython
|
|
26
25
|
oauth2client~=4.1.3
|
|
27
|
-
google-cloud-storage~=1.42.0
|
|
28
26
|
azure-storage-blob
|
|
29
|
-
|
|
27
|
+
azure-cli
|
|
28
|
+
google-cloud-storage~=1.42.0
|
|
30
29
|
google-api-python-client~=2.31.0
|
|
31
|
-
|
|
30
|
+
azure-identity
|
|
31
|
+
libdeeplake==0.0.129
|
|
32
32
|
|
|
33
33
|
[all:python_version >= "3.7" or sys_platform != "win32"]
|
|
34
34
|
av>=8.1.0
|
|
@@ -53,7 +53,7 @@ pydicom
|
|
|
53
53
|
nibabel
|
|
54
54
|
|
|
55
55
|
[enterprise]
|
|
56
|
-
libdeeplake==0.0.
|
|
56
|
+
libdeeplake==0.0.129
|
|
57
57
|
pyjwt
|
|
58
58
|
|
|
59
59
|
[gcp]
|
|
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
|
|
|
70
70
|
extras_require["all"] = [req_map[r] for r in all_extras]
|
|
71
71
|
|
|
72
72
|
if libdeeplake_available():
|
|
73
|
-
libdeeplake = "libdeeplake==0.0.
|
|
73
|
+
libdeeplake = "libdeeplake==0.0.129"
|
|
74
74
|
extras_require["enterprise"] = [libdeeplake, "pyjwt"]
|
|
75
75
|
extras_require["all"].append(libdeeplake)
|
|
76
76
|
install_requires.append(libdeeplake)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
BUGOUT_TOKEN = 'c21aa850-6619-47a7-9d31-54dd3595fd88'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|