deeplake 3.9.3__tar.gz → 3.9.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.3/deeplake.egg-info → deeplake-3.9.5}/PKG-INFO +1 -1
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/__init__.py +1 -1
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk_engine.py +4 -4
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/dataset.py +10 -2
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/index/index.py +2 -3
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/gcs.py +3 -1
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/s3.py +9 -8
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/dataloader.py +24 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/client_fixtures.py +1 -1
- deeplake-3.9.5/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/exceptions.py +14 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/storage.py +2 -0
- {deeplake-3.9.3 → deeplake-3.9.5/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake.egg-info/requires.txt +10 -10
- {deeplake-3.9.3 → deeplake-3.9.5}/setup.py +1 -1
- deeplake-3.9.3/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.3 → deeplake-3.9.5}/LICENSE +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/MANIFEST.in +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/README.md +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/info.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/link.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/read.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/client.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/config.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/log.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/compression.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/constants.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compression.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/io.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/sample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tensor.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/hooks.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/htype.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/image.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/json.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/link.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/path.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/split.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/token.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/video.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/pyproject.toml +0 -0
- {deeplake-3.9.3 → deeplake-3.9.5}/setup.cfg +0 -0
|
@@ -603,7 +603,7 @@ class ChunkEngine:
|
|
|
603
603
|
chunk = self.copy_chunk_to_new_commit(chunk, chunk_name)
|
|
604
604
|
return chunk
|
|
605
605
|
except Exception as e:
|
|
606
|
-
raise GetChunkError(chunk_key) from e
|
|
606
|
+
raise GetChunkError(chunk_key, cause=e) from e
|
|
607
607
|
|
|
608
608
|
def get_video_chunk(self, chunk_id, copy: bool = False):
|
|
609
609
|
"""Returns video chunks. Chunk will contain presigned url to the video instead of data if the chunk is large."""
|
|
@@ -2157,7 +2157,7 @@ class ChunkEngine:
|
|
|
2157
2157
|
return None, chunk_info
|
|
2158
2158
|
cache_used_percent = lambda: self.cache.cache_used / self.cache.cache_size
|
|
2159
2159
|
while cache_used_percent() > 0.9:
|
|
2160
|
-
|
|
2160
|
+
self.cache._pop_from_cache()
|
|
2161
2161
|
base_storage = storages.get(threading.get_ident())
|
|
2162
2162
|
if base_storage is None:
|
|
2163
2163
|
if isinstance(self.base_storage, MemoryProvider):
|
|
@@ -2298,7 +2298,7 @@ class ChunkEngine:
|
|
|
2298
2298
|
else:
|
|
2299
2299
|
sample = self.get_single_sample(idx, index, pad_tensor=pad_tensor)
|
|
2300
2300
|
except GetChunkError as e:
|
|
2301
|
-
raise GetChunkError(e.chunk_key, idx, self.name) from e
|
|
2301
|
+
raise GetChunkError(e.chunk_key, idx, self.name, e) from e
|
|
2302
2302
|
except ReadSampleFromChunkError as e:
|
|
2303
2303
|
raise ReadSampleFromChunkError(e.chunk_key, idx, self.name) from e
|
|
2304
2304
|
check_sample_shape(sample.shape, last_shape, self.key, index, aslist)
|
|
@@ -2396,7 +2396,7 @@ class ChunkEngine:
|
|
|
2396
2396
|
)
|
|
2397
2397
|
except GetChunkError as e:
|
|
2398
2398
|
raise GetChunkError(
|
|
2399
|
-
e.chunk_key, global_sample_index, self.name
|
|
2399
|
+
e.chunk_key, global_sample_index, self.name, e
|
|
2400
2400
|
) from e
|
|
2401
2401
|
except ReadSampleFromChunkError as e:
|
|
2402
2402
|
raise ReadSampleFromChunkError(
|
|
@@ -391,12 +391,20 @@ class Dataset:
|
|
|
391
391
|
@property
|
|
392
392
|
def max_len(self):
|
|
393
393
|
"""Return the maximum length of the tensor."""
|
|
394
|
-
return
|
|
394
|
+
return (
|
|
395
|
+
max([len(tensor) for tensor in self.tensors.values()])
|
|
396
|
+
if self.tensors
|
|
397
|
+
else 0
|
|
398
|
+
)
|
|
395
399
|
|
|
396
400
|
@property
|
|
397
401
|
def min_len(self):
|
|
398
402
|
"""Return the minimum length of the tensor."""
|
|
399
|
-
return
|
|
403
|
+
return (
|
|
404
|
+
min([len(tensor) for tensor in self.tensors.values()])
|
|
405
|
+
if self.tensors
|
|
406
|
+
else 0
|
|
407
|
+
)
|
|
400
408
|
|
|
401
409
|
def __getstate__(self) -> Dict[str, Any]:
|
|
402
410
|
"""Returns a dict that can be pickled and used to restore this dataset.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import sys
|
|
1
2
|
from typing import Union, List, Tuple, Iterable, Optional
|
|
2
3
|
from collections.abc import Iterable
|
|
3
4
|
import numpy as np
|
|
@@ -199,9 +200,7 @@ class IndexEntry:
|
|
|
199
200
|
def subscriptable(self):
|
|
200
201
|
"""Returns whether an IndexEntry can be further subscripted."""
|
|
201
202
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
if INDRA_INSTALLED:
|
|
203
|
+
if "indra" in sys.modules:
|
|
205
204
|
from indra import api # type: ignore
|
|
206
205
|
|
|
207
206
|
if isinstance(self.value, api.core.IndexMappingInt64):
|
|
@@ -528,5 +528,7 @@ class GCSProvider(StorageProvider):
|
|
|
528
528
|
|
|
529
529
|
def get_creds(self):
|
|
530
530
|
d = self.scoped_credentials.get_token_info()
|
|
531
|
-
d["expiration"] =
|
|
531
|
+
d["expiration"] = (
|
|
532
|
+
self.expiration if hasattr(self, "expiration") and self.expiration else ""
|
|
533
|
+
)
|
|
532
534
|
return d
|
|
@@ -6,7 +6,7 @@ import boto3
|
|
|
6
6
|
import botocore # type: ignore
|
|
7
7
|
import posixpath
|
|
8
8
|
import ssl
|
|
9
|
-
from typing import Dict, Optional, Tuple, Type
|
|
9
|
+
from typing import Dict, Optional, Tuple, Type, Any
|
|
10
10
|
from datetime import datetime, timezone
|
|
11
11
|
from botocore.session import ComponentLocator
|
|
12
12
|
from deeplake.client.client import DeepLakeBackendClient
|
|
@@ -96,6 +96,7 @@ class S3Provider(StorageProvider):
|
|
|
96
96
|
aws_region: Optional[str] = None,
|
|
97
97
|
profile_name: Optional[str] = None,
|
|
98
98
|
token: Optional[str] = None,
|
|
99
|
+
config: Optional[Any] = None,
|
|
99
100
|
**kwargs,
|
|
100
101
|
):
|
|
101
102
|
"""Initializes the S3Provider
|
|
@@ -118,6 +119,7 @@ class S3Provider(StorageProvider):
|
|
|
118
119
|
profile_name (str, optional): Specifies the AWS profile name to use.
|
|
119
120
|
token (str, optional): Activeloop token, used for fetching credentials for Deep Lake datasets (if this is underlying storage for Deep Lake dataset).
|
|
120
121
|
This is optional, tokens are normally autogenerated.
|
|
122
|
+
config (Any, Optional): s3 client configuration provided by the user. Defaults to None.
|
|
121
123
|
**kwargs: Additional arguments to pass to the S3 client. Includes: ``expiration``.
|
|
122
124
|
"""
|
|
123
125
|
self.root = root
|
|
@@ -132,7 +134,7 @@ class S3Provider(StorageProvider):
|
|
|
132
134
|
self.tag: Optional[str] = None
|
|
133
135
|
self.token: Optional[str] = token
|
|
134
136
|
self.loaded_creds_from_environment = False
|
|
135
|
-
self.client_config =
|
|
137
|
+
self.client_config = config
|
|
136
138
|
self.start_time = time.time()
|
|
137
139
|
self.profile_name = profile_name
|
|
138
140
|
self._initialize_s3_parameters()
|
|
@@ -234,17 +236,16 @@ class S3Provider(StorageProvider):
|
|
|
234
236
|
def _get_bytes(
|
|
235
237
|
self, path, start_byte: Optional[int] = None, end_byte: Optional[int] = None
|
|
236
238
|
):
|
|
239
|
+
range_kwarg = {}
|
|
237
240
|
if start_byte is not None and end_byte is not None:
|
|
238
241
|
if start_byte == end_byte:
|
|
239
242
|
return b""
|
|
240
|
-
|
|
243
|
+
range_kwarg["Range"] = f"bytes={start_byte}-{end_byte - 1}"
|
|
241
244
|
elif start_byte is not None:
|
|
242
|
-
|
|
245
|
+
range_kwarg["Range"] = f"bytes={start_byte}-"
|
|
243
246
|
elif end_byte is not None:
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
range = ""
|
|
247
|
-
resp = self.client.get_object(Bucket=self.bucket, Key=path, Range=range)
|
|
247
|
+
range_kwarg["Range"] = f"bytes=0-{end_byte - 1}"
|
|
248
|
+
resp = self.client.get_object(Bucket=self.bucket, Key=path, **range_kwarg)
|
|
248
249
|
return resp["Body"].read()
|
|
249
250
|
|
|
250
251
|
def get_bytes(
|
|
@@ -801,6 +801,27 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
801
801
|
info=info,
|
|
802
802
|
)
|
|
803
803
|
|
|
804
|
+
def _fill_sample_info_tensors(
|
|
805
|
+
self,
|
|
806
|
+
dataset,
|
|
807
|
+
sample_info_tensors,
|
|
808
|
+
json_tensors,
|
|
809
|
+
list_tensors,
|
|
810
|
+
):
|
|
811
|
+
for tensor_name in sample_info_tensors:
|
|
812
|
+
tensor = dataset._get_tensor_from_root(tensor_name)
|
|
813
|
+
if len(tensor) == 0:
|
|
814
|
+
raise EmptyTensorError(
|
|
815
|
+
f" the dataset has an empty tensor {tensor_name}, pytorch dataloader can't be created."
|
|
816
|
+
f" Please either populate the tensor or pass tensors argument to .pytorch that excludes this"
|
|
817
|
+
f" tensor."
|
|
818
|
+
)
|
|
819
|
+
meta = tensor.meta
|
|
820
|
+
if meta.htype == "json":
|
|
821
|
+
json_tensors.append(tensor_name)
|
|
822
|
+
elif meta.htype in ["list", "tag"]:
|
|
823
|
+
list_tensors.append(tensor_name)
|
|
824
|
+
|
|
804
825
|
def __iter__(self):
|
|
805
826
|
if self._dataloader is None:
|
|
806
827
|
dataset = self.dataset
|
|
@@ -825,6 +846,9 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
825
846
|
sample_info_tensors, tensor_info_tensors = find_additional_tensors_and_info(
|
|
826
847
|
dataset, data_tensors
|
|
827
848
|
)
|
|
849
|
+
self._fill_sample_info_tensors(
|
|
850
|
+
dataset, sample_info_tensors, json_tensors, list_tensors
|
|
851
|
+
)
|
|
828
852
|
tensors.extend(sample_info_tensors)
|
|
829
853
|
htype_dict, ndim_dict, tensor_info_dict = get_htype_ndim_tensor_info_dicts(
|
|
830
854
|
dataset, data_tensors, tensor_info_tensors
|
|
@@ -49,7 +49,7 @@ def hub_cloud_dev_token(hub_cloud_dev_credentials):
|
|
|
49
49
|
|
|
50
50
|
assert (
|
|
51
51
|
token is not None
|
|
52
|
-
), f"Deep Lake dev token was not found in the environment variable '{
|
|
52
|
+
), f"Deep Lake dev token was not found in the environment variable '{ENV_HUB_DEV_TOKEN}'. This is necessary for testing deeplake cloud datasets."
|
|
53
53
|
|
|
54
54
|
return token
|
|
55
55
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = 'c21aa850-6619-47a7-9d31-54dd3595fd88'
|
|
@@ -993,8 +993,14 @@ class GetChunkError(Exception):
|
|
|
993
993
|
chunk_key: Optional[str],
|
|
994
994
|
global_index: Optional[int] = None,
|
|
995
995
|
tensor_name: Optional[str] = None,
|
|
996
|
+
cause: Optional[Exception] = None,
|
|
996
997
|
):
|
|
998
|
+
self.root_cause = cause
|
|
997
999
|
self.chunk_key = chunk_key
|
|
1000
|
+
|
|
1001
|
+
if isinstance(cause, GetChunkError):
|
|
1002
|
+
self.root_cause = cause.root_cause
|
|
1003
|
+
|
|
998
1004
|
message = "Unable to get chunk"
|
|
999
1005
|
if chunk_key is not None:
|
|
1000
1006
|
message += f" '{chunk_key}'"
|
|
@@ -1003,6 +1009,14 @@ class GetChunkError(Exception):
|
|
|
1003
1009
|
if tensor_name is not None:
|
|
1004
1010
|
message += f" in tensor {tensor_name}"
|
|
1005
1011
|
message += "."
|
|
1012
|
+
|
|
1013
|
+
if cause is not None:
|
|
1014
|
+
cause_message = str(cause)
|
|
1015
|
+
if isinstance(cause, KeyError):
|
|
1016
|
+
cause_message = f"The file {cause} does not exist."
|
|
1017
|
+
|
|
1018
|
+
message += f" Root Cause: {cause_message}"
|
|
1019
|
+
|
|
1006
1020
|
super().__init__(message)
|
|
1007
1021
|
|
|
1008
1022
|
|
|
@@ -84,6 +84,7 @@ def storage_provider_from_path(
|
|
|
84
84
|
session_token = creds.get("aws_session_token")
|
|
85
85
|
endpoint_url = creds.get("endpoint_url")
|
|
86
86
|
region = creds.get("aws_region") or creds.get("region")
|
|
87
|
+
config = creds.get("config", None) or deeplake.config["s3"]
|
|
87
88
|
profile = creds.get("profile_name")
|
|
88
89
|
storage = S3Provider(
|
|
89
90
|
path,
|
|
@@ -94,6 +95,7 @@ def storage_provider_from_path(
|
|
|
94
95
|
region,
|
|
95
96
|
profile_name=profile,
|
|
96
97
|
token=token,
|
|
98
|
+
config=config,
|
|
97
99
|
)
|
|
98
100
|
storage.creds_used = creds_used
|
|
99
101
|
else:
|
|
@@ -8,27 +8,27 @@ tqdm
|
|
|
8
8
|
lz4
|
|
9
9
|
pyjwt
|
|
10
10
|
pydantic
|
|
11
|
-
libdeeplake==0.0.
|
|
11
|
+
libdeeplake==0.0.128
|
|
12
12
|
|
|
13
13
|
[:python_version >= "3.7" and sys_platform != "win32"]
|
|
14
14
|
aioboto3>=10.4.0
|
|
15
15
|
nest_asyncio
|
|
16
16
|
|
|
17
17
|
[all]
|
|
18
|
-
google-auth-oauthlib~=0.4.5
|
|
19
18
|
azure-identity
|
|
20
|
-
google-auth~=
|
|
19
|
+
google-auth-oauthlib~=0.4.5
|
|
20
|
+
nibabel
|
|
21
21
|
IPython
|
|
22
|
+
laspy
|
|
23
|
+
azure-cli
|
|
24
|
+
pydicom
|
|
25
|
+
google-auth~=2.0.1
|
|
22
26
|
oauth2client~=4.1.3
|
|
23
27
|
google-cloud-storage~=1.42.0
|
|
24
|
-
google-api-python-client~=2.31.0
|
|
25
|
-
pydicom
|
|
26
28
|
azure-storage-blob
|
|
27
|
-
laspy
|
|
28
|
-
nibabel
|
|
29
29
|
flask
|
|
30
|
-
|
|
31
|
-
libdeeplake==0.0.
|
|
30
|
+
google-api-python-client~=2.31.0
|
|
31
|
+
libdeeplake==0.0.128
|
|
32
32
|
|
|
33
33
|
[all:python_version >= "3.7" or sys_platform != "win32"]
|
|
34
34
|
av>=8.1.0
|
|
@@ -53,7 +53,7 @@ pydicom
|
|
|
53
53
|
nibabel
|
|
54
54
|
|
|
55
55
|
[enterprise]
|
|
56
|
-
libdeeplake==0.0.
|
|
56
|
+
libdeeplake==0.0.128
|
|
57
57
|
pyjwt
|
|
58
58
|
|
|
59
59
|
[gcp]
|
|
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
|
|
|
70
70
|
extras_require["all"] = [req_map[r] for r in all_extras]
|
|
71
71
|
|
|
72
72
|
if libdeeplake_available():
|
|
73
|
-
libdeeplake = "libdeeplake==0.0.
|
|
73
|
+
libdeeplake = "libdeeplake==0.0.128"
|
|
74
74
|
extras_require["enterprise"] = [libdeeplake, "pyjwt"]
|
|
75
75
|
extras_require["all"].append(libdeeplake)
|
|
76
76
|
install_requires.append(libdeeplake)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
BUGOUT_TOKEN = '75fda478-bbc3-4909-a3ff-4ace4e4ed2ad'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|