deeplake 3.8.26__tar.gz → 3.8.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.8.26/deeplake.egg-info → deeplake-3.8.28}/PKG-INFO +1 -1
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/__init__.py +1 -1
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/dataset.py +68 -9
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_api.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_hosted_datasets.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_views.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/structured/dataframe.py +2 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/tests/test_ingestion.py +4 -4
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/auth/activeloop.py +7 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/auth/auth_context.py +5 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/auth/azure.py +15 -2
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/client.py +3 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/constants.py +2 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compression.py +6 -6
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/dataset/dataset.py +63 -20
- deeplake-3.8.26/deeplake/core/dataset/deeplake_query_dataset.py → deeplake-3.8.28/deeplake/core/dataset/indra_dataset_view.py +77 -106
- deeplake-3.8.26/deeplake/core/dataset/deeplake_query_tensor.py → deeplake-3.8.28/deeplake/core/dataset/indra_tensor_view.py +21 -38
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/dataset/view_entry.py +1 -3
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/index/index.py +9 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/query/filter.py +10 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/sample.py +8 -3
- deeplake-3.8.28/deeplake/core/storage/indra.py +107 -0
- deeplake-3.8.26/deeplake/core/tests/test_deeplake_indra_dataset.py → deeplake-3.8.28/deeplake/core/tests/test_indra_dataset.py +25 -12
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_query.py +3 -1
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_serialize.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_vdb_indexes.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/transform/transform.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +1 -10
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/deep_memory/deep_memory.py +1 -7
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/deeplake_vectorstore.py +3 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +3 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +1 -10
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +3 -12
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +1 -8
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/utils.py +12 -17
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/__init__.py +0 -4
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/convert_to_libdeeplake.py +41 -18
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/dataloader.py +5 -7
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/libdeeplake_query.py +5 -5
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/test_query.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/util.py +3 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/pytorch/shuffle_buffer.py +2 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/requirements/common.txt +1 -1
- deeplake-3.8.28/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/storage.py +22 -4
- {deeplake-3.8.26 → deeplake-3.8.28/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake.egg-info/SOURCES.txt +4 -3
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake.egg-info/requires.txt +10 -10
- {deeplake-3.8.26 → deeplake-3.8.28}/setup.py +1 -1
- deeplake-3.8.26/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.8.26 → deeplake-3.8.28}/LICENSE +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/MANIFEST.in +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/README.md +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/info.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/link.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/read.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/api/tiled.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/config.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/log.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/test_client.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/client/utils.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/compression.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/io.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/ipc.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/lock.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/polygon.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/query/query.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/seed.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/serialize.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tensor.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/hooks.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/htype.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/common.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/access_method.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/agreement.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/array_list.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/auto.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/casting.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/class_label.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/compression.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/compute.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/diff.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/downsample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/encoder.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/exif.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/hash.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/htype.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/image.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/json.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/keys.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/link.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/logging.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/merge.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/modified.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/notebook.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/path.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/spinner.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/split.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tag.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/testing.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/threading.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/token.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/transform.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/version_control.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/video.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/util/warnings.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/pyproject.toml +0 -0
- {deeplake-3.8.26 → deeplake-3.8.28}/setup.cfg +0 -0
|
@@ -13,6 +13,7 @@ from deeplake.auto.unstructured.coco.coco import CocoDataset
|
|
|
13
13
|
from deeplake.auto.unstructured.yolo.yolo import YoloDataset
|
|
14
14
|
from deeplake.client.log import logger
|
|
15
15
|
from deeplake.core.dataset import Dataset, dataset_factory
|
|
16
|
+
from deeplake.core.dataset.indra_dataset_view import IndraDatasetView
|
|
16
17
|
from deeplake.core.tensor import Tensor
|
|
17
18
|
from deeplake.core.meta.dataset_meta import DatasetMeta
|
|
18
19
|
from deeplake.util.connect_dataset import connect_dataset_entry
|
|
@@ -43,6 +44,7 @@ from deeplake.constants import (
|
|
|
43
44
|
DEFAULT_READONLY,
|
|
44
45
|
DATASET_META_FILENAME,
|
|
45
46
|
DATASET_LOCK_FILENAME,
|
|
47
|
+
USE_INDRA,
|
|
46
48
|
)
|
|
47
49
|
from deeplake.util.access_method import (
|
|
48
50
|
check_access_method,
|
|
@@ -101,6 +103,7 @@ class dataset:
|
|
|
101
103
|
lock_enabled: Optional[bool] = True,
|
|
102
104
|
lock_timeout: Optional[int] = 0,
|
|
103
105
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
106
|
+
indra: bool = USE_INDRA,
|
|
104
107
|
):
|
|
105
108
|
"""Returns a :class:`~deeplake.core.dataset.Dataset` object referencing either a new or existing dataset.
|
|
106
109
|
|
|
@@ -173,6 +176,7 @@ class dataset:
|
|
|
173
176
|
lock_timeout (int): Number of seconds to wait before throwing a LockException. If None, wait indefinitely
|
|
174
177
|
lock_enabled (bool): If true, the dataset manages a write lock. NOTE: Only set to False if you are managing concurrent access externally
|
|
175
178
|
index_params: Optional[Dict[str, Union[int, str]]] = None : The index parameters used while creating vector store is passed down to dataset.
|
|
179
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
176
180
|
|
|
177
181
|
..
|
|
178
182
|
# noqa: DAR101
|
|
@@ -225,6 +229,7 @@ class dataset:
|
|
|
225
229
|
token=token,
|
|
226
230
|
memory_cache_size=memory_cache_size,
|
|
227
231
|
local_cache_size=local_cache_size,
|
|
232
|
+
indra=indra,
|
|
228
233
|
)
|
|
229
234
|
|
|
230
235
|
feature_report_path(path, "dataset", {"Overwrite": overwrite}, token=token)
|
|
@@ -378,6 +383,7 @@ class dataset:
|
|
|
378
383
|
lock_timeout: Optional[int] = 0,
|
|
379
384
|
verbose: bool = True,
|
|
380
385
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
386
|
+
indra: bool = USE_INDRA,
|
|
381
387
|
) -> Dataset:
|
|
382
388
|
"""Creates an empty dataset
|
|
383
389
|
|
|
@@ -402,6 +408,7 @@ class dataset:
|
|
|
402
408
|
lock_timeout (int): Number of seconds to wait before throwing a LockException. If None, wait indefinitely
|
|
403
409
|
lock_enabled (bool): If true, the dataset manages a write lock. NOTE: Only set to False if you are managing concurrent access externally.
|
|
404
410
|
index_params: Optional[Dict[str, Union[int, str]]]: Index parameters used while creating vector store, passed down to dataset.
|
|
411
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
405
412
|
|
|
406
413
|
Returns:
|
|
407
414
|
Dataset: Dataset created using the arguments provided.
|
|
@@ -441,6 +448,7 @@ class dataset:
|
|
|
441
448
|
token=token,
|
|
442
449
|
memory_cache_size=memory_cache_size,
|
|
443
450
|
local_cache_size=local_cache_size,
|
|
451
|
+
indra=indra,
|
|
444
452
|
)
|
|
445
453
|
|
|
446
454
|
feature_report_path(
|
|
@@ -508,6 +516,7 @@ class dataset:
|
|
|
508
516
|
access_method: str = "stream",
|
|
509
517
|
unlink: bool = False,
|
|
510
518
|
reset: bool = False,
|
|
519
|
+
indra: bool = USE_INDRA,
|
|
511
520
|
check_integrity: Optional[bool] = None,
|
|
512
521
|
lock_timeout: Optional[int] = 0,
|
|
513
522
|
lock_enabled: Optional[bool] = True,
|
|
@@ -578,6 +587,7 @@ class dataset:
|
|
|
578
587
|
setting ``reset=True`` will reset HEAD changes and load the previous version.
|
|
579
588
|
check_integrity (bool, Optional): Performs an integrity check by default (None) if the dataset has 20 or fewer tensors.
|
|
580
589
|
Set to ``True`` to force integrity check, ``False`` to skip integrity check.
|
|
590
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
581
591
|
|
|
582
592
|
..
|
|
583
593
|
# noqa: DAR101
|
|
@@ -624,6 +634,7 @@ class dataset:
|
|
|
624
634
|
token=token,
|
|
625
635
|
memory_cache_size=memory_cache_size,
|
|
626
636
|
local_cache_size=local_cache_size,
|
|
637
|
+
indra=indra,
|
|
627
638
|
)
|
|
628
639
|
feature_report_path(
|
|
629
640
|
path,
|
|
@@ -644,6 +655,12 @@ class dataset:
|
|
|
644
655
|
f"A Deep Lake dataset does not exist at the given path ({path}). Check the path provided or in case you want to create a new dataset, use deeplake.empty()."
|
|
645
656
|
)
|
|
646
657
|
|
|
658
|
+
if indra and read_only:
|
|
659
|
+
from indra import api # type: ignore
|
|
660
|
+
|
|
661
|
+
ids = api.load_from_storage(storage.core)
|
|
662
|
+
return IndraDatasetView(indra_ds=ids)
|
|
663
|
+
|
|
647
664
|
dataset_kwargs: Dict[str, Union[None, str, bool, int, Dict]] = {
|
|
648
665
|
"path": path,
|
|
649
666
|
"read_only": read_only,
|
|
@@ -812,10 +829,10 @@ class dataset:
|
|
|
812
829
|
|
|
813
830
|
feature_report_path(old_path, "rename", {}, token=token)
|
|
814
831
|
|
|
815
|
-
|
|
816
|
-
|
|
832
|
+
deeplake.deepcopy(old_path, new_path, verbose=False, token=token, creds=creds)
|
|
833
|
+
deeplake.delete(old_path, token=token, creds=creds)
|
|
817
834
|
|
|
818
|
-
return
|
|
835
|
+
return deeplake.load(new_path, verbose=False, token=token, creds=creds)
|
|
819
836
|
|
|
820
837
|
@staticmethod
|
|
821
838
|
@spinner
|
|
@@ -1491,6 +1508,7 @@ class dataset:
|
|
|
1491
1508
|
num_workers: int = 0,
|
|
1492
1509
|
token: Optional[str] = None,
|
|
1493
1510
|
connect_kwargs: Optional[Dict] = None,
|
|
1511
|
+
indra: bool = USE_INDRA,
|
|
1494
1512
|
**dataset_kwargs,
|
|
1495
1513
|
) -> Dataset:
|
|
1496
1514
|
"""Ingest images and annotations in COCO format to a Deep Lake Dataset. The source data can be stored locally or in the cloud.
|
|
@@ -1544,6 +1562,7 @@ class dataset:
|
|
|
1544
1562
|
num_workers (int): The number of workers to use for ingestion. Set to ``0`` by default.
|
|
1545
1563
|
token (Optional[str]): The token to use for accessing the dataset and/or connecting it to Deep Lake.
|
|
1546
1564
|
connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :meth:`Dataset.connect <deeplake.core.dataset.Dataset.connect>`.
|
|
1565
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
1547
1566
|
**dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.empty`.
|
|
1548
1567
|
|
|
1549
1568
|
Returns:
|
|
@@ -1582,7 +1601,12 @@ class dataset:
|
|
|
1582
1601
|
structure = unstructured.prepare_structure(inspect_limit)
|
|
1583
1602
|
|
|
1584
1603
|
ds = deeplake.empty(
|
|
1585
|
-
dest,
|
|
1604
|
+
dest,
|
|
1605
|
+
creds=dest_creds,
|
|
1606
|
+
verbose=False,
|
|
1607
|
+
token=token,
|
|
1608
|
+
indra=indra,
|
|
1609
|
+
**dataset_kwargs,
|
|
1586
1610
|
)
|
|
1587
1611
|
if connect_kwargs is not None:
|
|
1588
1612
|
connect_kwargs["token"] = token or connect_kwargs.get("token")
|
|
@@ -1613,6 +1637,7 @@ class dataset:
|
|
|
1613
1637
|
num_workers: int = 0,
|
|
1614
1638
|
token: Optional[str] = None,
|
|
1615
1639
|
connect_kwargs: Optional[Dict] = None,
|
|
1640
|
+
indra: bool = USE_INDRA,
|
|
1616
1641
|
**dataset_kwargs,
|
|
1617
1642
|
) -> Dataset:
|
|
1618
1643
|
"""Ingest images and annotations (bounding boxes or polygons) in YOLO format to a Deep Lake Dataset. The source data can be stored locally or in the cloud.
|
|
@@ -1661,6 +1686,7 @@ class dataset:
|
|
|
1661
1686
|
num_workers (int): The number of workers to use for ingestion. Set to ``0`` by default.
|
|
1662
1687
|
token (Optional[str]): The token to use for accessing the dataset and/or connecting it to Deep Lake.
|
|
1663
1688
|
connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :meth:`Dataset.connect <deeplake.core.dataset.Dataset.connect>`.
|
|
1689
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
1664
1690
|
**dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.empty`.
|
|
1665
1691
|
|
|
1666
1692
|
Returns:
|
|
@@ -1708,7 +1734,12 @@ class dataset:
|
|
|
1708
1734
|
structure = unstructured.prepare_structure()
|
|
1709
1735
|
|
|
1710
1736
|
ds = deeplake.empty(
|
|
1711
|
-
dest,
|
|
1737
|
+
dest,
|
|
1738
|
+
creds=dest_creds,
|
|
1739
|
+
verbose=False,
|
|
1740
|
+
token=token,
|
|
1741
|
+
indra=indra,
|
|
1742
|
+
**dataset_kwargs,
|
|
1712
1743
|
)
|
|
1713
1744
|
if connect_kwargs is not None:
|
|
1714
1745
|
connect_kwargs["token"] = token or connect_kwargs.get("token")
|
|
@@ -1738,6 +1769,7 @@ class dataset:
|
|
|
1738
1769
|
shuffle: bool = True,
|
|
1739
1770
|
token: Optional[str] = None,
|
|
1740
1771
|
connect_kwargs: Optional[Dict] = None,
|
|
1772
|
+
indra: bool = USE_INDRA,
|
|
1741
1773
|
**dataset_kwargs,
|
|
1742
1774
|
) -> Dataset:
|
|
1743
1775
|
"""Ingest a dataset of images from a local folder to a Deep Lake Dataset. Images should be stored in subfolders by class name.
|
|
@@ -1758,6 +1790,7 @@ class dataset:
|
|
|
1758
1790
|
shuffle (bool): Shuffles the input data prior to ingestion. Since data arranged in folders by class is highly non-random, shuffling is important in order to produce optimal results when training. Defaults to ``True``.
|
|
1759
1791
|
token (Optional[str]): The token to use for accessing the dataset.
|
|
1760
1792
|
connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :meth:`Dataset.connect <deeplake.core.dataset.Dataset.connect>`.
|
|
1793
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
1761
1794
|
**dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function see :func:`deeplake.empty`.
|
|
1762
1795
|
|
|
1763
1796
|
Returns:
|
|
@@ -1839,6 +1872,7 @@ class dataset:
|
|
|
1839
1872
|
dest_creds=dest_creds,
|
|
1840
1873
|
progressbar=progressbar,
|
|
1841
1874
|
token=token,
|
|
1875
|
+
indra=indra,
|
|
1842
1876
|
**dataset_kwargs,
|
|
1843
1877
|
)
|
|
1844
1878
|
return ds
|
|
@@ -1861,7 +1895,12 @@ class dataset:
|
|
|
1861
1895
|
unstructured = ImageClassification(source=src)
|
|
1862
1896
|
|
|
1863
1897
|
ds = deeplake.empty(
|
|
1864
|
-
dest,
|
|
1898
|
+
dest,
|
|
1899
|
+
creds=dest_creds,
|
|
1900
|
+
token=token,
|
|
1901
|
+
verbose=False,
|
|
1902
|
+
indra=indra,
|
|
1903
|
+
**dataset_kwargs,
|
|
1865
1904
|
)
|
|
1866
1905
|
if connect_kwargs is not None:
|
|
1867
1906
|
connect_kwargs["token"] = token or connect_kwargs.get("token")
|
|
@@ -1892,6 +1931,7 @@ class dataset:
|
|
|
1892
1931
|
progressbar: bool = True,
|
|
1893
1932
|
summary: bool = True,
|
|
1894
1933
|
shuffle: bool = True,
|
|
1934
|
+
indra: bool = USE_INDRA,
|
|
1895
1935
|
**dataset_kwargs,
|
|
1896
1936
|
) -> Dataset:
|
|
1897
1937
|
"""Download and ingest a kaggle dataset and store it as a structured dataset to destination.
|
|
@@ -1911,6 +1951,7 @@ class dataset:
|
|
|
1911
1951
|
progressbar (bool): Enables or disables ingestion progress bar. Set to ``True`` by default.
|
|
1912
1952
|
summary (bool): Generates ingestion summary. Set to ``True`` by default.
|
|
1913
1953
|
shuffle (bool): Shuffles the input data prior to ingestion. Since data arranged in folders by class is highly non-random, shuffling is important in order to produce optimal results when training. Defaults to ``True``.
|
|
1954
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
1914
1955
|
**dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.dataset`.
|
|
1915
1956
|
|
|
1916
1957
|
Returns:
|
|
@@ -1956,6 +1997,7 @@ class dataset:
|
|
|
1956
1997
|
progressbar=progressbar,
|
|
1957
1998
|
summary=summary,
|
|
1958
1999
|
shuffle=shuffle,
|
|
2000
|
+
indra=indra,
|
|
1959
2001
|
**dataset_kwargs,
|
|
1960
2002
|
)
|
|
1961
2003
|
|
|
@@ -1972,6 +2014,7 @@ class dataset:
|
|
|
1972
2014
|
progressbar: bool = True,
|
|
1973
2015
|
token: Optional[str] = None,
|
|
1974
2016
|
connect_kwargs: Optional[Dict] = None,
|
|
2017
|
+
indra: bool = USE_INDRA,
|
|
1975
2018
|
**dataset_kwargs,
|
|
1976
2019
|
):
|
|
1977
2020
|
"""Convert pandas dataframe to a Deep Lake Dataset. The contents of the dataframe can be parsed literally, or can be treated as links to local or cloud files.
|
|
@@ -2021,6 +2064,7 @@ class dataset:
|
|
|
2021
2064
|
progressbar (bool): Enables or disables ingestion progress bar. Set to ``True`` by default.
|
|
2022
2065
|
token (Optional[str]): The token to use for accessing the dataset.
|
|
2023
2066
|
connect_kwargs (Optional[Dict]): A dictionary containing arguments to be passed to the dataset connect method. See :meth:`Dataset.connect`.
|
|
2067
|
+
indra (bool): Flag indicating whether indra api should be used to create the dataset. Defaults to false
|
|
2024
2068
|
**dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.empty`.
|
|
2025
2069
|
|
|
2026
2070
|
Returns:
|
|
@@ -2045,15 +2089,30 @@ class dataset:
|
|
|
2045
2089
|
structured = DataFrame(src, column_params, src_creds, creds_key)
|
|
2046
2090
|
|
|
2047
2091
|
dest = convert_pathlib_to_string_if_needed(dest)
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2092
|
+
if indra:
|
|
2093
|
+
from indra import api
|
|
2094
|
+
|
|
2095
|
+
ds = api.dataset_writer(
|
|
2096
|
+
dest, creds=dest_creds, token=token, **dataset_kwargs
|
|
2097
|
+
)
|
|
2098
|
+
else:
|
|
2099
|
+
ds = deeplake.empty(
|
|
2100
|
+
dest,
|
|
2101
|
+
creds=dest_creds,
|
|
2102
|
+
token=token,
|
|
2103
|
+
verbose=False,
|
|
2104
|
+
**dataset_kwargs,
|
|
2105
|
+
)
|
|
2051
2106
|
if connect_kwargs is not None:
|
|
2052
2107
|
connect_kwargs["token"] = token or connect_kwargs.get("token")
|
|
2053
2108
|
ds.connect(**connect_kwargs)
|
|
2054
2109
|
|
|
2055
2110
|
structured.fill_dataset(ds, progressbar) # type: ignore
|
|
2056
2111
|
|
|
2112
|
+
if indra:
|
|
2113
|
+
ids = api.load_from_storage(ds.storage)
|
|
2114
|
+
return IndraDatasetView(indra_ds=ids)
|
|
2115
|
+
|
|
2057
2116
|
return ds # type: ignore
|
|
2058
2117
|
|
|
2059
2118
|
@staticmethod
|
|
@@ -36,6 +36,8 @@ class DataFrame(StructuredDataset):
|
|
|
36
36
|
if not isinstance(self.source, pd.DataFrame):
|
|
37
37
|
raise Exception("Source is not a pandas dataframe object.")
|
|
38
38
|
|
|
39
|
+
self.source = self.source.replace({np.nan: None})
|
|
40
|
+
|
|
39
41
|
self.creds = creds
|
|
40
42
|
self.creds_key = creds_key
|
|
41
43
|
self._initialize_params(column_params)
|
|
@@ -231,7 +231,7 @@ def test_csv(memory_ds: Dataset, dataframe_ingestion_data: dict):
|
|
|
231
231
|
assert ds[tensors_names[2]].htype == "text"
|
|
232
232
|
assert ds[tensors_names[2]].dtype == str
|
|
233
233
|
np.testing.assert_array_equal(
|
|
234
|
-
ds[tensors_names[2]].numpy().reshape(-1), df[df_keys[2]].values
|
|
234
|
+
np.array(ds[tensors_names[2]].numpy()).reshape(-1), df[df_keys[2]].values
|
|
235
235
|
)
|
|
236
236
|
|
|
237
237
|
|
|
@@ -273,7 +273,7 @@ def test_dataframe_basic(
|
|
|
273
273
|
assert ds[df_keys[2]].htype == "text"
|
|
274
274
|
assert ds[df_keys[2]].dtype == str
|
|
275
275
|
np.testing.assert_array_equal(
|
|
276
|
-
ds[df_keys[2]].numpy().reshape(-1), df[df_keys[2]].values
|
|
276
|
+
np.array(ds[df_keys[2]].numpy()).reshape(-1), df[df_keys[2]].values
|
|
277
277
|
)
|
|
278
278
|
|
|
279
279
|
|
|
@@ -306,7 +306,7 @@ def test_dataframe_files(memory_ds: Dataset, dataframe_ingestion_data):
|
|
|
306
306
|
|
|
307
307
|
def test_dataframe_array(memory_ds: Dataset):
|
|
308
308
|
data = {
|
|
309
|
-
"AA": ["Alice", "Bob",
|
|
309
|
+
"AA": ["Alice", "Bob", np.nan, None],
|
|
310
310
|
"BB": [
|
|
311
311
|
np.array([3, 22, 1, 3]),
|
|
312
312
|
np.array([1, 22, 10, 1]),
|
|
@@ -342,7 +342,7 @@ def test_dataframe_array(memory_ds: Dataset):
|
|
|
342
342
|
)
|
|
343
343
|
|
|
344
344
|
np.testing.assert_array_equal(
|
|
345
|
-
ds[df_keys[2]].numpy().reshape(-1), df[df_keys[2]].values
|
|
345
|
+
np.array(ds[df_keys[2]][0:3].numpy()).reshape(-1), df[df_keys[2]].values[0:3]
|
|
346
346
|
)
|
|
347
347
|
assert ds[df_keys[2]].dtype == df[df_keys[2]].dtype
|
|
348
348
|
|
|
@@ -15,6 +15,13 @@ class ActiveLoopAuthContext(AuthContext):
|
|
|
15
15
|
|
|
16
16
|
return self.token
|
|
17
17
|
|
|
18
|
+
def is_public_user(self) -> bool:
|
|
19
|
+
token = self.get_token()
|
|
20
|
+
if token is None:
|
|
21
|
+
return False
|
|
22
|
+
|
|
23
|
+
return token.startswith("PUBLIC_TOKEN_")
|
|
24
|
+
|
|
18
25
|
def authenticate(self) -> None:
|
|
19
26
|
self.token = (
|
|
20
27
|
self.token
|
|
@@ -19,6 +19,11 @@ class AuthContext(ABC):
|
|
|
19
19
|
def get_token(self) -> Optional[str]:
|
|
20
20
|
pass
|
|
21
21
|
|
|
22
|
+
# Return True if the user has no authentication token
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def is_public_user(self) -> bool:
|
|
25
|
+
pass
|
|
26
|
+
|
|
22
27
|
@abstractmethod
|
|
23
28
|
def authenticate(self) -> None:
|
|
24
29
|
"""
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from datetime import datetime, timedelta
|
|
3
|
-
from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
|
|
4
|
-
from azure.core.exceptions import ClientAuthenticationError
|
|
5
3
|
|
|
6
4
|
from deeplake.client.auth.auth_context import AuthContext, AuthProviderType
|
|
7
5
|
from deeplake.util.exceptions import InvalidAuthContextError
|
|
@@ -11,11 +9,21 @@ ID_TOKEN_CACHE_MINUTES = 5
|
|
|
11
9
|
|
|
12
10
|
class AzureAuthContext(AuthContext):
|
|
13
11
|
def __init__(self):
|
|
12
|
+
try:
|
|
13
|
+
from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
|
|
14
|
+
from azure.core.exceptions import ClientAuthenticationError
|
|
15
|
+
except ImportError:
|
|
16
|
+
raise ImportError(
|
|
17
|
+
"Azure packages not installed. Run `pip install deeplake[azure]`."
|
|
18
|
+
)
|
|
19
|
+
|
|
14
20
|
self.credential = self._get_azure_credential()
|
|
15
21
|
self.token = None
|
|
16
22
|
self._last_auth_time = None
|
|
17
23
|
|
|
18
24
|
def _get_azure_credential(self):
|
|
25
|
+
from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
|
|
26
|
+
|
|
19
27
|
azure_keys = [i for i in os.environ if i.startswith("AZURE_")]
|
|
20
28
|
if "AZURE_CLIENT_ID" in azure_keys and len(azure_keys) == 1:
|
|
21
29
|
# Explicitly set client_id, to avoid any warnings coming from DefaultAzureCredential
|
|
@@ -25,6 +33,9 @@ class AzureAuthContext(AuthContext):
|
|
|
25
33
|
|
|
26
34
|
return DefaultAzureCredential()
|
|
27
35
|
|
|
36
|
+
def is_public_user(self) -> bool:
|
|
37
|
+
return False
|
|
38
|
+
|
|
28
39
|
def get_token(self) -> str:
|
|
29
40
|
self.authenticate()
|
|
30
41
|
|
|
@@ -38,6 +49,8 @@ class AzureAuthContext(AuthContext):
|
|
|
38
49
|
):
|
|
39
50
|
return
|
|
40
51
|
|
|
52
|
+
from azure.core.exceptions import ClientAuthenticationError
|
|
53
|
+
|
|
41
54
|
try:
|
|
42
55
|
response = self.credential.get_token(
|
|
43
56
|
"https://management.azure.com/.default"
|
|
@@ -856,7 +856,7 @@ def _open_video(file: Union[str, bytes, memoryview]):
|
|
|
856
856
|
)
|
|
857
857
|
|
|
858
858
|
if isinstance(file, str):
|
|
859
|
-
container = av.open(
|
|
859
|
+
container: Union[av.InputContainer, av.OutputContainer] = av.open(
|
|
860
860
|
file, options={"protocol_whitelist": "file,http,https,tcp,tls,subfile"}
|
|
861
861
|
)
|
|
862
862
|
else:
|
|
@@ -941,7 +941,7 @@ def _decompress_video(
|
|
|
941
941
|
seekable = True
|
|
942
942
|
try:
|
|
943
943
|
container.seek(seek_target, stream=vstream)
|
|
944
|
-
except av.error.
|
|
944
|
+
except av.error.FFmpegError:
|
|
945
945
|
seekable = False
|
|
946
946
|
container, vstream = _open_video(file) # try again but this time don't seek
|
|
947
947
|
warning(
|
|
@@ -985,7 +985,7 @@ def _read_timestamps(
|
|
|
985
985
|
|
|
986
986
|
stamps = []
|
|
987
987
|
if vstream.duration is None:
|
|
988
|
-
time_base = 1 / av.time_base
|
|
988
|
+
time_base = 1 / av.time_base # type: ignore
|
|
989
989
|
else:
|
|
990
990
|
time_base = vstream.time_base.numerator / vstream.time_base.denominator
|
|
991
991
|
|
|
@@ -1000,7 +1000,7 @@ def _read_timestamps(
|
|
|
1000
1000
|
seekable = True
|
|
1001
1001
|
try:
|
|
1002
1002
|
container.seek(seek_target, stream=vstream)
|
|
1003
|
-
except av.error.
|
|
1003
|
+
except av.error.FFmpegError:
|
|
1004
1004
|
seekable = False
|
|
1005
1005
|
container, vstream = _open_video(file) # try again but this time don't seek
|
|
1006
1006
|
warning(
|
|
@@ -1040,7 +1040,7 @@ def _open_audio(file: Union[str, bytes, memoryview]):
|
|
|
1040
1040
|
)
|
|
1041
1041
|
|
|
1042
1042
|
if isinstance(file, str):
|
|
1043
|
-
container = av.open(
|
|
1043
|
+
container: Union[av.InputContainer, av.OutputContainer] = av.open(
|
|
1044
1044
|
file, options={"protocol_whitelist": "file,http,https,tcp,tls,subfile"}
|
|
1045
1045
|
)
|
|
1046
1046
|
else:
|
|
@@ -1097,7 +1097,7 @@ def _read_audio_meta(
|
|
|
1097
1097
|
meta["time_base"] = astream.time_base.numerator / astream.time_base.denominator
|
|
1098
1098
|
else:
|
|
1099
1099
|
meta["duration"] = container.duration
|
|
1100
|
-
meta["time_base"] = 1 / av.time_base
|
|
1100
|
+
meta["time_base"] = 1 / av.time_base # type: ignore
|
|
1101
1101
|
meta["sample_rate"] = astream.sample_rate
|
|
1102
1102
|
meta["duration"] = astream.duration or container.duration
|
|
1103
1103
|
meta["frame_size"] = astream.frame_size
|
|
@@ -295,6 +295,7 @@ class Dataset:
|
|
|
295
295
|
self._initial_autoflush: List[bool] = (
|
|
296
296
|
[]
|
|
297
297
|
) # This is a stack to support nested with contexts
|
|
298
|
+
|
|
298
299
|
self._indexing_history: List[int] = []
|
|
299
300
|
|
|
300
301
|
if not self.read_only:
|
|
@@ -2242,7 +2243,7 @@ class Dataset:
|
|
|
2242
2243
|
... pass
|
|
2243
2244
|
|
|
2244
2245
|
"""
|
|
2245
|
-
from deeplake.enterprise import dataloader
|
|
2246
|
+
from deeplake.enterprise.dataloader import dataloader
|
|
2246
2247
|
|
|
2247
2248
|
deeplake_reporter.feature_report(feature_name="dataloader", parameters={})
|
|
2248
2249
|
|
|
@@ -2278,10 +2279,26 @@ class Dataset:
|
|
|
2278
2279
|
|
|
2279
2280
|
|
|
2280
2281
|
Example:
|
|
2281
|
-
|
|
2282
|
+
Return dataset view where all the samples have label equals to 2:
|
|
2282
2283
|
|
|
2283
2284
|
>>> dataset.filter(lambda sample: sample.labels.numpy() == 2)
|
|
2284
|
-
|
|
2285
|
+
|
|
2286
|
+
|
|
2287
|
+
Append one dataset onto another (only works if their structure is identical):
|
|
2288
|
+
|
|
2289
|
+
>>> @deeplake.compute
|
|
2290
|
+
>>> def dataset_append(sample_in, sample_out):
|
|
2291
|
+
>>>
|
|
2292
|
+
>>> sample_out.append(sample_in.tensors)
|
|
2293
|
+
>>>
|
|
2294
|
+
>>> return sample_out
|
|
2295
|
+
>>>
|
|
2296
|
+
>>>
|
|
2297
|
+
>>> dataset_append().eval(
|
|
2298
|
+
>>> ds_in,
|
|
2299
|
+
>>> ds_out,
|
|
2300
|
+
>>> num_workers = 2
|
|
2301
|
+
>>> )
|
|
2285
2302
|
"""
|
|
2286
2303
|
from deeplake.core.query import filter_dataset, query_dataset
|
|
2287
2304
|
|
|
@@ -3784,7 +3801,7 @@ class Dataset:
|
|
|
3784
3801
|
) from e
|
|
3785
3802
|
else:
|
|
3786
3803
|
raise ReadOnlyModeError(
|
|
3787
|
-
"Cannot save view in read only dataset.
|
|
3804
|
+
"Cannot save view in read only dataset. Specify a path to save the view in a different location."
|
|
3788
3805
|
)
|
|
3789
3806
|
else:
|
|
3790
3807
|
vds = self._save_view_in_subdir(
|
|
@@ -4588,6 +4605,40 @@ class Dataset:
|
|
|
4588
4605
|
def __contains__(self, tensor: str):
|
|
4589
4606
|
return tensor in self.tensors
|
|
4590
4607
|
|
|
4608
|
+
def _optimize_and_copy_view(
|
|
4609
|
+
self,
|
|
4610
|
+
info,
|
|
4611
|
+
path: str,
|
|
4612
|
+
tensors: Optional[List[str]] = None,
|
|
4613
|
+
external=False,
|
|
4614
|
+
unlink=True,
|
|
4615
|
+
num_workers=0,
|
|
4616
|
+
scheduler="threaded",
|
|
4617
|
+
progressbar=True,
|
|
4618
|
+
):
|
|
4619
|
+
tql_query = info.get("tql_query")
|
|
4620
|
+
vds = self._sub_ds(".queries/" + path, verbose=False)
|
|
4621
|
+
view = vds._get_view(not external)
|
|
4622
|
+
new_path = path + "_OPTIMIZED"
|
|
4623
|
+
if tql_query is not None:
|
|
4624
|
+
view = view.query(tql_query)
|
|
4625
|
+
view.indra_ds.materialize(new_path, tensors, True)
|
|
4626
|
+
optimized = self._sub_ds(".queries/" + new_path, empty=False, verbose=False)
|
|
4627
|
+
else:
|
|
4628
|
+
optimized = self._sub_ds(".queries/" + new_path, empty=True, verbose=False)
|
|
4629
|
+
view._copy(
|
|
4630
|
+
optimized,
|
|
4631
|
+
tensors=tensors,
|
|
4632
|
+
overwrite=True,
|
|
4633
|
+
unlink=unlink,
|
|
4634
|
+
create_vds_index_tensor=True,
|
|
4635
|
+
num_workers=num_workers,
|
|
4636
|
+
scheduler=scheduler,
|
|
4637
|
+
progressbar=progressbar,
|
|
4638
|
+
)
|
|
4639
|
+
optimized.info.update(vds.info.__getstate__())
|
|
4640
|
+
return (vds, optimized, new_path)
|
|
4641
|
+
|
|
4591
4642
|
def _optimize_saved_view(
|
|
4592
4643
|
self,
|
|
4593
4644
|
id: str,
|
|
@@ -4614,32 +4665,24 @@ class Dataset:
|
|
|
4614
4665
|
# Already optimized
|
|
4615
4666
|
return info
|
|
4616
4667
|
path = info.get("path", info["id"])
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
optimized = self._sub_ds(
|
|
4621
|
-
".queries/" + new_path, empty=True, verbose=False
|
|
4622
|
-
)
|
|
4623
|
-
view._copy(
|
|
4624
|
-
optimized,
|
|
4668
|
+
old, new, new_path = self._optimize_and_copy_view(
|
|
4669
|
+
info,
|
|
4670
|
+
path,
|
|
4625
4671
|
tensors=tensors,
|
|
4626
|
-
overwrite=True,
|
|
4627
4672
|
unlink=unlink,
|
|
4628
|
-
create_vds_index_tensor=True,
|
|
4629
4673
|
num_workers=num_workers,
|
|
4630
4674
|
scheduler=scheduler,
|
|
4631
4675
|
progressbar=progressbar,
|
|
4632
4676
|
)
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
optimized.flush()
|
|
4677
|
+
new.info["virtual-datasource"] = False
|
|
4678
|
+
new.info["path"] = new_path
|
|
4679
|
+
new.flush()
|
|
4637
4680
|
info["virtual-datasource"] = False
|
|
4638
4681
|
info["path"] = new_path
|
|
4639
4682
|
self._write_queries_json(qjson)
|
|
4640
|
-
|
|
4683
|
+
old.base_storage.disable_readonly()
|
|
4641
4684
|
try:
|
|
4642
|
-
|
|
4685
|
+
old.base_storage.clear()
|
|
4643
4686
|
except Exception as e:
|
|
4644
4687
|
warnings.warn(
|
|
4645
4688
|
f"Error while deleting old view after writing optimized version: {e}"
|