deeplake 3.8.0__tar.gz → 3.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.8.0/deeplake.egg-info → deeplake-3.8.2}/PKG-INFO +1 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/__init__.py +1 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api.py +11 -2
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api_with_compression.py +2 -2
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_views.py +58 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/constants.py +2 -2
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk_engine.py +28 -4
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compression.py +5 -16
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/dataset.py +26 -4
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_dataset.py +9 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_tensor.py +7 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tensor.py +10 -14
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_deeplake_indra_dataset.py +16 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform_dataset.py +19 -6
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deep_memory.py +1 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deeplake_vectorstore.py +18 -14
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +24 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/index.py +3 -19
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/utils.py +2 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/dataloader.py +70 -19
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/htype.py +1 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/common.txt +1 -1
- deeplake-3.8.2/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.8.0 → deeplake-3.8.2/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/requires.txt +11 -11
- {deeplake-3.8.0 → deeplake-3.8.2}/setup.py +1 -1
- deeplake-3.8.0/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.8.0 → deeplake-3.8.2}/LICENSE +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/MANIFEST.in +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/README.md +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/info.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/link.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/read.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/api/tiled.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/auth.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/commands.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/cli/test_cli.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/client.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/config.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/log.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/test_client.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/client/utils.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/compression.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/index/index.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/io.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/ipc.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/lock.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/polygon.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/query/query.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/sample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/seed.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/serialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/deepmemory_vectorstore.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deepmemory.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/vectorstore/vectorstore_factory.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/hooks.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/common.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/access_method.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/agreement.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/array_list.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/auto.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/casting.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/class_label.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/compression.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/compute.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/diff.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/downsample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/encoder.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/exif.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/hash.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/htype.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/image.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/json.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/keys.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/link.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/logging.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/merge.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/modified.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/notebook.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/path.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/spinner.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/split.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/storage.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tag.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/testing.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/threading.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/token.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/transform.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/version_control.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/video.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/util/warnings.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/entry_points.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/pyproject.toml +0 -0
- {deeplake-3.8.0 → deeplake-3.8.2}/setup.cfg +0 -0
|
@@ -2680,12 +2680,12 @@ def test_tensor_dtype_bug(local_path):
|
|
|
2680
2680
|
ds.abc.append(deeplake.link(f"{data_path}/standard.nii.gz"))
|
|
2681
2681
|
|
|
2682
2682
|
assert ds.abc[0].numpy().shape == (4, 5, 7)
|
|
2683
|
-
assert ds.abc.dtype == np.dtype("
|
|
2683
|
+
assert ds.abc.dtype == np.dtype("uint8")
|
|
2684
2684
|
|
|
2685
2685
|
ds2 = ds.copy(f"{local_path}_2", overwrite=True)
|
|
2686
2686
|
|
|
2687
2687
|
assert ds2.abc[0].numpy().shape == (4, 5, 7)
|
|
2688
|
-
assert ds2.abc.dtype == np.dtype("
|
|
2688
|
+
assert ds2.abc.dtype == np.dtype("uint8")
|
|
2689
2689
|
|
|
2690
2690
|
|
|
2691
2691
|
def test_iterate_with_groups(memory_ds):
|
|
@@ -3043,3 +3043,12 @@ def test_change_htype_fail(local_ds_generator):
|
|
|
3043
3043
|
ds.create_tensor("images3", htype="image", sample_compression="jpg")
|
|
3044
3044
|
with pytest.raises(UnsupportedCompressionError):
|
|
3045
3045
|
ds.images3.htype = "embedding"
|
|
3046
|
+
|
|
3047
|
+
|
|
3048
|
+
def test_append_non_uint8_to_image(local_ds):
|
|
3049
|
+
with local_ds as ds:
|
|
3050
|
+
ds.create_tensor("images", htype="image", sample_compression="png")
|
|
3051
|
+
ds.images.append(np.zeros((40, 40, 1), dtype=np.int16))
|
|
3052
|
+
ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
|
|
3053
|
+
|
|
3054
|
+
assert ds.images.dtype.name == "int16"
|
|
@@ -43,7 +43,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
|
|
|
43
43
|
tiling_threshold=1 * MB,
|
|
44
44
|
)
|
|
45
45
|
|
|
46
|
-
assert images.meta.dtype ==
|
|
46
|
+
assert images.meta.dtype == None
|
|
47
47
|
assert images.meta.sample_compression == "png"
|
|
48
48
|
|
|
49
49
|
_populate_compressed_samples(images, cat_path, flower_path)
|
|
@@ -73,7 +73,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
|
|
|
73
73
|
def test_iterate_compressed_samples(local_ds, cat_path, flower_path):
|
|
74
74
|
images = local_ds.create_tensor(TENSOR_KEY, htype="image", sample_compression="png")
|
|
75
75
|
|
|
76
|
-
assert images.meta.dtype ==
|
|
76
|
+
assert images.meta.dtype == None
|
|
77
77
|
assert images.meta.sample_compression == "png"
|
|
78
78
|
|
|
79
79
|
_populate_compressed_samples(images, cat_path, flower_path)
|
|
@@ -175,3 +175,61 @@ def test_save_view_ignore_errors(local_ds):
|
|
|
175
175
|
assert view.images.shape == (8, 30, 20, 3)
|
|
176
176
|
|
|
177
177
|
np.testing.assert_array_equal(view.labels.numpy(), np.array([[0]] * 8))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@pytest.mark.parametrize("optimize_first_view", [True, False])
|
|
181
|
+
@pytest.mark.parametrize("optimize_second_view", [True, False])
|
|
182
|
+
def test_save_view_of_view(
|
|
183
|
+
local_ds_generator, optimize_first_view, optimize_second_view
|
|
184
|
+
):
|
|
185
|
+
with local_ds_generator() as ds:
|
|
186
|
+
ds.create_tensor("abc")
|
|
187
|
+
ds.abc.extend(list(range(100)))
|
|
188
|
+
|
|
189
|
+
ds.commit()
|
|
190
|
+
|
|
191
|
+
ds[:20].save_view(id="first_20", optimize=optimize_first_view)
|
|
192
|
+
|
|
193
|
+
view = ds.load_view("first_20")
|
|
194
|
+
|
|
195
|
+
view[:10].save_view(id="first_10", optimize=optimize_second_view)
|
|
196
|
+
view[10:].save_view(id="second_10", optimize=optimize_second_view)
|
|
197
|
+
|
|
198
|
+
first_10, second_10 = ds.load_view("first_10"), ds.load_view("second_10")
|
|
199
|
+
np.testing.assert_array_equal(first_10.abc.numpy(), ds[:10].abc.numpy())
|
|
200
|
+
np.testing.assert_array_equal(second_10.abc.numpy(), ds[10:20].abc.numpy())
|
|
201
|
+
|
|
202
|
+
with local_ds_generator() as ds:
|
|
203
|
+
first_10, second_10 = ds.load_view("first_10"), ds.load_view("second_10")
|
|
204
|
+
np.testing.assert_array_equal(first_10.abc.numpy(), ds[:10].abc.numpy())
|
|
205
|
+
np.testing.assert_array_equal(second_10.abc.numpy(), ds[10:20].abc.numpy())
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@pytest.mark.parametrize("optimize_first_view", [True, False])
|
|
209
|
+
@pytest.mark.parametrize("optimize_second_view", [True, False])
|
|
210
|
+
@pytest.mark.parametrize("optimize_third_view", [True, False])
|
|
211
|
+
def test_save_view_of_view_of_view(
|
|
212
|
+
local_ds_generator, optimize_first_view, optimize_second_view, optimize_third_view
|
|
213
|
+
):
|
|
214
|
+
with local_ds_generator() as ds:
|
|
215
|
+
ds.create_tensor("abc")
|
|
216
|
+
ds.abc.extend(list(range(100)))
|
|
217
|
+
|
|
218
|
+
ds.commit()
|
|
219
|
+
|
|
220
|
+
ds[:20].save_view(id="first_20", optimize=optimize_first_view)
|
|
221
|
+
|
|
222
|
+
view = ds.load_view("first_20")
|
|
223
|
+
|
|
224
|
+
view[:10].save_view(id="first_10", optimize=optimize_second_view)
|
|
225
|
+
|
|
226
|
+
first_10 = ds.load_view("first_10")
|
|
227
|
+
|
|
228
|
+
first_10[:5].save_view(id="first_5", optimize=optimize_third_view)
|
|
229
|
+
|
|
230
|
+
first_5 = ds.load_view("first_5")
|
|
231
|
+
np.testing.assert_array_equal(first_5.abc.numpy(), ds[:5].abc.numpy())
|
|
232
|
+
|
|
233
|
+
with local_ds_generator() as ds:
|
|
234
|
+
first_5 = ds.load_view("first_5")
|
|
235
|
+
np.testing.assert_array_equal(first_5.abc.numpy(), ds[:5].abc.numpy())
|
|
@@ -15,7 +15,7 @@ from typing import (
|
|
|
15
15
|
)
|
|
16
16
|
from deeplake.api.info import Info
|
|
17
17
|
from deeplake.core.link_creds import LinkCreds
|
|
18
|
-
from deeplake.core.linked_sample import LinkedSample
|
|
18
|
+
from deeplake.core.linked_sample import LinkedSample, read_linked_sample
|
|
19
19
|
from deeplake.core.meta.encode.base_encoder import LAST_SEEN_INDEX_COLUMN
|
|
20
20
|
from deeplake.core.serialize import HEADER_SIZE_BYTES, text_to_bytes
|
|
21
21
|
from deeplake.core.tensor_link import (
|
|
@@ -692,9 +692,33 @@ class ChunkEngine:
|
|
|
692
692
|
if tensor_meta.htype is None and not all_empty:
|
|
693
693
|
tensor_meta.set_htype(get_htype(samples))
|
|
694
694
|
if tensor_meta.dtype is None and not all_empty:
|
|
695
|
-
tensor_meta.
|
|
696
|
-
|
|
697
|
-
|
|
695
|
+
if tensor_meta.is_link:
|
|
696
|
+
try:
|
|
697
|
+
# download one sample to get dtype
|
|
698
|
+
sample = next(filter(lambda x: x is not None, samples))
|
|
699
|
+
assert isinstance(
|
|
700
|
+
sample, LinkedSample
|
|
701
|
+
), "Sample must be LinkedSample"
|
|
702
|
+
dtype = np.dtype(
|
|
703
|
+
read_linked_sample(
|
|
704
|
+
sample.path, sample.creds_key, self.link_creds, True
|
|
705
|
+
)._typestr
|
|
706
|
+
)
|
|
707
|
+
except:
|
|
708
|
+
# assume uint8 if download fails
|
|
709
|
+
dtype = np.dtype("uint8")
|
|
710
|
+
else:
|
|
711
|
+
non_empty_samples = list(filter(lambda x: x is not None, samples))
|
|
712
|
+
for sample in non_empty_samples:
|
|
713
|
+
try:
|
|
714
|
+
dtype = get_dtype(sample)
|
|
715
|
+
break
|
|
716
|
+
except:
|
|
717
|
+
pass
|
|
718
|
+
else:
|
|
719
|
+
if not ignore_errors:
|
|
720
|
+
raise ValueError("Could not determine dtype of samples")
|
|
721
|
+
tensor_meta.set_dtype(dtype)
|
|
698
722
|
if self._convert_to_list(samples):
|
|
699
723
|
samples = list(samples)
|
|
700
724
|
if self._is_temp_label_tensor:
|
|
@@ -25,11 +25,12 @@ from pathlib import Path
|
|
|
25
25
|
from PIL import Image # type: ignore
|
|
26
26
|
from io import BytesIO
|
|
27
27
|
|
|
28
|
+
import lz4.frame # type: ignore
|
|
29
|
+
import lz4.block # type: ignore
|
|
28
30
|
import mmap
|
|
29
31
|
import struct
|
|
30
32
|
import sys
|
|
31
33
|
import re
|
|
32
|
-
import numcodecs.lz4 # type: ignore
|
|
33
34
|
from numpy.core.fromnumeric import compress # type: ignore
|
|
34
35
|
import math
|
|
35
36
|
from pathlib import Path
|
|
@@ -42,12 +43,6 @@ try:
|
|
|
42
43
|
except ImportError:
|
|
43
44
|
_PYAV_INSTALLED = False
|
|
44
45
|
|
|
45
|
-
try:
|
|
46
|
-
import lz4.frame # type: ignore
|
|
47
|
-
|
|
48
|
-
_LZ4_INSTALLED = True
|
|
49
|
-
except ImportError:
|
|
50
|
-
_LZ4_INSTALLED = False
|
|
51
46
|
|
|
52
47
|
try:
|
|
53
48
|
import nibabel as nib # type: ignore
|
|
@@ -157,7 +152,7 @@ def compress_bytes(
|
|
|
157
152
|
if not buffer:
|
|
158
153
|
return b""
|
|
159
154
|
if compression == "lz4":
|
|
160
|
-
return
|
|
155
|
+
return lz4.block.compress(buffer)
|
|
161
156
|
else:
|
|
162
157
|
raise SampleCompressionError(
|
|
163
158
|
(len(buffer),), compression, f"Not a byte compression: {compression}"
|
|
@@ -173,15 +168,9 @@ def decompress_bytes(
|
|
|
173
168
|
# weird edge case of lz4 + empty string
|
|
174
169
|
if buffer == b"\x00\x00\x00\x00\x00":
|
|
175
170
|
return b""
|
|
176
|
-
if
|
|
177
|
-
buffer[:4] == b'\x04"M\x18'
|
|
178
|
-
): # python-lz4 magic number (backward compatiblity)
|
|
179
|
-
if not _LZ4_INSTALLED:
|
|
180
|
-
raise ModuleNotFoundError(
|
|
181
|
-
"Module lz4 not found. Install using `pip install lz4`."
|
|
182
|
-
)
|
|
171
|
+
if buffer[:4] == b'\x04"M\x18': # python-lz4 magic number
|
|
183
172
|
return lz4.frame.decompress(buffer)
|
|
184
|
-
return
|
|
173
|
+
return lz4.block.decompress(buffer)
|
|
185
174
|
else:
|
|
186
175
|
raise SampleDecompressionError()
|
|
187
176
|
|
|
@@ -3361,7 +3361,7 @@ class Dataset:
|
|
|
3361
3361
|
message: Optional[str] = None,
|
|
3362
3362
|
copy: bool = False,
|
|
3363
3363
|
):
|
|
3364
|
-
if self.has_head_changes:
|
|
3364
|
+
if self.has_head_changes and not self.is_optimized:
|
|
3365
3365
|
raise DatasetViewSavingError(
|
|
3366
3366
|
"The dataset's HEAD node has uncommitted changes. Please create a commit on"
|
|
3367
3367
|
" the dataset object [ds.commit(<insert optional message>)] prior to saving the view."
|
|
@@ -3386,6 +3386,8 @@ class Dataset:
|
|
|
3386
3386
|
if tql_query:
|
|
3387
3387
|
info["tql_query"] = tql_query
|
|
3388
3388
|
info["source-dataset-index"] = getattr(self, "_source_ds_idx", None)
|
|
3389
|
+
if not (query or tql_query):
|
|
3390
|
+
info["source-dataset-index"] = self.index.to_json()
|
|
3389
3391
|
return info
|
|
3390
3392
|
|
|
3391
3393
|
def _lock_queries_json(self):
|
|
@@ -3505,10 +3507,29 @@ class Dataset:
|
|
|
3505
3507
|
"""Saves this view under ".queries" sub directory of same storage."""
|
|
3506
3508
|
info = self._get_view_info(id, message, copy)
|
|
3507
3509
|
hash = info["id"]
|
|
3510
|
+
# creating sub-view of optimized view
|
|
3511
|
+
if self.is_optimized:
|
|
3512
|
+
view_entry = self._view_entry
|
|
3513
|
+
ds = view_entry._src_ds.no_view_dataset
|
|
3514
|
+
|
|
3515
|
+
if copy:
|
|
3516
|
+
view_info = view_entry.info
|
|
3517
|
+
info["source-dataset"] = ds.path
|
|
3518
|
+
info["source-dataset-version"] = view_info["source-dataset-version"]
|
|
3519
|
+
if "source-dataset-index" in view_info:
|
|
3520
|
+
original_idx = Index.from_json(view_info["source-dataset-index"])
|
|
3521
|
+
combined_idx = original_idx[self.index]
|
|
3522
|
+
info["source-dataset-index"] = combined_idx.to_json()
|
|
3523
|
+
else:
|
|
3524
|
+
info["source-dataset-version"] = (
|
|
3525
|
+
info["source-dataset-version"] or FIRST_COMMIT_ID
|
|
3526
|
+
)
|
|
3527
|
+
else:
|
|
3528
|
+
ds = self
|
|
3508
3529
|
path = f".queries/{hash}"
|
|
3509
|
-
vds =
|
|
3530
|
+
vds = ds._sub_ds(path, empty=True, verbose=False)
|
|
3510
3531
|
self._write_vds(vds, info, copy, tensors, num_workers, scheduler, ignore_errors)
|
|
3511
|
-
|
|
3532
|
+
ds._append_to_queries_json(info)
|
|
3512
3533
|
return vds
|
|
3513
3534
|
|
|
3514
3535
|
def _save_view_in_path(
|
|
@@ -3683,7 +3704,8 @@ class Dataset:
|
|
|
3683
3704
|
raise NotImplementedError(
|
|
3684
3705
|
"Saving views inplace is not supported for in-memory datasets."
|
|
3685
3706
|
)
|
|
3686
|
-
|
|
3707
|
+
|
|
3708
|
+
if self.read_only and not base._locked_out and not self.is_optimized:
|
|
3687
3709
|
if isinstance(self, deeplake.core.dataset.DeepLakeCloudDataset):
|
|
3688
3710
|
try:
|
|
3689
3711
|
with self._temp_write_access():
|
|
@@ -284,6 +284,15 @@ class DeepLakeQueryDataset(Dataset):
|
|
|
284
284
|
def index(self):
|
|
285
285
|
return self._index
|
|
286
286
|
|
|
287
|
+
@property
|
|
288
|
+
def sample_indices(self):
|
|
289
|
+
for t in self.tensors.values():
|
|
290
|
+
try:
|
|
291
|
+
return t.indra_tensor.indexes
|
|
292
|
+
except RuntimeError:
|
|
293
|
+
pass
|
|
294
|
+
return range(self.num_samples)
|
|
295
|
+
|
|
287
296
|
def _tensors(
|
|
288
297
|
self, include_hidden: bool = True, include_disabled=True
|
|
289
298
|
) -> Dict[str, Tensor]:
|
|
@@ -137,6 +137,13 @@ class DeepLakeQueryTensor(tensor.Tensor):
|
|
|
137
137
|
def chunk_engine(self):
|
|
138
138
|
raise NotImplementedError("Virtual tensor does not have chunk engine.")
|
|
139
139
|
|
|
140
|
+
@property
|
|
141
|
+
def sample_indices(self):
|
|
142
|
+
try:
|
|
143
|
+
return self.indra_tensor.indexes
|
|
144
|
+
except RuntimeError:
|
|
145
|
+
return range(self.num_samples)
|
|
146
|
+
|
|
140
147
|
@property
|
|
141
148
|
def shape(self):
|
|
142
149
|
if (
|
|
@@ -1485,20 +1485,16 @@ class Tensor:
|
|
|
1485
1485
|
|
|
1486
1486
|
def _regenerate_vdb_indexes(self):
|
|
1487
1487
|
try:
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
distance = vdb_index["distance"]
|
|
1499
|
-
self.delete_vdb_index(id)
|
|
1500
|
-
# Recreate it back.
|
|
1501
|
-
self.create_vdb_index(id, distance)
|
|
1488
|
+
vdb_indexes = self.get_vdb_indexes()
|
|
1489
|
+
|
|
1490
|
+
for vdb_index in vdb_indexes:
|
|
1491
|
+
self.delete_vdb_index(vdb_index["id"])
|
|
1492
|
+
# Recreate it back.
|
|
1493
|
+
self.create_vdb_index(
|
|
1494
|
+
vdb_index["id"],
|
|
1495
|
+
vdb_index["distance"],
|
|
1496
|
+
additional_params=vdb_index.get("additional_params", None),
|
|
1497
|
+
)
|
|
1502
1498
|
except Exception as e:
|
|
1503
1499
|
raise Exception(f"An error occurred while regenerating VDB indexes: {e}")
|
|
1504
1500
|
|
|
@@ -423,3 +423,19 @@ def test_virtual_tensors(local_auth_ds_generator):
|
|
|
423
423
|
assert deeplake_indra_ds.score[100 - i].numpy() == [
|
|
424
424
|
np.sqrt(2.0 / (i + 1) / (i + 1))
|
|
425
425
|
]
|
|
426
|
+
|
|
427
|
+
assert list(deeplake_indra_ds.sample_indices) == list(range(100))
|
|
428
|
+
deeplake_indra_ds = deeplake_ds.query(
|
|
429
|
+
"SELECT *, l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
|
|
430
|
+
)
|
|
431
|
+
assert list(deeplake_indra_ds.sample_indices) == list(reversed(range(100)))
|
|
432
|
+
assert list(deeplake_indra_ds.embeddings.sample_indices) == list(
|
|
433
|
+
reversed(range(100))
|
|
434
|
+
)
|
|
435
|
+
assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
|
|
436
|
+
|
|
437
|
+
deeplake_indra_ds = deeplake_ds.query(
|
|
438
|
+
"SELECT l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
|
|
439
|
+
)
|
|
440
|
+
assert list(deeplake_indra_ds.sample_indices) == list(range(100))
|
|
441
|
+
assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
|
|
@@ -161,7 +161,7 @@ class TransformDataset:
|
|
|
161
161
|
# First sample in tensor
|
|
162
162
|
# Flush to set meta attributes
|
|
163
163
|
if dtype is None:
|
|
164
|
-
self.flush()
|
|
164
|
+
self.flush(clear_on_fail=False)
|
|
165
165
|
return
|
|
166
166
|
sizeof_item = self._calculate_sample_size(item, dtype, htype)
|
|
167
167
|
except:
|
|
@@ -201,22 +201,29 @@ class TransformDataset:
|
|
|
201
201
|
updated_tensors[full_name] = len(items)
|
|
202
202
|
tensor.items.clear()
|
|
203
203
|
|
|
204
|
-
def _rollback(self, updated_tensors):
|
|
204
|
+
def _rollback(self, updated_tensors, no_dtype_tensors):
|
|
205
205
|
for t in updated_tensors:
|
|
206
206
|
chunk_engine = self.all_chunk_engines[t]
|
|
207
207
|
num_samples = updated_tensors[t]
|
|
208
208
|
for _ in range(num_samples):
|
|
209
209
|
chunk_engine.pop(link_callback=chunk_engine._transform_pop_callback)
|
|
210
210
|
|
|
211
|
+
if t in no_dtype_tensors:
|
|
212
|
+
meta = chunk_engine.tensor_meta
|
|
213
|
+
meta.dtype = None
|
|
214
|
+
meta.typestr = None
|
|
215
|
+
meta.is_dirty = True
|
|
216
|
+
|
|
211
217
|
def _clear(self):
|
|
212
218
|
for tensor in self.data.values():
|
|
213
219
|
tensor.items.clear()
|
|
214
220
|
self.cache_used = 0
|
|
215
221
|
|
|
216
|
-
def flush(self):
|
|
222
|
+
def flush(self, clear_on_fail=True):
|
|
217
223
|
all_chunk_engines = self.all_chunk_engines
|
|
218
224
|
label_temp_tensors = self.label_temp_tensors
|
|
219
225
|
updated_tensors = {}
|
|
226
|
+
no_dtype_tensors = []
|
|
220
227
|
try:
|
|
221
228
|
for name, tensor in self.data.items():
|
|
222
229
|
if not tensor.is_group:
|
|
@@ -225,6 +232,11 @@ class TransformDataset:
|
|
|
225
232
|
chunk_engine = all_chunk_engines[name]
|
|
226
233
|
callback = chunk_engine._transform_callback
|
|
227
234
|
|
|
235
|
+
meta = chunk_engine.tensor_meta
|
|
236
|
+
if meta.length == 0 and meta.dtype is None:
|
|
237
|
+
# for rolling back dtype change
|
|
238
|
+
no_dtype_tensors.append(name)
|
|
239
|
+
|
|
228
240
|
if tensor.numpy_only:
|
|
229
241
|
self._flush_numpy_tensor_to_chunk_engine(
|
|
230
242
|
name, tensor, chunk_engine, callback, updated_tensors
|
|
@@ -234,9 +246,10 @@ class TransformDataset:
|
|
|
234
246
|
name, tensor, chunk_engine, callback, updated_tensors
|
|
235
247
|
)
|
|
236
248
|
self.start_input_idx = None
|
|
249
|
+
self._clear()
|
|
237
250
|
except Exception as e:
|
|
238
|
-
self._rollback(updated_tensors)
|
|
251
|
+
self._rollback(updated_tensors, no_dtype_tensors)
|
|
252
|
+
if clear_on_fail:
|
|
253
|
+
self._clear()
|
|
239
254
|
e = e.__cause__ if isinstance(e, SampleAppendError) else e # type: ignore
|
|
240
255
|
raise SampleAppendError(name) from e
|
|
241
|
-
finally:
|
|
242
|
-
self._clear()
|
|
@@ -393,7 +393,7 @@ class DeepMemory:
|
|
|
393
393
|
queries_data = {
|
|
394
394
|
"text": queries,
|
|
395
395
|
"metadata": [
|
|
396
|
-
{"
|
|
396
|
+
{"relevance": relevance_per_doc} for relevance_per_doc in relevance
|
|
397
397
|
],
|
|
398
398
|
"embedding": query_embs,
|
|
399
399
|
"id": [uuid.uuid4().hex for _ in range(len(queries))],
|
|
@@ -330,13 +330,7 @@ class VectorStore:
|
|
|
330
330
|
)
|
|
331
331
|
|
|
332
332
|
assert id_ is not None
|
|
333
|
-
utils.check_length_of_each_tensor(processed_tensors)
|
|
334
|
-
|
|
335
|
-
# In Case prefilled dataset which is already having index defined
|
|
336
|
-
# regenerate the index post ingestion.
|
|
337
|
-
index_regeneration = False
|
|
338
|
-
if len(self.dataset) > 0 and index.check_vdb_indexes(self.dataset):
|
|
339
|
-
index_regeneration = True
|
|
333
|
+
data_length = utils.check_length_of_each_tensor(processed_tensors)
|
|
340
334
|
|
|
341
335
|
dataset_utils.extend_or_ingest_dataset(
|
|
342
336
|
processed_tensors=processed_tensors,
|
|
@@ -348,13 +342,7 @@ class VectorStore:
|
|
|
348
342
|
rate_limiter=rate_limiter,
|
|
349
343
|
)
|
|
350
344
|
|
|
351
|
-
|
|
352
|
-
index.index_cache_cleanup(self.dataset)
|
|
353
|
-
self.distance_metric_index = index.validate_and_create_vector_index(
|
|
354
|
-
dataset=self.dataset,
|
|
355
|
-
index_params=self.index_params,
|
|
356
|
-
regenerate_index=index_regeneration,
|
|
357
|
-
)
|
|
345
|
+
self._update_index(regenerate_index=data_length > 0)
|
|
358
346
|
|
|
359
347
|
try_flushing(self.dataset)
|
|
360
348
|
|
|
@@ -612,7 +600,11 @@ class VectorStore:
|
|
|
612
600
|
return True
|
|
613
601
|
|
|
614
602
|
dataset_utils.delete_and_without_commit(self.dataset, row_ids)
|
|
603
|
+
|
|
604
|
+
self._update_index(regenerate_index=len(row_ids) > 0 if row_ids else False)
|
|
605
|
+
|
|
615
606
|
try_flushing(self.dataset)
|
|
607
|
+
|
|
616
608
|
return True
|
|
617
609
|
|
|
618
610
|
def update_embedding(
|
|
@@ -718,6 +710,9 @@ class VectorStore:
|
|
|
718
710
|
)
|
|
719
711
|
|
|
720
712
|
self.dataset[row_ids].update(embedding_tensor_data)
|
|
713
|
+
|
|
714
|
+
self._update_index(regenerate_index=len(row_ids) > 0 if row_ids else False)
|
|
715
|
+
|
|
721
716
|
try_flushing(self.dataset)
|
|
722
717
|
|
|
723
718
|
@staticmethod
|
|
@@ -784,5 +779,14 @@ class VectorStore:
|
|
|
784
779
|
"""Length of the dataset"""
|
|
785
780
|
return len(self.dataset)
|
|
786
781
|
|
|
782
|
+
def _update_index(self, regenerate_index=False):
|
|
783
|
+
if utils.index_used(self.exec_option):
|
|
784
|
+
index.index_cache_cleanup(self.dataset)
|
|
785
|
+
self.distance_metric_index = index.validate_and_create_vector_index(
|
|
786
|
+
dataset=self.dataset,
|
|
787
|
+
index_params=self.index_params,
|
|
788
|
+
regenerate_index=regenerate_index,
|
|
789
|
+
)
|
|
790
|
+
|
|
787
791
|
|
|
788
792
|
DeepLakeVectorStore = VectorStore
|
|
@@ -514,6 +514,7 @@ def test_search_basic(local_path, hub_cloud_dev_token):
|
|
|
514
514
|
assert len(result) == 4
|
|
515
515
|
|
|
516
516
|
|
|
517
|
+
@pytest.mark.slow
|
|
517
518
|
@requires_libdeeplake
|
|
518
519
|
def test_index_basic(local_path, hub_cloud_dev_token):
|
|
519
520
|
# Start by testing behavior without an index
|
|
@@ -545,6 +546,29 @@ def test_index_basic(local_path, hub_cloud_dev_token):
|
|
|
545
546
|
== METRIC_TO_INDEX_METRIC[DEFAULT_VECTORSTORE_DISTANCE_METRIC]
|
|
546
547
|
)
|
|
547
548
|
|
|
549
|
+
# Test index with sample updates
|
|
550
|
+
pre_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
551
|
+
vector_store.add(
|
|
552
|
+
embedding=[embeddings[0]], text=[texts[0]], metadata=[metadatas[0]]
|
|
553
|
+
)
|
|
554
|
+
post_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
555
|
+
|
|
556
|
+
assert pre_update_index == post_update_index
|
|
557
|
+
|
|
558
|
+
# Test index with sample deletion
|
|
559
|
+
pre_delete_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
560
|
+
vector_store.delete(row_ids=[len(vector_store) - 1])
|
|
561
|
+
post_delete_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
562
|
+
|
|
563
|
+
assert pre_delete_index == post_delete_index
|
|
564
|
+
|
|
565
|
+
# Test index with sample updating
|
|
566
|
+
pre_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
567
|
+
vector_store.update_embedding(row_ids=[0], embedding_function=embedding_fn)
|
|
568
|
+
post_update_index = vector_store.dataset.embedding.get_vdb_indexes()[0]
|
|
569
|
+
|
|
570
|
+
assert pre_update_index == post_update_index
|
|
571
|
+
|
|
548
572
|
# Check that distance metric throws a warning when there is an index
|
|
549
573
|
with pytest.warns(None):
|
|
550
574
|
vector_store.search(embedding=query_embedding, distance_metric="l1")
|
|
@@ -116,25 +116,6 @@ def validate_and_create_vector_index(dataset, index_params, regenerate_index=Fal
|
|
|
116
116
|
|
|
117
117
|
tensors = dataset.tensors
|
|
118
118
|
|
|
119
|
-
# TODO: BRING BACK WHEN IT IS IN USE
|
|
120
|
-
|
|
121
|
-
# index_regen = False
|
|
122
|
-
# Check if regenerate_index is true.
|
|
123
|
-
# if regenerate_index:
|
|
124
|
-
# for _, tensor in tensors.items():
|
|
125
|
-
# is_embedding = utils.is_embedding_tensor(tensor)
|
|
126
|
-
# has_vdb_indexes = hasattr(tensor.meta, "vdb_indexes")
|
|
127
|
-
# try:
|
|
128
|
-
# vdb_index_ids_present = len(tensor.get_vdb_indexes()) > 0
|
|
129
|
-
# except AttributeError:
|
|
130
|
-
# vdb_index_ids_present = False
|
|
131
|
-
|
|
132
|
-
# if is_embedding and has_vdb_indexes and vdb_index_ids_present:
|
|
133
|
-
# tensor._regenerate_vdb_indexes()
|
|
134
|
-
# index_regen = True
|
|
135
|
-
# if index_regen:
|
|
136
|
-
# return
|
|
137
|
-
|
|
138
119
|
# Check all tensors from the dataset.
|
|
139
120
|
for _, tensor in tensors.items():
|
|
140
121
|
is_embedding = utils.is_embedding_tensor(tensor)
|
|
@@ -159,6 +140,9 @@ def validate_and_create_vector_index(dataset, index_params, regenerate_index=Fal
|
|
|
159
140
|
except ValueError as e:
|
|
160
141
|
raise e
|
|
161
142
|
elif len(vdb_indexes) > 0:
|
|
143
|
+
if regenerate_index:
|
|
144
|
+
tensor._regenerate_vdb_indexes()
|
|
145
|
+
|
|
162
146
|
return vdb_indexes[0]["distance"]
|
|
163
147
|
|
|
164
148
|
return None
|
|
@@ -158,6 +158,8 @@ def check_length_of_each_tensor(tensors):
|
|
|
158
158
|
f"All of the tensors should have equal length. Currently tensors have different length: {tensor_lengths}"
|
|
159
159
|
)
|
|
160
160
|
|
|
161
|
+
return tensor_length
|
|
162
|
+
|
|
161
163
|
|
|
162
164
|
def create_tensor_to_length_str(tensors):
|
|
163
165
|
tensor_lengths = "\n"
|