deeplake 3.5.0__tar.gz → 3.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.5.0/deeplake.egg-info → deeplake-3.5.2}/PKG-INFO +1 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/__init__.py +1 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/dataset.py +13 -4
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk_engine.py +1 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/s3.py +2 -2
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tensor_link.py +3 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/deeplake_vectorstore.py +18 -8
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +1 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +2 -2
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/indra_vector_search.py +3 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/query.py +4 -2
- deeplake-3.5.2/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.5.0 → deeplake-3.5.2/deeplake.egg-info}/PKG-INFO +1 -1
- deeplake-3.5.0/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.5.0 → deeplake-3.5.2}/LICENSE +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/MANIFEST.in +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/README.md +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/info.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/link.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/read.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/api/tiled.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/cli/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/cli/auth.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/cli/commands.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/cli/test_cli.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/client.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/config.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/log.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/test_client.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/client/utils.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/compression.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/constants.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compression.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/deeplake_query_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/index/index.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/io.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/ipc.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/lock.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/polygon.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/query/query.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/sample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/serialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tensor.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_deeplake_indra_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/remote_engine_search.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/dataloader.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/hooks.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/htype.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/common.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/access_method.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/agreement.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/array_list.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/auto.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/casting.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/class_label.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/compression.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/compute.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/diff.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/downsample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/encoder.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/exif.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/hash.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/htype.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/image.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/json.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/keys.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/link.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/logging.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/merge.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/modified.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/notebook.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/path.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/spinner.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/split.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/storage.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tag.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/testing.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/threading.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/token.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/transform.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/version_control.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/video.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/util/warnings.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/entry_points.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/requires.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/setup.cfg +0 -0
- {deeplake-3.5.0 → deeplake-3.5.2}/setup.py +0 -0
|
@@ -90,6 +90,7 @@ class dataset:
|
|
|
90
90
|
verbose: bool = True,
|
|
91
91
|
access_method: str = "stream",
|
|
92
92
|
reset: bool = False,
|
|
93
|
+
check_integrity: bool = True,
|
|
93
94
|
):
|
|
94
95
|
"""Returns a :class:`~deeplake.core.dataset.Dataset` object referencing either a new or existing dataset.
|
|
95
96
|
|
|
@@ -156,6 +157,7 @@ class dataset:
|
|
|
156
157
|
and default scheduler (threaded), and 'local:processed' will use a single worker and use processed scheduler.
|
|
157
158
|
reset (bool): If the specified dataset cannot be loaded due to a corrupted HEAD state of the branch being loaded,
|
|
158
159
|
setting ``reset=True`` will reset HEAD changes and load the previous version.
|
|
160
|
+
check_integrity (bool): If the param is True it will do integrity check during dataset loading otherwise the check is not performed
|
|
159
161
|
|
|
160
162
|
..
|
|
161
163
|
# noqa: DAR101
|
|
@@ -258,7 +260,9 @@ class dataset:
|
|
|
258
260
|
)
|
|
259
261
|
|
|
260
262
|
try:
|
|
261
|
-
return dataset._load(
|
|
263
|
+
return dataset._load(
|
|
264
|
+
dataset_kwargs, access_method, create, check_integrity=check_integrity
|
|
265
|
+
)
|
|
262
266
|
except (AgreementError, CheckoutError, LockedException) as e:
|
|
263
267
|
raise e from None
|
|
264
268
|
except Exception as e:
|
|
@@ -436,6 +440,7 @@ class dataset:
|
|
|
436
440
|
verbose: bool = True,
|
|
437
441
|
access_method: str = "stream",
|
|
438
442
|
reset: bool = False,
|
|
443
|
+
check_integrity: bool = True,
|
|
439
444
|
) -> Dataset:
|
|
440
445
|
"""Loads an existing dataset
|
|
441
446
|
|
|
@@ -499,6 +504,7 @@ class dataset:
|
|
|
499
504
|
and default scheduler (threaded), and 'local:processed' will use a single worker and use processed scheduler.
|
|
500
505
|
reset (bool): If the specified dataset cannot be loaded due to a corrupted HEAD state of the branch being loaded,
|
|
501
506
|
setting ``reset=True`` will reset HEAD changes and load the previous version.
|
|
507
|
+
check_integrity (bool): If the param is True it will do integrity check during dataset loading otherwise the check is not performed
|
|
502
508
|
|
|
503
509
|
..
|
|
504
510
|
# noqa: DAR101
|
|
@@ -581,7 +587,9 @@ class dataset:
|
|
|
581
587
|
)
|
|
582
588
|
|
|
583
589
|
try:
|
|
584
|
-
return dataset._load(
|
|
590
|
+
return dataset._load(
|
|
591
|
+
dataset_kwargs, access_method, check_integrity=check_integrity
|
|
592
|
+
)
|
|
585
593
|
except (AgreementError, CheckoutError, LockedException) as e:
|
|
586
594
|
raise e from None
|
|
587
595
|
except Exception as e:
|
|
@@ -648,7 +656,7 @@ class dataset:
|
|
|
648
656
|
return ds
|
|
649
657
|
|
|
650
658
|
@staticmethod
|
|
651
|
-
def _load(dataset_kwargs, access_method=None, create=False):
|
|
659
|
+
def _load(dataset_kwargs, access_method=None, create=False, check_integrity=True):
|
|
652
660
|
if access_method in ("stream", None):
|
|
653
661
|
ret = dataset_factory(**dataset_kwargs)
|
|
654
662
|
if create:
|
|
@@ -656,7 +664,8 @@ class dataset:
|
|
|
656
664
|
else:
|
|
657
665
|
dataset_loaded(ret)
|
|
658
666
|
|
|
659
|
-
|
|
667
|
+
if check_integrity:
|
|
668
|
+
integrity_check(ret)
|
|
660
669
|
|
|
661
670
|
verbose = dataset_kwargs.get("verbose")
|
|
662
671
|
path = dataset_kwargs.get("path")
|
|
@@ -2539,7 +2539,7 @@ class ChunkEngine:
|
|
|
2539
2539
|
try:
|
|
2540
2540
|
shape = sample_shape_provider(idx) # type: ignore
|
|
2541
2541
|
if isinstance(shape, tuple) and shape == ():
|
|
2542
|
-
shape = (
|
|
2542
|
+
shape = (0,)
|
|
2543
2543
|
if self.is_sequence:
|
|
2544
2544
|
if sample_index and not sample_index[0].subscriptable():
|
|
2545
2545
|
shape = (1, *tuple(shape[sample_index[0].value].tolist())) # type: ignore
|
|
@@ -5,7 +5,7 @@ import boto3
|
|
|
5
5
|
import botocore # type: ignore
|
|
6
6
|
import posixpath
|
|
7
7
|
from typing import Dict, Optional, Tuple, Type
|
|
8
|
-
from datetime import datetime
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
9
|
from botocore.session import ComponentLocator
|
|
10
10
|
from deeplake.client.client import DeepLakeBackendClient
|
|
11
11
|
from deeplake.core.storage.provider import StorageProvider
|
|
@@ -506,7 +506,7 @@ class S3Provider(StorageProvider):
|
|
|
506
506
|
This would only happen for datasets stored on Deep Lake storage for which temporary 12 hour credentials are generated.
|
|
507
507
|
"""
|
|
508
508
|
if self.expiration and (
|
|
509
|
-
force or float(self.expiration) < datetime.
|
|
509
|
+
force or float(self.expiration) < datetime.now(timezone.utc).timestamp()
|
|
510
510
|
):
|
|
511
511
|
client = DeepLakeBackendClient(self.token)
|
|
512
512
|
org_id, ds_name = self.tag.split("/")
|
|
@@ -109,7 +109,9 @@ def update_shape(new_sample, link_creds=None, tensor_meta=None):
|
|
|
109
109
|
new_sample = read_linked_sample(
|
|
110
110
|
new_sample.path, new_sample.creds_key, link_creds, verify=False
|
|
111
111
|
)
|
|
112
|
-
if np.isscalar(new_sample)
|
|
112
|
+
if np.isscalar(new_sample) or (
|
|
113
|
+
isinstance(new_sample, np.ndarray) and new_sample.shape == ()
|
|
114
|
+
):
|
|
113
115
|
ret = np.array([1], dtype=np.int64)
|
|
114
116
|
else:
|
|
115
117
|
ret = np.array(
|
|
@@ -67,18 +67,20 @@ class DeepLakeVectorStore:
|
|
|
67
67
|
def add(
|
|
68
68
|
self,
|
|
69
69
|
texts: Iterable[str],
|
|
70
|
+
embedding_function: Optional[Callable] = None,
|
|
70
71
|
metadatas: Optional[List[dict]] = None,
|
|
71
72
|
ids: Optional[List[str]] = None,
|
|
72
|
-
embeddings: Optional[np.ndarray] = None,
|
|
73
|
+
embeddings: Optional[Union[List[float], np.ndarray]] = None,
|
|
73
74
|
total_samples_processed: int = 0,
|
|
74
75
|
) -> List[str]:
|
|
75
76
|
"""Adding elements to deeplake vector store
|
|
76
77
|
|
|
77
78
|
Args:
|
|
78
79
|
texts (Iterable[str]): texts to add to deeplake vector store
|
|
80
|
+
embedding_function (callable, optional): embedding function used to convert document texts into embeddings.
|
|
79
81
|
metadatas (List[dict], optional): List of metadatas. Defaults to None.
|
|
80
82
|
ids (List[str], optional): List of document IDs. Defaults to None.
|
|
81
|
-
embeddings (np.ndarray, optional): embedding of texts. Defaults to None.
|
|
83
|
+
embeddings (Union[List[float], np.ndarray], optional): embedding of texts. Defaults to None.
|
|
82
84
|
total_samples_processed (int): Total number of samples processed before transforms stopped.
|
|
83
85
|
|
|
84
86
|
Returns:
|
|
@@ -93,7 +95,7 @@ class DeepLakeVectorStore:
|
|
|
93
95
|
ingest_data.run_data_ingestion(
|
|
94
96
|
elements=elements,
|
|
95
97
|
dataset=self.dataset,
|
|
96
|
-
embedding_function=self.embedding_function,
|
|
98
|
+
embedding_function=embedding_function or self.embedding_function,
|
|
97
99
|
ingestion_batch_size=self.ingestion_batch_size,
|
|
98
100
|
num_workers=self.num_workers,
|
|
99
101
|
total_samples_processed=total_samples_processed,
|
|
@@ -105,8 +107,9 @@ class DeepLakeVectorStore:
|
|
|
105
107
|
|
|
106
108
|
def search(
|
|
107
109
|
self,
|
|
110
|
+
embedding_function: Optional[Callable] = None,
|
|
108
111
|
query: Optional[str] = None,
|
|
109
|
-
embedding: Optional[np.ndarray] = None,
|
|
112
|
+
embedding: Optional[Union[List[float], np.ndarray]] = None,
|
|
110
113
|
k: int = 4,
|
|
111
114
|
distance_metric: str = "L2",
|
|
112
115
|
filter: Optional[Any] = None,
|
|
@@ -116,7 +119,8 @@ class DeepLakeVectorStore:
|
|
|
116
119
|
|
|
117
120
|
Args:
|
|
118
121
|
query (str, optional): String representation of the query to run. Defaults to None.
|
|
119
|
-
|
|
122
|
+
embedding_function (callable, optional): function for converting query into embedding.
|
|
123
|
+
embedding (Union[np.ndarray, List[float]], optional): Embedding representation of the query to run. Defaults to None.
|
|
120
124
|
k (int): Number of elements to return after running query. Defaults to 4.
|
|
121
125
|
distance_metric (str): Type of distance metric to use for sorting the data. Avaliable options are: "L1", "L2", "COS", "MAX". Defaults to "L2".
|
|
122
126
|
filter (Any, optional): Metadata dictionary for exact search. Defaults to None.
|
|
@@ -140,6 +144,7 @@ class DeepLakeVectorStore:
|
|
|
140
144
|
utils.check_indra_installation(exec_option, indra_installed=_INDRA_INSTALLED)
|
|
141
145
|
|
|
142
146
|
return self._search(
|
|
147
|
+
embedding_function=embedding_function or self.embedding_function,
|
|
143
148
|
view=view,
|
|
144
149
|
exec_option=exec_option,
|
|
145
150
|
embedding=embedding,
|
|
@@ -152,6 +157,7 @@ class DeepLakeVectorStore:
|
|
|
152
157
|
self,
|
|
153
158
|
view: DeepLakeDataset,
|
|
154
159
|
exec_option: str,
|
|
160
|
+
embedding_function: Optional[Callable] = None,
|
|
155
161
|
embedding: Optional[Union[List[float], np.ndarray]] = None,
|
|
156
162
|
query: Optional[str] = None,
|
|
157
163
|
k: int = 4,
|
|
@@ -162,7 +168,8 @@ class DeepLakeVectorStore:
|
|
|
162
168
|
Args:
|
|
163
169
|
view (DeepLakeDataset): DeepLakeDataset object to run query inside.
|
|
164
170
|
query (Optional[str], optional): String representation of the query to run. Defaults to None.
|
|
165
|
-
embedding (
|
|
171
|
+
embedding (Union[List[float], np.ndarray], optional): Embedding representation of the query to run. Defaults to None.
|
|
172
|
+
embedding_function (callable, optional): function for converting query into embedding.
|
|
166
173
|
k (int): Number of elements to return after running query. Defaults to 4.
|
|
167
174
|
distance_metric (str): Type of distance metric to use for sorting the data. Avaliable options are: "L1", "L2", "COS", "MAX". Defaults to "L2".
|
|
168
175
|
exec_option (str): Type of query execution. It could be either "python", "compute_engine" or "tensor_db".
|
|
@@ -173,11 +180,13 @@ class DeepLakeVectorStore:
|
|
|
173
180
|
Returns:
|
|
174
181
|
tuple (view, indices, scores): View is the dataset view generated from the queried samples, indices are the indices of the ordered samples, scores are respectively the scores of the ordered samples
|
|
175
182
|
"""
|
|
176
|
-
if
|
|
183
|
+
if embedding_function is None and embedding is None:
|
|
177
184
|
view, scores, indices = filter_utils.exact_text_search(view, query)
|
|
178
185
|
else:
|
|
179
186
|
query_emb = dataset_utils.get_embedding(
|
|
180
|
-
embedding,
|
|
187
|
+
embedding,
|
|
188
|
+
query,
|
|
189
|
+
embedding_function=embedding_function,
|
|
181
190
|
)
|
|
182
191
|
exec_option = exec_option or self._exec_option
|
|
183
192
|
embeddings = dataset_utils.fetch_embeddings(
|
|
@@ -192,6 +201,7 @@ class DeepLakeVectorStore:
|
|
|
192
201
|
exec_option=exec_option,
|
|
193
202
|
deeplake_dataset=self.dataset,
|
|
194
203
|
)
|
|
204
|
+
view = view[indices]
|
|
195
205
|
return (view, indices, scores)
|
|
196
206
|
|
|
197
207
|
def delete(
|
{deeplake-3.5.0 → deeplake-3.5.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py
RENAMED
|
@@ -212,7 +212,7 @@ def get_embedding(embedding, query, embedding_function=None):
|
|
|
212
212
|
always_warn("both embedding and embedding_function are specified. ")
|
|
213
213
|
embedding = embedding_function(query) # type: ignore
|
|
214
214
|
|
|
215
|
-
if embedding.dtype != "float32":
|
|
215
|
+
if isinstance(embedding, list) or embedding.dtype != "float32":
|
|
216
216
|
embedding = np.array(embedding, dtype=np.float32)
|
|
217
217
|
|
|
218
218
|
return embedding
|
|
@@ -245,7 +245,7 @@ def create_elements(
|
|
|
245
245
|
texts: Iterable[str],
|
|
246
246
|
ids: Optional[List[str]] = None,
|
|
247
247
|
metadatas: Optional[List[dict]] = None,
|
|
248
|
-
embeddings: Optional[np.ndarray] = None,
|
|
248
|
+
embeddings: Optional[Union[List[float], np.ndarray]] = None,
|
|
249
249
|
):
|
|
250
250
|
processed_tensors, ids = preprocess_tensors(ids, texts, metadatas, embeddings)
|
|
251
251
|
utils.check_length_of_each_tensor(processed_tensors)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
from typing import Union, List, Any, Optional, Tuple
|
|
3
3
|
|
|
4
|
+
from deeplake.enterprise.convert_to_libdeeplake import dataset_to_libdeeplake
|
|
5
|
+
|
|
4
6
|
from deeplake.core.vectorstore.vector_search.indra import query
|
|
5
7
|
from deeplake.core.vectorstore.vector_search import utils
|
|
6
8
|
from deeplake.core.dataset import Dataset as DeepLakeDataset
|
|
@@ -30,7 +32,7 @@ def vector_search(
|
|
|
30
32
|
from indra import api # type: ignore
|
|
31
33
|
|
|
32
34
|
tql_query = query.parse_query(distance_metric, k, query_embedding, embedding_tensor)
|
|
33
|
-
indra_ds =
|
|
35
|
+
indra_ds = dataset_to_libdeeplake(deeplake_dataset)
|
|
34
36
|
|
|
35
37
|
view = indra_ds.query(tql_query)
|
|
36
38
|
indices = view.indexes
|
|
@@ -55,12 +55,14 @@ def convert_tensor_to_str(query_embedding: np.ndarray):
|
|
|
55
55
|
Args:
|
|
56
56
|
query_embedding (Union[List[float], np.ndarray]) - embedding representation of the query string.
|
|
57
57
|
"""
|
|
58
|
-
query_embedding
|
|
58
|
+
if len(query_embedding.shape) > 1:
|
|
59
|
+
query_embedding = query_embedding.transpose(1, 0)
|
|
60
|
+
query_embedding = query_embedding[:, 0]
|
|
59
61
|
|
|
60
62
|
query_embedding_str = ""
|
|
61
63
|
|
|
62
64
|
for item in query_embedding:
|
|
63
|
-
query_embedding_str += f"{item
|
|
65
|
+
query_embedding_str += f"{item}, "
|
|
64
66
|
|
|
65
67
|
return f"ARRAY[{query_embedding_str[:-2]}]"
|
|
66
68
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = "800025eb-3253-41d5-a3b2-6884a6ffd90f"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
BUGOUT_TOKEN = "0555e1ab-17e8-437b-80a5-a57b68fbc8e7"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|