deeplake 3.6.18__tar.gz → 3.6.20__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.6.18/deeplake.egg-info → deeplake-3.6.20}/PKG-INFO +11 -7
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/__init__.py +4 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api.py +14 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/constants.py +11 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/dataset.py +55 -10
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_query_dataset.py +1 -0
- deeplake-3.6.20/deeplake/core/seed.py +25 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/azure.py +4 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/lru_cache.py +2 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_deeplake_indra_dataset.py +36 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/test_transform.py +4 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform.py +4 -2
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/deeplake_vectorstore.py +2 -7
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +23 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +2 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +111 -18
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +64 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +2 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/utils.py +1 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/test_version_control.py +89 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/convert_to_libdeeplake.py +14 -3
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/dataloader.py +22 -2
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_pytorch.py +14 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_query.py +28 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/common.py +1 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -4
- deeplake-3.6.20/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/exceptions.py +18 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/scheduling.py +4 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/shuffle.py +5 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/spinner.py +37 -29
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_shuffle.py +2 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/transform.py +7 -2
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/version_control.py +111 -0
- {deeplake-3.6.18 → deeplake-3.6.20/deeplake.egg-info}/PKG-INFO +11 -7
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/SOURCES.txt +1 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/entry_points.txt +1 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/requires.txt +20 -20
- {deeplake-3.6.18 → deeplake-3.6.20}/setup.py +1 -1
- deeplake-3.6.18/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.6.18 → deeplake-3.6.20}/LICENSE +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/MANIFEST.in +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/README.md +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/info.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/link.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/read.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/api/tiled.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/auth.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/commands.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/cli/test_cli.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/client.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/config.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/log.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/test_client.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/client/utils.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/compression.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compression.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/index/index.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/io.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/ipc.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/lock.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/polygon.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/query/query.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/sample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/serialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tensor.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/hooks.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/htype.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/common.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/access_method.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/agreement.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/array_list.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/auto.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/casting.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/class_label.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/compression.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/compute.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/diff.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/downsample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/encoder.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/exif.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/hash.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/htype.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/image.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/json.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/keys.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/link.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/logging.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/merge.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/modified.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/notebook.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/path.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/split.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/storage.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tag.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/testing.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/threading.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/token.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/video.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/util/warnings.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.6.18 → deeplake-3.6.20}/setup.cfg +0 -0
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: deeplake
|
|
3
|
-
Version: 3.6.
|
|
3
|
+
Version: 3.6.20
|
|
4
4
|
Summary: Activeloop Deep Lake
|
|
5
|
+
Home-page: UNKNOWN
|
|
5
6
|
Author: activeloop.ai
|
|
6
7
|
Author-email: support@activeloop.ai
|
|
7
8
|
License: MPL-2.0
|
|
8
9
|
Project-URL: Documentation, https://docs.activeloop.ai/
|
|
9
10
|
Project-URL: Source, https://github.com/activeloopai/deeplake
|
|
11
|
+
Platform: UNKNOWN
|
|
10
12
|
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
|
11
13
|
Description-Content-Type: text/markdown
|
|
14
|
+
Provides-Extra: all
|
|
12
15
|
Provides-Extra: audio
|
|
13
|
-
Provides-Extra: video
|
|
14
16
|
Provides-Extra: av
|
|
15
|
-
Provides-Extra: gcp
|
|
16
17
|
Provides-Extra: azure
|
|
17
18
|
Provides-Extra: dicom
|
|
18
|
-
Provides-Extra:
|
|
19
|
-
Provides-Extra:
|
|
19
|
+
Provides-Extra: enterprise
|
|
20
|
+
Provides-Extra: gcp
|
|
20
21
|
Provides-Extra: gdrive
|
|
22
|
+
Provides-Extra: medical
|
|
21
23
|
Provides-Extra: point_cloud
|
|
22
|
-
Provides-Extra:
|
|
23
|
-
Provides-Extra:
|
|
24
|
+
Provides-Extra: video
|
|
25
|
+
Provides-Extra: visualizer
|
|
24
26
|
License-File: LICENSE
|
|
25
27
|
|
|
26
28
|
<img src="https://static.scarf.sh/a.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82" /><p align="center">
|
|
@@ -288,3 +290,5 @@ If you use Deep Lake in your research, please cite Activeloop using:
|
|
|
288
290
|
## Acknowledgment
|
|
289
291
|
|
|
290
292
|
This technology was inspired by our research work at Princeton University. We would like to thank William Silversmith @SeungLab for his awesome [cloud-volume](https://github.com/seung-lab/cloud-volume) tool.
|
|
293
|
+
|
|
294
|
+
|
|
@@ -24,6 +24,7 @@ from .api.tiled import tiled
|
|
|
24
24
|
from .core.dataset import Dataset
|
|
25
25
|
from .core.transform import compute, compose
|
|
26
26
|
from .core.tensor import Tensor
|
|
27
|
+
from .core.seed import DeeplakeRandom
|
|
27
28
|
from .util.bugout_reporter import deeplake_reporter
|
|
28
29
|
from .compression import SUPPORTED_COMPRESSIONS
|
|
29
30
|
from .htype import HTYPE_CONFIGURATIONS
|
|
@@ -50,6 +51,7 @@ ingest_dataframe = api_dataset.ingest_dataframe
|
|
|
50
51
|
ingest_huggingface = huggingface.ingest_huggingface
|
|
51
52
|
dataset = api_dataset.init # type: ignore
|
|
52
53
|
tensor = Tensor
|
|
54
|
+
random = DeeplakeRandom()
|
|
53
55
|
|
|
54
56
|
__all__ = [
|
|
55
57
|
"tensor",
|
|
@@ -76,10 +78,11 @@ __all__ = [
|
|
|
76
78
|
"delete",
|
|
77
79
|
"copy",
|
|
78
80
|
"rename",
|
|
81
|
+
"random",
|
|
79
82
|
]
|
|
80
83
|
|
|
81
84
|
|
|
82
|
-
__version__ = "3.6.
|
|
85
|
+
__version__ = "3.6.20"
|
|
83
86
|
warn_if_update_required(__version__)
|
|
84
87
|
__encoded_version__ = np.array(__version__)
|
|
85
88
|
config = {"s3": Config(max_pool_connections=50, connect_timeout=300, read_timeout=300)}
|
|
@@ -40,6 +40,7 @@ from deeplake.util.exceptions import (
|
|
|
40
40
|
InvalidDatasetNameException,
|
|
41
41
|
UnsupportedParameterException,
|
|
42
42
|
DynamicTensorNumpyError,
|
|
43
|
+
SampleExtendError,
|
|
43
44
|
)
|
|
44
45
|
from deeplake.util.path import convert_string_to_pathlib_if_needed, verify_dataset_name
|
|
45
46
|
from deeplake.util.testing import assert_array_equal
|
|
@@ -2766,3 +2767,16 @@ def test_non_local_org_id():
|
|
|
2766
2767
|
def test_azure_bad_path():
|
|
2767
2768
|
with pytest.raises(ValueError):
|
|
2768
2769
|
ds = deeplake.empty("az://storage_account")
|
|
2770
|
+
|
|
2771
|
+
|
|
2772
|
+
def test_dataset_extend_error_suggestion(local_ds):
|
|
2773
|
+
with local_ds as ds:
|
|
2774
|
+
ds.create_tensor("abc")
|
|
2775
|
+
|
|
2776
|
+
with pytest.raises(SampleExtendError) as e:
|
|
2777
|
+
ds.extend({"abc": [1, 2, 3, 4, "abcd", 5]})
|
|
2778
|
+
|
|
2779
|
+
assert (
|
|
2780
|
+
"If you wish to skip the samples that cause errors,"
|
|
2781
|
+
" please specify `ignore_errors=True`."
|
|
2782
|
+
) in str(e)
|
|
@@ -246,3 +246,14 @@ DEFAULT_VECTORSTORE_TENSORS = [
|
|
|
246
246
|
]
|
|
247
247
|
|
|
248
248
|
VIEW_SUMMARY_SAFE_LIMIT = 10000
|
|
249
|
+
|
|
250
|
+
# openai constants
|
|
251
|
+
OPENAI_TOKEN_SIZE = 2 * B
|
|
252
|
+
OPENAI_ADA_MAX_TOKENS_PER_MINUTE = 1000000
|
|
253
|
+
MAX_BYTES_PER_MINUTE = (
|
|
254
|
+
0.9 * OPENAI_TOKEN_SIZE * OPENAI_ADA_MAX_TOKENS_PER_MINUTE
|
|
255
|
+
) # 0.9 is a safety factor
|
|
256
|
+
TARGET_BYTE_SIZE = 10000
|
|
257
|
+
|
|
258
|
+
# Maximum default message length for saving query views
|
|
259
|
+
QUERY_MESSAGE_MAX_SIZE = 1000
|
|
@@ -24,6 +24,7 @@ from deeplake.util.version_control import (
|
|
|
24
24
|
integrity_check,
|
|
25
25
|
save_commit_info,
|
|
26
26
|
rebuild_version_info,
|
|
27
|
+
_squash_main,
|
|
27
28
|
)
|
|
28
29
|
from deeplake.util.invalid_view_op import invalid_view_op
|
|
29
30
|
from deeplake.util.spinner import spinner
|
|
@@ -44,6 +45,7 @@ from deeplake.constants import (
|
|
|
44
45
|
SAMPLE_INFO_TENSOR_MAX_CHUNK_SIZE,
|
|
45
46
|
DEFAULT_READONLY,
|
|
46
47
|
ENV_HUB_DEV_USERNAME,
|
|
48
|
+
QUERY_MESSAGE_MAX_SIZE,
|
|
47
49
|
)
|
|
48
50
|
from deeplake.core.fast_forwarding import ffw_dataset_meta
|
|
49
51
|
from deeplake.core.index import Index
|
|
@@ -104,6 +106,8 @@ from deeplake.util.exceptions import (
|
|
|
104
106
|
CheckoutError,
|
|
105
107
|
DatasetCorruptError,
|
|
106
108
|
BadRequestException,
|
|
109
|
+
SampleAppendError,
|
|
110
|
+
SampleExtendError,
|
|
107
111
|
)
|
|
108
112
|
from deeplake.util.keys import (
|
|
109
113
|
dataset_exists,
|
|
@@ -245,6 +249,7 @@ class Dataset:
|
|
|
245
249
|
d["_lock_timeout"] = lock_timeout
|
|
246
250
|
d["_temp_tensors"] = []
|
|
247
251
|
d["_vc_info_updated"] = True
|
|
252
|
+
d["_query_string"] = None
|
|
248
253
|
dct = self.__dict__
|
|
249
254
|
dct.update(d)
|
|
250
255
|
|
|
@@ -1722,7 +1727,7 @@ class Dataset:
|
|
|
1722
1727
|
Raises:
|
|
1723
1728
|
CommitError: If ``branch`` could not be found.
|
|
1724
1729
|
ReadOnlyModeError: If branch deletion is attempted in read-only mode.
|
|
1725
|
-
Exception: If you have
|
|
1730
|
+
Exception: If you have the given branch currently checked out.
|
|
1726
1731
|
|
|
1727
1732
|
Examples:
|
|
1728
1733
|
|
|
@@ -1767,6 +1772,38 @@ class Dataset:
|
|
|
1767
1772
|
self._set_read_only(read_only, err=True)
|
|
1768
1773
|
self.storage.autoflush = self._initial_autoflush.pop()
|
|
1769
1774
|
|
|
1775
|
+
@invalid_view_op
|
|
1776
|
+
def _squash_main(self) -> None:
|
|
1777
|
+
"""
|
|
1778
|
+
DEPRECATED: This method is deprecated and will be removed in a future release.
|
|
1779
|
+
|
|
1780
|
+
Squashes all commits in current branch into one commit.
|
|
1781
|
+
NOTE: This cannot be run if there are any branches besides ``main``
|
|
1782
|
+
|
|
1783
|
+
Raises:
|
|
1784
|
+
ReadOnlyModeError: If branch deletion is attempted in read-only mode.
|
|
1785
|
+
VersionControlError: If the branch cannot be squashed.
|
|
1786
|
+
"""
|
|
1787
|
+
if self._is_filtered_view:
|
|
1788
|
+
raise Exception(
|
|
1789
|
+
"Cannot perform version control operations on a filtered dataset view."
|
|
1790
|
+
)
|
|
1791
|
+
read_only = self._read_only
|
|
1792
|
+
if read_only:
|
|
1793
|
+
raise ReadOnlyModeError()
|
|
1794
|
+
|
|
1795
|
+
try_flushing(self)
|
|
1796
|
+
|
|
1797
|
+
self._initial_autoflush.append(self.storage.autoflush)
|
|
1798
|
+
self.storage.autoflush = False
|
|
1799
|
+
try:
|
|
1800
|
+
self._unlock()
|
|
1801
|
+
_squash_main(self)
|
|
1802
|
+
finally:
|
|
1803
|
+
self._set_read_only(read_only, err=True)
|
|
1804
|
+
self.libdeeplake_dataset = None
|
|
1805
|
+
self.storage.autoflush = self._initial_autoflush.pop()
|
|
1806
|
+
|
|
1770
1807
|
def log(self):
|
|
1771
1808
|
"""Displays the details of all the past commits."""
|
|
1772
1809
|
|
|
@@ -2257,7 +2294,15 @@ class Dataset:
|
|
|
2257
2294
|
|
|
2258
2295
|
from deeplake.enterprise import query
|
|
2259
2296
|
|
|
2260
|
-
|
|
2297
|
+
result = query(self, query_string)
|
|
2298
|
+
|
|
2299
|
+
if len(query_string) > QUERY_MESSAGE_MAX_SIZE:
|
|
2300
|
+
message = query_string[: QUERY_MESSAGE_MAX_SIZE - 3] + "..."
|
|
2301
|
+
else:
|
|
2302
|
+
message = query_string
|
|
2303
|
+
result._query_string = message
|
|
2304
|
+
|
|
2305
|
+
return result
|
|
2261
2306
|
|
|
2262
2307
|
def sample_by(
|
|
2263
2308
|
self,
|
|
@@ -3037,6 +3082,7 @@ class Dataset:
|
|
|
3037
3082
|
TensorDoesNotExistError: If tensor in ``samples`` does not exist.
|
|
3038
3083
|
ValueError: If all tensors being updated are not of the same length.
|
|
3039
3084
|
NotImplementedError: If an error occurs while writing tiles.
|
|
3085
|
+
SampleExtendError: If the extend failed while appending a sample.
|
|
3040
3086
|
Exception: Error while attempting to rollback appends.
|
|
3041
3087
|
"""
|
|
3042
3088
|
extend = False
|
|
@@ -3079,6 +3125,8 @@ class Dataset:
|
|
|
3079
3125
|
if ignore_errors:
|
|
3080
3126
|
continue
|
|
3081
3127
|
else:
|
|
3128
|
+
if isinstance(e, SampleAppendError):
|
|
3129
|
+
raise SampleExtendError(str(e)) from e.__cause__
|
|
3082
3130
|
raise e
|
|
3083
3131
|
|
|
3084
3132
|
@invalid_view_op
|
|
@@ -3479,7 +3527,7 @@ class Dataset:
|
|
|
3479
3527
|
return self._save_view(
|
|
3480
3528
|
path,
|
|
3481
3529
|
id,
|
|
3482
|
-
message,
|
|
3530
|
+
message or self._query_string,
|
|
3483
3531
|
optimize,
|
|
3484
3532
|
tensors,
|
|
3485
3533
|
num_workers,
|
|
@@ -4463,8 +4511,6 @@ class Dataset:
|
|
|
4463
4511
|
Raises:
|
|
4464
4512
|
IndexError: If the index is out of range.
|
|
4465
4513
|
"""
|
|
4466
|
-
self._initial_autoflush.append(self.storage.autoflush)
|
|
4467
|
-
self.storage.autoflush = False
|
|
4468
4514
|
max_len = max((t.num_samples for t in self.tensors.values()), default=0)
|
|
4469
4515
|
if max_len == 0:
|
|
4470
4516
|
raise IndexError("Can't pop from empty dataset.")
|
|
@@ -4479,11 +4525,10 @@ class Dataset:
|
|
|
4479
4525
|
f"Index {index} is out of range. The longest tensor has {max_len} samples."
|
|
4480
4526
|
)
|
|
4481
4527
|
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
tensor.
|
|
4485
|
-
|
|
4486
|
-
self.storage.autoflush = self._initial_autoflush.pop()
|
|
4528
|
+
with self:
|
|
4529
|
+
for tensor in self.tensors.values():
|
|
4530
|
+
if tensor.num_samples > index:
|
|
4531
|
+
tensor.pop(index)
|
|
4487
4532
|
|
|
4488
4533
|
@property
|
|
4489
4534
|
def is_view(self) -> bool:
|
|
@@ -331,5 +331,6 @@ class DeepLakeQueryDataset(Dataset):
|
|
|
331
331
|
def random_split(self, lengths: Sequence[Union[int, float]]):
|
|
332
332
|
if math.isclose(sum(lengths), 1) and sum(lengths) <= 1:
|
|
333
333
|
lengths = calculate_absolute_lengths(lengths, len(self))
|
|
334
|
+
|
|
334
335
|
vs = self.indra_ds.random_split(lengths)
|
|
335
336
|
return [DeepLakeQueryDataset(self.deeplake_ds, v) for v in vs]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
class DeeplakeRandom(object):
|
|
5
|
+
def __new__(cls):
|
|
6
|
+
if not hasattr(cls, 'instance'):
|
|
7
|
+
cls.instance = super(DeeplakeRandom, cls).__new__(cls)
|
|
8
|
+
cls.instance.internal_seed = None
|
|
9
|
+
cls.instance.indra_api = None
|
|
10
|
+
return cls.instance
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def seed(self, seed: Optional[int] = None):
|
|
14
|
+
if isinstance(seed, Optional[int]):
|
|
15
|
+
self.internal_seed = seed
|
|
16
|
+
if self.indra_api is None:
|
|
17
|
+
from deeplake.enterprise.convert_to_libdeeplake import import_indra_api_silent
|
|
18
|
+
self.indra_api = import_indra_api_silent()
|
|
19
|
+
if self.indra_api is not None:
|
|
20
|
+
self.indra_api.set_seed(self.internal_seed)
|
|
21
|
+
else:
|
|
22
|
+
raise TypeError(f"provided seed type `{type(seed)}` is increect seed must be an integer")
|
|
23
|
+
|
|
24
|
+
def get_seed(self) -> Optional[int]:
|
|
25
|
+
return self.internal_seed
|
|
@@ -150,7 +150,10 @@ class AzureProvider(StorageProvider):
|
|
|
150
150
|
blobs = [
|
|
151
151
|
posixpath.join(self.root_folder, key) for key in self._all_keys(prefix)
|
|
152
152
|
]
|
|
153
|
-
|
|
153
|
+
# delete_blobs can only delete 256 blobs at a time
|
|
154
|
+
batches = [blobs[i : i + 256] for i in range(0, len(blobs), 256)]
|
|
155
|
+
for batch in batches:
|
|
156
|
+
self.container_client.delete_blobs(*batch)
|
|
154
157
|
|
|
155
158
|
def get_sas_token(self):
|
|
156
159
|
self._check_update_creds()
|
|
@@ -451,6 +451,7 @@ class LRUCache(StorageProvider):
|
|
|
451
451
|
"next_storage": self.next_storage,
|
|
452
452
|
"cache_storage": self.cache_storage,
|
|
453
453
|
"cache_size": self.cache_size,
|
|
454
|
+
"use_async": self.use_async,
|
|
454
455
|
}
|
|
455
456
|
|
|
456
457
|
def __setstate__(self, state: Dict[str, Any]):
|
|
@@ -469,6 +470,7 @@ class LRUCache(StorageProvider):
|
|
|
469
470
|
self.next_storage = state["next_storage"]
|
|
470
471
|
self.cache_storage = state["cache_storage"]
|
|
471
472
|
self.cache_size = state["cache_size"]
|
|
473
|
+
self.use_async = state["use_async"]
|
|
472
474
|
self.lru_sizes = OrderedDict()
|
|
473
475
|
self.dirty_keys = OrderedDict()
|
|
474
476
|
self.cache_used = 0
|
|
@@ -293,6 +293,41 @@ def test_query_tensors_polygon_htype_consistency(local_auth_ds_generator):
|
|
|
293
293
|
assert np.all(i == j)
|
|
294
294
|
|
|
295
295
|
|
|
296
|
+
@requires_libdeeplake
|
|
297
|
+
def test_random_split_with_seed(local_auth_ds_generator):
|
|
298
|
+
deeplake_ds = local_auth_ds_generator()
|
|
299
|
+
from deeplake.core.seed import DeeplakeRandom
|
|
300
|
+
|
|
301
|
+
with deeplake_ds:
|
|
302
|
+
deeplake_ds.create_tensor("label", htype="generic", dtype=np.int32)
|
|
303
|
+
for i in range(1000):
|
|
304
|
+
deeplake_ds.label.append(int(i % 100))
|
|
305
|
+
|
|
306
|
+
deeplake_indra_ds = deeplake_ds.query("SELECT * GROUP BY label")
|
|
307
|
+
|
|
308
|
+
initial_state = np.random.get_state()
|
|
309
|
+
DeeplakeRandom().seed(100)
|
|
310
|
+
split1 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
|
|
311
|
+
assert len(split1) == 3
|
|
312
|
+
assert len(split1[0]) == 20
|
|
313
|
+
|
|
314
|
+
DeeplakeRandom().seed(101)
|
|
315
|
+
split2 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
|
|
316
|
+
assert len(split2) == 3
|
|
317
|
+
assert len(split2[0]) == 20
|
|
318
|
+
|
|
319
|
+
DeeplakeRandom().seed(100)
|
|
320
|
+
split3 = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
|
|
321
|
+
assert len(split3) == 3
|
|
322
|
+
assert len(split3[0]) == 20
|
|
323
|
+
|
|
324
|
+
for i in range(len(split1)):
|
|
325
|
+
assert np.all(split1[i].label.numpy() == split3[i].label.numpy())
|
|
326
|
+
assert not np.all(split1[i].label.numpy() == split2[i].label.numpy())
|
|
327
|
+
|
|
328
|
+
np.random.set_state(initial_state)
|
|
329
|
+
|
|
330
|
+
|
|
296
331
|
@requires_libdeeplake
|
|
297
332
|
def test_random_split(local_auth_ds_generator):
|
|
298
333
|
deeplake_ds = local_auth_ds_generator()
|
|
@@ -306,6 +341,7 @@ def test_random_split(local_auth_ds_generator):
|
|
|
306
341
|
split = deeplake_indra_ds.random_split([0.2, 0.2, 0.6])
|
|
307
342
|
assert len(split) == 3
|
|
308
343
|
assert len(split[0]) == 20
|
|
344
|
+
|
|
309
345
|
l = split[0].dataloader().pytorch()
|
|
310
346
|
for b in l:
|
|
311
347
|
pass
|
|
@@ -1428,7 +1428,10 @@ def test_ds_append_errors(
|
|
|
1428
1428
|
err_msg += re.escape(
|
|
1429
1429
|
" Last checkpoint: 10 samples processed. You can slice the input to resume from this point."
|
|
1430
1430
|
)
|
|
1431
|
-
err_msg += re.escape(
|
|
1431
|
+
err_msg += re.escape(
|
|
1432
|
+
" See traceback for more details."
|
|
1433
|
+
" If you wish to skip the samples that cause errors, please specify `ignore_errors=True`."
|
|
1434
|
+
)
|
|
1432
1435
|
|
|
1433
1436
|
samples.insert(17, bad_sample)
|
|
1434
1437
|
|
|
@@ -33,6 +33,7 @@ from deeplake.util.exceptions import (
|
|
|
33
33
|
HubComposeEmptyListError,
|
|
34
34
|
HubComposeIncompatibleFunction,
|
|
35
35
|
TransformError,
|
|
36
|
+
SampleAppendError,
|
|
36
37
|
)
|
|
37
38
|
from deeplake.hooks import dataset_written, dataset_read
|
|
38
39
|
from deeplake.util.version_control import auto_checkout
|
|
@@ -305,9 +306,9 @@ class Pipeline:
|
|
|
305
306
|
target_ds.reset(force=True)
|
|
306
307
|
target_ds._send_compute_progress(**progress_args, status="failed")
|
|
307
308
|
close_states(compute_provider, pbar, pqueue)
|
|
308
|
-
index, sample = None, None
|
|
309
|
+
index, sample, suggest = None, None, False
|
|
309
310
|
if isinstance(e, TransformError):
|
|
310
|
-
index, sample = e.index, e.sample
|
|
311
|
+
index, sample, suggest = e.index, e.sample, e.suggest
|
|
311
312
|
if checkpointing_enabled and isinstance(index, int):
|
|
312
313
|
index = samples_processed + index
|
|
313
314
|
e = e.__cause__ # type: ignore
|
|
@@ -317,6 +318,7 @@ class Pipeline:
|
|
|
317
318
|
index=index,
|
|
318
319
|
sample=sample,
|
|
319
320
|
samples_processed=samples_processed,
|
|
321
|
+
suggest=suggest,
|
|
320
322
|
) from e
|
|
321
323
|
finally:
|
|
322
324
|
reload_and_rechunk(
|
|
@@ -301,7 +301,7 @@ class VectorStore:
|
|
|
301
301
|
|
|
302
302
|
def search(
|
|
303
303
|
self,
|
|
304
|
-
embedding_data=None,
|
|
304
|
+
embedding_data: Union[str, List[str]] = None,
|
|
305
305
|
embedding_function: Optional[Callable] = None,
|
|
306
306
|
embedding: Optional[Union[List[float], np.ndarray]] = None,
|
|
307
307
|
k: int = 4,
|
|
@@ -344,7 +344,7 @@ class VectorStore:
|
|
|
344
344
|
|
|
345
345
|
Args:
|
|
346
346
|
embedding (Union[np.ndarray, List[float]], optional): Embedding representation for performing the search. Defaults to None. The ``embedding_data`` and ``embedding`` cannot both be specified.
|
|
347
|
-
embedding_data: Data against which the search will be performed by embedding it using the `embedding_function`. Defaults to None. The `embedding_data` and `embedding` cannot both be specified.
|
|
347
|
+
embedding_data (List[str]): Data against which the search will be performed by embedding it using the `embedding_function`. Defaults to None. The `embedding_data` and `embedding` cannot both be specified.
|
|
348
348
|
embedding_function (Optional[Callable], optional): function for converting `embedding_data` into embedding. Only valid if `embedding_data` is specified
|
|
349
349
|
k (int): Number of elements to return after running query. Defaults to 4.
|
|
350
350
|
distance_metric (str): Type of distance metric to use for sorting the data. Avaliable options are: ``"L1", "L2", "COS", "MAX"``. Defaults to ``"COS"``.
|
|
@@ -424,11 +424,6 @@ class VectorStore:
|
|
|
424
424
|
embedding_data,
|
|
425
425
|
embedding_function=embedding_function or self.embedding_function,
|
|
426
426
|
)
|
|
427
|
-
if isinstance(query_emb, np.ndarray):
|
|
428
|
-
assert (
|
|
429
|
-
query_emb.ndim == 1 or query_emb.shape[0] == 1
|
|
430
|
-
), "Query embedding must be 1-dimensional. Please consider using another embedding function for converting query string to embedding."
|
|
431
|
-
|
|
432
427
|
return vector_search.search(
|
|
433
428
|
query=query,
|
|
434
429
|
logger=logger,
|
|
@@ -433,6 +433,24 @@ def test_search_basic(local_path, hub_cloud_dev_token):
|
|
|
433
433
|
assert isinstance(data.text[0].data()["value"], str)
|
|
434
434
|
assert data.embedding[0].numpy().size > 0
|
|
435
435
|
|
|
436
|
+
data = vector_store.search(
|
|
437
|
+
embedding_function=embedding_fn3,
|
|
438
|
+
embedding_data="dummy",
|
|
439
|
+
return_view=True,
|
|
440
|
+
k=2,
|
|
441
|
+
)
|
|
442
|
+
assert len(data) == 2
|
|
443
|
+
assert isinstance(data.text[0].data()["value"], str)
|
|
444
|
+
assert data.embedding[0].numpy().size > 0
|
|
445
|
+
|
|
446
|
+
with pytest.raises(NotImplementedError):
|
|
447
|
+
data = vector_store.search(
|
|
448
|
+
embedding_function=embedding_fn3,
|
|
449
|
+
embedding_data=["dummy", "dummy2"],
|
|
450
|
+
return_view=True,
|
|
451
|
+
k=2,
|
|
452
|
+
)
|
|
453
|
+
|
|
436
454
|
data = vector_store.search(
|
|
437
455
|
filter={"metadata": {"abcdefh": 1}},
|
|
438
456
|
embedding=None,
|
|
@@ -552,6 +570,8 @@ def test_search_managed(hub_cloud_dev_token):
|
|
|
552
570
|
exec_option="tensor_db",
|
|
553
571
|
)
|
|
554
572
|
|
|
573
|
+
assert "vectordb/" in vector_store.dataset.base_storage.path
|
|
574
|
+
|
|
555
575
|
assert len(data_ce["score"]) == len(data_db["score"])
|
|
556
576
|
assert all(
|
|
557
577
|
[
|
|
@@ -1661,9 +1681,11 @@ def test_multiple_embeddings(local_path, capsys):
|
|
|
1661
1681
|
embedding_data=[texts, texts],
|
|
1662
1682
|
embedding_tensor=["embedding_1", "embedding_2"],
|
|
1663
1683
|
)
|
|
1684
|
+
|
|
1664
1685
|
vector_store.add(
|
|
1665
1686
|
text=texts, embedding_1=(embedding_fn, texts), embedding_2=(embedding_fn, texts)
|
|
1666
1687
|
)
|
|
1688
|
+
|
|
1667
1689
|
vector_store.add(
|
|
1668
1690
|
text=texts,
|
|
1669
1691
|
embedding_function=embedding_fn,
|
|
@@ -1736,7 +1758,7 @@ def test_query_dim(local_path):
|
|
|
1736
1758
|
)
|
|
1737
1759
|
|
|
1738
1760
|
vector_store.add(text=texts, embedding=embeddings)
|
|
1739
|
-
with pytest.raises(
|
|
1761
|
+
with pytest.raises(NotImplementedError):
|
|
1740
1762
|
vector_store.search([texts[0], texts[0]], embedding_fn3, k=1)
|
|
1741
1763
|
|
|
1742
1764
|
vector_store.search([texts[0]], embedding_fn4, k=1)
|