deeplake 3.8.1__tar.gz → 3.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.8.1/deeplake.egg-info → deeplake-3.8.2}/PKG-INFO +1 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/__init__.py +1 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api.py +11 -2
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api_with_compression.py +2 -2
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk_engine.py +28 -4
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compression.py +1 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_deeplake_indra_dataset.py +4 -4
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform_dataset.py +19 -6
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/dataloader.py +70 -19
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/htype.py +1 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/shuffle_buffer.py +5 -0
- deeplake-3.8.2/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.8.1 → deeplake-3.8.2/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/requires.txt +11 -11
- {deeplake-3.8.1 → deeplake-3.8.2}/setup.py +1 -1
- deeplake-3.8.1/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.8.1 → deeplake-3.8.2}/LICENSE +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/MANIFEST.in +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/README.md +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/info.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/link.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/read.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/api/tiled.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/auth.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/commands.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/cli/test_cli.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/client.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/config.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/log.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/test_client.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/client/utils.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/compression.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/constants.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/deeplake_query_tensor.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/index/index.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/io.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/ipc.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/lock.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/polygon.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/query/query.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/sample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/seed.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/serialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tensor.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deep_memory.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/deepmemory_vectorstore.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/test_deepmemory.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/index.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/vectorstore/vectorstore_factory.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/hooks.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/common.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/access_method.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/agreement.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/array_list.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/auto.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/casting.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/class_label.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/compression.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/compute.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/diff.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/downsample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/encoder.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/exif.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/hash.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/htype.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/image.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/json.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/keys.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/link.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/logging.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/merge.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/modified.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/notebook.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/path.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/spinner.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/split.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/storage.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tag.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/testing.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/threading.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/token.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/transform.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/version_control.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/video.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/util/warnings.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/entry_points.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/pyproject.toml +0 -0
- {deeplake-3.8.1 → deeplake-3.8.2}/setup.cfg +0 -0
|
@@ -2680,12 +2680,12 @@ def test_tensor_dtype_bug(local_path):
|
|
|
2680
2680
|
ds.abc.append(deeplake.link(f"{data_path}/standard.nii.gz"))
|
|
2681
2681
|
|
|
2682
2682
|
assert ds.abc[0].numpy().shape == (4, 5, 7)
|
|
2683
|
-
assert ds.abc.dtype == np.dtype("
|
|
2683
|
+
assert ds.abc.dtype == np.dtype("uint8")
|
|
2684
2684
|
|
|
2685
2685
|
ds2 = ds.copy(f"{local_path}_2", overwrite=True)
|
|
2686
2686
|
|
|
2687
2687
|
assert ds2.abc[0].numpy().shape == (4, 5, 7)
|
|
2688
|
-
assert ds2.abc.dtype == np.dtype("
|
|
2688
|
+
assert ds2.abc.dtype == np.dtype("uint8")
|
|
2689
2689
|
|
|
2690
2690
|
|
|
2691
2691
|
def test_iterate_with_groups(memory_ds):
|
|
@@ -3043,3 +3043,12 @@ def test_change_htype_fail(local_ds_generator):
|
|
|
3043
3043
|
ds.create_tensor("images3", htype="image", sample_compression="jpg")
|
|
3044
3044
|
with pytest.raises(UnsupportedCompressionError):
|
|
3045
3045
|
ds.images3.htype = "embedding"
|
|
3046
|
+
|
|
3047
|
+
|
|
3048
|
+
def test_append_non_uint8_to_image(local_ds):
|
|
3049
|
+
with local_ds as ds:
|
|
3050
|
+
ds.create_tensor("images", htype="image", sample_compression="png")
|
|
3051
|
+
ds.images.append(np.zeros((40, 40, 1), dtype=np.int16))
|
|
3052
|
+
ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
|
|
3053
|
+
|
|
3054
|
+
assert ds.images.dtype.name == "int16"
|
|
@@ -43,7 +43,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
|
|
|
43
43
|
tiling_threshold=1 * MB,
|
|
44
44
|
)
|
|
45
45
|
|
|
46
|
-
assert images.meta.dtype ==
|
|
46
|
+
assert images.meta.dtype == None
|
|
47
47
|
assert images.meta.sample_compression == "png"
|
|
48
48
|
|
|
49
49
|
_populate_compressed_samples(images, cat_path, flower_path)
|
|
@@ -73,7 +73,7 @@ def test_populate_compressed_samples(local_ds, cat_path, flower_path):
|
|
|
73
73
|
def test_iterate_compressed_samples(local_ds, cat_path, flower_path):
|
|
74
74
|
images = local_ds.create_tensor(TENSOR_KEY, htype="image", sample_compression="png")
|
|
75
75
|
|
|
76
|
-
assert images.meta.dtype ==
|
|
76
|
+
assert images.meta.dtype == None
|
|
77
77
|
assert images.meta.sample_compression == "png"
|
|
78
78
|
|
|
79
79
|
_populate_compressed_samples(images, cat_path, flower_path)
|
|
@@ -15,7 +15,7 @@ from typing import (
|
|
|
15
15
|
)
|
|
16
16
|
from deeplake.api.info import Info
|
|
17
17
|
from deeplake.core.link_creds import LinkCreds
|
|
18
|
-
from deeplake.core.linked_sample import LinkedSample
|
|
18
|
+
from deeplake.core.linked_sample import LinkedSample, read_linked_sample
|
|
19
19
|
from deeplake.core.meta.encode.base_encoder import LAST_SEEN_INDEX_COLUMN
|
|
20
20
|
from deeplake.core.serialize import HEADER_SIZE_BYTES, text_to_bytes
|
|
21
21
|
from deeplake.core.tensor_link import (
|
|
@@ -692,9 +692,33 @@ class ChunkEngine:
|
|
|
692
692
|
if tensor_meta.htype is None and not all_empty:
|
|
693
693
|
tensor_meta.set_htype(get_htype(samples))
|
|
694
694
|
if tensor_meta.dtype is None and not all_empty:
|
|
695
|
-
tensor_meta.
|
|
696
|
-
|
|
697
|
-
|
|
695
|
+
if tensor_meta.is_link:
|
|
696
|
+
try:
|
|
697
|
+
# download one sample to get dtype
|
|
698
|
+
sample = next(filter(lambda x: x is not None, samples))
|
|
699
|
+
assert isinstance(
|
|
700
|
+
sample, LinkedSample
|
|
701
|
+
), "Sample must be LinkedSample"
|
|
702
|
+
dtype = np.dtype(
|
|
703
|
+
read_linked_sample(
|
|
704
|
+
sample.path, sample.creds_key, self.link_creds, True
|
|
705
|
+
)._typestr
|
|
706
|
+
)
|
|
707
|
+
except:
|
|
708
|
+
# assume uint8 if download fails
|
|
709
|
+
dtype = np.dtype("uint8")
|
|
710
|
+
else:
|
|
711
|
+
non_empty_samples = list(filter(lambda x: x is not None, samples))
|
|
712
|
+
for sample in non_empty_samples:
|
|
713
|
+
try:
|
|
714
|
+
dtype = get_dtype(sample)
|
|
715
|
+
break
|
|
716
|
+
except:
|
|
717
|
+
pass
|
|
718
|
+
else:
|
|
719
|
+
if not ignore_errors:
|
|
720
|
+
raise ValueError("Could not determine dtype of samples")
|
|
721
|
+
tensor_meta.set_dtype(dtype)
|
|
698
722
|
if self._convert_to_list(samples):
|
|
699
723
|
samples = list(samples)
|
|
700
724
|
if self._is_temp_label_tensor:
|
|
@@ -152,7 +152,7 @@ def compress_bytes(
|
|
|
152
152
|
if not buffer:
|
|
153
153
|
return b""
|
|
154
154
|
if compression == "lz4":
|
|
155
|
-
return lz4.
|
|
155
|
+
return lz4.block.compress(buffer)
|
|
156
156
|
else:
|
|
157
157
|
raise SampleCompressionError(
|
|
158
158
|
(len(buffer),), compression, f"Not a byte compression: {compression}"
|
|
@@ -424,7 +424,7 @@ def test_virtual_tensors(local_auth_ds_generator):
|
|
|
424
424
|
np.sqrt(2.0 / (i + 1) / (i + 1))
|
|
425
425
|
]
|
|
426
426
|
|
|
427
|
-
assert deeplake_indra_ds.sample_indices ==
|
|
427
|
+
assert list(deeplake_indra_ds.sample_indices) == list(range(100))
|
|
428
428
|
deeplake_indra_ds = deeplake_ds.query(
|
|
429
429
|
"SELECT *, l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
|
|
430
430
|
)
|
|
@@ -432,10 +432,10 @@ def test_virtual_tensors(local_auth_ds_generator):
|
|
|
432
432
|
assert list(deeplake_indra_ds.embeddings.sample_indices) == list(
|
|
433
433
|
reversed(range(100))
|
|
434
434
|
)
|
|
435
|
-
assert deeplake_indra_ds.score.sample_indices ==
|
|
435
|
+
assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
|
|
436
436
|
|
|
437
437
|
deeplake_indra_ds = deeplake_ds.query(
|
|
438
438
|
"SELECT l2_norm(embeddings - ARRAY[0, 0, 0]) as score order by l2_norm(embeddings - ARRAY[0, 0, 0]) asc"
|
|
439
439
|
)
|
|
440
|
-
assert deeplake_indra_ds.sample_indices ==
|
|
441
|
-
assert deeplake_indra_ds.score.sample_indices ==
|
|
440
|
+
assert list(deeplake_indra_ds.sample_indices) == list(range(100))
|
|
441
|
+
assert list(deeplake_indra_ds.score.sample_indices) == list(range(100))
|
|
@@ -161,7 +161,7 @@ class TransformDataset:
|
|
|
161
161
|
# First sample in tensor
|
|
162
162
|
# Flush to set meta attributes
|
|
163
163
|
if dtype is None:
|
|
164
|
-
self.flush()
|
|
164
|
+
self.flush(clear_on_fail=False)
|
|
165
165
|
return
|
|
166
166
|
sizeof_item = self._calculate_sample_size(item, dtype, htype)
|
|
167
167
|
except:
|
|
@@ -201,22 +201,29 @@ class TransformDataset:
|
|
|
201
201
|
updated_tensors[full_name] = len(items)
|
|
202
202
|
tensor.items.clear()
|
|
203
203
|
|
|
204
|
-
def _rollback(self, updated_tensors):
|
|
204
|
+
def _rollback(self, updated_tensors, no_dtype_tensors):
|
|
205
205
|
for t in updated_tensors:
|
|
206
206
|
chunk_engine = self.all_chunk_engines[t]
|
|
207
207
|
num_samples = updated_tensors[t]
|
|
208
208
|
for _ in range(num_samples):
|
|
209
209
|
chunk_engine.pop(link_callback=chunk_engine._transform_pop_callback)
|
|
210
210
|
|
|
211
|
+
if t in no_dtype_tensors:
|
|
212
|
+
meta = chunk_engine.tensor_meta
|
|
213
|
+
meta.dtype = None
|
|
214
|
+
meta.typestr = None
|
|
215
|
+
meta.is_dirty = True
|
|
216
|
+
|
|
211
217
|
def _clear(self):
|
|
212
218
|
for tensor in self.data.values():
|
|
213
219
|
tensor.items.clear()
|
|
214
220
|
self.cache_used = 0
|
|
215
221
|
|
|
216
|
-
def flush(self):
|
|
222
|
+
def flush(self, clear_on_fail=True):
|
|
217
223
|
all_chunk_engines = self.all_chunk_engines
|
|
218
224
|
label_temp_tensors = self.label_temp_tensors
|
|
219
225
|
updated_tensors = {}
|
|
226
|
+
no_dtype_tensors = []
|
|
220
227
|
try:
|
|
221
228
|
for name, tensor in self.data.items():
|
|
222
229
|
if not tensor.is_group:
|
|
@@ -225,6 +232,11 @@ class TransformDataset:
|
|
|
225
232
|
chunk_engine = all_chunk_engines[name]
|
|
226
233
|
callback = chunk_engine._transform_callback
|
|
227
234
|
|
|
235
|
+
meta = chunk_engine.tensor_meta
|
|
236
|
+
if meta.length == 0 and meta.dtype is None:
|
|
237
|
+
# for rolling back dtype change
|
|
238
|
+
no_dtype_tensors.append(name)
|
|
239
|
+
|
|
228
240
|
if tensor.numpy_only:
|
|
229
241
|
self._flush_numpy_tensor_to_chunk_engine(
|
|
230
242
|
name, tensor, chunk_engine, callback, updated_tensors
|
|
@@ -234,9 +246,10 @@ class TransformDataset:
|
|
|
234
246
|
name, tensor, chunk_engine, callback, updated_tensors
|
|
235
247
|
)
|
|
236
248
|
self.start_input_idx = None
|
|
249
|
+
self._clear()
|
|
237
250
|
except Exception as e:
|
|
238
|
-
self._rollback(updated_tensors)
|
|
251
|
+
self._rollback(updated_tensors, no_dtype_tensors)
|
|
252
|
+
if clear_on_fail:
|
|
253
|
+
self._clear()
|
|
239
254
|
e = e.__cause__ if isinstance(e, SampleAppendError) else e # type: ignore
|
|
240
255
|
raise SampleAppendError(name) from e
|
|
241
|
-
finally:
|
|
242
|
-
self._clear()
|
|
@@ -40,6 +40,7 @@ import warnings
|
|
|
40
40
|
import math
|
|
41
41
|
|
|
42
42
|
import itertools
|
|
43
|
+
import multiprocessing
|
|
43
44
|
|
|
44
45
|
original_islice = itertools.islice
|
|
45
46
|
|
|
@@ -146,6 +147,7 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
146
147
|
self._IterableDataset_len_called = None
|
|
147
148
|
self._iterator = None
|
|
148
149
|
self._worker_init_fn = None
|
|
150
|
+
self.__multiprocessing_context = None
|
|
149
151
|
|
|
150
152
|
@property
|
|
151
153
|
def batch_size(self):
|
|
@@ -187,7 +189,46 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
187
189
|
|
|
188
190
|
@property # type: ignore
|
|
189
191
|
def multiprocessing_context(self):
|
|
190
|
-
return
|
|
192
|
+
return self.__multiprocessing_context
|
|
193
|
+
|
|
194
|
+
@multiprocessing_context.setter
|
|
195
|
+
def multiprocessing_context(self, multiprocessing_context):
|
|
196
|
+
if multiprocessing_context is not None:
|
|
197
|
+
if self.num_workers > 0:
|
|
198
|
+
if isinstance(multiprocessing_context, str):
|
|
199
|
+
valid_start_methods = multiprocessing.get_all_start_methods()
|
|
200
|
+
if multiprocessing_context not in valid_start_methods:
|
|
201
|
+
raise ValueError(
|
|
202
|
+
(
|
|
203
|
+
"multiprocessing_context option "
|
|
204
|
+
"should specify a valid start method in {!r}, but got "
|
|
205
|
+
"multiprocessing_context={!r}"
|
|
206
|
+
).format(valid_start_methods, multiprocessing_context)
|
|
207
|
+
)
|
|
208
|
+
multiprocessing_context = multiprocessing.get_context(
|
|
209
|
+
multiprocessing_context
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
if not isinstance(
|
|
213
|
+
multiprocessing_context, multiprocessing.context.BaseContext
|
|
214
|
+
):
|
|
215
|
+
raise TypeError(
|
|
216
|
+
(
|
|
217
|
+
"multiprocessing_context option should be a valid context "
|
|
218
|
+
"object or a string specifying the start method, but got "
|
|
219
|
+
"multiprocessing_context={}"
|
|
220
|
+
).format(multiprocessing_context)
|
|
221
|
+
)
|
|
222
|
+
else:
|
|
223
|
+
raise ValueError(
|
|
224
|
+
(
|
|
225
|
+
"multiprocessing_context can only be used with "
|
|
226
|
+
"multi-process loading (num_workers > 0), but got "
|
|
227
|
+
"num_workers={}"
|
|
228
|
+
).format(self.num_workers)
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
self.__multiprocessing_context = multiprocessing_context
|
|
191
232
|
|
|
192
233
|
@property
|
|
193
234
|
def _dataset_kind(self):
|
|
@@ -424,9 +465,9 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
424
465
|
|
|
425
466
|
def close(self):
|
|
426
467
|
"""Shuts down the workers and releases the resources."""
|
|
427
|
-
if self._iterator is not None:
|
|
468
|
+
if hasattr(self, "_iterator") and self._iterator is not None:
|
|
428
469
|
self._iterator.close()
|
|
429
|
-
if self._dataloader is not None:
|
|
470
|
+
if hasattr(self, "_dataloader") and self._dataloader is not None:
|
|
430
471
|
self._dataloader = None
|
|
431
472
|
|
|
432
473
|
def pytorch(
|
|
@@ -697,6 +738,30 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
697
738
|
"not equal after each initialization and iteration through the dataloader. If you intend to shuffle data while "
|
|
698
739
|
"preserving the offset for resuming iteration at a predictable index and order, please set a random seed using deeplake.random()"
|
|
699
740
|
)
|
|
741
|
+
from indra.pytorch.tensorinfo import TensorsInfo, LoaderMetaInfo # type:ignore
|
|
742
|
+
|
|
743
|
+
info = TensorsInfo(
|
|
744
|
+
raw_tensors=raw_tensors or [],
|
|
745
|
+
htype_dict=htype_dict or {},
|
|
746
|
+
ndim_dict=ndim_dict or {},
|
|
747
|
+
tensor_info_dict=tensor_info_dict or {},
|
|
748
|
+
pil_compressed_tensors=pil_compressed_tensors or [],
|
|
749
|
+
json_tensors=json_tensors or [],
|
|
750
|
+
list_tensors=list_tensors or [],
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
loader_meta = LoaderMetaInfo(
|
|
754
|
+
context=self.multiprocessing_context,
|
|
755
|
+
distributed=self._distributed,
|
|
756
|
+
upcast=self._mode == "pytorch", # upcast to handle unsupported dtypes,
|
|
757
|
+
return_index=self._return_index,
|
|
758
|
+
verbose=self._verbose,
|
|
759
|
+
ignore_errors=self._ignore_errors,
|
|
760
|
+
prefetch_factor=self._prefetch_factor,
|
|
761
|
+
offset=self._offset,
|
|
762
|
+
primary_tensor=self._primary_tensor_name,
|
|
763
|
+
worker_init_fn=self.worker_init_fn,
|
|
764
|
+
)
|
|
700
765
|
|
|
701
766
|
return INDRA_LOADER( # type: ignore [misc]
|
|
702
767
|
indra_dataset,
|
|
@@ -706,26 +771,12 @@ class DeepLakeDataLoader(DataLoader):
|
|
|
706
771
|
num_workers=self._num_workers,
|
|
707
772
|
collate_fn=self.collate_fn,
|
|
708
773
|
transform_fn=self._transform,
|
|
709
|
-
distributed=self._distributed,
|
|
710
|
-
prefetch_factor=self._prefetch_factor,
|
|
711
774
|
tensors=tensors,
|
|
712
775
|
drop_last=self._drop_last,
|
|
713
|
-
ignore_errors=self._ignore_errors,
|
|
714
|
-
upcast=self._mode == "pytorch", # upcast to handle unsupported dtypes,
|
|
715
|
-
return_index=self._return_index,
|
|
716
|
-
primary_tensor=self._primary_tensor_name,
|
|
717
776
|
buffer_size=self._buffer_size,
|
|
718
|
-
raw_tensors=raw_tensors,
|
|
719
|
-
pil_compressed_tensors=pil_compressed_tensors,
|
|
720
|
-
json_tensors=json_tensors,
|
|
721
|
-
list_tensors=list_tensors,
|
|
722
777
|
persistent_workers=self._persistent_workers,
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
tensor_info_dict=tensor_info_dict,
|
|
726
|
-
verbose=self._verbose,
|
|
727
|
-
offset=self._offset,
|
|
728
|
-
worker_init_fn=self.worker_init_fn,
|
|
778
|
+
loader_meta=loader_meta,
|
|
779
|
+
info=info,
|
|
729
780
|
)
|
|
730
781
|
|
|
731
782
|
def __iter__(self):
|
|
@@ -46,6 +46,11 @@ class ShuffleBuffer:
|
|
|
46
46
|
)
|
|
47
47
|
self.pbar_closed = False
|
|
48
48
|
|
|
49
|
+
def reset(self):
|
|
50
|
+
self.buffer = list()
|
|
51
|
+
self.buffer_used = 0
|
|
52
|
+
self.num_torch_tensors = 0
|
|
53
|
+
|
|
49
54
|
def exchange(self, sample):
|
|
50
55
|
"""Shuffle with existing elements in a buffer and return value if buffer is full or if `None` is provided as argument.
|
|
51
56
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = '120ee6dc-8a93-4c96-8032-edafd33d2dac'
|
|
@@ -7,27 +7,27 @@ humbug>=0.3.1
|
|
|
7
7
|
tqdm
|
|
8
8
|
lz4
|
|
9
9
|
pyjwt
|
|
10
|
-
libdeeplake==0.0.
|
|
10
|
+
libdeeplake==0.0.84
|
|
11
11
|
|
|
12
12
|
[:python_version >= "3.7" and sys_platform != "win32"]
|
|
13
13
|
aioboto3>=10.4.0
|
|
14
14
|
nest_asyncio
|
|
15
15
|
|
|
16
16
|
[all]
|
|
17
|
-
google-
|
|
17
|
+
google-auth-oauthlib~=0.4.5
|
|
18
|
+
azure-cli
|
|
19
|
+
azure-identity
|
|
20
|
+
IPython
|
|
18
21
|
google-api-python-client~=2.31.0
|
|
22
|
+
flask
|
|
19
23
|
azure-storage-blob
|
|
20
|
-
|
|
24
|
+
oauth2client~=4.1.3
|
|
25
|
+
pydicom
|
|
21
26
|
laspy
|
|
22
|
-
google-
|
|
27
|
+
google-cloud-storage~=1.42.0
|
|
23
28
|
google-auth~=2.0.1
|
|
24
|
-
azure-cli
|
|
25
29
|
nibabel
|
|
26
|
-
|
|
27
|
-
pydicom
|
|
28
|
-
IPython
|
|
29
|
-
flask
|
|
30
|
-
libdeeplake==0.0.83
|
|
30
|
+
libdeeplake==0.0.84
|
|
31
31
|
|
|
32
32
|
[all:python_version >= "3.7" or sys_platform != "win32"]
|
|
33
33
|
av>=8.1.0
|
|
@@ -52,7 +52,7 @@ pydicom
|
|
|
52
52
|
nibabel
|
|
53
53
|
|
|
54
54
|
[enterprise]
|
|
55
|
-
libdeeplake==0.0.
|
|
55
|
+
libdeeplake==0.0.84
|
|
56
56
|
pyjwt
|
|
57
57
|
|
|
58
58
|
[gcp]
|
|
@@ -70,7 +70,7 @@ extras_require = {k: [req_map[r] for r in v] for k, v in extras.items()}
|
|
|
70
70
|
extras_require["all"] = [req_map[r] for r in all_extras]
|
|
71
71
|
|
|
72
72
|
if libdeeplake_available():
|
|
73
|
-
libdeeplake = "libdeeplake==0.0.
|
|
73
|
+
libdeeplake = "libdeeplake==0.0.84"
|
|
74
74
|
extras_require["enterprise"] = [libdeeplake, "pyjwt"]
|
|
75
75
|
extras_require["all"].append(libdeeplake)
|
|
76
76
|
install_requires.append(libdeeplake)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
BUGOUT_TOKEN = '718a213b-6509-495b-aadb-f9e63c4c7240'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|