deeplake 3.9.24__tar.gz → 3.9.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.24/deeplake.egg-info → deeplake-3.9.26}/PKG-INFO +1 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/__init__.py +1 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/dataset.py +30 -36
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/structured/dataframe.py +3 -3
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/tests/test_ingestion.py +1 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/utils.py +0 -12
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/chunk_compressed_chunk.py +6 -7
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/uncompressed_chunk.py +9 -11
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk_engine.py +35 -37
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tensor.py +5 -6
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_indra_dataset.py +4 -4
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mmseg/mmseg_.py +9 -18
- deeplake-3.9.26/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/exceptions.py +16 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/version_control.py +0 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/video.py +1 -1
- {deeplake-3.9.24 → deeplake-3.9.26/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake.egg-info/requires.txt +10 -10
- {deeplake-3.9.24 → deeplake-3.9.26}/pyproject.toml +19 -6
- {deeplake-3.9.24 → deeplake-3.9.26}/setup.py +1 -1
- deeplake-3.9.24/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.24 → deeplake-3.9.26}/LICENSE +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/MANIFEST.in +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/README.md +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/info.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/link.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/read.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/client.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/config.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/log.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/compression.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/constants.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compression.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/io.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/sample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_inverted_indexes.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/transform/transform.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/dataloader.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/hooks.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/htype.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mm/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mm/mm_common.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mm/mm_runners.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/mmseg/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/image.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/json.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/link.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/path.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/split.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/storage.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/token.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.24 → deeplake-3.9.26}/setup.cfg +0 -0
|
@@ -72,6 +72,7 @@ from deeplake.util.exceptions import (
|
|
|
72
72
|
LockedException,
|
|
73
73
|
BadRequestException,
|
|
74
74
|
RenameError,
|
|
75
|
+
InvalidPandasDataframeError,
|
|
75
76
|
)
|
|
76
77
|
from deeplake.util.storage import (
|
|
77
78
|
get_storage_and_cache_chain,
|
|
@@ -82,6 +83,17 @@ from deeplake.util.remove_cache import get_base_storage
|
|
|
82
83
|
from deeplake.util.cache_chain import generate_chain
|
|
83
84
|
from deeplake.core.storage.deeplake_memory_object import DeepLakeMemoryObject
|
|
84
85
|
|
|
86
|
+
allow_delete_error_message = "Dataset overwrite failed. The dataset is marked as allow_delete=false. To allow overwrite, you must first run `allow_delete = True` on the dataset."
|
|
87
|
+
see_traceback_error_message = (
|
|
88
|
+
"Dataset overwrite failed. See traceback for more information."
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
dataset_corrupted_error_message = (
|
|
92
|
+
"The source dataset is corrupted."
|
|
93
|
+
"You can try to fix this by loading the dataset with `reset=True` "
|
|
94
|
+
"which will attempt to reset uncommitted HEAD changes and load the previous version."
|
|
95
|
+
)
|
|
96
|
+
|
|
85
97
|
|
|
86
98
|
def _check_indra_and_read_only_flags(indra: bool, read_only: Optional[bool]):
|
|
87
99
|
if indra == False:
|
|
@@ -258,7 +270,7 @@ class dataset:
|
|
|
258
270
|
db_engine = parse_runtime_parameters(path, runtime)["tensor_db"]
|
|
259
271
|
|
|
260
272
|
try:
|
|
261
|
-
|
|
273
|
+
_, cache_chain = get_storage_and_cache_chain(
|
|
262
274
|
path=path,
|
|
263
275
|
db_engine=db_engine,
|
|
264
276
|
read_only=read_only,
|
|
@@ -279,16 +291,12 @@ class dataset:
|
|
|
279
291
|
if ds_exists:
|
|
280
292
|
if overwrite:
|
|
281
293
|
if not dataset._allow_delete(cache_chain):
|
|
282
|
-
raise DatasetHandlerError(
|
|
283
|
-
"Dataset overwrite failed. The dataset is marked as allow_delete=false. To allow overwrite, you must first run `allow_delete = True` on the dataset."
|
|
284
|
-
)
|
|
294
|
+
raise DatasetHandlerError(allow_delete_error_message)
|
|
285
295
|
|
|
286
296
|
try:
|
|
287
297
|
cache_chain.clear()
|
|
288
298
|
except Exception as e:
|
|
289
|
-
raise DatasetHandlerError(
|
|
290
|
-
"Dataset overwrite failed. See traceback for more information."
|
|
291
|
-
) from e
|
|
299
|
+
raise DatasetHandlerError(see_traceback_error_message) from e
|
|
292
300
|
create = True
|
|
293
301
|
else:
|
|
294
302
|
create = False
|
|
@@ -403,7 +411,7 @@ class dataset:
|
|
|
403
411
|
_fetch_creds_from_key(creds, org_id, token)
|
|
404
412
|
|
|
405
413
|
try:
|
|
406
|
-
storage,
|
|
414
|
+
storage, _ = get_storage_and_cache_chain(
|
|
407
415
|
path=path,
|
|
408
416
|
read_only=True,
|
|
409
417
|
creds=creds,
|
|
@@ -523,16 +531,12 @@ class dataset:
|
|
|
523
531
|
|
|
524
532
|
if overwrite and dataset_exists(cache_chain):
|
|
525
533
|
if not dataset._allow_delete(cache_chain):
|
|
526
|
-
raise DatasetHandlerError(
|
|
527
|
-
"Dataset overwrite failed. The dataset is marked as allow_delete=false. To allow overwrite, you must first run `allow_delete = True` on the dataset."
|
|
528
|
-
)
|
|
534
|
+
raise DatasetHandlerError(allow_delete_error_message)
|
|
529
535
|
|
|
530
536
|
try:
|
|
531
537
|
cache_chain.clear()
|
|
532
538
|
except Exception as e:
|
|
533
|
-
raise DatasetHandlerError(
|
|
534
|
-
"Dataset overwrite failed. See traceback for more information."
|
|
535
|
-
) from e
|
|
539
|
+
raise DatasetHandlerError(see_traceback_error_message) from e
|
|
536
540
|
elif dataset_exists(cache_chain):
|
|
537
541
|
raise DatasetHandlerError(
|
|
538
542
|
f"A dataset already exists at the given path ({path}). If you want to create"
|
|
@@ -1143,11 +1147,11 @@ class dataset:
|
|
|
1143
1147
|
if dest_path == src_path:
|
|
1144
1148
|
# load tensor data to memory before deleting
|
|
1145
1149
|
# in case of in-place deeplake.like
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1150
|
+
_ = source_tensor.meta
|
|
1151
|
+
_ = source_tensor.info
|
|
1152
|
+
_ = source_tensor._sample_shape_tensor
|
|
1153
|
+
_ = source_tensor._sample_id_tensor
|
|
1154
|
+
_ = source_tensor._sample_info_tensor
|
|
1151
1155
|
destination_ds.delete_tensor(tensor_name)
|
|
1152
1156
|
destination_ds.create_tensor_like(tensor_name, source_tensor, unlink=tensor_name in unlink) # type: ignore
|
|
1153
1157
|
|
|
@@ -1218,9 +1222,7 @@ class dataset:
|
|
|
1218
1222
|
)
|
|
1219
1223
|
except DatasetCorruptError as e:
|
|
1220
1224
|
raise DatasetCorruptError(
|
|
1221
|
-
|
|
1222
|
-
"You can try to fix this by loading the dataset with `reset=True` "
|
|
1223
|
-
"which will attempt to reset uncommitted HEAD changes and load the previous version.",
|
|
1225
|
+
dataset_corrupted_error_message,
|
|
1224
1226
|
e.__cause__,
|
|
1225
1227
|
)
|
|
1226
1228
|
else:
|
|
@@ -1323,9 +1325,7 @@ class dataset:
|
|
|
1323
1325
|
)
|
|
1324
1326
|
except DatasetCorruptError as e:
|
|
1325
1327
|
raise DatasetCorruptError(
|
|
1326
|
-
|
|
1327
|
-
"You can try to fix this by loading the dataset with `reset=True` "
|
|
1328
|
-
"which will attempt to reset uncommitted HEAD changes and load the previous version.",
|
|
1328
|
+
dataset_corrupted_error_message,
|
|
1329
1329
|
e.__cause__,
|
|
1330
1330
|
)
|
|
1331
1331
|
else:
|
|
@@ -1364,16 +1364,12 @@ class dataset:
|
|
|
1364
1364
|
if dataset_exists(cache_chain):
|
|
1365
1365
|
if overwrite:
|
|
1366
1366
|
if not dataset._allow_delete(cache_chain):
|
|
1367
|
-
raise DatasetHandlerError(
|
|
1368
|
-
"Dataset overwrite failed. The dataset is marked as allow_delete=false. To allow overwrite, you must first run `allow_delete = True` on the dataset."
|
|
1369
|
-
)
|
|
1367
|
+
raise DatasetHandlerError(allow_delete_error_message)
|
|
1370
1368
|
|
|
1371
1369
|
try:
|
|
1372
1370
|
cache_chain.clear()
|
|
1373
1371
|
except Exception as e:
|
|
1374
|
-
raise DatasetHandlerError(
|
|
1375
|
-
"Dataset overwrite failed. See traceback for more information."
|
|
1376
|
-
) from e
|
|
1372
|
+
raise DatasetHandlerError(see_traceback_error_message) from e
|
|
1377
1373
|
else:
|
|
1378
1374
|
raise DatasetHandlerError(
|
|
1379
1375
|
f"A dataset already exists at the given path ({dest}). If you want to copy to a new dataset, either specify another path or use overwrite=True."
|
|
@@ -2125,7 +2121,7 @@ class dataset:
|
|
|
2125
2121
|
Dataset: New dataset created from the dataframe.
|
|
2126
2122
|
|
|
2127
2123
|
Raises:
|
|
2128
|
-
|
|
2124
|
+
InvalidPandasDataframeError: If ``src`` is not a valid pandas dataframe object.
|
|
2129
2125
|
"""
|
|
2130
2126
|
import pandas as pd
|
|
2131
2127
|
from deeplake.auto.structured.dataframe import DataFrame
|
|
@@ -2138,7 +2134,7 @@ class dataset:
|
|
|
2138
2134
|
)
|
|
2139
2135
|
|
|
2140
2136
|
if not isinstance(src, pd.DataFrame):
|
|
2141
|
-
raise
|
|
2137
|
+
raise InvalidPandasDataframeError()
|
|
2142
2138
|
|
|
2143
2139
|
structured = DataFrame(src, column_params, src_creds, creds_key)
|
|
2144
2140
|
|
|
@@ -2244,9 +2240,7 @@ class dataset:
|
|
|
2244
2240
|
)
|
|
2245
2241
|
except DatasetCorruptError as e:
|
|
2246
2242
|
raise DatasetCorruptError(
|
|
2247
|
-
|
|
2248
|
-
"You can try to fix this by loading the dataset with `reset=True` "
|
|
2249
|
-
"which will attempt to reset uncommitted HEAD changes and load the previous version.",
|
|
2243
|
+
dataset_corrupted_error_message,
|
|
2250
2244
|
e.__cause__,
|
|
2251
2245
|
)
|
|
2252
2246
|
else:
|
|
@@ -3,7 +3,7 @@ from .base import StructuredDataset
|
|
|
3
3
|
from deeplake import Dataset
|
|
4
4
|
from deeplake import read, link
|
|
5
5
|
from deeplake.htype import HTYPE_SUPPORTED_COMPRESSIONS
|
|
6
|
-
from deeplake.util.exceptions import IngestionError
|
|
6
|
+
from deeplake.util.exceptions import IngestionError, InvalidPandasDataframeError
|
|
7
7
|
from deeplake.util.dataset import sanitize_tensor_name
|
|
8
8
|
|
|
9
9
|
from collections import defaultdict
|
|
@@ -26,14 +26,14 @@ class DataFrame(StructuredDataset):
|
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
Raises:
|
|
29
|
-
|
|
29
|
+
InvalidPandasDataframeError: If source is not a pandas dataframe object.
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
32
|
import pandas as pd # type: ignore
|
|
33
33
|
|
|
34
34
|
super().__init__(source)
|
|
35
35
|
if not isinstance(self.source, pd.DataFrame):
|
|
36
|
-
raise
|
|
36
|
+
raise InvalidPandasDataframeError()
|
|
37
37
|
|
|
38
38
|
self.creds = creds
|
|
39
39
|
self.creds_key = creds_key
|
|
@@ -29,18 +29,6 @@ PENDING_STATUS = "not available yet"
|
|
|
29
29
|
BEST_RECALL = "best_recall@10"
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def remove_username_from_config():
|
|
33
|
-
try:
|
|
34
|
-
config = {}
|
|
35
|
-
with open(REPORTING_CONFIG_FILE_PATH, "r") as f:
|
|
36
|
-
config = json.load(f)
|
|
37
|
-
config["username"] = "public"
|
|
38
|
-
with open(REPORTING_CONFIG_FILE_PATH, "w") as f:
|
|
39
|
-
json.dump(config, f)
|
|
40
|
-
except (FileNotFoundError, KeyError):
|
|
41
|
-
return
|
|
42
|
-
|
|
43
|
-
|
|
44
32
|
def check_response_status(response: requests.Response):
|
|
45
33
|
"""Check response status and throw corresponding exception on failure."""
|
|
46
34
|
code = response.status_code
|
|
@@ -160,13 +160,12 @@ class ChunkCompressedChunk(BaseChunk):
|
|
|
160
160
|
cast = False
|
|
161
161
|
sample_nbytes = sample.nbytes
|
|
162
162
|
else:
|
|
163
|
-
if sample.size:
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
)
|
|
163
|
+
if sample.size and not np.can_cast(sample_dtype, chunk_dtype):
|
|
164
|
+
raise TensorDtypeMismatchError(
|
|
165
|
+
chunk_dtype,
|
|
166
|
+
sample_dtype,
|
|
167
|
+
self.htype,
|
|
168
|
+
)
|
|
170
169
|
cast = True
|
|
171
170
|
sample_nbytes = np.dtype(chunk_dtype).itemsize * sample.size
|
|
172
171
|
min_chunk_size = self.min_chunk_size
|
|
@@ -48,9 +48,8 @@ class UncompressedChunk(BaseChunk):
|
|
|
48
48
|
num_data_bytes = self.num_data_bytes
|
|
49
49
|
space_left = min_chunk_size - num_data_bytes
|
|
50
50
|
idx = np.searchsorted(csum, space_left)
|
|
51
|
-
if not idx and csum[0] > space_left:
|
|
52
|
-
|
|
53
|
-
return 0
|
|
51
|
+
if not idx and csum[0] > space_left and self._data_bytes:
|
|
52
|
+
return 0
|
|
54
53
|
num_samples = int(min(len(incoming_samples), idx + 1)) # type: ignore
|
|
55
54
|
bts = list(
|
|
56
55
|
map(self._text_sample_to_byte_string, incoming_samples[:num_samples])
|
|
@@ -131,13 +130,12 @@ class UncompressedChunk(BaseChunk):
|
|
|
131
130
|
chunk_dtype = self.dtype
|
|
132
131
|
samples_dtype = incoming_samples.dtype
|
|
133
132
|
if samples_dtype != chunk_dtype:
|
|
134
|
-
if size:
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
)
|
|
133
|
+
if size and not np.can_cast(samples_dtype, chunk_dtype):
|
|
134
|
+
raise TensorDtypeMismatchError(
|
|
135
|
+
chunk_dtype,
|
|
136
|
+
samples_dtype,
|
|
137
|
+
self.htype,
|
|
138
|
+
)
|
|
141
139
|
samples = samples.astype(chunk_dtype)
|
|
142
140
|
self._data_bytes += samples.tobytes() # type: ignore
|
|
143
141
|
self.register_in_meta_and_headers(
|
|
@@ -163,7 +161,7 @@ class UncompressedChunk(BaseChunk):
|
|
|
163
161
|
if shape is not None and not self.tensor_meta.is_link:
|
|
164
162
|
self.num_dims = self.num_dims or len(shape)
|
|
165
163
|
check_sample_shape(shape, self.num_dims)
|
|
166
|
-
except Exception
|
|
164
|
+
except Exception:
|
|
167
165
|
if ignore_errors:
|
|
168
166
|
skipped.append(i)
|
|
169
167
|
continue
|
|
@@ -838,26 +838,27 @@ class ChunkEngine:
|
|
|
838
838
|
incoming_num_samples = len(samples)
|
|
839
839
|
enc_ids: List[Optional[str]] = []
|
|
840
840
|
enc_count = [0]
|
|
841
|
-
if
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
)
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
841
|
+
if (
|
|
842
|
+
extending
|
|
843
|
+
and self.tensor_meta.htype == "text"
|
|
844
|
+
and (self.chunk_class != SampleCompressedChunk)
|
|
845
|
+
):
|
|
846
|
+
lengths = np.zeros(len(samples), dtype=np.uint32)
|
|
847
|
+
for i, s in enumerate(samples):
|
|
848
|
+
try:
|
|
849
|
+
s = s.numpy()
|
|
850
|
+
except AttributeError:
|
|
851
|
+
pass
|
|
852
|
+
try:
|
|
853
|
+
if s.dtype.name[:3] == "str":
|
|
854
|
+
lengths[i] = len(str(s.reshape(())))
|
|
855
|
+
except AttributeError:
|
|
851
856
|
try:
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
except AttributeError: # None
|
|
858
|
-
lengths[i] = 0
|
|
859
|
-
except TypeError: # Numpy scalar str
|
|
860
|
-
lengths[i] = str(s).__len__()
|
|
857
|
+
lengths[i] = s.__len__()
|
|
858
|
+
except AttributeError: # None
|
|
859
|
+
lengths[i] = 0
|
|
860
|
+
except TypeError: # Numpy scalar str
|
|
861
|
+
lengths[i] = str(s).__len__()
|
|
861
862
|
extra_args = {"lengths": lengths}
|
|
862
863
|
current_chunk = start_chunk
|
|
863
864
|
updated_chunks: List[Optional[str]] = []
|
|
@@ -1039,12 +1040,11 @@ class ChunkEngine:
|
|
|
1039
1040
|
lengths,
|
|
1040
1041
|
):
|
|
1041
1042
|
sample = samples[0]
|
|
1042
|
-
if sample.is_first_write:
|
|
1043
|
-
if
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
enc_count[-1] += 1
|
|
1043
|
+
if sample.is_first_write and register:
|
|
1044
|
+
if start_chunk_row is not None:
|
|
1045
|
+
enc.register_samples(1)
|
|
1046
|
+
else:
|
|
1047
|
+
enc_count[-1] += 1
|
|
1048
1048
|
if sample.is_last_write:
|
|
1049
1049
|
tiles[
|
|
1050
1050
|
incoming_num_samples - len(samples) + bool(register) * orig_meta_length
|
|
@@ -1775,7 +1775,7 @@ class ChunkEngine:
|
|
|
1775
1775
|
samples: Union[np.ndarray, Sequence[InputSample], InputSample],
|
|
1776
1776
|
operator: str,
|
|
1777
1777
|
):
|
|
1778
|
-
"""Update data at `index` with the output of elem-wise
|
|
1778
|
+
"""Update data at `index` with the output of elem-wise operation with samples"""
|
|
1779
1779
|
try:
|
|
1780
1780
|
if isinstance(samples, deeplake.core.tensor.Tensor):
|
|
1781
1781
|
samples = samples.numpy()
|
|
@@ -1891,7 +1891,7 @@ class ChunkEngine:
|
|
|
1891
1891
|
"""
|
|
1892
1892
|
threshold = 10
|
|
1893
1893
|
|
|
1894
|
-
if
|
|
1894
|
+
if isinstance(index.values[0].value, slice):
|
|
1895
1895
|
start = index.values[0].value.start or 0
|
|
1896
1896
|
stop = index.values[0].value.stop or self.num_samples
|
|
1897
1897
|
step = index.values[0].value.step or 1
|
|
@@ -2218,11 +2218,10 @@ class ChunkEngine:
|
|
|
2218
2218
|
if exception:
|
|
2219
2219
|
raise exception
|
|
2220
2220
|
chunk, chunk_info = future.result()
|
|
2221
|
-
if chunk:
|
|
2222
|
-
|
|
2223
|
-
self.
|
|
2224
|
-
|
|
2225
|
-
)
|
|
2221
|
+
if chunk and _get_nbytes(chunk) <= self.cache.cache_size:
|
|
2222
|
+
self.cache._insert_in_cache(
|
|
2223
|
+
self.get_chunk_key_for_id(chunk_info[0]), chunk
|
|
2224
|
+
)
|
|
2226
2225
|
yield chunk_info
|
|
2227
2226
|
else:
|
|
2228
2227
|
with ThreadPoolExecutor() as executor:
|
|
@@ -2232,11 +2231,10 @@ class ChunkEngine:
|
|
|
2232
2231
|
repeat(storages),
|
|
2233
2232
|
):
|
|
2234
2233
|
chunk, chunk_info = result
|
|
2235
|
-
if chunk:
|
|
2236
|
-
|
|
2237
|
-
self.
|
|
2238
|
-
|
|
2239
|
-
)
|
|
2234
|
+
if chunk and _get_nbytes(chunk) <= self.cache.cache_size:
|
|
2235
|
+
self.cache._insert_in_cache(
|
|
2236
|
+
self.get_chunk_key_for_id(chunk_info[0]), chunk
|
|
2237
|
+
)
|
|
2240
2238
|
yield chunk_info
|
|
2241
2239
|
|
|
2242
2240
|
def _get_samples(
|
|
@@ -1546,11 +1546,11 @@ class Tensor:
|
|
|
1546
1546
|
stream = BytesIO(serialized_data)
|
|
1547
1547
|
|
|
1548
1548
|
# Read number of partitions
|
|
1549
|
-
|
|
1549
|
+
metadata_size = int.from_bytes(
|
|
1550
1550
|
stream.read(8), "little"
|
|
1551
1551
|
) # Assuming size_t is 8 bytes
|
|
1552
1552
|
|
|
1553
|
-
metadata_bytes = stream.read(
|
|
1553
|
+
metadata_bytes = stream.read(metadata_size)
|
|
1554
1554
|
metadata = json.loads(metadata_bytes.decode("utf-8"))
|
|
1555
1555
|
|
|
1556
1556
|
temp_paths_size = int.from_bytes(stream.read(8), "little")
|
|
@@ -1847,7 +1847,6 @@ class Tensor:
|
|
|
1847
1847
|
metadata_json = json.dumps(metadata)
|
|
1848
1848
|
metadata_bytes = metadata_json.encode("utf-8")
|
|
1849
1849
|
self.storage[inverted_meta_key] = metadata_bytes
|
|
1850
|
-
temp_serialized_paths_count = len(temp_serialized_paths)
|
|
1851
1850
|
temp_serialized_paths = [str(path) for path in temp_serialized_paths]
|
|
1852
1851
|
for i, path in enumerate(temp_serialized_paths):
|
|
1853
1852
|
file_name = pathlib.Path(path).name
|
|
@@ -1985,7 +1984,7 @@ class Tensor:
|
|
|
1985
1984
|
def load_vdb_index(self, id: str):
|
|
1986
1985
|
if not self.meta.contains_vdb_index(id):
|
|
1987
1986
|
raise ValueError(f"Tensor meta has no vdb index with name '{id}'.")
|
|
1988
|
-
if
|
|
1987
|
+
if self.dataset.libdeeplake_dataset is not None:
|
|
1989
1988
|
ds = self.dataset.libdeeplake_dataset
|
|
1990
1989
|
else:
|
|
1991
1990
|
from deeplake.enterprise.convert_to_libdeeplake import (
|
|
@@ -2003,7 +2002,7 @@ class Tensor:
|
|
|
2003
2002
|
raise ValueError(f"An error occurred while loading the VDB index {id}: {e}")
|
|
2004
2003
|
|
|
2005
2004
|
def unload_vdb_index_cache(self):
|
|
2006
|
-
if
|
|
2005
|
+
if self.dataset.libdeeplake_dataset is not None:
|
|
2007
2006
|
ds = self.dataset.libdeeplake_dataset
|
|
2008
2007
|
else:
|
|
2009
2008
|
from deeplake.enterprise.convert_to_libdeeplake import (
|
|
@@ -2028,7 +2027,7 @@ class Tensor:
|
|
|
2028
2027
|
def fetch_vdb_indexes(self) -> List[Dict[str, str]]:
|
|
2029
2028
|
vdb_indexes = []
|
|
2030
2029
|
if self.meta.htype == "embedding" or self.meta.htype == "text":
|
|
2031
|
-
if (
|
|
2030
|
+
if (self.meta.vdb_indexes is not None) and len(self.meta.vdb_indexes) > 0:
|
|
2032
2031
|
vdb_indexes.extend(self.meta.vdb_indexes)
|
|
2033
2032
|
return vdb_indexes
|
|
2034
2033
|
|
|
@@ -17,7 +17,7 @@ def test_indexing(local_auth_ds_generator):
|
|
|
17
17
|
deeplake_ds.create_tensor(
|
|
18
18
|
"label", htype="generic", dtype=np.int32, **disabale_hidden_tensors_config
|
|
19
19
|
)
|
|
20
|
-
for
|
|
20
|
+
for _ in range(1000):
|
|
21
21
|
deeplake_ds.label.append(int(100 * random.uniform(0.0, 1.0)))
|
|
22
22
|
|
|
23
23
|
indra_ds = dataset_to_libdeeplake(deeplake_ds)
|
|
@@ -64,7 +64,7 @@ def test_save_view(local_auth_ds_generator):
|
|
|
64
64
|
deeplake_ds.create_tensor(
|
|
65
65
|
"label", htype="generic", dtype=np.int32, **disabale_hidden_tensors_config
|
|
66
66
|
)
|
|
67
|
-
for
|
|
67
|
+
for _ in range(1000):
|
|
68
68
|
deeplake_ds.label.append(int(100 * random.uniform(0.0, 1.0)))
|
|
69
69
|
|
|
70
70
|
deeplake_ds.commit("First")
|
|
@@ -104,7 +104,7 @@ def test_load_view(local_auth_ds_generator):
|
|
|
104
104
|
deeplake_indra_ds = IndraDatasetView(indra_ds=indra_ds)
|
|
105
105
|
|
|
106
106
|
with pytest.raises(Exception):
|
|
107
|
-
|
|
107
|
+
deeplake_indra_ds.pytorch()
|
|
108
108
|
|
|
109
109
|
query_str = "select * group by label"
|
|
110
110
|
view = deeplake_ds.query(query_str)
|
|
@@ -233,7 +233,7 @@ def test_accessing_data(local_auth_ds_generator):
|
|
|
233
233
|
deeplake_ds.create_tensor(
|
|
234
234
|
"label", htype="generic", dtype=np.int32, **disabale_hidden_tensors_config
|
|
235
235
|
)
|
|
236
|
-
for
|
|
236
|
+
for _ in range(1000):
|
|
237
237
|
deeplake_ds.label.append(int(100 * random.uniform(0.0, 1.0)))
|
|
238
238
|
|
|
239
239
|
indra_ds = dataset_to_libdeeplake(deeplake_ds)
|
|
@@ -164,6 +164,7 @@ from collections import OrderedDict
|
|
|
164
164
|
from typing import Callable, Optional, List, Dict
|
|
165
165
|
from prettytable import PrettyTable # type: ignore
|
|
166
166
|
from PIL import Image # type: ignore
|
|
167
|
+
from deeplake.util.exceptions import ValidationDatasetMissingError
|
|
167
168
|
|
|
168
169
|
from mmseg.core import DistEvalHook, EvalHook # type: ignore
|
|
169
170
|
from mmseg.core import build_optimizer
|
|
@@ -611,7 +612,6 @@ def _train_segmentor(
|
|
|
611
612
|
eval_cfg = cfg.get("evaluation", {})
|
|
612
613
|
dl_impl = cfg.get("deeplake_dataloader_type", "auto").lower()
|
|
613
614
|
|
|
614
|
-
# TODO: check whether dataset is actually supported by enterprise dataloader if c++
|
|
615
615
|
if dl_impl == "auto":
|
|
616
616
|
dl_impl = "c++" if indra_available() else "python"
|
|
617
617
|
elif dl_impl == "cpp":
|
|
@@ -733,9 +733,8 @@ def _train_segmentor(
|
|
|
733
733
|
cfg.get("momentum_config", None),
|
|
734
734
|
)
|
|
735
735
|
|
|
736
|
-
if distributed:
|
|
737
|
-
|
|
738
|
-
runner.register_hook(DistSamplerSeedHook())
|
|
736
|
+
if distributed and isinstance(runner, EpochBasedRunner):
|
|
737
|
+
runner.register_hook(DistSamplerSeedHook())
|
|
739
738
|
|
|
740
739
|
# register eval hooks
|
|
741
740
|
if validate:
|
|
@@ -764,14 +763,8 @@ def _train_segmentor(
|
|
|
764
763
|
|
|
765
764
|
if ds_val is None:
|
|
766
765
|
cfg_ds_val = cfg.data.get("val")
|
|
767
|
-
if cfg_ds_val is None:
|
|
768
|
-
raise
|
|
769
|
-
"Validation dataset is not specified even though validate = True. Please set validate = False or specify a validation dataset."
|
|
770
|
-
)
|
|
771
|
-
elif cfg_ds_val.get("deeplake_path") is None:
|
|
772
|
-
raise Exception(
|
|
773
|
-
"Validation dataset is not specified even though validate = True. Please set validate = False or specify a validation dataset."
|
|
774
|
-
)
|
|
766
|
+
if cfg_ds_val is None or cfg_ds_val.get("deeplake_path") is None:
|
|
767
|
+
raise ValidationDatasetMissingError()
|
|
775
768
|
|
|
776
769
|
ds_val = load_ds_from_cfg(cfg.data.val)
|
|
777
770
|
ds_val_tensors = cfg.data.val.get("deeplake_tensors", {})
|
|
@@ -779,13 +772,11 @@ def _train_segmentor(
|
|
|
779
772
|
cfg_data = cfg.data.val.get("deeplake_path")
|
|
780
773
|
if cfg_data is not None:
|
|
781
774
|
always_warn(
|
|
782
|
-
"A Deep Lake dataset was specified in the cfg as well as
|
|
775
|
+
"A Deep Lake dataset was specified in the cfg as well as in the dataset input to train_segmentor. The dataset input to train_segmentor will be used in the workflow."
|
|
783
776
|
)
|
|
784
777
|
|
|
785
778
|
if ds_val is None:
|
|
786
|
-
raise
|
|
787
|
-
"Validation dataset is not specified even though validate = True. Please set validate = False or specify a validation dataset."
|
|
788
|
-
)
|
|
779
|
+
raise ValidationDatasetMissingError()
|
|
789
780
|
|
|
790
781
|
if ds_val_tensors:
|
|
791
782
|
val_images_tensor = ds_val_tensors["img"]
|
|
@@ -871,8 +862,8 @@ def build_dataloader(
|
|
|
871
862
|
dataset.CLASSES = classes
|
|
872
863
|
pipeline = build_pipeline(pipeline)
|
|
873
864
|
persistent_workers = train_loader_config.get("persistent_workers", False)
|
|
874
|
-
|
|
875
|
-
|
|
865
|
+
_ = train_loader_config.get("ignore_index")
|
|
866
|
+
_ = train_loader_config.get("reduce_zero_label")
|
|
876
867
|
dist = train_loader_config["dist"]
|
|
877
868
|
if dist and implementation == "python":
|
|
878
869
|
raise NotImplementedError(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
BUGOUT_TOKEN = '7a7147bb-ad3b-4070-8a7e-d6fd5aebf264'
|
|
@@ -593,7 +593,7 @@ class SampleAppendError(Exception):
|
|
|
593
593
|
|
|
594
594
|
if print_item:
|
|
595
595
|
msg += str(sample) + " "
|
|
596
|
-
|
|
596
|
+
else:
|
|
597
597
|
msg += f"at path '{sample.path}' "
|
|
598
598
|
else:
|
|
599
599
|
msg = f"Failed to append a sample "
|
|
@@ -897,6 +897,7 @@ class SampleExtendingError(Exception):
|
|
|
897
897
|
"`ds.extend({'image_tensor': samples, 'label_tensor': samples})` or you need to call `extend` method of the required tensor, "
|
|
898
898
|
"like: `ds.image_tensor.extend(samples)`"
|
|
899
899
|
)
|
|
900
|
+
super().__init__(message)
|
|
900
901
|
|
|
901
902
|
|
|
902
903
|
class DatasetTooLargeToDelete(Exception):
|
|
@@ -1143,9 +1144,23 @@ class IncorrectQueriesTypeError(Exception):
|
|
|
1143
1144
|
super().__init__(msg)
|
|
1144
1145
|
|
|
1145
1146
|
|
|
1147
|
+
class ValidationDatasetMissingError(Exception):
|
|
1148
|
+
def __init__(self):
|
|
1149
|
+
msg = (
|
|
1150
|
+
"Validation dataset is not specified even though validate = True. "
|
|
1151
|
+
"Please set validate = False or specify a validation dataset."
|
|
1152
|
+
)
|
|
1153
|
+
super().__init__(msg)
|
|
1154
|
+
|
|
1155
|
+
|
|
1146
1156
|
class InvalidAuthContextError(Exception):
|
|
1147
1157
|
def __init__(
|
|
1148
1158
|
self,
|
|
1149
1159
|
message: str = "Authentication failed due to invalid or insufficient configuration.",
|
|
1150
1160
|
):
|
|
1151
1161
|
super().__init__(message)
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
class InvalidPandasDataframeError(Exception):
|
|
1165
|
+
def __init__(self):
|
|
1166
|
+
super().__init__("Source provided is not a valid pandas dataframe object")
|
|
@@ -35,7 +35,7 @@ def normalize_index(index, nframes):
|
|
|
35
35
|
start, stop = stop + 1, start + 1
|
|
36
36
|
elif isinstance(index, list):
|
|
37
37
|
raise IndexError(
|
|
38
|
-
|
|
38
|
+
"Cannot specify a list video frames. You must specify a range with an optional step such as [5:10] or [0:100:5]"
|
|
39
39
|
)
|
|
40
40
|
else:
|
|
41
41
|
raise IndexError(
|