deeplake 3.2.19__tar.gz → 3.2.21__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.2.19 → deeplake-3.2.21}/PKG-INFO +1 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/__init__.py +1 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/dataset.py +1 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_link.py +19 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk_engine.py +11 -1
- deeplake-3.2.21/deeplake/core/compute/provider.py +85 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/serial.py +12 -4
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/dataset.py +9 -3
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/deeplake_cloud_dataset.py +6 -2
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/link_creds.py +15 -4
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_chunk_engine.py +12 -19
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/filter.py +2 -2
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tensor.py +9 -4
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_compute.py +1 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/test_transform.py +50 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform.py +107 -52
- deeplake-3.2.21/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/exceptions.py +12 -7
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/iterable_ordered_dict.py +1 -2
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/link.py +3 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/transform.py +73 -5
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/PKG-INFO +1 -1
- deeplake-3.2.19/deeplake/core/compute/provider.py +0 -78
- deeplake-3.2.19/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.2.19 → deeplake-3.2.21}/MANIFEST.in +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/README.md +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/info.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/link.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/read.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/api/tiled.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/auth.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/commands.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/cli/test_cli.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/client.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/config.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/log.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/test_client.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/client/utils.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/compression.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/constants.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compression.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/index/index.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/io.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/ipc.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/lock.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/polygon.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/query/query.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/sample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/serialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/dataloader.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/hooks.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/htype.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/common.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/access_method.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/agreement.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/array_list.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/auto.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/casting.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/class_label.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/compression.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/compute.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/diff.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/downsample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/encoder.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/exif.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/hash.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/htype.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/image.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/json.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/keys.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/logging.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/merge.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/modified.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/notebook.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/path.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/spinner.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/split.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/storage.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tag.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/testing.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/threading.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/token.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/version_control.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/video.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/util/warnings.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/entry_points.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/requires.txt +6 -6
- {deeplake-3.2.19 → deeplake-3.2.21}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/setup.cfg +0 -0
- {deeplake-3.2.19 → deeplake-3.2.21}/setup.py +0 -0
|
@@ -1175,7 +1175,7 @@ class dataset:
|
|
|
1175
1175
|
compute_provider = get_compute_provider(scheduler, num_workers)
|
|
1176
1176
|
try:
|
|
1177
1177
|
if progressbar:
|
|
1178
|
-
compute_provider.
|
|
1178
|
+
compute_provider.map_with_progress_bar(
|
|
1179
1179
|
copy_func_with_progress_bar,
|
|
1180
1180
|
keys,
|
|
1181
1181
|
len_keys,
|
|
@@ -644,3 +644,22 @@ def test_rgb_gray(local_ds, cat_path, hopper_gray_path):
|
|
|
644
644
|
assert len(ds.abc[1].shape) == 3
|
|
645
645
|
assert len(ds.abc[0].numpy().shape) == 3
|
|
646
646
|
assert len(ds.abc[1].numpy().shape) == 3
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def test_creds(hub_cloud_ds_generator, cat_path):
|
|
650
|
+
creds_key = "ENV"
|
|
651
|
+
ds = hub_cloud_ds_generator()
|
|
652
|
+
ds.add_creds_key(creds_key)
|
|
653
|
+
ds.populate_creds(creds_key, from_environment=True)
|
|
654
|
+
with ds:
|
|
655
|
+
tensor = ds.create_tensor("abc", "link[image]", sample_compression="jpeg")
|
|
656
|
+
tensor.append(deeplake.link(cat_path, creds_key))
|
|
657
|
+
|
|
658
|
+
assert tensor[0].creds_key() == creds_key
|
|
659
|
+
ds.add_creds_key("my_s3_creds", True)
|
|
660
|
+
assert ds.get_managed_creds_keys() == ["my_s3_creds"]
|
|
661
|
+
assert set(ds.get_creds_keys()) == {"my_s3_creds", "ENV"}
|
|
662
|
+
ds.update_creds_key("my_s3_creds", managed=True)
|
|
663
|
+
ds = hub_cloud_ds_generator()
|
|
664
|
+
assert ds.get_managed_creds_keys() == ["my_s3_creds"]
|
|
665
|
+
assert set(ds.get_creds_keys()) == {"my_s3_creds", "ENV"}
|
|
@@ -1399,6 +1399,7 @@ class ChunkEngine:
|
|
|
1399
1399
|
del self.cache[from_chunk.key] # type: ignore
|
|
1400
1400
|
except KeyError:
|
|
1401
1401
|
pass
|
|
1402
|
+
self.cache[to_chunk.key] = to_chunk # type: ignore
|
|
1402
1403
|
return True
|
|
1403
1404
|
|
|
1404
1405
|
def _is_tiled(self, row: int) -> bool:
|
|
@@ -1436,7 +1437,11 @@ class ChunkEngine:
|
|
|
1436
1437
|
if next_chunk_size + chunk.num_data_bytes < next_chunk.min_chunk_size:
|
|
1437
1438
|
if next_chunk_commit_id != self.commit_id:
|
|
1438
1439
|
next_chunk = self.copy_chunk_to_new_commit(next_chunk, next_chunk_name)
|
|
1439
|
-
|
|
1440
|
+
chunk_id = chunk.id
|
|
1441
|
+
chunk_name = ChunkIdEncoder.name_from_id(chunk_id)
|
|
1442
|
+
chunk_commit_id, tkey = self.get_chunk_commit(chunk_name)
|
|
1443
|
+
if chunk_commit_id != self.commit_id:
|
|
1444
|
+
chunk = self.copy_chunk_to_new_commit(chunk, chunk_name)
|
|
1440
1445
|
return self._merge_chunks(
|
|
1441
1446
|
from_chunk=next_chunk,
|
|
1442
1447
|
from_chunk_row=next_chunk_row,
|
|
@@ -1462,6 +1467,11 @@ class ChunkEngine:
|
|
|
1462
1467
|
if prev_chunk_size + chunk.num_data_bytes < prev_chunk.min_chunk_size:
|
|
1463
1468
|
if prev_chunk_commit_id != self.commit_id:
|
|
1464
1469
|
prev_chunk = self.copy_chunk_to_new_commit(prev_chunk, prev_chunk_name)
|
|
1470
|
+
chunk_id = chunk.id
|
|
1471
|
+
chunk_name = ChunkIdEncoder.name_from_id(chunk_id)
|
|
1472
|
+
chunk_commit_id, tkey = self.get_chunk_commit(chunk_name)
|
|
1473
|
+
if chunk_commit_id != self.commit_id:
|
|
1474
|
+
chunk = self.copy_chunk_to_new_commit(chunk, chunk_name)
|
|
1465
1475
|
# merge with previous chunk
|
|
1466
1476
|
return self._merge_chunks(
|
|
1467
1477
|
from_chunk=chunk,
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Optional
|
|
3
|
+
import threading
|
|
4
|
+
import warnings
|
|
5
|
+
from tqdm.std import tqdm # type: ignore
|
|
6
|
+
from tqdm import TqdmWarning # type: ignore
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_progress_bar(total_length, desc):
|
|
10
|
+
warnings.simplefilter("ignore", TqdmWarning)
|
|
11
|
+
|
|
12
|
+
return tqdm(
|
|
13
|
+
total=total_length,
|
|
14
|
+
desc=desc,
|
|
15
|
+
bar_format="{desc}: {percentage:.0f}%|{bar}| {n:.0f}/{total_fmt} [{elapsed}<{remaining}",
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_progress_thread(progress_bar, progress_queue):
|
|
20
|
+
def update_pg(bar, queue):
|
|
21
|
+
while True:
|
|
22
|
+
r = queue.get()
|
|
23
|
+
if r is not None:
|
|
24
|
+
bar.update(r)
|
|
25
|
+
else:
|
|
26
|
+
break
|
|
27
|
+
|
|
28
|
+
progress_thread = threading.Thread(
|
|
29
|
+
target=update_pg, args=(progress_bar, progress_queue)
|
|
30
|
+
)
|
|
31
|
+
progress_thread.start()
|
|
32
|
+
return progress_thread
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ComputeProvider(ABC):
|
|
36
|
+
"""An abstract base class for implementing a compute provider."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, workers):
|
|
39
|
+
self.workers = workers
|
|
40
|
+
|
|
41
|
+
def map_with_progress_bar(
|
|
42
|
+
self,
|
|
43
|
+
func,
|
|
44
|
+
iterable,
|
|
45
|
+
total_length: int,
|
|
46
|
+
desc: Optional[str] = None,
|
|
47
|
+
pbar=None,
|
|
48
|
+
pqueue=None,
|
|
49
|
+
):
|
|
50
|
+
progress_bar = pbar or get_progress_bar(total_length, desc)
|
|
51
|
+
progress_queue = pqueue or self.create_queue()
|
|
52
|
+
|
|
53
|
+
def sub_func(*args, **kwargs):
|
|
54
|
+
def pg_callback(value: int):
|
|
55
|
+
progress_queue.put(value) # type: ignore[trust]
|
|
56
|
+
|
|
57
|
+
return func(pg_callback, *args, **kwargs)
|
|
58
|
+
|
|
59
|
+
progress_thread = get_progress_thread(progress_bar, progress_queue)
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
result = self.map(sub_func, iterable)
|
|
63
|
+
finally:
|
|
64
|
+
progress_queue.put(None) # type: ignore[trust]
|
|
65
|
+
if pqueue is None and hasattr(progress_queue, "close"):
|
|
66
|
+
progress_queue.close()
|
|
67
|
+
progress_thread.join()
|
|
68
|
+
if pbar is None:
|
|
69
|
+
progress_bar.close()
|
|
70
|
+
|
|
71
|
+
return result
|
|
72
|
+
|
|
73
|
+
@abstractmethod
|
|
74
|
+
def create_queue(self):
|
|
75
|
+
"""Creates queue for specific provider"""
|
|
76
|
+
|
|
77
|
+
@abstractmethod
|
|
78
|
+
def map(self, func, iterable):
|
|
79
|
+
"""Applies 'func' to each element in 'iterable', collecting the results
|
|
80
|
+
in a list that is returned.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
@abstractmethod
|
|
84
|
+
def close(self):
|
|
85
|
+
"""Closes the provider."""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from deeplake.core.compute.provider import ComputeProvider
|
|
1
|
+
from deeplake.core.compute.provider import ComputeProvider, get_progress_bar
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class SerialProvider(ComputeProvider):
|
|
@@ -8,8 +8,16 @@ class SerialProvider(ComputeProvider):
|
|
|
8
8
|
def map(self, func, iterable):
|
|
9
9
|
return list(map(func, iterable))
|
|
10
10
|
|
|
11
|
-
def
|
|
12
|
-
|
|
11
|
+
def map_with_progress_bar(
|
|
12
|
+
self,
|
|
13
|
+
func,
|
|
14
|
+
iterable,
|
|
15
|
+
total_length: int,
|
|
16
|
+
desc=None,
|
|
17
|
+
pbar=None,
|
|
18
|
+
pqueue=None,
|
|
19
|
+
):
|
|
20
|
+
progress_bar = pbar or get_progress_bar(total_length, desc)
|
|
13
21
|
|
|
14
22
|
def sub_func(*args, **kwargs):
|
|
15
23
|
def pg_callback(value: int):
|
|
@@ -22,7 +30,7 @@ class SerialProvider(ComputeProvider):
|
|
|
22
30
|
return result
|
|
23
31
|
|
|
24
32
|
def create_queue(self):
|
|
25
|
-
|
|
33
|
+
return None
|
|
26
34
|
|
|
27
35
|
def close(self):
|
|
28
36
|
return
|
|
@@ -3537,7 +3537,7 @@ class Dataset:
|
|
|
3537
3537
|
|
|
3538
3538
|
@invalid_view_op
|
|
3539
3539
|
@spinner
|
|
3540
|
-
def reset(self):
|
|
3540
|
+
def reset(self, force: bool = False):
|
|
3541
3541
|
"""Resets the uncommitted changes present in the branch.
|
|
3542
3542
|
|
|
3543
3543
|
Note:
|
|
@@ -3547,7 +3547,7 @@ class Dataset:
|
|
|
3547
3547
|
if version_state["commit_node"].children:
|
|
3548
3548
|
print("You are not at the head node of the branch, cannot reset.")
|
|
3549
3549
|
return
|
|
3550
|
-
if not self.has_head_changes:
|
|
3550
|
+
if not self.has_head_changes and not force:
|
|
3551
3551
|
print("There are no uncommitted changes on this branch.")
|
|
3552
3552
|
return
|
|
3553
3553
|
|
|
@@ -3716,7 +3716,13 @@ class Dataset:
|
|
|
3716
3716
|
|
|
3717
3717
|
def get_creds_keys(self) -> List[str]:
|
|
3718
3718
|
"""Returns the list of creds keys added to the dataset. These are used to fetch external data in linked tensors"""
|
|
3719
|
-
return self.link_creds.creds_keys
|
|
3719
|
+
return list(self.link_creds.creds_keys)
|
|
3720
|
+
|
|
3721
|
+
def get_managed_creds_keys(self) -> List[str]:
|
|
3722
|
+
"""Returns the list of creds keys added to the dataset that are managed by Activeloop platform. These are used to fetch external data in linked tensors."""
|
|
3723
|
+
raise ValueError(
|
|
3724
|
+
"Managed creds are not supported for datasets that are not connected to activeloop platform."
|
|
3725
|
+
)
|
|
3720
3726
|
|
|
3721
3727
|
def visualize(
|
|
3722
3728
|
self, width: Union[int, str, None] = None, height: Union[int, str, None] = None
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import posixpath
|
|
2
|
-
from typing import Any, Dict, Optional, Union
|
|
2
|
+
from typing import Any, Dict, List, Optional, Union
|
|
3
3
|
from deeplake.client.utils import get_user_name
|
|
4
4
|
from deeplake.constants import HUB_CLOUD_DEV_USERNAME
|
|
5
5
|
from deeplake.core.dataset import Dataset
|
|
@@ -336,7 +336,7 @@ class DeepLakeCloudDataset(Dataset):
|
|
|
336
336
|
if management_changed:
|
|
337
337
|
managed_info = (managed, key_index)
|
|
338
338
|
if original_key_is_managed and managed is not False:
|
|
339
|
-
self.link_creds.
|
|
339
|
+
self.link_creds.populate_single_managed_creds(creds_key)
|
|
340
340
|
except Exception:
|
|
341
341
|
if replaced_index is not None:
|
|
342
342
|
# revert the change
|
|
@@ -383,3 +383,7 @@ class DeepLakeCloudDataset(Dataset):
|
|
|
383
383
|
raise InvalidSourcePathError(
|
|
384
384
|
f"The dataset being connected is already accessible via Deep Lake path {self.path}"
|
|
385
385
|
)
|
|
386
|
+
|
|
387
|
+
def get_managed_creds_keys(self) -> List[str]:
|
|
388
|
+
"""Returns the list of creds keys added to the dataset that are managed by Activeloop platform. These are used to fetch external data in linked tensors."""
|
|
389
|
+
return list(self.link_creds.managed_creds_keys)
|
|
@@ -5,7 +5,11 @@ from deeplake.constants import ALL_CLOUD_PREFIXES
|
|
|
5
5
|
from deeplake.core.storage.deeplake_memory_object import DeepLakeMemoryObject
|
|
6
6
|
from deeplake.core.storage import StorageProvider, storage_factory
|
|
7
7
|
from deeplake.core.storage.s3 import S3Provider
|
|
8
|
-
from deeplake.util.exceptions import
|
|
8
|
+
from deeplake.util.exceptions import (
|
|
9
|
+
ManagedCredentialsNotFoundError,
|
|
10
|
+
MissingCredsError,
|
|
11
|
+
MissingManagedCredsError,
|
|
12
|
+
)
|
|
9
13
|
from deeplake.util.token import expires_in_to_expires_at, is_expired_token
|
|
10
14
|
from deeplake.client.log import logger
|
|
11
15
|
|
|
@@ -168,7 +172,7 @@ class LinkCreds(DeepLakeMemoryObject):
|
|
|
168
172
|
obj.creds_mapping = {k: i + 1 for i, k in enumerate(obj.creds_keys)}
|
|
169
173
|
obj.managed_creds_keys = set(d["managed_creds_keys"])
|
|
170
174
|
obj.used_creds_keys = set(d["used_creds_keys"])
|
|
171
|
-
if "ENV" in obj.used_creds_keys:
|
|
175
|
+
if "ENV" in obj.used_creds_keys and "ENV" not in obj.creds_keys:
|
|
172
176
|
obj.creds_keys = ["ENV"] + obj.creds_keys
|
|
173
177
|
obj.creds_mapping["ENV"] = 0
|
|
174
178
|
obj.is_dirty = False
|
|
@@ -224,9 +228,16 @@ class LinkCreds(DeepLakeMemoryObject):
|
|
|
224
228
|
assert self.client is not None
|
|
225
229
|
assert self.org_id is not None
|
|
226
230
|
for creds_key in self.managed_creds_keys:
|
|
227
|
-
|
|
231
|
+
try:
|
|
232
|
+
self.populate_single_managed_creds(creds_key, verbose=verbose)
|
|
233
|
+
except ManagedCredentialsNotFoundError:
|
|
234
|
+
logger.warning(
|
|
235
|
+
f"Credentials '{creds_key}' not found in Activeloop platform. "
|
|
236
|
+
f"Please make sure the credentials are added to the platform and reload the dataset. "
|
|
237
|
+
f"Alternatively, use ds.update_creds_key(key_name, managed=False) to disable the managed credentials.)"
|
|
238
|
+
)
|
|
228
239
|
|
|
229
|
-
def
|
|
240
|
+
def populate_single_managed_creds(self, creds_key: str, verbose: bool = True):
|
|
230
241
|
assert self.client is not None
|
|
231
242
|
assert self.org_id is not None
|
|
232
243
|
creds = self.fetch_managed_creds(creds_key, verbose=verbose)
|
|
@@ -96,9 +96,7 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
96
96
|
def linked_sample(
|
|
97
97
|
self, global_sample_index: int
|
|
98
98
|
) -> Union[LinkedSample, LinkedTiledSample]:
|
|
99
|
-
|
|
100
|
-
sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
|
|
101
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
|
|
99
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
102
100
|
if self._is_tiled_sample(global_sample_index):
|
|
103
101
|
path_array: np.ndarray = (
|
|
104
102
|
super()
|
|
@@ -114,11 +112,15 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
114
112
|
sample_path = self.get_path(global_sample_index, fetch_chunks=True)
|
|
115
113
|
return LinkedSample(sample_path, sample_creds_key)
|
|
116
114
|
|
|
115
|
+
def creds_key(self, global_sample_index: int):
|
|
116
|
+
sample_creds_encoded = self.creds_encoder.get_encoded_creds_key(
|
|
117
|
+
global_sample_index
|
|
118
|
+
)
|
|
119
|
+
return self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
|
|
120
|
+
|
|
117
121
|
def get_video_url(self, global_sample_index):
|
|
118
|
-
creds_encoder = self.creds_encoder
|
|
119
122
|
sample_path = self.get_path(global_sample_index)
|
|
120
|
-
|
|
121
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
|
|
123
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
122
124
|
storage = None
|
|
123
125
|
if sample_path.startswith(("gcs://", "gcp://", "s3://")):
|
|
124
126
|
provider_type = "s3" if sample_path.startswith("s3://") else "gcs"
|
|
@@ -166,10 +168,7 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
166
168
|
.path_array
|
|
167
169
|
)
|
|
168
170
|
|
|
169
|
-
|
|
170
|
-
global_sample_index
|
|
171
|
-
)
|
|
172
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
|
|
171
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
173
172
|
tiled_arrays = [
|
|
174
173
|
read_linked_sample(path, sample_creds_key, self.link_creds, False).array
|
|
175
174
|
for path in iter(path_array.flat)
|
|
@@ -195,10 +194,7 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
195
194
|
)
|
|
196
195
|
required_tile_paths = ordered_tile_paths[tiles_index]
|
|
197
196
|
|
|
198
|
-
|
|
199
|
-
global_sample_index
|
|
200
|
-
)
|
|
201
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
|
|
197
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
202
198
|
|
|
203
199
|
tiles = np.vectorize(
|
|
204
200
|
lambda path: read_linked_sample(
|
|
@@ -230,8 +226,7 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
230
226
|
sample_path = self.get_path(global_sample_index, fetch_chunks)
|
|
231
227
|
if not sample_path:
|
|
232
228
|
return None
|
|
233
|
-
|
|
234
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
|
|
229
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
235
230
|
return read_linked_sample(sample_path, sample_creds_key, self.link_creds, False)
|
|
236
231
|
|
|
237
232
|
@property
|
|
@@ -329,11 +324,9 @@ class LinkedChunkEngine(ChunkEngine):
|
|
|
329
324
|
local_sample_index, cast=cast, copy=copy, decompress=decompress
|
|
330
325
|
)[0]
|
|
331
326
|
|
|
332
|
-
creds_encoder = self.creds_encoder
|
|
333
327
|
if not sample_path:
|
|
334
328
|
return self.get_empty_sample()
|
|
335
|
-
|
|
336
|
-
sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded) # type: ignore
|
|
329
|
+
sample_creds_key = self.creds_key(global_sample_index)
|
|
337
330
|
read_sample = read_linked_sample(
|
|
338
331
|
sample_path, sample_creds_key, self.link_creds, False
|
|
339
332
|
)
|
|
@@ -219,7 +219,7 @@ def filter_with_compute(
|
|
|
219
219
|
|
|
220
220
|
try:
|
|
221
221
|
if progressbar:
|
|
222
|
-
result = compute.
|
|
222
|
+
result = compute.map_with_progress_bar(pg_filter_slice, idx, total_length=len(dataset)) # type: ignore
|
|
223
223
|
else:
|
|
224
224
|
result = compute.map(filter_slice, idx) # type: ignore
|
|
225
225
|
index_map = [k for x in result for k in x] # unfold the result map
|
|
@@ -446,7 +446,7 @@ def query_inplace(
|
|
|
446
446
|
]
|
|
447
447
|
|
|
448
448
|
if progressbar:
|
|
449
|
-
result = compute.
|
|
449
|
+
result = compute.map_with_progress_bar(pg_subquery, subdatasets, total_length=num_samples) # type: ignore
|
|
450
450
|
else:
|
|
451
451
|
result = compute.map(subquery, subdatasets) # type: ignore
|
|
452
452
|
|
|
@@ -406,7 +406,6 @@ class Tensor:
|
|
|
406
406
|
def clear(self):
|
|
407
407
|
"""Deletes all samples from the tensor"""
|
|
408
408
|
self.chunk_engine.clear()
|
|
409
|
-
sample_id_key = get_sample_id_tensor_key(self.key)
|
|
410
409
|
try:
|
|
411
410
|
for t in self._all_tensor_links():
|
|
412
411
|
t.chunk_engine.clear()
|
|
@@ -1039,9 +1038,6 @@ class Tensor:
|
|
|
1039
1038
|
else:
|
|
1040
1039
|
val = cast_to_type(val, tensor.dtype)
|
|
1041
1040
|
tensor[global_sample_index] = val
|
|
1042
|
-
# if self.meta.is_link and not has_shape_tensor:
|
|
1043
|
-
# func = get_link_transform("update_shape")
|
|
1044
|
-
# func(new_sample, link_creds=self.link_creds, tensor_meta=self.meta)
|
|
1045
1041
|
|
|
1046
1042
|
@invalid_view_op
|
|
1047
1043
|
def pop(self, index: Optional[int] = None):
|
|
@@ -1316,6 +1312,15 @@ class Tensor:
|
|
|
1316
1312
|
assert isinstance(self.chunk_engine, LinkedChunkEngine)
|
|
1317
1313
|
return self.chunk_engine.path(self.index, fetch_chunks=fetch_chunks)
|
|
1318
1314
|
|
|
1315
|
+
def creds_key(self):
|
|
1316
|
+
"""Return path data. Only applicable for linked tensors"""
|
|
1317
|
+
if not self.is_link:
|
|
1318
|
+
raise Exception(f"Only supported for linked tensors.")
|
|
1319
|
+
if self.index.values[0].subscriptable() or len(self.index.values) > 1:
|
|
1320
|
+
raise ValueError("_linked_sample can be used only on exatcly 1 sample.")
|
|
1321
|
+
assert isinstance(self.chunk_engine, LinkedChunkEngine)
|
|
1322
|
+
return self.chunk_engine.creds_key(self.index.values[0].value)
|
|
1323
|
+
|
|
1319
1324
|
def invalidate_libdeeplake_dataset(self):
|
|
1320
1325
|
"""Invalidates the libdeeplake dataset object."""
|
|
1321
1326
|
self.dataset.libdeeplake_dataset = None
|
|
@@ -15,7 +15,7 @@ def test_compute_with_progress_bar(scheduler):
|
|
|
15
15
|
|
|
16
16
|
compute = get_compute_provider(scheduler=scheduler, num_workers=2)
|
|
17
17
|
try:
|
|
18
|
-
r = compute.
|
|
18
|
+
r = compute.map_with_progress_bar(f, range(1000), 1000)
|
|
19
19
|
|
|
20
20
|
assert r is not None
|
|
21
21
|
assert len(r) == 1000
|
|
@@ -1280,6 +1280,56 @@ def test_none_rechunk_post_transform(local_ds):
|
|
|
1280
1280
|
assert num_chunks == 2
|
|
1281
1281
|
|
|
1282
1282
|
|
|
1283
|
+
@pytest.mark.parametrize("scheduler", ["serial", "threaded", "processed"])
|
|
1284
|
+
def test_transform_checkpointing(local_ds, scheduler):
|
|
1285
|
+
@deeplake.compute
|
|
1286
|
+
def upload(i, ds):
|
|
1287
|
+
if i == 45:
|
|
1288
|
+
raise Exception("test")
|
|
1289
|
+
ds.abc.append(i)
|
|
1290
|
+
|
|
1291
|
+
@deeplake.compute
|
|
1292
|
+
def double(data_in, ds):
|
|
1293
|
+
ds.abc.append(data_in.abc * 2)
|
|
1294
|
+
|
|
1295
|
+
data_in = list(range(100))
|
|
1296
|
+
|
|
1297
|
+
with local_ds as ds:
|
|
1298
|
+
ds.create_tensor("abc")
|
|
1299
|
+
|
|
1300
|
+
# not divisible by num_workers
|
|
1301
|
+
with pytest.raises(ValueError):
|
|
1302
|
+
upload().eval(
|
|
1303
|
+
data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=51
|
|
1304
|
+
)
|
|
1305
|
+
|
|
1306
|
+
# greater than len(data_in)
|
|
1307
|
+
with pytest.raises(ValueError):
|
|
1308
|
+
upload().eval(
|
|
1309
|
+
data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=102
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
# less than 10% of data_in, shows warning
|
|
1313
|
+
with pytest.warns(UserWarning, match="10%"):
|
|
1314
|
+
with pytest.raises(TransformError):
|
|
1315
|
+
upload().eval(
|
|
1316
|
+
data_in, ds, num_workers=2, scheduler=scheduler, checkpoint_interval=8
|
|
1317
|
+
)
|
|
1318
|
+
|
|
1319
|
+
assert len(ds.abc) == 40
|
|
1320
|
+
assert ds.abc.numpy(aslist=True) == list(range(40))
|
|
1321
|
+
with pytest.raises(ValueError):
|
|
1322
|
+
double().eval(ds, num_workers=2, scheduler=scheduler, checkpoint_interval=10)
|
|
1323
|
+
|
|
1324
|
+
# fix input data
|
|
1325
|
+
data_in[45] = 0
|
|
1326
|
+
|
|
1327
|
+
upload().eval(
|
|
1328
|
+
data_in[40:], ds, num_workers=2, scheduler=scheduler, checkpoint_interval=10
|
|
1329
|
+
)
|
|
1330
|
+
assert ds.abc.numpy(aslist=True) == data_in
|
|
1331
|
+
|
|
1332
|
+
|
|
1283
1333
|
def create_test_ds(path):
|
|
1284
1334
|
ds = deeplake.empty(path, overwrite=True)
|
|
1285
1335
|
ds.create_tensor("images", htype="image", sample_compression="jpg")
|