deeplake 3.9.7__tar.gz → 3.9.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.7/deeplake.egg-info → deeplake-3.9.9}/PKG-INFO +1 -1
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/__init__.py +1 -1
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/dataset.py +103 -53
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_api.py +56 -49
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_connect_datasets.py +27 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/client.py +10 -16
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/dataset.py +51 -33
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/deeplake_cloud_dataset.py +10 -6
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/view_entry.py +1 -1
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_compute.py +0 -2
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/transform/test_transform.py +0 -2
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/transform/transform.py +4 -4
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/requirements/plugins.txt +0 -1
- deeplake-3.9.9/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/check_installation.py +0 -4
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/compute.py +0 -10
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/exceptions.py +6 -19
- {deeplake-3.9.7 → deeplake-3.9.9/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake.egg-info/SOURCES.txt +0 -1
- deeplake-3.9.7/deeplake/core/compute/ray.py +0 -24
- deeplake-3.9.7/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.7 → deeplake-3.9.9}/LICENSE +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/MANIFEST.in +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/README.md +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/info.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/link.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/read.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/config.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/log.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/compression.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/constants.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compression.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/io.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/sample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tensor.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/convert_to_libdeeplake.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/dataloader.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/hooks.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/htype.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/image.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/json.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/link.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/path.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/split.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/storage.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/token.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/video.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake.egg-info/requires.txt +8 -8
- {deeplake-3.9.7 → deeplake-3.9.9}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/pyproject.toml +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/setup.cfg +0 -0
- {deeplake-3.9.7 → deeplake-3.9.9}/setup.py +0 -0
|
@@ -11,6 +11,7 @@ from deeplake.auto.unstructured.kaggle import download_kaggle_dataset
|
|
|
11
11
|
from deeplake.auto.unstructured.image_classification import ImageClassification
|
|
12
12
|
from deeplake.auto.unstructured.coco.coco import CocoDataset
|
|
13
13
|
from deeplake.auto.unstructured.yolo.yolo import YoloDataset
|
|
14
|
+
from deeplake.client.client import DeepLakeBackendClient
|
|
14
15
|
from deeplake.client.log import logger
|
|
15
16
|
from deeplake.core.dataset import Dataset, dataset_factory
|
|
16
17
|
from deeplake.core.dataset.indra_dataset_view import IndraDatasetView
|
|
@@ -70,6 +71,7 @@ from deeplake.util.exceptions import (
|
|
|
70
71
|
ReadOnlyModeError,
|
|
71
72
|
LockedException,
|
|
72
73
|
BadRequestException,
|
|
74
|
+
RenameError,
|
|
73
75
|
)
|
|
74
76
|
from deeplake.util.storage import (
|
|
75
77
|
get_storage_and_cache_chain,
|
|
@@ -91,6 +93,30 @@ def _check_indra_and_read_only_flags(indra: bool, read_only: Optional[bool]):
|
|
|
91
93
|
)
|
|
92
94
|
|
|
93
95
|
|
|
96
|
+
def _fetch_creds_from_key(
|
|
97
|
+
creds: Union[dict, str], org_id: Optional[str], token: Optional[str]
|
|
98
|
+
) -> Optional[str]:
|
|
99
|
+
if not isinstance(creds, dict):
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
if "creds_key" in creds:
|
|
103
|
+
if len(creds) != 1:
|
|
104
|
+
raise ValueError("Other creds values are not allowed with creds_key")
|
|
105
|
+
|
|
106
|
+
client = DeepLakeBackendClient(token)
|
|
107
|
+
creds_key = creds["creds_key"]
|
|
108
|
+
|
|
109
|
+
if not org_id:
|
|
110
|
+
raise ValueError("Please specify org_id when using creds_key")
|
|
111
|
+
|
|
112
|
+
creds.update(client.get_managed_creds(org_id, creds_key))
|
|
113
|
+
del creds["creds_key"]
|
|
114
|
+
|
|
115
|
+
return creds_key
|
|
116
|
+
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
94
120
|
class dataset:
|
|
95
121
|
@staticmethod
|
|
96
122
|
@spinner
|
|
@@ -119,28 +145,28 @@ class dataset:
|
|
|
119
145
|
|
|
120
146
|
Examples:
|
|
121
147
|
|
|
122
|
-
>>> ds = deeplake.dataset("hub://
|
|
123
|
-
>>> ds = deeplake.dataset("s3://mybucket/my_dataset")
|
|
124
|
-
>>> ds = deeplake.dataset("./datasets/my_dataset", overwrite=True)
|
|
148
|
+
>>> ds = deeplake.dataset("hub://org_id/dataset") # Initialize dataset managed by Deep Lake.
|
|
149
|
+
>>> ds = deeplake.dataset("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Initialize dataset stored in your cloud using your own credentials.
|
|
150
|
+
>>> ds = deeplake.dataset("./datasets/my_dataset", overwrite=True) # Overwrite the dataset currently at the path
|
|
125
151
|
|
|
126
152
|
Loading to a specfic version:
|
|
127
153
|
|
|
128
|
-
>>> ds = deeplake.dataset("hub://
|
|
129
|
-
>>> ds = deeplake.dataset("hub://
|
|
154
|
+
>>> ds = deeplake.dataset("hub://org_id/dataset@new_branch")
|
|
155
|
+
>>> ds = deeplake.dataset("hub://org_id/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
|
|
130
156
|
|
|
131
157
|
>>> my_commit_id = "3e49cded62b6b335c74ff07e97f8451a37aca7b2"
|
|
132
|
-
>>> ds = deeplake.dataset(f"hub://
|
|
158
|
+
>>> ds = deeplake.dataset(f"hub://org_id/dataset@{my_commit_id}")
|
|
133
159
|
|
|
134
160
|
Args:
|
|
135
161
|
path (str, pathlib.Path): - The full path to the dataset. Can be:
|
|
136
|
-
- a Deep Lake cloud path of the form ``hub://
|
|
162
|
+
- a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
|
|
137
163
|
- an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
|
|
138
164
|
- a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
|
|
139
165
|
- a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
|
|
140
166
|
- Loading to a specific version:
|
|
141
167
|
|
|
142
168
|
- You can also specify a ``commit_id`` or ``branch`` to load the dataset to that version directly by using the ``@`` symbol.
|
|
143
|
-
- The path will then be of the form ``hub://
|
|
169
|
+
- The path will then be of the form ``hub://org_id/dataset@{branch}`` or ``hub://org_id/dataset@{commit_id}``.
|
|
144
170
|
- See examples above.
|
|
145
171
|
runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub:// paths.
|
|
146
172
|
read_only (bool, optional): Opens dataset in read only mode if this is passed as ``True``. Defaults to ``False``.
|
|
@@ -152,9 +178,10 @@ class dataset:
|
|
|
152
178
|
creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
|
|
153
179
|
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
154
180
|
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
181
|
+
- To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
|
|
155
182
|
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
156
183
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
157
|
-
org_id (str, Optional): Organization id to be used for enabling high-performance features
|
|
184
|
+
org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
|
|
158
185
|
verbose (bool): If ``True``, logs will be printed. Defaults to ``True``.
|
|
159
186
|
access_method (str): The access method to use for the dataset. Can be:
|
|
160
187
|
|
|
@@ -223,12 +250,11 @@ class dataset:
|
|
|
223
250
|
path, address = process_dataset_path(path)
|
|
224
251
|
verify_dataset_name(path)
|
|
225
252
|
|
|
226
|
-
if org_id is not None and get_path_type(path) != "local":
|
|
227
|
-
raise ValueError("org_id parameter can only be used with local datasets")
|
|
228
|
-
|
|
229
253
|
if creds is None:
|
|
230
254
|
creds = {}
|
|
231
255
|
|
|
256
|
+
dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
|
|
257
|
+
|
|
232
258
|
db_engine = parse_runtime_parameters(path, runtime)["tensor_db"]
|
|
233
259
|
|
|
234
260
|
try:
|
|
@@ -285,6 +311,11 @@ class dataset:
|
|
|
285
311
|
"index_params": index_params,
|
|
286
312
|
}
|
|
287
313
|
|
|
314
|
+
if dataset_creds_key:
|
|
315
|
+
dataset_kwargs["dataset_creds_key"] = dataset_creds_key
|
|
316
|
+
dataset_kwargs["dataset_creds_key_org_id"] = org_id
|
|
317
|
+
dataset_kwargs["dataset_creds_key_token"] = token
|
|
318
|
+
|
|
288
319
|
if access_method == "stream":
|
|
289
320
|
dataset_kwargs.update(
|
|
290
321
|
{
|
|
@@ -339,6 +370,7 @@ class dataset:
|
|
|
339
370
|
path: Union[str, pathlib.Path],
|
|
340
371
|
creds: Optional[Union[Dict, str]] = None,
|
|
341
372
|
token: Optional[str] = None,
|
|
373
|
+
org_id: Optional[str] = None,
|
|
342
374
|
) -> bool:
|
|
343
375
|
"""Checks if a dataset exists at the given ``path``.
|
|
344
376
|
|
|
@@ -347,8 +379,10 @@ class dataset:
|
|
|
347
379
|
creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
|
|
348
380
|
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
349
381
|
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
382
|
+
- To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
|
|
350
383
|
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
351
384
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
385
|
+
org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
|
|
352
386
|
|
|
353
387
|
Returns:
|
|
354
388
|
A boolean confirming whether the dataset exists or not at the given path.
|
|
@@ -365,6 +399,9 @@ class dataset:
|
|
|
365
399
|
|
|
366
400
|
if creds is None:
|
|
367
401
|
creds = {}
|
|
402
|
+
|
|
403
|
+
_fetch_creds_from_key(creds, org_id, token)
|
|
404
|
+
|
|
368
405
|
try:
|
|
369
406
|
storage, cache_chain = get_storage_and_cache_chain(
|
|
370
407
|
path=path,
|
|
@@ -397,6 +434,13 @@ class dataset:
|
|
|
397
434
|
) -> Dataset:
|
|
398
435
|
"""Creates an empty Deep Lake dataset.
|
|
399
436
|
|
|
437
|
+
Examples:
|
|
438
|
+
|
|
439
|
+
>>> ds = deeplake.empty("hub://org_id/dataset") # Create dataset in the default storage for your organization.
|
|
440
|
+
>>> ds = deeplake.empty("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Create dataset stored in your cloud using your own credentials.
|
|
441
|
+
>>> ds = deeplake.empty("s3://mybucket/my_dataset", creds = {"creds_key": "managed_creds_key"}, org_id = "my_org_id") # Create dataset stored in your cloud using Deep Lake managed credentials.
|
|
442
|
+
>>> ds = deeplake.empty("./datasets/my_dataset", overwrite=True) # Overwrite the dataset currently at the path
|
|
443
|
+
|
|
400
444
|
Args:
|
|
401
445
|
path (str, pathlib.Path): - The full path to the dataset. It can be:
|
|
402
446
|
- a Deep Lake cloud path of the form ``hub://org_id/dataset_name``. Requires registration with Deep Lake.
|
|
@@ -411,9 +455,10 @@ class dataset:
|
|
|
411
455
|
creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
|
|
412
456
|
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
413
457
|
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
458
|
+
- To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
|
|
414
459
|
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
415
460
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
416
|
-
org_id (str, Optional): Organization id to be used for enabling high-performance features
|
|
461
|
+
org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
|
|
417
462
|
verbose (bool): If True, logs will be printed. Defaults to True.
|
|
418
463
|
lock_timeout (int): Number of seconds to wait before throwing a LockException. If None, wait indefinitely
|
|
419
464
|
lock_enabled (bool): If true, the dataset manages a write lock. NOTE: Only set to False if you are managing concurrent access externally.
|
|
@@ -434,8 +479,6 @@ class dataset:
|
|
|
434
479
|
"""
|
|
435
480
|
path, address = process_dataset_path(path)
|
|
436
481
|
|
|
437
|
-
if org_id is not None and get_path_type(path) != "local":
|
|
438
|
-
raise ValueError("org_id parameter can only be used with local datasets")
|
|
439
482
|
db_engine = parse_runtime_parameters(path, runtime)["tensor_db"]
|
|
440
483
|
|
|
441
484
|
if address:
|
|
@@ -448,6 +491,8 @@ class dataset:
|
|
|
448
491
|
if creds is None:
|
|
449
492
|
creds = {}
|
|
450
493
|
|
|
494
|
+
dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
|
|
495
|
+
|
|
451
496
|
try:
|
|
452
497
|
storage, cache_chain = get_storage_and_cache_chain(
|
|
453
498
|
path=path,
|
|
@@ -506,6 +551,9 @@ class dataset:
|
|
|
506
551
|
"lock_timeout": lock_timeout,
|
|
507
552
|
"lock_enabled": lock_enabled,
|
|
508
553
|
"index_params": index_params,
|
|
554
|
+
"dataset_creds_key": dataset_creds_key,
|
|
555
|
+
"dataset_creds_key_org_id": org_id,
|
|
556
|
+
"dataset_creds_key_token": token,
|
|
509
557
|
}
|
|
510
558
|
ret = dataset._load(dataset_kwargs, create=True)
|
|
511
559
|
return ret
|
|
@@ -530,32 +578,32 @@ class dataset:
|
|
|
530
578
|
lock_enabled: Optional[bool] = True,
|
|
531
579
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
532
580
|
) -> Dataset:
|
|
533
|
-
"""Loads an existing Deep Lake dataset
|
|
581
|
+
"""Loads an existing Deep Lake dataset.
|
|
534
582
|
|
|
535
583
|
Examples:
|
|
536
584
|
|
|
537
|
-
>>> ds = deeplake.load("hub://
|
|
538
|
-
>>> ds = deeplake.load("s3://mybucket/my_dataset")
|
|
539
|
-
>>> ds = deeplake.load("
|
|
585
|
+
>>> ds = deeplake.load("hub://org_id/dataset") # Load dataset managed by Deep Lake.
|
|
586
|
+
>>> ds = deeplake.load("s3://mybucket/my_dataset", creds = {"aws_access_key_id": ..., ...}) # Load dataset stored in your cloud using your own credentials.
|
|
587
|
+
>>> ds = deeplake.load("s3://mybucket/my_dataset", creds = {"creds_key": "managed_creds_key"}, org_id = "my_org_id") # Load dataset stored in your cloud using Deep Lake managed credentials.
|
|
540
588
|
|
|
541
589
|
Loading to a specfic version:
|
|
542
590
|
|
|
543
|
-
>>> ds = deeplake.load("hub://
|
|
544
|
-
>>> ds = deeplake.load("hub://
|
|
591
|
+
>>> ds = deeplake.load("hub://org_id/dataset@new_branch")
|
|
592
|
+
>>> ds = deeplake.load("hub://org_id/dataset@3e49cded62b6b335c74ff07e97f8451a37aca7b2)
|
|
545
593
|
|
|
546
594
|
>>> my_commit_id = "3e49cded62b6b335c74ff07e97f8451a37aca7b2"
|
|
547
|
-
>>> ds = deeplake.load(f"hub://
|
|
595
|
+
>>> ds = deeplake.load(f"hub://org_id/dataset@{my_commit_id}")
|
|
548
596
|
|
|
549
597
|
Args:
|
|
550
598
|
path (str, pathlib.Path): - The full path to the dataset. Can be:
|
|
551
|
-
- a Deep Lake cloud path of the form ``hub://
|
|
599
|
+
- a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
|
|
552
600
|
- an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
|
|
553
601
|
- a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
|
|
554
602
|
- a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
|
|
555
603
|
- Loading to a specific version:
|
|
556
604
|
|
|
557
605
|
- You can also specify a ``commit_id`` or ``branch`` to load the dataset to that version directly by using the ``@`` symbol.
|
|
558
|
-
- The path will then be of the form ``hub://
|
|
606
|
+
- The path will then be of the form ``hub://org_id/dataset@{branch}`` or ``hub://org_id/dataset@{commit_id}``.
|
|
559
607
|
- See examples above.
|
|
560
608
|
read_only (bool, optional): Opens dataset in read only mode if this is passed as ``True``. Defaults to ``False``.
|
|
561
609
|
Datasets stored on Deep Lake cloud that your account does not have write access to will automatically open in read mode.
|
|
@@ -564,6 +612,7 @@ class dataset:
|
|
|
564
612
|
creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
|
|
565
613
|
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
566
614
|
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
615
|
+
- To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
|
|
567
616
|
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
568
617
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
569
618
|
org_id (str, Optional): Organization id to be used for enabling high-performance features. Only applicable for local datasets.
|
|
@@ -632,8 +681,7 @@ class dataset:
|
|
|
632
681
|
if creds is None:
|
|
633
682
|
creds = {}
|
|
634
683
|
|
|
635
|
-
|
|
636
|
-
raise ValueError("org_id parameter can only be used with local datasets")
|
|
684
|
+
dataset_creds_key = _fetch_creds_from_key(creds, org_id, token)
|
|
637
685
|
|
|
638
686
|
try:
|
|
639
687
|
storage, cache_chain = get_storage_and_cache_chain(
|
|
@@ -681,6 +729,11 @@ class dataset:
|
|
|
681
729
|
"index_params": index_params,
|
|
682
730
|
}
|
|
683
731
|
|
|
732
|
+
if dataset_creds_key:
|
|
733
|
+
dataset_kwargs["dataset_creds_key"] = dataset_creds_key
|
|
734
|
+
dataset_kwargs["dataset_creds_key_org_id"] = org_id
|
|
735
|
+
dataset_kwargs["dataset_creds_key_token"] = token
|
|
736
|
+
|
|
684
737
|
if access_method == "stream":
|
|
685
738
|
dataset_kwargs.update(
|
|
686
739
|
{
|
|
@@ -803,45 +856,37 @@ class dataset:
|
|
|
803
856
|
|
|
804
857
|
@staticmethod
|
|
805
858
|
def rename(
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
creds: Optional[Union[dict, str]] = None,
|
|
859
|
+
path: str,
|
|
860
|
+
new_name: str,
|
|
809
861
|
token: Optional[str] = None,
|
|
810
862
|
) -> Dataset:
|
|
811
|
-
"""Renames dataset at ``
|
|
863
|
+
"""Renames managed dataset at ``path`` to ``new_name``.
|
|
812
864
|
|
|
813
865
|
Examples:
|
|
814
|
-
|
|
815
|
-
>>> deeplake.rename("hub://
|
|
816
|
-
>>> deeplake.rename("s3://mybucket/my_ds", "s3://mybucket/renamed_ds")
|
|
866
|
+
>>> deeplake.rename("hub://username/image_ds", "new_ds")
|
|
867
|
+
>>> deeplake.rename("hub://org_id/image_ds", "new_ds") ## New dataset path is `hub://org_id/new_ds`
|
|
817
868
|
|
|
818
869
|
Args:
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
823
|
-
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
824
|
-
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
825
|
-
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
870
|
+
path (str): The path to the dataset to be renamed.
|
|
871
|
+
new_name (str): New name of the dataset.
|
|
872
|
+
token (str, optional): Activeloop token for accessing the dataset
|
|
826
873
|
|
|
827
874
|
Returns:
|
|
828
875
|
Dataset: The renamed Dataset.
|
|
829
876
|
|
|
830
877
|
Raises:
|
|
831
|
-
|
|
878
|
+
RenameError: If a Dataset does not exist at the given path or if new path is not a managed dataset.
|
|
832
879
|
"""
|
|
833
|
-
old_path = convert_pathlib_to_string_if_needed(old_path)
|
|
834
|
-
new_path = convert_pathlib_to_string_if_needed(new_path)
|
|
835
880
|
|
|
836
|
-
if
|
|
837
|
-
|
|
881
|
+
if not path.startswith("hub://"):
|
|
882
|
+
raise RenameError("Rename is not available for non-managed datasets")
|
|
838
883
|
|
|
839
|
-
feature_report_path(
|
|
884
|
+
feature_report_path(path, "rename", {}, token=token)
|
|
840
885
|
|
|
841
|
-
deeplake.
|
|
842
|
-
|
|
886
|
+
ds = deeplake.load(path, verbose=False, token=token)
|
|
887
|
+
ds.rename(new_name)
|
|
843
888
|
|
|
844
|
-
return
|
|
889
|
+
return ds
|
|
845
890
|
|
|
846
891
|
@staticmethod
|
|
847
892
|
@spinner
|
|
@@ -852,6 +897,7 @@ class dataset:
|
|
|
852
897
|
creds: Optional[Union[dict, str]] = None,
|
|
853
898
|
token: Optional[str] = None,
|
|
854
899
|
verbose: bool = False,
|
|
900
|
+
org_id: Optional[str] = None,
|
|
855
901
|
) -> None:
|
|
856
902
|
"""Deletes a dataset at a given path.
|
|
857
903
|
|
|
@@ -863,9 +909,11 @@ class dataset:
|
|
|
863
909
|
creds (dict, str, optional): The string ``ENV`` or a dictionary containing credentials used to access the dataset at the path.
|
|
864
910
|
- If 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token' are present, these take precedence over credentials present in the environment or in credentials file. Currently only works with s3 paths.
|
|
865
911
|
- It supports 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'endpoint_url', 'aws_region', 'profile_name' as keys.
|
|
912
|
+
- To use credentials managed in your Activeloop organization, use they key 'creds_key': 'managed_key_name'. This requires the org_id dataset argument to be set.
|
|
866
913
|
- If 'ENV' is passed, credentials are fetched from the environment variables. This is also the case when creds is not passed for cloud datasets. For datasets connected to hub cloud, specifying 'ENV' will override the credentials fetched from Activeloop and use local ones.
|
|
867
914
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
868
915
|
verbose (bool): If True, logs will be printed. Defaults to True.
|
|
916
|
+
org_id (str, Optional): Organization id to be used for enabling high-performance features and credential lookup.
|
|
869
917
|
|
|
870
918
|
Raises:
|
|
871
919
|
DatasetHandlerError: If a Dataset does not exist at the given path and ``force = False``.
|
|
@@ -886,6 +934,8 @@ class dataset:
|
|
|
886
934
|
if creds is None:
|
|
887
935
|
creds = {}
|
|
888
936
|
|
|
937
|
+
_fetch_creds_from_key(creds, org_id, token)
|
|
938
|
+
|
|
889
939
|
feature_report_path(
|
|
890
940
|
path, "delete", {"Force": force, "Large_OK": large_ok}, token=token
|
|
891
941
|
)
|
|
@@ -1136,7 +1186,7 @@ class dataset:
|
|
|
1136
1186
|
dest_creds (dict, optional): creds required to create / overwrite datasets at ``dest``.
|
|
1137
1187
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
1138
1188
|
num_workers (int): The number of workers to use for copying. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
|
|
1139
|
-
scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded',
|
|
1189
|
+
scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', and 'processed'.
|
|
1140
1190
|
Defaults to 'threaded'.
|
|
1141
1191
|
progressbar (bool): Displays a progress bar if True (default).
|
|
1142
1192
|
**kwargs (dict): Additional keyword arguments
|
|
@@ -1222,7 +1272,7 @@ class dataset:
|
|
|
1222
1272
|
dest_creds (dict, optional): creds required to create / overwrite datasets at ``dest``.
|
|
1223
1273
|
token (str, optional): Activeloop token, used for fetching credentials to the dataset at path if it is a Deep Lake dataset. This is optional, tokens are normally autogenerated.
|
|
1224
1274
|
num_workers (int): The number of workers to use for copying. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
|
|
1225
|
-
scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded',
|
|
1275
|
+
scheduler (str): The scheduler to be used for copying. Supported values include: 'serial', 'threaded', and 'processed'.
|
|
1226
1276
|
Defaults to 'threaded'.
|
|
1227
1277
|
progressbar (bool): Displays a progress bar if True (default).
|
|
1228
1278
|
public (bool): Defines if the dataset will have public access. Applicable only if Deep Lake cloud storage is used and a new Dataset is being created. Defaults to ``False``.
|
|
@@ -1943,7 +1993,7 @@ class dataset:
|
|
|
1943
1993
|
tag (str): Kaggle dataset tag. Example: ``"coloradokb/dandelionimages"`` points to https://www.kaggle.com/coloradokb/dandelionimages
|
|
1944
1994
|
src (str, pathlib.Path): Local path to where the raw kaggle dataset will be downlaoded to.
|
|
1945
1995
|
dest (str, pathlib.Path): - The full path to the dataset. Can be:
|
|
1946
|
-
- a Deep Lake cloud path of the form ``hub://
|
|
1996
|
+
- a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
|
|
1947
1997
|
- an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
|
|
1948
1998
|
- a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
|
|
1949
1999
|
- a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
|
|
@@ -2056,7 +2106,7 @@ class dataset:
|
|
|
2056
2106
|
src (pd.DataFrame): The pandas dataframe to be converted.
|
|
2057
2107
|
dest (str, pathlib.Path):
|
|
2058
2108
|
- A Dataset or The full path to the dataset. Can be:
|
|
2059
|
-
- a Deep Lake cloud path of the form ``hub://
|
|
2109
|
+
- a Deep Lake cloud path of the form ``hub://org_id/datasetname``. To write to Deep Lake cloud datasets, ensure that you are authenticated to Deep Lake (pass in a token using the 'token' parameter).
|
|
2060
2110
|
- an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
|
|
2061
2111
|
- a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
|
|
2062
2112
|
- a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import sys
|
|
3
|
+
import time
|
|
4
|
+
|
|
3
5
|
import numpy as np
|
|
4
6
|
import pathlib
|
|
5
7
|
import pytest
|
|
@@ -924,63 +926,28 @@ def test_invalid_token():
|
|
|
924
926
|
)
|
|
925
927
|
|
|
926
928
|
|
|
927
|
-
@pytest.mark.
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
("local_ds_generator", "local_path", "hub_cloud_dev_token"),
|
|
931
|
-
pytest.param(
|
|
932
|
-
"s3_ds_generator", "s3_path", "hub_cloud_dev_token", marks=pytest.mark.slow
|
|
933
|
-
),
|
|
934
|
-
pytest.param(
|
|
935
|
-
"gcs_ds_generator",
|
|
936
|
-
"gcs_path",
|
|
937
|
-
"hub_cloud_dev_token",
|
|
938
|
-
marks=pytest.mark.slow,
|
|
939
|
-
),
|
|
940
|
-
pytest.param(
|
|
941
|
-
"azure_ds_generator",
|
|
942
|
-
"azure_path",
|
|
943
|
-
"hub_cloud_dev_token",
|
|
944
|
-
marks=pytest.mark.slow,
|
|
945
|
-
),
|
|
946
|
-
pytest.param(
|
|
947
|
-
"hub_cloud_ds_generator",
|
|
948
|
-
"hub_cloud_path",
|
|
949
|
-
"hub_cloud_dev_token",
|
|
950
|
-
marks=pytest.mark.slow,
|
|
951
|
-
),
|
|
952
|
-
],
|
|
953
|
-
indirect=True,
|
|
954
|
-
)
|
|
955
|
-
@pytest.mark.parametrize("convert_to_pathlib", [True, False])
|
|
956
|
-
def test_dataset_rename(ds_generator, path, hub_token, convert_to_pathlib):
|
|
957
|
-
ds = ds_generator()
|
|
929
|
+
@pytest.mark.slow
|
|
930
|
+
def test_dataset_rename(hub_cloud_ds_generator, hub_cloud_path, hub_cloud_dev_token):
|
|
931
|
+
ds = hub_cloud_ds_generator()
|
|
958
932
|
ds.create_tensor("abc")
|
|
959
933
|
ds.abc.append([1, 2, 3, 4])
|
|
934
|
+
original_path = ds.path
|
|
960
935
|
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
ds.path = convert_string_to_pathlib_if_needed(ds.path, convert_to_pathlib)
|
|
964
|
-
new_path = convert_string_to_pathlib_if_needed(new_path, convert_to_pathlib)
|
|
936
|
+
new_name = f"renamed-{time.time()}"
|
|
937
|
+
new_path = original_path.rsplit("/", 1)[0] + "/" + new_name
|
|
965
938
|
|
|
966
939
|
with pytest.raises(RenameError):
|
|
967
|
-
ds.rename("
|
|
940
|
+
ds.rename("hub://wrong_org/test")
|
|
968
941
|
|
|
969
|
-
|
|
970
|
-
with pytest.raises(BadRequestException):
|
|
971
|
-
ds.rename(ds.path)
|
|
972
|
-
else:
|
|
973
|
-
with pytest.raises(PathNotEmptyException):
|
|
974
|
-
ds.rename(ds.path)
|
|
942
|
+
ds.rename(new_name)
|
|
975
943
|
|
|
976
|
-
|
|
977
|
-
assert ds.path == str(new_path)
|
|
944
|
+
assert ds.path.endswith("/" + new_name)
|
|
978
945
|
assert_array_equal(ds.abc.numpy(), np.array([[1, 2, 3, 4]]))
|
|
979
946
|
|
|
980
|
-
ds = deeplake.load(new_path, token=
|
|
947
|
+
ds = deeplake.load(new_path, token=hub_cloud_dev_token)
|
|
981
948
|
assert_array_equal(ds.abc.numpy(), np.array([[1, 2, 3, 4]]))
|
|
982
949
|
|
|
983
|
-
deeplake.delete(new_path, token=
|
|
950
|
+
deeplake.delete(new_path, token=hub_cloud_dev_token)
|
|
984
951
|
|
|
985
952
|
|
|
986
953
|
@pytest.mark.parametrize(
|
|
@@ -2807,13 +2774,13 @@ def test_slice_shape_interval(memory_ds):
|
|
|
2807
2774
|
|
|
2808
2775
|
|
|
2809
2776
|
def test_non_local_org_id():
|
|
2810
|
-
with pytest.raises(
|
|
2777
|
+
with pytest.raises(InvalidTokenException):
|
|
2811
2778
|
ds = deeplake.dataset("hub://test/test_dataset", org_id="test")
|
|
2812
2779
|
|
|
2813
|
-
with pytest.raises(
|
|
2780
|
+
with pytest.raises(InvalidTokenException):
|
|
2814
2781
|
ds = deeplake.empty("hub://test/test_dataset", org_id="test")
|
|
2815
2782
|
|
|
2816
|
-
with pytest.raises(
|
|
2783
|
+
with pytest.raises(InvalidTokenException):
|
|
2817
2784
|
ds = deeplake.load("hub://test/test_dataset", org_id="test")
|
|
2818
2785
|
|
|
2819
2786
|
with pytest.raises(ValueError):
|
|
@@ -3030,3 +2997,43 @@ def test_append_non_uint8_to_image(local_ds):
|
|
|
3030
2997
|
ds.images.append(np.zeros((40, 40, 1), dtype=np.uint8))
|
|
3031
2998
|
|
|
3032
2999
|
assert ds.images.dtype.name == "int16"
|
|
3000
|
+
|
|
3001
|
+
|
|
3002
|
+
@pytest.mark.slow
|
|
3003
|
+
def test_create_and_load_with_managed_credentials(
|
|
3004
|
+
hub_cloud_path: str, hub_cloud_dev_token
|
|
3005
|
+
):
|
|
3006
|
+
old_environ = dict(os.environ)
|
|
3007
|
+
os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
|
3008
|
+
os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
|
3009
|
+
os.environ.pop("AWS_SESSION_TOKEN", None)
|
|
3010
|
+
|
|
3011
|
+
try:
|
|
3012
|
+
dir_name = hub_cloud_path.rsplit("/", 1)[1]
|
|
3013
|
+
ds = deeplake.empty(
|
|
3014
|
+
f"s3://deeplake-tests/{dir_name}",
|
|
3015
|
+
creds={"creds_key": "aws_creds"},
|
|
3016
|
+
org_id="testingacc2",
|
|
3017
|
+
token=hub_cloud_dev_token,
|
|
3018
|
+
)
|
|
3019
|
+
ds.create_tensor("id", htype="text")
|
|
3020
|
+
|
|
3021
|
+
assert ds.path == f"s3://deeplake-tests/{dir_name}"
|
|
3022
|
+
|
|
3023
|
+
ds = deeplake.load(
|
|
3024
|
+
f"s3://deeplake-tests/{dir_name}",
|
|
3025
|
+
creds={"creds_key": "aws_creds"},
|
|
3026
|
+
org_id="testingacc2",
|
|
3027
|
+
token=hub_cloud_dev_token,
|
|
3028
|
+
)
|
|
3029
|
+
assert "id" in ds.tensors
|
|
3030
|
+
|
|
3031
|
+
deeplake.delete(
|
|
3032
|
+
f"s3://deeplake-tests/{dir_name}",
|
|
3033
|
+
creds={"creds_key": "aws_creds"},
|
|
3034
|
+
org_id="testingacc2",
|
|
3035
|
+
token=hub_cloud_dev_token,
|
|
3036
|
+
)
|
|
3037
|
+
finally:
|
|
3038
|
+
os.environ.clear()
|
|
3039
|
+
os.environ.update(old_environ)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
import pytest
|
|
2
4
|
|
|
3
5
|
import deeplake
|
|
@@ -87,3 +89,28 @@ def test_connect_user_not_in_org(s3_ds_generator, hub_cloud_dev_token):
|
|
|
87
89
|
token=hub_cloud_dev_token,
|
|
88
90
|
)
|
|
89
91
|
assert "organization id" in str(e)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# @pytest.mark.slow
|
|
95
|
+
def test_connect_from_managed_credentials(hub_cloud_path: str, hub_cloud_dev_token):
|
|
96
|
+
old_environ = dict(os.environ)
|
|
97
|
+
os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
|
98
|
+
os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
|
99
|
+
os.environ.pop("AWS_SESSION_TOKEN", None)
|
|
100
|
+
|
|
101
|
+
try:
|
|
102
|
+
dir_name = hub_cloud_path.rsplit("/", 1)[1]
|
|
103
|
+
ds = deeplake.empty(
|
|
104
|
+
f"s3://deeplake-tests/{dir_name}",
|
|
105
|
+
creds={"creds_key": "aws_creds"},
|
|
106
|
+
org_id="testingacc2",
|
|
107
|
+
token=hub_cloud_dev_token,
|
|
108
|
+
)
|
|
109
|
+
ds.create_tensor("id", htype="text")
|
|
110
|
+
|
|
111
|
+
ds.connect()
|
|
112
|
+
assert ds.path == f"hub://testingacc2/{dir_name}"
|
|
113
|
+
|
|
114
|
+
finally:
|
|
115
|
+
os.environ.clear()
|
|
116
|
+
os.environ.update(old_environ)
|