deeplake 3.9.6__tar.gz → 3.9.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.6/deeplake.egg-info → deeplake-3.9.7}/PKG-INFO +1 -1
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/__init__.py +1 -1
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/dataset.py +8 -7
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/dataset.py +90 -67
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tensor.py +2 -4
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform.py +73 -40
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/convert_to_libdeeplake.py +1 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/dataloader.py +5 -3
- deeplake-3.9.7/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.6 → deeplake-3.9.7/deeplake.egg-info}/PKG-INFO +1 -1
- deeplake-3.9.6/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.6 → deeplake-3.9.7}/LICENSE +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/MANIFEST.in +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/README.md +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/info.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/link.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/read.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/client.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/config.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/log.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/compression.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/constants.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compression.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/io.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/sample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/hooks.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/htype.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/image.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/json.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/link.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/path.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/split.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/storage.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/token.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/video.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/requires.txt +7 -7
- {deeplake-3.9.6 → deeplake-3.9.7}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/pyproject.toml +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/setup.cfg +0 -0
- {deeplake-3.9.6 → deeplake-3.9.7}/setup.py +0 -0
|
@@ -395,7 +395,7 @@ class dataset:
|
|
|
395
395
|
verbose: bool = True,
|
|
396
396
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
397
397
|
) -> Dataset:
|
|
398
|
-
"""Creates an empty dataset
|
|
398
|
+
"""Creates an empty Deep Lake dataset.
|
|
399
399
|
|
|
400
400
|
Args:
|
|
401
401
|
path (str, pathlib.Path): - The full path to the dataset. It can be:
|
|
@@ -530,7 +530,7 @@ class dataset:
|
|
|
530
530
|
lock_enabled: Optional[bool] = True,
|
|
531
531
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
532
532
|
) -> Dataset:
|
|
533
|
-
"""Loads an existing dataset
|
|
533
|
+
"""Loads an existing Deep Lake dataset
|
|
534
534
|
|
|
535
535
|
Examples:
|
|
536
536
|
|
|
@@ -1120,7 +1120,8 @@ class dataset:
|
|
|
1120
1120
|
progressbar=True,
|
|
1121
1121
|
**kwargs,
|
|
1122
1122
|
):
|
|
1123
|
-
"""Copies dataset at ``src`` to ``dest``. Version control history is not included.
|
|
1123
|
+
"""Copies dataset at ``src`` to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
|
|
1124
|
+
For fast copying, we recommend using ``deepake.deepcopy()`` instead.
|
|
1124
1125
|
|
|
1125
1126
|
Args:
|
|
1126
1127
|
src (str, Dataset, pathlib.Path): The Dataset or the path to the dataset to be copied.
|
|
@@ -1206,7 +1207,7 @@ class dataset:
|
|
|
1206
1207
|
verbose: bool = True,
|
|
1207
1208
|
**kwargs,
|
|
1208
1209
|
):
|
|
1209
|
-
"""Copies dataset at ``src`` to ``dest`` including version control history.
|
|
1210
|
+
"""Copies dataset at ``src`` to ``dest`` including version control history. This is the fastest method for copying datasets.
|
|
1210
1211
|
|
|
1211
1212
|
Args:
|
|
1212
1213
|
src (str, pathlib.Path, Dataset): The Dataset or the path to the dataset to be copied.
|
|
@@ -1452,9 +1453,9 @@ class dataset:
|
|
|
1452
1453
|
|
|
1453
1454
|
Examples:
|
|
1454
1455
|
>>> # Connect an s3 dataset
|
|
1455
|
-
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key"
|
|
1456
|
+
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key")
|
|
1456
1457
|
>>> # or
|
|
1457
|
-
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key"
|
|
1458
|
+
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key")
|
|
1458
1459
|
|
|
1459
1460
|
Args:
|
|
1460
1461
|
src_path (str): Cloud path to the source dataset. Can be:
|
|
@@ -1653,9 +1654,9 @@ class dataset:
|
|
|
1653
1654
|
>>> "path/to/data/directory",
|
|
1654
1655
|
>>> dest="hub://org_id/dataset",
|
|
1655
1656
|
>>> allow_no_annotation=True,
|
|
1656
|
-
>>> token="my_activeloop_token",
|
|
1657
1657
|
>>> num_workers=4,
|
|
1658
1658
|
>>> )
|
|
1659
|
+
|
|
1659
1660
|
>>> # Ingest data from your cloud into another Deep Lake dataset in your cloud, and connect that dataset to the Deep Lake backend.
|
|
1660
1661
|
>>> ds = deeplake.ingest_yolo(
|
|
1661
1662
|
>>> "s3://bucket/data_directory",
|
|
@@ -372,7 +372,7 @@ class Dataset:
|
|
|
372
372
|
return self._client
|
|
373
373
|
|
|
374
374
|
def __len__(self, warn: bool = True):
|
|
375
|
-
"""Returns the length of the
|
|
375
|
+
"""Returns the length (number of rows) of the shortest tensor in the dataset."""
|
|
376
376
|
tensor_lengths = [len(tensor) for tensor in self.tensors.values()]
|
|
377
377
|
pad_tensors = self._pad_tensors
|
|
378
378
|
if (
|
|
@@ -390,7 +390,7 @@ class Dataset:
|
|
|
390
390
|
|
|
391
391
|
@property
|
|
392
392
|
def max_len(self):
|
|
393
|
-
"""
|
|
393
|
+
"""Returns the length (number of rows) of the longest tensor in the dataset."""
|
|
394
394
|
return (
|
|
395
395
|
max([len(tensor) for tensor in self.tensors.values()])
|
|
396
396
|
if self.tensors
|
|
@@ -399,7 +399,7 @@ class Dataset:
|
|
|
399
399
|
|
|
400
400
|
@property
|
|
401
401
|
def min_len(self):
|
|
402
|
-
"""
|
|
402
|
+
"""Returns the length (number of rows) of the shortest tensor in the dataset."""
|
|
403
403
|
return (
|
|
404
404
|
min([len(tensor) for tensor in self.tensors.values()])
|
|
405
405
|
if self.tensors
|
|
@@ -668,32 +668,34 @@ class Dataset:
|
|
|
668
668
|
tiling_threshold: Optional[int] = None,
|
|
669
669
|
**kwargs,
|
|
670
670
|
):
|
|
671
|
-
"""Creates a new tensor in the dataset.
|
|
671
|
+
"""Creates a new tensor in the Deep Lake dataset. Specifying the tensor's htype is highly recommended for complex data such as images, video, dicom, text, json, etc.
|
|
672
|
+
Specifying htype is not necessary for simple numeric date such as arrays and scalars.
|
|
672
673
|
|
|
673
674
|
Examples:
|
|
674
|
-
>>> #
|
|
675
|
-
>>> ds = deeplake.
|
|
676
|
-
|
|
677
|
-
>>> # create tensors
|
|
678
|
-
>>> ds.create_tensor("images", htype="image", sample_compression="jpg")
|
|
679
|
-
>>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
|
|
680
|
-
>>> ds.create_tensor("data")
|
|
681
|
-
>>> ds.create_tensor("point_clouds", htype="point_cloud")
|
|
675
|
+
>>> # Create dataset
|
|
676
|
+
>>> ds = deeplake.empty("path/to/dataset")
|
|
682
677
|
|
|
683
|
-
>>> #
|
|
684
|
-
>>> ds
|
|
685
|
-
>>>
|
|
686
|
-
>>>
|
|
678
|
+
>>> # Create tensors
|
|
679
|
+
>>> with ds:
|
|
680
|
+
>>> ds.create_tensor("images", htype="image", sample_compression="jpg")
|
|
681
|
+
>>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
|
|
682
|
+
>>> ds.create_tensor("data")
|
|
683
|
+
>>> ds.create_tensor("point_clouds", htype="point_cloud")
|
|
684
|
+
>>>
|
|
685
|
+
>>> # Append data
|
|
686
|
+
>>> ds.images.append(np.ones((400, 400, 3), dtype='uint8'))
|
|
687
|
+
>>> ds.videos.append(deeplake.read("videos/sample_video.mp4"))
|
|
688
|
+
>>> ds.data.append(np.zeros((100, 100, 2)))
|
|
687
689
|
|
|
688
690
|
Args:
|
|
689
691
|
name (str): The name of the tensor to be created.
|
|
690
692
|
htype (str):
|
|
691
|
-
- The class of data for the tensor.
|
|
692
|
-
- The defaults for other parameters are determined in terms of this value.
|
|
693
|
+
- The class of data for the tensor. Specifying ``htype`` is highly recommended for complex data such as images, video, dicom, text, json, etc. Specifying htype is not necessary for simple numeric date such as arrays and scalars.
|
|
694
|
+
- The defaults for other parameters are determined in terms of this value, thus improving error checking and visualization.
|
|
693
695
|
- For example, ``htype="image"`` would have ``dtype`` default to ``uint8``.
|
|
694
696
|
- These defaults can be overridden by explicitly passing any of the other parameters to this function.
|
|
695
697
|
- May also modify the defaults for other parameters.
|
|
696
|
-
dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``.
|
|
698
|
+
dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``. Specifying ``dtype`` is typically not necessary unless you are dealing with mixed numerical data in a single tensor (column).
|
|
697
699
|
sample_compression (str): All samples will be compressed in the provided format. If ``None``, samples are uncompressed. For ``link[]`` tensors, ``sample_compression`` is used only for optimizing dataset views.
|
|
698
700
|
chunk_compression (str): All chunks will be compressed in the provided format. If ``None``, chunks are uncompressed. For ``link[]`` tensors, ``chunk_compression`` is used only for optimizing dataset views.
|
|
699
701
|
hidden (bool): If ``True``, the tensor will be hidden from ds.tensors but can still be accessed via ``ds[tensor_name]``.
|
|
@@ -1178,7 +1180,7 @@ class Dataset:
|
|
|
1178
1180
|
def create_tensor_like(
|
|
1179
1181
|
self, name: str, source: "Tensor", unlink: bool = False
|
|
1180
1182
|
) -> "Tensor":
|
|
1181
|
-
"""
|
|
1183
|
+
"""Creates a tensor with the same properties as ``source``. No samples or data copied, other than the tensor meta/info.
|
|
1182
1184
|
|
|
1183
1185
|
Examples:
|
|
1184
1186
|
>>> ds.create_tensor_like("cats", ds["images"])
|
|
@@ -2093,13 +2095,16 @@ class Dataset:
|
|
|
2093
2095
|
*args,
|
|
2094
2096
|
**kwargs,
|
|
2095
2097
|
):
|
|
2096
|
-
"""
|
|
2098
|
+
"""Creates a PyTorch Dataloader from the Deep Lake dataset. During iteration, the data from all tensors will be streamed on-the-fly from the storage location.
|
|
2099
|
+
Understanding the parameters below is critical for achieving fast streaming for your use-case
|
|
2097
2100
|
|
|
2098
2101
|
Args:
|
|
2099
2102
|
*args: Additional args to be passed to torch_dataset
|
|
2100
2103
|
**kwargs: Additional kwargs to be passed to torch_dataset
|
|
2101
2104
|
transform (Callable, Optional): Transformation function to be applied to each sample.
|
|
2102
|
-
tensors (List, Optional):
|
|
2105
|
+
tensors (List, Optional): List of tensors to load. If ``None``, all tensors are loaded. Defaults to ``None``.
|
|
2106
|
+
For datasets with many tensors, its extremely important to stream only the data that is needed for training the model, in order to avoid bottlenecks associated with streaming unused data.
|
|
2107
|
+
For example, if you have a dataset that has ``image``, ``label``, and ``metadata`` tensors, if ``tensors=["image", "label"]``, the Data Loader will only stream the ``image`` and ``label`` tensors.
|
|
2103
2108
|
num_workers (int): The number of workers to use for fetching data in parallel.
|
|
2104
2109
|
batch_size (int): Number of samples per batch to load. Default value is 1.
|
|
2105
2110
|
drop_last (bool): Set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
|
|
@@ -2564,10 +2569,10 @@ class Dataset:
|
|
|
2564
2569
|
|
|
2565
2570
|
@spinner
|
|
2566
2571
|
def flush(self):
|
|
2567
|
-
"""
|
|
2568
|
-
Writes all the
|
|
2569
|
-
|
|
2570
|
-
|
|
2572
|
+
"""
|
|
2573
|
+
Writes all the data that has been changed/assigned from the cache layers (if any) to the underlying storage.
|
|
2574
|
+
|
|
2575
|
+
NOTE: The high-level APIs flush the cache automatically and users generally do not have explicitly run the ``flush`` command.
|
|
2571
2576
|
"""
|
|
2572
2577
|
self._flush_vc_info()
|
|
2573
2578
|
self.storage.flush()
|
|
@@ -2582,11 +2587,13 @@ class Dataset:
|
|
|
2582
2587
|
|
|
2583
2588
|
def clear_cache(self):
|
|
2584
2589
|
"""
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2590
|
+
Flushes (see :func:`Dataset.flush`) the contents of the cache layers (if any) and then deletes contents of all the layers of it.
|
|
2591
|
+
This doesn't delete data from the actual storage.
|
|
2592
|
+
This is useful if you have multiple datasets with memory caches open, taking up too much RAM, or when local cache is no longer needed for certain datasets and is taking up storage space.
|
|
2593
|
+
|
|
2594
|
+
NOTE: The high-level APIs clear the cache automatically and users generally do not have explicitly run the ``clear_cache`` command.
|
|
2589
2595
|
"""
|
|
2596
|
+
|
|
2590
2597
|
if hasattr(self.storage, "clear_cache"):
|
|
2591
2598
|
self.storage.clear_cache()
|
|
2592
2599
|
|
|
@@ -2628,7 +2635,7 @@ class Dataset:
|
|
|
2628
2635
|
|
|
2629
2636
|
@invalid_view_op
|
|
2630
2637
|
def delete(self, large_ok=False):
|
|
2631
|
-
"""Deletes the entire dataset from the cache layers (if any)
|
|
2638
|
+
"""Deletes the entire dataset from the underlying storage and cache layers (if any).
|
|
2632
2639
|
This is an **IRREVERSIBLE** operation. Data once deleted can not be recovered.
|
|
2633
2640
|
|
|
2634
2641
|
Args:
|
|
@@ -2672,10 +2679,10 @@ class Dataset:
|
|
|
2672
2679
|
self.storage.clear()
|
|
2673
2680
|
|
|
2674
2681
|
def summary(self, force: bool = False):
|
|
2675
|
-
"""Prints a summary of the dataset.
|
|
2682
|
+
"""Prints a summary of the dataset, including the tensor names and their lengths, shapes, htypes, dtypes, compressions, and other relevant information.
|
|
2676
2683
|
|
|
2677
2684
|
Args:
|
|
2678
|
-
force (bool): Dataset views with more than 10000 samples might take
|
|
2685
|
+
force (bool): Dataset views with more than 10000 samples might take several seconds of minutes to summarize. If `force=True`,
|
|
2679
2686
|
the summary will be printed regardless. An error will be raised otherwise.
|
|
2680
2687
|
|
|
2681
2688
|
Raises:
|
|
@@ -2976,7 +2983,8 @@ class Dataset:
|
|
|
2976
2983
|
return self[fullname]
|
|
2977
2984
|
|
|
2978
2985
|
def create_group(self, name: str, exist_ok=False) -> "Dataset":
|
|
2979
|
-
"""Creates a tensor group
|
|
2986
|
+
"""Creates a tensor group, which is a collection of tensors that can be reference together.
|
|
2987
|
+
Groups are recommended for use-cases with many tensors in order to organize and reference data more easily.
|
|
2980
2988
|
|
|
2981
2989
|
Args:
|
|
2982
2990
|
name: The name of the group to create.
|
|
@@ -2992,11 +3000,17 @@ class Dataset:
|
|
|
2992
3000
|
Examples:
|
|
2993
3001
|
|
|
2994
3002
|
>>> ds.create_group("images")
|
|
2995
|
-
>>> ds
|
|
3003
|
+
>>> ds.images.create_tensor("left_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3004
|
+
>>> ds.images.create_tensor("center_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3005
|
+
>>> ds.images.create_tensor("right_camera", htype = "image", "sample_compression" = "jpeg")
|
|
2996
3006
|
|
|
2997
|
-
>>>
|
|
2998
|
-
>>> ds[
|
|
2999
|
-
>>>
|
|
3007
|
+
>>> # Data from a tensors in groups is referenced using:
|
|
3008
|
+
>>> ds.images.right_camera[0].numpy()
|
|
3009
|
+
>>> # OR
|
|
3010
|
+
>>> ds["images/right_camera"].numpy()
|
|
3011
|
+
|
|
3012
|
+
>>> # "/" Notation can also be used to create groups with ``create_tensor``
|
|
3013
|
+
>>> ds.create_tensor("images/right_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3000
3014
|
"""
|
|
3001
3015
|
|
|
3002
3016
|
deeplake_reporter.feature_report(
|
|
@@ -3104,15 +3118,6 @@ class Dataset:
|
|
|
3104
3118
|
Exception: Error while attempting to rollback appends.
|
|
3105
3119
|
SampleAppendingError: Error that occurs when someone tries to append a tensor value directly to the dataset without specifying tensor name.
|
|
3106
3120
|
|
|
3107
|
-
Examples:
|
|
3108
|
-
|
|
3109
|
-
>>> ds = deeplake.empty("../test/test_ds")
|
|
3110
|
-
>>> ds.create_tensor('data')
|
|
3111
|
-
Tensor(key='data')
|
|
3112
|
-
>>> ds.create_tensor('labels')
|
|
3113
|
-
Tensor(key='labels')
|
|
3114
|
-
>>> ds.append({"data": [1, 2, 3, 4], "labels":[0, 1, 2, 3]})
|
|
3115
|
-
|
|
3116
3121
|
"""
|
|
3117
3122
|
tensors = self.tensors
|
|
3118
3123
|
if isinstance(sample, Dataset):
|
|
@@ -3205,10 +3210,10 @@ class Dataset:
|
|
|
3205
3210
|
ignore_errors: bool = False,
|
|
3206
3211
|
progressbar: bool = False,
|
|
3207
3212
|
):
|
|
3208
|
-
"""Appends multiple rows
|
|
3213
|
+
"""Appends multiple samples (rows) to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
|
|
3209
3214
|
|
|
3210
3215
|
Args:
|
|
3211
|
-
samples (Dict[str, Any]): Dictionary with tensor names as keys and
|
|
3216
|
+
samples (Dict[str, Any]): Dictionary with tensor names as keys and data as values. The values can be a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
|
|
3212
3217
|
skip_ok (bool): Skip tensors not in ``samples`` if set to True.
|
|
3213
3218
|
append_empty (bool): Append empty samples to tensors not specified in ``sample`` if set to ``True``. If True, ``skip_ok`` is ignored.
|
|
3214
3219
|
ignore_errors (bool): Skip samples that cause errors while extending, if set to ``True``.
|
|
@@ -3221,6 +3226,18 @@ class Dataset:
|
|
|
3221
3226
|
NotImplementedError: If an error occurs while writing tiles.
|
|
3222
3227
|
SampleExtendError: If the extend failed while appending a sample.
|
|
3223
3228
|
Exception: Error while attempting to rollback appends.
|
|
3229
|
+
|
|
3230
|
+
Examples:
|
|
3231
|
+
|
|
3232
|
+
>>> ds = deeplake.empty("../test/test_ds")
|
|
3233
|
+
|
|
3234
|
+
>>> with ds:
|
|
3235
|
+
>>> ds.create_tensor('data')
|
|
3236
|
+
>>> ds.create_tensor('labels')
|
|
3237
|
+
|
|
3238
|
+
>>> # This operation will append 4 samples (rows) to the Deep Lake dataset
|
|
3239
|
+
>>> ds.extend({"data": [1, 2, 3, 4], "labels":["table", "chair", "desk", "table"]})
|
|
3240
|
+
|
|
3224
3241
|
"""
|
|
3225
3242
|
extend = False
|
|
3226
3243
|
if isinstance(samples, Dataset):
|
|
@@ -3283,7 +3300,7 @@ class Dataset:
|
|
|
3283
3300
|
skip_ok: bool = False,
|
|
3284
3301
|
append_empty: bool = False,
|
|
3285
3302
|
):
|
|
3286
|
-
"""Append
|
|
3303
|
+
"""Append a single sample (row) to multiple tensors at once.
|
|
3287
3304
|
|
|
3288
3305
|
Args:
|
|
3289
3306
|
sample (dict): Dictionary with tensor names as keys and samples as values.
|
|
@@ -3301,11 +3318,13 @@ class Dataset:
|
|
|
3301
3318
|
Examples:
|
|
3302
3319
|
|
|
3303
3320
|
>>> ds = deeplake.empty("../test/test_ds")
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
>>>
|
|
3307
|
-
|
|
3308
|
-
|
|
3321
|
+
|
|
3322
|
+
>>> with ds:
|
|
3323
|
+
>>> ds.create_tensor('data')
|
|
3324
|
+
>>> ds.create_tensor('labels')
|
|
3325
|
+
|
|
3326
|
+
>>> # This operation will append 1 sample (row) to the Deep Lake dataset
|
|
3327
|
+
>>> ds.append({"data": 1, "labels": "table"})
|
|
3309
3328
|
|
|
3310
3329
|
"""
|
|
3311
3330
|
new_row_ids = [self.__len__(warn=False)]
|
|
@@ -4300,12 +4319,12 @@ class Dataset:
|
|
|
4300
4319
|
progressbar=True,
|
|
4301
4320
|
public: bool = False,
|
|
4302
4321
|
):
|
|
4303
|
-
"""Copies this dataset or dataset view to ``dest``. Version control history is not included.
|
|
4322
|
+
"""Copies this dataset or dataset view to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
|
|
4304
4323
|
|
|
4305
4324
|
Args:
|
|
4306
4325
|
dest (str, pathlib.Path): Destination dataset or path to copy to. If a Dataset instance is provided, it is expected to be empty.
|
|
4307
4326
|
tensors (List[str], optional): Names of tensors (and groups) to be copied. If not specified all tensors are copied.
|
|
4308
|
-
runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub
|
|
4327
|
+
runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub://... paths.
|
|
4309
4328
|
overwrite (bool): If ``True`` and a dataset exists at `destination`, it will be overwritten. Defaults to False.
|
|
4310
4329
|
creds (dict, Optional): creds required to create / overwrite datasets at `dest`.
|
|
4311
4330
|
token (str, Optional): token used to for fetching credentials to `dest`.
|
|
@@ -4398,21 +4417,25 @@ class Dataset:
|
|
|
4398
4417
|
ds_name: Optional[str] = None,
|
|
4399
4418
|
token: Optional[str] = None,
|
|
4400
4419
|
):
|
|
4401
|
-
"""Connect a Deep Lake cloud
|
|
4420
|
+
"""Connect a Deep Lake dataset stored in your cloud to the Deep Lake App.
|
|
4402
4421
|
|
|
4403
4422
|
Examples:
|
|
4404
|
-
>>> #
|
|
4405
|
-
>>> s3_ds = deeplake.
|
|
4406
|
-
>>>
|
|
4407
|
-
>>> #
|
|
4408
|
-
>>> ds = s3_ds.connect(org_id="
|
|
4423
|
+
>>> # Load an s3 dataset
|
|
4424
|
+
>>> s3_ds = deeplake.load("s3://bucket/dataset")
|
|
4425
|
+
>>>
|
|
4426
|
+
>>> # Specify the org_id and managed creds_key, and the dataset name is inferred from the currently storage location path
|
|
4427
|
+
>>> ds = s3_ds.connect(org_id="my_org_id", creds_key="managed_creds_key")
|
|
4428
|
+
>>>
|
|
4429
|
+
>>> # Or if you want to explicitly specify the full path, which contains the org_id
|
|
4430
|
+
>>> ds = s3_ds.connect(dest_path="hub://my_org/dataset", creds_key="managed_creds_key")
|
|
4431
|
+
|
|
4409
4432
|
|
|
4410
4433
|
Args:
|
|
4411
4434
|
creds_key (str): The managed credentials to be used for accessing the source path.
|
|
4412
|
-
dest_path (str, optional): The full path to
|
|
4413
|
-
a Deep Lake path like ``hub
|
|
4414
|
-
org_id (str, optional): The organization to
|
|
4415
|
-
ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or
|
|
4435
|
+
dest_path (str, optional): The full path to which the connected Deep Lake dataset will reside. Can be:
|
|
4436
|
+
a Deep Lake path like ``hub://<org_id>/<dataset_name>``
|
|
4437
|
+
org_id (str, optional): The organization to which the connected Deep Lake dataset will be added. The dataset name will be the same as parent folder for the dataset in the cloud storage location.
|
|
4438
|
+
ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or storage location path if not provided.
|
|
4416
4439
|
token (str, optional): Activeloop token used to fetch the managed credentials.
|
|
4417
4440
|
|
|
4418
4441
|
Raises:
|
|
@@ -326,8 +326,7 @@ class Tensor:
|
|
|
326
326
|
progressbar: bool = False,
|
|
327
327
|
ignore_errors: bool = False,
|
|
328
328
|
):
|
|
329
|
-
"""Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence
|
|
330
|
-
or a sequence of :func:`deeplake.read` outputs, which can be used to load files. See examples down below.
|
|
329
|
+
"""Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
|
|
331
330
|
|
|
332
331
|
Example:
|
|
333
332
|
Numpy input:
|
|
@@ -429,8 +428,7 @@ class Tensor:
|
|
|
429
428
|
|
|
430
429
|
@invalid_view_op
|
|
431
430
|
def append(self, sample: InputSample):
|
|
432
|
-
"""Appends a single sample to the end of the tensor.
|
|
433
|
-
which can be used to load files. See examples down below.
|
|
431
|
+
"""Appends a single sample (row) to the end of the tensor.
|
|
434
432
|
|
|
435
433
|
Examples:
|
|
436
434
|
Numpy input:
|
|
@@ -34,19 +34,18 @@ from deeplake.util.exceptions import (
|
|
|
34
34
|
HubComposeEmptyListError,
|
|
35
35
|
HubComposeIncompatibleFunction,
|
|
36
36
|
TransformError,
|
|
37
|
-
SampleAppendError,
|
|
38
37
|
)
|
|
39
38
|
from deeplake.hooks import dataset_written, dataset_read
|
|
40
39
|
from deeplake.util.version_control import auto_checkout
|
|
41
40
|
from deeplake.util.class_label import sync_labels
|
|
42
41
|
from deeplake.constants import DEFAULT_TRANSFORM_SAMPLE_CACHE_SIZE
|
|
43
42
|
|
|
44
|
-
import posixpath
|
|
45
|
-
|
|
46
43
|
|
|
47
44
|
class ComputeFunction:
|
|
48
45
|
def __init__(self, func, args, kwargs, name: Optional[str] = None):
|
|
49
|
-
"""Creates a ComputeFunction object that can be evaluated using
|
|
46
|
+
"""Creates a ComputeFunction object that can be evaluated using ``.eval()`` or used as a part of a Pipeline.
|
|
47
|
+
Compute Functions are evaluated in parallel, where the input data iterable is devided between the workers for processing.
|
|
48
|
+
"""
|
|
50
49
|
self.func = func
|
|
51
50
|
self.args = args
|
|
52
51
|
self.kwargs = kwargs
|
|
@@ -68,37 +67,58 @@ class ComputeFunction:
|
|
|
68
67
|
ignore_errors: bool = False,
|
|
69
68
|
**kwargs,
|
|
70
69
|
):
|
|
71
|
-
"""
|
|
70
|
+
"""
|
|
71
|
+
Evaluates the ComputeFunction on ``data_in`` to produce an output dataset ``ds_out``. The purpose of compute functions is to process the input data in parallel,
|
|
72
|
+
which is useful when rapidly ingesting data to a Deep Lake dataset. Compute Functions can also be executed in-place, where it modifies the input dataset (see ``ds_out`` parameters below) instead of writing to a new dataset.
|
|
72
73
|
|
|
73
74
|
Args:
|
|
74
|
-
data_in: Input passed to the transform to generate output dataset. Should support
|
|
75
|
-
ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided,
|
|
76
|
-
|
|
75
|
+
data_in: Input passed to the transform to generate output dataset. Should support ``__getitem__`` and ``__len__`` operations. Can be a Deep Lake dataset.
|
|
76
|
+
ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, the ComputeFunction will operate in-place, which means that data will be written to tensors in ``data_in`` .
|
|
77
|
+
All tensors modified in the ComputeFunction should already be defined in ``ds_out``. It's initial state should be either:
|
|
77
78
|
- Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
|
|
78
79
|
- All tensors are populated and have same length. In this case new samples are appended to the dataset.
|
|
79
80
|
num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
|
|
80
|
-
scheduler (str): The scheduler to be used to compute the transformation. Supported values include:
|
|
81
|
-
Defaults to
|
|
81
|
+
scheduler (str): The scheduler to be used to compute the transformation. Supported values include: ``serial``, ``threaded``, and ``processed``.
|
|
82
|
+
Defaults to ``threaded``.
|
|
82
83
|
progressbar (bool): Displays a progress bar if True (default).
|
|
83
|
-
skip_ok (bool): If True
|
|
84
|
+
skip_ok (bool): If ``True``, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
|
|
84
85
|
This is especially useful for inplace transformations in which certain tensors are not modified. Defaults to False.
|
|
85
|
-
check_lengths (bool): If True
|
|
86
|
-
pad_data_in (bool): NOTE: This is only applicable if data_in is a Deep Lake dataset. If True, pads tensors of data_in to match the length of the largest tensor in data_in.
|
|
86
|
+
check_lengths (bool): If ``True``, checks whether ds_out has tensors of same lengths initially.
|
|
87
|
+
pad_data_in (bool): NOTE: This is only applicable if ``data_in`` is a Deep Lake dataset. If True, pads tensors of data_in to match the length of the largest tensor in data_in.
|
|
87
88
|
Defaults to False.
|
|
88
89
|
read_only_ok (bool): If ``True`` and output dataset is same as input dataset, the read-only check is skipped. This can be used to read data in parallel without making changes to underlying dataset.
|
|
89
90
|
Defaults to False.
|
|
90
91
|
cache_size (int): Cache size to be used by transform per worker.
|
|
91
|
-
checkpoint_interval (int): If > 0, the
|
|
92
|
-
If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if num_workers > 0. Defaults to 0.
|
|
92
|
+
checkpoint_interval (int): If > 0, the ComputeFunction will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
|
|
93
|
+
If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if ``num_workers`` > 0. Defaults to 0.
|
|
93
94
|
ignore_errors (bool): If ``True``, input samples that causes transform to fail will be skipped and the errors will be ignored **if possible**.
|
|
94
95
|
**kwargs: Additional arguments.
|
|
95
96
|
|
|
96
97
|
Raises:
|
|
97
|
-
InvalidInputDataError: If data_in passed to transform is invalid. It should support
|
|
98
|
+
InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as data_in will also raise this.
|
|
98
99
|
InvalidOutputDatasetError: If all the tensors of ds_out passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ds_out will also raise this.
|
|
99
100
|
TensorMismatchError: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
|
|
100
|
-
UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include:
|
|
101
|
+
UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: ``serial``, ``threaded``, and ``processed``.
|
|
102
|
+
TransformError: All other exceptions raised if there are problems while running the ComputeFunction.
|
|
101
103
|
ValueError: If ``num_workers`` > 0 and ``checkpoint_interval`` is not a multiple of ``num_workers`` or if ``checkpoint_interval`` > 0 and ds_out is None.
|
|
104
|
+
AllSamplesSkippedError: If all samples are skipped during execution of the Pipeline.
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
# Suppose we have a list of dictionaries that we want to upload in parallel to a Deep Lake dataset.
|
|
108
|
+
data_in = [{"label": "cat", "score": 0.9}, {"label": "dog", "score": 0.8}]
|
|
109
|
+
|
|
110
|
+
# First, we define a function that takes a single element of the `data_in` list and uploads it to a Deep Lake dataset
|
|
111
|
+
@deeplake.compute
|
|
112
|
+
def data_upload(data_dict_in, sample_out, score_multiplier):
|
|
113
|
+
label = data_dict_in["label"]
|
|
114
|
+
score = data_dict_in["score"] * score_multiplier
|
|
115
|
+
sample_out.append({"label": label, "score": score})
|
|
116
|
+
|
|
117
|
+
# To evaluate the function, static parameters (if any) are specified to the function input, and the input data iterable and output dataset are specified to `.eval(...)`
|
|
118
|
+
data_upload(score_multiplier).eval(data_in, ds_out, scheduler="threaded", num_workers=4)
|
|
119
|
+
|
|
120
|
+
Note:
|
|
121
|
+
``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.
|
|
102
122
|
"""
|
|
103
123
|
|
|
104
124
|
pipeline = Pipeline([self])
|
|
@@ -124,7 +144,7 @@ class ComputeFunction:
|
|
|
124
144
|
|
|
125
145
|
class Pipeline:
|
|
126
146
|
def __init__(self, functions: List[ComputeFunction]):
|
|
127
|
-
"""Takes a list of functions decorated using :func:`deeplake.compute` and creates a pipeline that can be evaluated using
|
|
147
|
+
"""Takes a list of functions decorated using :func:`deeplake.compute` and creates a pipeline that can be evaluated using ``.eval()``"""
|
|
128
148
|
self.functions = functions
|
|
129
149
|
|
|
130
150
|
def __len__(self):
|
|
@@ -147,16 +167,18 @@ class Pipeline:
|
|
|
147
167
|
verbose: bool = True,
|
|
148
168
|
**kwargs,
|
|
149
169
|
):
|
|
150
|
-
"""
|
|
170
|
+
"""
|
|
171
|
+
Evaluates the Pipeline of ComputeFunctions on ``data_in`` to produce an output dataset ``ds_out``. The purpose of compute functions is to process the input data in parallel,
|
|
172
|
+
which is useful when rapidly ingesting data to a Deep Lake dataset. Pipelines can also be executed in-place, where it modifies the input dataset (see ``ds_out`` parameters below) instead of writing to a new dataset.
|
|
151
173
|
|
|
152
174
|
Args:
|
|
153
|
-
data_in: Input passed to the transform to generate output dataset. Should support
|
|
154
|
-
ds_out (Dataset, optional):
|
|
155
|
-
|
|
156
|
-
-
|
|
157
|
-
-
|
|
175
|
+
data_in: Input passed to the transform to generate output dataset. Should support ``__getitem__`` and ``__len__`` operations. Can be a Deep Lake dataset.
|
|
176
|
+
ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, the ComputeFunction will operate in-place, which means that data will be written to tensors in ``data_in`` .
|
|
177
|
+
All tensors modified in the ComputeFunction should already be defined in ``ds_out``. It's initial state should be either:
|
|
178
|
+
- Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
|
|
179
|
+
- All tensors are populated and have same length. In this case new samples are appended to the dataset.
|
|
158
180
|
num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
|
|
159
|
-
scheduler (str): The scheduler to be used to compute the transformation. Supported values include:
|
|
181
|
+
scheduler (str): The scheduler to be used to compute the transformation. Supported values include: ``serial``, ``threaded``, and ``processed``.
|
|
160
182
|
Defaults to 'threaded'.
|
|
161
183
|
progressbar (bool): Displays a progress bar if ``True`` (default).
|
|
162
184
|
skip_ok (bool): If ``True``, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
|
|
@@ -167,38 +189,49 @@ class Pipeline:
|
|
|
167
189
|
read_only_ok (bool): If ``True`` and output dataset is same as input dataset, the read-only check is skipped.
|
|
168
190
|
Defaults to False.
|
|
169
191
|
cache_size (int): Cache size to be used by transform per worker.
|
|
170
|
-
checkpoint_interval (int): If > 0, the
|
|
171
|
-
If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if num_workers > 0. Defaults to 0.
|
|
192
|
+
checkpoint_interval (int): If > 0, the ComputeFunction will be checkpointed with a commit every ``checkpoint_interval`` input samples to avoid restarting full transform due to intermitten failures. If the transform is interrupted, the intermediate data is deleted and the dataset is reset to the last commit.
|
|
193
|
+
If <= 0, no checkpointing is done. Checkpoint interval should be a multiple of num_workers if ``num_workers`` > 0. Defaults to 0.
|
|
172
194
|
ignore_errors (bool): If ``True``, input samples that causes transform to fail will be skipped and the errors will be ignored **if possible**.
|
|
173
195
|
verbose (bool): If ``True``, prints additional information about the transform.
|
|
174
196
|
**kwargs: Additional arguments.
|
|
175
197
|
|
|
176
198
|
Raises:
|
|
177
|
-
InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support
|
|
199
|
+
InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than ``threaded`` with deeplake dataset having base storage as memory as ``data_in`` will also raise this.
|
|
178
200
|
InvalidOutputDatasetError: If all the tensors of ``ds_out`` passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``ds_out`` will also raise this.
|
|
179
201
|
TensorMismatchError: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
|
|
180
|
-
UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include:
|
|
202
|
+
UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: ``serial``, ``threaded``, and ``processed``.
|
|
181
203
|
TransformError: All other exceptions raised if there are problems while running the pipeline.
|
|
182
204
|
ValueError: If ``num_workers`` > 0 and ``checkpoint_interval`` is not a multiple of ``num_workers`` or if ``checkpoint_interval`` > 0 and ds_out is None.
|
|
183
|
-
|
|
205
|
+
AllSamplesSkippedError: If all samples are skipped during execution of the Pipeline.
|
|
206
|
+
ModuleNotInstalledException: If the module ``ray`` is not installed and the scheduler is set to ``ray``.
|
|
184
207
|
|
|
185
208
|
# noqa: DAR401
|
|
186
209
|
|
|
187
|
-
Example
|
|
210
|
+
Example:
|
|
211
|
+
|
|
212
|
+
# Suppose we have a series of operations that we want to perform in parallel on images using reusable pipelines.
|
|
213
|
+
# We use the pipeline to ingest the transfomed data from one dataset to another dataset.
|
|
214
|
+
|
|
215
|
+
# First, we define the ComputeFunctions that will be used in the pipeline
|
|
216
|
+
@deeplake.compute
|
|
217
|
+
def flip_vertical(sample_in, sample_out):
|
|
218
|
+
sample_out.append({'labels': sample_in.labels.numpy(),
|
|
219
|
+
'images': np.flip(sample_in.images.numpy(), axis = 0)})
|
|
188
220
|
|
|
189
221
|
@deeplake.compute
|
|
190
|
-
def
|
|
191
|
-
|
|
222
|
+
def resize(sample_in, sample_out, new_size):
|
|
223
|
+
sample_out.append({"labels": sample_in.labels.numpy(),
|
|
224
|
+
"images": np.array(Image.fromarray(sample_in.images.numpy()).resize(new_size))})
|
|
192
225
|
|
|
193
|
-
#
|
|
226
|
+
# Append the label and image to the output sample
|
|
227
|
+
sample_out.labels.append(sample_in.labels.numpy())
|
|
228
|
+
sample_out.images.append(np.array(Image.fromarray(sample_in.images.numpy()).resize(new_size)))
|
|
194
229
|
|
|
195
|
-
#
|
|
196
|
-
|
|
197
|
-
my_fn(arg0, arg1).eval(data_in, ds_out, scheduler="threaded", num_workers=5)
|
|
230
|
+
# We can define the pipeline using:
|
|
231
|
+
pipeline = deeplake.compose([flip_vertical(), resize(new_size = (64,64))])
|
|
198
232
|
|
|
199
|
-
#
|
|
200
|
-
pipeline
|
|
201
|
-
pipeline.eval(data_in, ds_out, scheduler="processed", num_workers=2)
|
|
233
|
+
# Finally, we can evaluate the pipeline using:
|
|
234
|
+
pipeline.eval(ds_in, ds_out, num_workers = 4)
|
|
202
235
|
|
|
203
236
|
Note:
|
|
204
237
|
``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.
|
|
@@ -149,6 +149,7 @@ def _get_indra_ds_from_s3_provider(
|
|
|
149
149
|
read_only=provider.read_only,
|
|
150
150
|
token=token,
|
|
151
151
|
origin_path=provider.root,
|
|
152
|
+
profile_name=provider.profile_name,
|
|
152
153
|
aws_access_key_id=provider.aws_access_key_id,
|
|
153
154
|
aws_secret_access_key=provider.aws_secret_access_key,
|
|
154
155
|
aws_session_token=provider.aws_session_token,
|