deeplake 3.9.5__tar.gz → 3.9.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deeplake-3.9.5/deeplake.egg-info → deeplake-3.9.7}/PKG-INFO +1 -1
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/__init__.py +1 -1
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/dataset.py +8 -7
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compression.py +28 -3
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/dataset.py +90 -67
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/sample.py +10 -3
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tensor.py +2 -4
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/transform/transform.py +73 -40
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/test_deepmemory.py +9 -9
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/convert_to_libdeeplake.py +1 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/dataloader.py +25 -19
- deeplake-3.9.7/deeplake/util/bugout_token.py +1 -0
- {deeplake-3.9.5 → deeplake-3.9.7/deeplake.egg-info}/PKG-INFO +1 -1
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake.egg-info/requires.txt +11 -11
- {deeplake-3.9.5 → deeplake-3.9.7}/setup.py +1 -1
- deeplake-3.9.5/deeplake/util/bugout_token.py +0 -1
- {deeplake-3.9.5 → deeplake-3.9.7}/LICENSE +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/MANIFEST.in +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/README.md +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/link_tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/read.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_access_method.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_agreement.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_api.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_api_tiling.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_api_with_compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_chunk_sizes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_connect_datasets.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_dicom.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_downsample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_events.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_grayscale.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_hosted_datasets.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_insertion_out_of_order.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_json.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_link_tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_linking.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_mesh.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_nifti.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_none.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_partial_upload.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_pickle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_point_cloud.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_polygons.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_pop.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_readonly.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_rechunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_reset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_sample_info.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_tag.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_text.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_update_samples.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tests/test_views.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/api/tiled.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/structured/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/structured/base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/structured/dataframe.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/tests/test_coco_template.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/tests/test_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/tests/test_kaggle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/tests/test_yolo_template.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/coco.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/convert.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/coco/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/image_classification.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/kaggle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/util.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/auto/unstructured/yolo/yolo.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/auth/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/auth/activeloop.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/auth/auth_context.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/auth/azure.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/auth/test_auth_context.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/client.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/config.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/log.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/test_client.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/client/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/constants.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/base_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/chunk_compressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/sample_compressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/test_chunk_compressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/test_sample_compressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/test_uncompressed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk/uncompressed_chunk.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/process.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/provider.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/ray.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/serial.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/compute/thread.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/deeplake_cloud_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/indra_dataset_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/indra_tensor_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/invalid_view.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/dataset/view_entry.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/distance_type.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/fast_forwarding.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/index/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/index/index.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/index_maintenance.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/io.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/ipc.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/link_creds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/linked_chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/linked_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/linked_tiled_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/lock.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/dataset_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/base_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/byte_positions.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/chunk_id.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/creds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/pad.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/sequence.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/shape.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_byte_positions_encoder_updates.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_chunk_id_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tests/test_shape_encoder_updates.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/encode/tile.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/meta/tensor_meta.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/partial_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/partial_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/polygon.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/query/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/query/autocomplete.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/query/filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/query/query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/seed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/azure.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/deeplake_memory_object.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/gcs.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/google_drive.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/indra.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/local.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/lru_cache.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/memory.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/provider.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/storage/s3.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tensor_link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_compute.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_indra_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_io.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_locking.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_readonly.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_seed.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tests/test_vdb_indexes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/deserialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/optimizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/sample_tiles.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/test_optimizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/tiling/test_serialize.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/transform/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/transform/test_transform.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/transform/transform_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/transform/transform_tensor.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/client_side_dataset_handler.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/dataset_handlers/dataset_handler_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/deep_memory/deep_memory.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/deeplake_vectorstore.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/embedder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/embeddings/test_embedder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/test_deeplake_vectorstore.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/dataset/test_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/filter/test_filter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/search_algorithm.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/test_indra.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/distance_metric.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/tql_distance_metrics/order.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/indra/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/data_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/ingest_data.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/ingestion/test_data_ingestion.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/search_algorithm.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/test_vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/python/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/vectorstore/vector_search/vector_search.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/commit_chunk_map.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/commit_diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/commit_node.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/dataset_diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/test_merge.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/core/version_control/test_version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/dummy_dataloader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/libdeeplake_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/test_pytorch.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/test_query.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/test_tensorflow.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/enterprise/util.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/hooks.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/htype.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/huggingface/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/huggingface/huggingface.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/mmdet/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_runners.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/mmdet/mmdet_utils.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/pytorch/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/pytorch/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/pytorch/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/pytorch/pytorch.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/pytorch/shuffle_buffer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/tf/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/tf/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/tf/datasettotensorflow.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/tf/deeplake_tensorflow_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/wandb/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/integrations/wandb/wandb.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/requirements/common.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/requirements/docs.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/requirements/plugins.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/requirements/tests.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/cache_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/client_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/common.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/dataset_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/path_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/tests/storage_fixtures.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/access_method.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/agreement.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/array_list.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/assert_byte_indexes.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/auto.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/bugout_reporter.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/cache_chain.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/casting.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/check_installation.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/check_latest_version.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/chunk_engine.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/class_label.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/compression.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/compute.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/connect_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/delete_entry.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/diff.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/downsample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/empty_sample.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/encoder.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/exceptions.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/exif.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/from_tfds.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/generate_id.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/hash.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/htype.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/image.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/invalid_view_op.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/iterable_ordered_dict.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/iteration_warning.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/join_chunks.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/json.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/keys.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/link.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/logging.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/merge.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/modified.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/notebook.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/mesh.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_parser.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/mesh_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/object_base_3d.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parser_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_parsers.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_reader_base.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/ply_readers.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/point_cloud.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/object_3d/read_3d_data.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/path.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/pretty_print.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/remove_cache.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/scheduling.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/shape_interval.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/shuffle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/spinner.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/split.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/storage.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tag.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tensor_db.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/testing.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_auto.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_connect_dataset.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_iterable_ordered_dict.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_keys.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_read.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_shape_interval.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_shuffle.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_split.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_tensor_db.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_token.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/tests/test_video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/threading.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/token.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/transform.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/version_control.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/video.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/util/warnings.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/visualizer/__init__.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/visualizer/video_streaming.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake/visualizer/visualizer.py +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake.egg-info/SOURCES.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake.egg-info/dependency_links.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake.egg-info/not-zip-safe +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/deeplake.egg-info/top_level.txt +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/pyproject.toml +0 -0
- {deeplake-3.9.5 → deeplake-3.9.7}/setup.cfg +0 -0
|
@@ -395,7 +395,7 @@ class dataset:
|
|
|
395
395
|
verbose: bool = True,
|
|
396
396
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
397
397
|
) -> Dataset:
|
|
398
|
-
"""Creates an empty dataset
|
|
398
|
+
"""Creates an empty Deep Lake dataset.
|
|
399
399
|
|
|
400
400
|
Args:
|
|
401
401
|
path (str, pathlib.Path): - The full path to the dataset. It can be:
|
|
@@ -530,7 +530,7 @@ class dataset:
|
|
|
530
530
|
lock_enabled: Optional[bool] = True,
|
|
531
531
|
index_params: Optional[Dict[str, Union[int, str]]] = None,
|
|
532
532
|
) -> Dataset:
|
|
533
|
-
"""Loads an existing dataset
|
|
533
|
+
"""Loads an existing Deep Lake dataset
|
|
534
534
|
|
|
535
535
|
Examples:
|
|
536
536
|
|
|
@@ -1120,7 +1120,8 @@ class dataset:
|
|
|
1120
1120
|
progressbar=True,
|
|
1121
1121
|
**kwargs,
|
|
1122
1122
|
):
|
|
1123
|
-
"""Copies dataset at ``src`` to ``dest``. Version control history is not included.
|
|
1123
|
+
"""Copies dataset at ``src`` to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
|
|
1124
|
+
For fast copying, we recommend using ``deepake.deepcopy()`` instead.
|
|
1124
1125
|
|
|
1125
1126
|
Args:
|
|
1126
1127
|
src (str, Dataset, pathlib.Path): The Dataset or the path to the dataset to be copied.
|
|
@@ -1206,7 +1207,7 @@ class dataset:
|
|
|
1206
1207
|
verbose: bool = True,
|
|
1207
1208
|
**kwargs,
|
|
1208
1209
|
):
|
|
1209
|
-
"""Copies dataset at ``src`` to ``dest`` including version control history.
|
|
1210
|
+
"""Copies dataset at ``src`` to ``dest`` including version control history. This is the fastest method for copying datasets.
|
|
1210
1211
|
|
|
1211
1212
|
Args:
|
|
1212
1213
|
src (str, pathlib.Path, Dataset): The Dataset or the path to the dataset to be copied.
|
|
@@ -1452,9 +1453,9 @@ class dataset:
|
|
|
1452
1453
|
|
|
1453
1454
|
Examples:
|
|
1454
1455
|
>>> # Connect an s3 dataset
|
|
1455
|
-
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key"
|
|
1456
|
+
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", dest_path="hub://my_org/dataset", creds_key="my_managed_credentials_key")
|
|
1456
1457
|
>>> # or
|
|
1457
|
-
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key"
|
|
1458
|
+
>>> ds = deeplake.connect(src_path="s3://bucket/dataset", org_id="my_org", creds_key="my_managed_credentials_key")
|
|
1458
1459
|
|
|
1459
1460
|
Args:
|
|
1460
1461
|
src_path (str): Cloud path to the source dataset. Can be:
|
|
@@ -1653,9 +1654,9 @@ class dataset:
|
|
|
1653
1654
|
>>> "path/to/data/directory",
|
|
1654
1655
|
>>> dest="hub://org_id/dataset",
|
|
1655
1656
|
>>> allow_no_annotation=True,
|
|
1656
|
-
>>> token="my_activeloop_token",
|
|
1657
1657
|
>>> num_workers=4,
|
|
1658
1658
|
>>> )
|
|
1659
|
+
|
|
1659
1660
|
>>> # Ingest data from your cloud into another Deep Lake dataset in your cloud, and connect that dataset to the Deep Lake backend.
|
|
1660
1661
|
>>> ds = deeplake.ingest_yolo(
|
|
1661
1662
|
>>> "s3://bucket/data_directory",
|
|
@@ -43,6 +43,8 @@ else:
|
|
|
43
43
|
_NATIVE_INT32 = ">i4"
|
|
44
44
|
_NATIVE_FLOAT32 = ">f4"
|
|
45
45
|
|
|
46
|
+
HEADER_MAX_BYTES = 132
|
|
47
|
+
|
|
46
48
|
DIMS_RE = re.compile(rb" ([0-9]+)x([0-9]+)")
|
|
47
49
|
FPS_RE = re.compile(rb" ([0-9]+) fps,")
|
|
48
50
|
DURATION_RE = re.compile(rb"Duration: ([0-9:.]+),")
|
|
@@ -489,11 +491,34 @@ def get_compression(header=None, path=None):
|
|
|
489
491
|
".nii",
|
|
490
492
|
".nii.gz",
|
|
491
493
|
]
|
|
492
|
-
path = str(path).lower()
|
|
494
|
+
path = str(path).lower().partition("?")[0].partition("#")[0].partition(";")[0]
|
|
493
495
|
for fmt in file_formats:
|
|
494
496
|
if path.endswith(fmt):
|
|
495
497
|
return fmt[1:]
|
|
496
498
|
if header:
|
|
499
|
+
if (
|
|
500
|
+
header[4:12] == b"\x66\x74\x79\x70\x4D\x53\x4E\x56"
|
|
501
|
+
or header[4:12] == b"\x66\x74\x79\x70\x69\x73\x6F\x6D"
|
|
502
|
+
):
|
|
503
|
+
return "mp4"
|
|
504
|
+
if header[0:4] == b"\x1A\x45\xDF\xA3":
|
|
505
|
+
return "mkv"
|
|
506
|
+
if (
|
|
507
|
+
header[0:2] == b"\xff\xfb"
|
|
508
|
+
or header[0:2] == b"\xff\xf3"
|
|
509
|
+
or header[0:2] == b"\xff\xf2"
|
|
510
|
+
):
|
|
511
|
+
return "mp3"
|
|
512
|
+
if header[0:2] == b"\x66\x4c\x61\x43":
|
|
513
|
+
return "flac"
|
|
514
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x57\x41\x56\x45":
|
|
515
|
+
return "wav"
|
|
516
|
+
if header[0:4] == b"\x52\x49\x46\x46" and header[8:12] == b"\x41\x56\x49\x20":
|
|
517
|
+
return "avi"
|
|
518
|
+
if header[128:132] == b"\x44\x49\x43\x4D":
|
|
519
|
+
return "dcm"
|
|
520
|
+
if header[0:4] == b"\x6e\x2b\x31\x00":
|
|
521
|
+
return "nii"
|
|
497
522
|
if not Image.OPEN:
|
|
498
523
|
Image.init()
|
|
499
524
|
for fmt in Image.OPEN:
|
|
@@ -651,10 +676,10 @@ def read_meta_from_compressed_file(
|
|
|
651
676
|
try:
|
|
652
677
|
if compression is None:
|
|
653
678
|
if hasattr(f, "read"):
|
|
654
|
-
compression = get_compression(f.read(
|
|
679
|
+
compression = get_compression(f.read(HEADER_MAX_BYTES), path)
|
|
655
680
|
f.seek(0)
|
|
656
681
|
else:
|
|
657
|
-
compression = get_compression(f[:
|
|
682
|
+
compression = get_compression(f[:HEADER_MAX_BYTES], path) # type: ignore
|
|
658
683
|
if compression == "jpeg":
|
|
659
684
|
try:
|
|
660
685
|
shape, typestr = _read_jpeg_shape(f), "|u1"
|
|
@@ -372,7 +372,7 @@ class Dataset:
|
|
|
372
372
|
return self._client
|
|
373
373
|
|
|
374
374
|
def __len__(self, warn: bool = True):
|
|
375
|
-
"""Returns the length of the
|
|
375
|
+
"""Returns the length (number of rows) of the shortest tensor in the dataset."""
|
|
376
376
|
tensor_lengths = [len(tensor) for tensor in self.tensors.values()]
|
|
377
377
|
pad_tensors = self._pad_tensors
|
|
378
378
|
if (
|
|
@@ -390,7 +390,7 @@ class Dataset:
|
|
|
390
390
|
|
|
391
391
|
@property
|
|
392
392
|
def max_len(self):
|
|
393
|
-
"""
|
|
393
|
+
"""Returns the length (number of rows) of the longest tensor in the dataset."""
|
|
394
394
|
return (
|
|
395
395
|
max([len(tensor) for tensor in self.tensors.values()])
|
|
396
396
|
if self.tensors
|
|
@@ -399,7 +399,7 @@ class Dataset:
|
|
|
399
399
|
|
|
400
400
|
@property
|
|
401
401
|
def min_len(self):
|
|
402
|
-
"""
|
|
402
|
+
"""Returns the length (number of rows) of the shortest tensor in the dataset."""
|
|
403
403
|
return (
|
|
404
404
|
min([len(tensor) for tensor in self.tensors.values()])
|
|
405
405
|
if self.tensors
|
|
@@ -668,32 +668,34 @@ class Dataset:
|
|
|
668
668
|
tiling_threshold: Optional[int] = None,
|
|
669
669
|
**kwargs,
|
|
670
670
|
):
|
|
671
|
-
"""Creates a new tensor in the dataset.
|
|
671
|
+
"""Creates a new tensor in the Deep Lake dataset. Specifying the tensor's htype is highly recommended for complex data such as images, video, dicom, text, json, etc.
|
|
672
|
+
Specifying htype is not necessary for simple numeric date such as arrays and scalars.
|
|
672
673
|
|
|
673
674
|
Examples:
|
|
674
|
-
>>> #
|
|
675
|
-
>>> ds = deeplake.
|
|
676
|
-
|
|
677
|
-
>>> # create tensors
|
|
678
|
-
>>> ds.create_tensor("images", htype="image", sample_compression="jpg")
|
|
679
|
-
>>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
|
|
680
|
-
>>> ds.create_tensor("data")
|
|
681
|
-
>>> ds.create_tensor("point_clouds", htype="point_cloud")
|
|
675
|
+
>>> # Create dataset
|
|
676
|
+
>>> ds = deeplake.empty("path/to/dataset")
|
|
682
677
|
|
|
683
|
-
>>> #
|
|
684
|
-
>>> ds
|
|
685
|
-
>>>
|
|
686
|
-
>>>
|
|
678
|
+
>>> # Create tensors
|
|
679
|
+
>>> with ds:
|
|
680
|
+
>>> ds.create_tensor("images", htype="image", sample_compression="jpg")
|
|
681
|
+
>>> ds.create_tensor("videos", htype="video", sample_compression="mp4")
|
|
682
|
+
>>> ds.create_tensor("data")
|
|
683
|
+
>>> ds.create_tensor("point_clouds", htype="point_cloud")
|
|
684
|
+
>>>
|
|
685
|
+
>>> # Append data
|
|
686
|
+
>>> ds.images.append(np.ones((400, 400, 3), dtype='uint8'))
|
|
687
|
+
>>> ds.videos.append(deeplake.read("videos/sample_video.mp4"))
|
|
688
|
+
>>> ds.data.append(np.zeros((100, 100, 2)))
|
|
687
689
|
|
|
688
690
|
Args:
|
|
689
691
|
name (str): The name of the tensor to be created.
|
|
690
692
|
htype (str):
|
|
691
|
-
- The class of data for the tensor.
|
|
692
|
-
- The defaults for other parameters are determined in terms of this value.
|
|
693
|
+
- The class of data for the tensor. Specifying ``htype`` is highly recommended for complex data such as images, video, dicom, text, json, etc. Specifying htype is not necessary for simple numeric date such as arrays and scalars.
|
|
694
|
+
- The defaults for other parameters are determined in terms of this value, thus improving error checking and visualization.
|
|
693
695
|
- For example, ``htype="image"`` would have ``dtype`` default to ``uint8``.
|
|
694
696
|
- These defaults can be overridden by explicitly passing any of the other parameters to this function.
|
|
695
697
|
- May also modify the defaults for other parameters.
|
|
696
|
-
dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``.
|
|
698
|
+
dtype (str): Optionally override this tensor's ``dtype``. All subsequent samples are required to have this ``dtype``. Specifying ``dtype`` is typically not necessary unless you are dealing with mixed numerical data in a single tensor (column).
|
|
697
699
|
sample_compression (str): All samples will be compressed in the provided format. If ``None``, samples are uncompressed. For ``link[]`` tensors, ``sample_compression`` is used only for optimizing dataset views.
|
|
698
700
|
chunk_compression (str): All chunks will be compressed in the provided format. If ``None``, chunks are uncompressed. For ``link[]`` tensors, ``chunk_compression`` is used only for optimizing dataset views.
|
|
699
701
|
hidden (bool): If ``True``, the tensor will be hidden from ds.tensors but can still be accessed via ``ds[tensor_name]``.
|
|
@@ -1178,7 +1180,7 @@ class Dataset:
|
|
|
1178
1180
|
def create_tensor_like(
|
|
1179
1181
|
self, name: str, source: "Tensor", unlink: bool = False
|
|
1180
1182
|
) -> "Tensor":
|
|
1181
|
-
"""
|
|
1183
|
+
"""Creates a tensor with the same properties as ``source``. No samples or data copied, other than the tensor meta/info.
|
|
1182
1184
|
|
|
1183
1185
|
Examples:
|
|
1184
1186
|
>>> ds.create_tensor_like("cats", ds["images"])
|
|
@@ -2093,13 +2095,16 @@ class Dataset:
|
|
|
2093
2095
|
*args,
|
|
2094
2096
|
**kwargs,
|
|
2095
2097
|
):
|
|
2096
|
-
"""
|
|
2098
|
+
"""Creates a PyTorch Dataloader from the Deep Lake dataset. During iteration, the data from all tensors will be streamed on-the-fly from the storage location.
|
|
2099
|
+
Understanding the parameters below is critical for achieving fast streaming for your use-case
|
|
2097
2100
|
|
|
2098
2101
|
Args:
|
|
2099
2102
|
*args: Additional args to be passed to torch_dataset
|
|
2100
2103
|
**kwargs: Additional kwargs to be passed to torch_dataset
|
|
2101
2104
|
transform (Callable, Optional): Transformation function to be applied to each sample.
|
|
2102
|
-
tensors (List, Optional):
|
|
2105
|
+
tensors (List, Optional): List of tensors to load. If ``None``, all tensors are loaded. Defaults to ``None``.
|
|
2106
|
+
For datasets with many tensors, its extremely important to stream only the data that is needed for training the model, in order to avoid bottlenecks associated with streaming unused data.
|
|
2107
|
+
For example, if you have a dataset that has ``image``, ``label``, and ``metadata`` tensors, if ``tensors=["image", "label"]``, the Data Loader will only stream the ``image`` and ``label`` tensors.
|
|
2103
2108
|
num_workers (int): The number of workers to use for fetching data in parallel.
|
|
2104
2109
|
batch_size (int): Number of samples per batch to load. Default value is 1.
|
|
2105
2110
|
drop_last (bool): Set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
|
|
@@ -2564,10 +2569,10 @@ class Dataset:
|
|
|
2564
2569
|
|
|
2565
2570
|
@spinner
|
|
2566
2571
|
def flush(self):
|
|
2567
|
-
"""
|
|
2568
|
-
Writes all the
|
|
2569
|
-
|
|
2570
|
-
|
|
2572
|
+
"""
|
|
2573
|
+
Writes all the data that has been changed/assigned from the cache layers (if any) to the underlying storage.
|
|
2574
|
+
|
|
2575
|
+
NOTE: The high-level APIs flush the cache automatically and users generally do not have explicitly run the ``flush`` command.
|
|
2571
2576
|
"""
|
|
2572
2577
|
self._flush_vc_info()
|
|
2573
2578
|
self.storage.flush()
|
|
@@ -2582,11 +2587,13 @@ class Dataset:
|
|
|
2582
2587
|
|
|
2583
2588
|
def clear_cache(self):
|
|
2584
2589
|
"""
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2590
|
+
Flushes (see :func:`Dataset.flush`) the contents of the cache layers (if any) and then deletes contents of all the layers of it.
|
|
2591
|
+
This doesn't delete data from the actual storage.
|
|
2592
|
+
This is useful if you have multiple datasets with memory caches open, taking up too much RAM, or when local cache is no longer needed for certain datasets and is taking up storage space.
|
|
2593
|
+
|
|
2594
|
+
NOTE: The high-level APIs clear the cache automatically and users generally do not have explicitly run the ``clear_cache`` command.
|
|
2589
2595
|
"""
|
|
2596
|
+
|
|
2590
2597
|
if hasattr(self.storage, "clear_cache"):
|
|
2591
2598
|
self.storage.clear_cache()
|
|
2592
2599
|
|
|
@@ -2628,7 +2635,7 @@ class Dataset:
|
|
|
2628
2635
|
|
|
2629
2636
|
@invalid_view_op
|
|
2630
2637
|
def delete(self, large_ok=False):
|
|
2631
|
-
"""Deletes the entire dataset from the cache layers (if any)
|
|
2638
|
+
"""Deletes the entire dataset from the underlying storage and cache layers (if any).
|
|
2632
2639
|
This is an **IRREVERSIBLE** operation. Data once deleted can not be recovered.
|
|
2633
2640
|
|
|
2634
2641
|
Args:
|
|
@@ -2672,10 +2679,10 @@ class Dataset:
|
|
|
2672
2679
|
self.storage.clear()
|
|
2673
2680
|
|
|
2674
2681
|
def summary(self, force: bool = False):
|
|
2675
|
-
"""Prints a summary of the dataset.
|
|
2682
|
+
"""Prints a summary of the dataset, including the tensor names and their lengths, shapes, htypes, dtypes, compressions, and other relevant information.
|
|
2676
2683
|
|
|
2677
2684
|
Args:
|
|
2678
|
-
force (bool): Dataset views with more than 10000 samples might take
|
|
2685
|
+
force (bool): Dataset views with more than 10000 samples might take several seconds of minutes to summarize. If `force=True`,
|
|
2679
2686
|
the summary will be printed regardless. An error will be raised otherwise.
|
|
2680
2687
|
|
|
2681
2688
|
Raises:
|
|
@@ -2976,7 +2983,8 @@ class Dataset:
|
|
|
2976
2983
|
return self[fullname]
|
|
2977
2984
|
|
|
2978
2985
|
def create_group(self, name: str, exist_ok=False) -> "Dataset":
|
|
2979
|
-
"""Creates a tensor group
|
|
2986
|
+
"""Creates a tensor group, which is a collection of tensors that can be reference together.
|
|
2987
|
+
Groups are recommended for use-cases with many tensors in order to organize and reference data more easily.
|
|
2980
2988
|
|
|
2981
2989
|
Args:
|
|
2982
2990
|
name: The name of the group to create.
|
|
@@ -2992,11 +3000,17 @@ class Dataset:
|
|
|
2992
3000
|
Examples:
|
|
2993
3001
|
|
|
2994
3002
|
>>> ds.create_group("images")
|
|
2995
|
-
>>> ds
|
|
3003
|
+
>>> ds.images.create_tensor("left_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3004
|
+
>>> ds.images.create_tensor("center_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3005
|
+
>>> ds.images.create_tensor("right_camera", htype = "image", "sample_compression" = "jpeg")
|
|
2996
3006
|
|
|
2997
|
-
>>>
|
|
2998
|
-
>>> ds[
|
|
2999
|
-
>>>
|
|
3007
|
+
>>> # Data from a tensors in groups is referenced using:
|
|
3008
|
+
>>> ds.images.right_camera[0].numpy()
|
|
3009
|
+
>>> # OR
|
|
3010
|
+
>>> ds["images/right_camera"].numpy()
|
|
3011
|
+
|
|
3012
|
+
>>> # "/" Notation can also be used to create groups with ``create_tensor``
|
|
3013
|
+
>>> ds.create_tensor("images/right_camera", htype = "image", "sample_compression" = "jpeg")
|
|
3000
3014
|
"""
|
|
3001
3015
|
|
|
3002
3016
|
deeplake_reporter.feature_report(
|
|
@@ -3104,15 +3118,6 @@ class Dataset:
|
|
|
3104
3118
|
Exception: Error while attempting to rollback appends.
|
|
3105
3119
|
SampleAppendingError: Error that occurs when someone tries to append a tensor value directly to the dataset without specifying tensor name.
|
|
3106
3120
|
|
|
3107
|
-
Examples:
|
|
3108
|
-
|
|
3109
|
-
>>> ds = deeplake.empty("../test/test_ds")
|
|
3110
|
-
>>> ds.create_tensor('data')
|
|
3111
|
-
Tensor(key='data')
|
|
3112
|
-
>>> ds.create_tensor('labels')
|
|
3113
|
-
Tensor(key='labels')
|
|
3114
|
-
>>> ds.append({"data": [1, 2, 3, 4], "labels":[0, 1, 2, 3]})
|
|
3115
|
-
|
|
3116
3121
|
"""
|
|
3117
3122
|
tensors = self.tensors
|
|
3118
3123
|
if isinstance(sample, Dataset):
|
|
@@ -3205,10 +3210,10 @@ class Dataset:
|
|
|
3205
3210
|
ignore_errors: bool = False,
|
|
3206
3211
|
progressbar: bool = False,
|
|
3207
3212
|
):
|
|
3208
|
-
"""Appends multiple rows
|
|
3213
|
+
"""Appends multiple samples (rows) to mutliple tensors at once. This method expects all tensors being updated to be of the same length.
|
|
3209
3214
|
|
|
3210
3215
|
Args:
|
|
3211
|
-
samples (Dict[str, Any]): Dictionary with tensor names as keys and
|
|
3216
|
+
samples (Dict[str, Any]): Dictionary with tensor names as keys and data as values. The values can be a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
|
|
3212
3217
|
skip_ok (bool): Skip tensors not in ``samples`` if set to True.
|
|
3213
3218
|
append_empty (bool): Append empty samples to tensors not specified in ``sample`` if set to ``True``. If True, ``skip_ok`` is ignored.
|
|
3214
3219
|
ignore_errors (bool): Skip samples that cause errors while extending, if set to ``True``.
|
|
@@ -3221,6 +3226,18 @@ class Dataset:
|
|
|
3221
3226
|
NotImplementedError: If an error occurs while writing tiles.
|
|
3222
3227
|
SampleExtendError: If the extend failed while appending a sample.
|
|
3223
3228
|
Exception: Error while attempting to rollback appends.
|
|
3229
|
+
|
|
3230
|
+
Examples:
|
|
3231
|
+
|
|
3232
|
+
>>> ds = deeplake.empty("../test/test_ds")
|
|
3233
|
+
|
|
3234
|
+
>>> with ds:
|
|
3235
|
+
>>> ds.create_tensor('data')
|
|
3236
|
+
>>> ds.create_tensor('labels')
|
|
3237
|
+
|
|
3238
|
+
>>> # This operation will append 4 samples (rows) to the Deep Lake dataset
|
|
3239
|
+
>>> ds.extend({"data": [1, 2, 3, 4], "labels":["table", "chair", "desk", "table"]})
|
|
3240
|
+
|
|
3224
3241
|
"""
|
|
3225
3242
|
extend = False
|
|
3226
3243
|
if isinstance(samples, Dataset):
|
|
@@ -3283,7 +3300,7 @@ class Dataset:
|
|
|
3283
3300
|
skip_ok: bool = False,
|
|
3284
3301
|
append_empty: bool = False,
|
|
3285
3302
|
):
|
|
3286
|
-
"""Append
|
|
3303
|
+
"""Append a single sample (row) to multiple tensors at once.
|
|
3287
3304
|
|
|
3288
3305
|
Args:
|
|
3289
3306
|
sample (dict): Dictionary with tensor names as keys and samples as values.
|
|
@@ -3301,11 +3318,13 @@ class Dataset:
|
|
|
3301
3318
|
Examples:
|
|
3302
3319
|
|
|
3303
3320
|
>>> ds = deeplake.empty("../test/test_ds")
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
>>>
|
|
3307
|
-
|
|
3308
|
-
|
|
3321
|
+
|
|
3322
|
+
>>> with ds:
|
|
3323
|
+
>>> ds.create_tensor('data')
|
|
3324
|
+
>>> ds.create_tensor('labels')
|
|
3325
|
+
|
|
3326
|
+
>>> # This operation will append 1 sample (row) to the Deep Lake dataset
|
|
3327
|
+
>>> ds.append({"data": 1, "labels": "table"})
|
|
3309
3328
|
|
|
3310
3329
|
"""
|
|
3311
3330
|
new_row_ids = [self.__len__(warn=False)]
|
|
@@ -4300,12 +4319,12 @@ class Dataset:
|
|
|
4300
4319
|
progressbar=True,
|
|
4301
4320
|
public: bool = False,
|
|
4302
4321
|
):
|
|
4303
|
-
"""Copies this dataset or dataset view to ``dest``. Version control history is not included.
|
|
4322
|
+
"""Copies this dataset or dataset view to ``dest``. Version control history is not included, and this operation copies data from the latest commit on the main branch.
|
|
4304
4323
|
|
|
4305
4324
|
Args:
|
|
4306
4325
|
dest (str, pathlib.Path): Destination dataset or path to copy to. If a Dataset instance is provided, it is expected to be empty.
|
|
4307
4326
|
tensors (List[str], optional): Names of tensors (and groups) to be copied. If not specified all tensors are copied.
|
|
4308
|
-
runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub
|
|
4327
|
+
runtime (dict): Parameters for Activeloop DB Engine. Only applicable for hub://... paths.
|
|
4309
4328
|
overwrite (bool): If ``True`` and a dataset exists at `destination`, it will be overwritten. Defaults to False.
|
|
4310
4329
|
creds (dict, Optional): creds required to create / overwrite datasets at `dest`.
|
|
4311
4330
|
token (str, Optional): token used to for fetching credentials to `dest`.
|
|
@@ -4398,21 +4417,25 @@ class Dataset:
|
|
|
4398
4417
|
ds_name: Optional[str] = None,
|
|
4399
4418
|
token: Optional[str] = None,
|
|
4400
4419
|
):
|
|
4401
|
-
"""Connect a Deep Lake cloud
|
|
4420
|
+
"""Connect a Deep Lake dataset stored in your cloud to the Deep Lake App.
|
|
4402
4421
|
|
|
4403
4422
|
Examples:
|
|
4404
|
-
>>> #
|
|
4405
|
-
>>> s3_ds = deeplake.
|
|
4406
|
-
>>>
|
|
4407
|
-
>>> #
|
|
4408
|
-
>>> ds = s3_ds.connect(org_id="
|
|
4423
|
+
>>> # Load an s3 dataset
|
|
4424
|
+
>>> s3_ds = deeplake.load("s3://bucket/dataset")
|
|
4425
|
+
>>>
|
|
4426
|
+
>>> # Specify the org_id and managed creds_key, and the dataset name is inferred from the currently storage location path
|
|
4427
|
+
>>> ds = s3_ds.connect(org_id="my_org_id", creds_key="managed_creds_key")
|
|
4428
|
+
>>>
|
|
4429
|
+
>>> # Or if you want to explicitly specify the full path, which contains the org_id
|
|
4430
|
+
>>> ds = s3_ds.connect(dest_path="hub://my_org/dataset", creds_key="managed_creds_key")
|
|
4431
|
+
|
|
4409
4432
|
|
|
4410
4433
|
Args:
|
|
4411
4434
|
creds_key (str): The managed credentials to be used for accessing the source path.
|
|
4412
|
-
dest_path (str, optional): The full path to
|
|
4413
|
-
a Deep Lake path like ``hub
|
|
4414
|
-
org_id (str, optional): The organization to
|
|
4415
|
-
ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or
|
|
4435
|
+
dest_path (str, optional): The full path to which the connected Deep Lake dataset will reside. Can be:
|
|
4436
|
+
a Deep Lake path like ``hub://<org_id>/<dataset_name>``
|
|
4437
|
+
org_id (str, optional): The organization to which the connected Deep Lake dataset will be added. The dataset name will be the same as parent folder for the dataset in the cloud storage location.
|
|
4438
|
+
ds_name (str, optional): The name of the connected Deep Lake dataset. Will be infered from ``dest_path`` or storage location path if not provided.
|
|
4416
4439
|
token (str, optional): Activeloop token used to fetch the managed credentials.
|
|
4417
4440
|
|
|
4418
4441
|
Raises:
|
|
@@ -10,6 +10,7 @@ from deeplake.core.compression import (
|
|
|
10
10
|
_read_audio_meta,
|
|
11
11
|
_read_3d_data_meta,
|
|
12
12
|
_open_nifti,
|
|
13
|
+
HEADER_MAX_BYTES,
|
|
13
14
|
)
|
|
14
15
|
from deeplake.compression import (
|
|
15
16
|
get_compression_type,
|
|
@@ -104,7 +105,9 @@ class Sample:
|
|
|
104
105
|
self._compression = get_compression(path=self.path)
|
|
105
106
|
compressed_bytes = self._read_from_path()
|
|
106
107
|
if self._compression is None:
|
|
107
|
-
self._compression = get_compression(
|
|
108
|
+
self._compression = get_compression(
|
|
109
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
110
|
+
)
|
|
108
111
|
self._shape, self._typestr = verify_compressed_file(compressed_bytes, self._compression) # type: ignore
|
|
109
112
|
|
|
110
113
|
if array is not None:
|
|
@@ -318,7 +321,9 @@ class Sample:
|
|
|
318
321
|
self._compression = get_compression(path=self.path)
|
|
319
322
|
compressed_bytes = self._read_from_path()
|
|
320
323
|
if self._compression is None:
|
|
321
|
-
self._compression = get_compression(
|
|
324
|
+
self._compression = get_compression(
|
|
325
|
+
header=compressed_bytes[:HEADER_MAX_BYTES]
|
|
326
|
+
)
|
|
322
327
|
if self._compression == compression:
|
|
323
328
|
if self._shape is None:
|
|
324
329
|
_, self._shape, self._typestr = read_meta_from_compressed_file(
|
|
@@ -328,7 +333,9 @@ class Sample:
|
|
|
328
333
|
compressed_bytes = self._recompress(compressed_bytes, compression)
|
|
329
334
|
elif self._buffer is not None:
|
|
330
335
|
if self._compression is None:
|
|
331
|
-
self._compression = get_compression(
|
|
336
|
+
self._compression = get_compression(
|
|
337
|
+
header=self._buffer[:HEADER_MAX_BYTES]
|
|
338
|
+
)
|
|
332
339
|
if self._compression == compression:
|
|
333
340
|
compressed_bytes = self._buffer
|
|
334
341
|
else:
|
|
@@ -326,8 +326,7 @@ class Tensor:
|
|
|
326
326
|
progressbar: bool = False,
|
|
327
327
|
ignore_errors: bool = False,
|
|
328
328
|
):
|
|
329
|
-
"""Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence
|
|
330
|
-
or a sequence of :func:`deeplake.read` outputs, which can be used to load files. See examples down below.
|
|
329
|
+
"""Extends the end of the tensor by appending multiple elements from a sequence. Accepts a sequence (i.e. a list) or a single numpy array (the first axis in the array is treated as the row axis).
|
|
331
330
|
|
|
332
331
|
Example:
|
|
333
332
|
Numpy input:
|
|
@@ -429,8 +428,7 @@ class Tensor:
|
|
|
429
428
|
|
|
430
429
|
@invalid_view_op
|
|
431
430
|
def append(self, sample: InputSample):
|
|
432
|
-
"""Appends a single sample to the end of the tensor.
|
|
433
|
-
which can be used to load files. See examples down below.
|
|
431
|
+
"""Appends a single sample (row) to the end of the tensor.
|
|
434
432
|
|
|
435
433
|
Examples:
|
|
436
434
|
Numpy input:
|