dataverse-sdk 2.5.3__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/PKG-INFO +22 -15
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/README.md +21 -14
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/backend.py +9 -5
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/client.py +9 -9
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/constants.py +3 -3
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/api.py +1 -2
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/client.py +4 -8
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/common.py +0 -1
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/PKG-INFO +22 -15
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/setup.py +1 -1
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/third_party.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/connections.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/exceptions/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/exceptions/client.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/base.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/coco.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/constant.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/exporter.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/utils.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/visionai.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/vqa.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/export/yolo.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/format.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/utils/__init__.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk/utils/utils.py +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/requires.txt +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
- {dataverse_sdk-2.5.3 → dataverse_sdk-2.6.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -30,7 +30,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
|
|
|
30
30
|
Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
|
|
31
31
|
- Create Project with your input ontology and sensors
|
|
32
32
|
- Get Project by project-id
|
|
33
|
-
- Create Dataset from your AWS
|
|
33
|
+
- Create Dataset from your AWS storage or local
|
|
34
34
|
- Get Dataset by dataset-id
|
|
35
35
|
- List models for your selected project-id
|
|
36
36
|
- Get and download your model
|
|
@@ -332,14 +332,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
|
|
|
332
332
|
|
|
333
333
|
### Create Dataset
|
|
334
334
|
|
|
335
|
+
**Required fields by `data_source`:**
|
|
336
|
+
|
|
337
|
+
| `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
|
|
338
|
+
| :--- | :---: | :---: | :---: | :--- |
|
|
339
|
+
| `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
|
|
340
|
+
| `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
|
|
341
|
+
| `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
|
|
342
|
+
|
|
343
|
+
`*--`: required for this `data_source` · `-`: not used (can be omitted)
|
|
344
|
+
|
|
345
|
+
> `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
|
|
346
|
+
>
|
|
347
|
+
> `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
|
|
348
|
+
|
|
335
349
|
#### Use `create_dataset` to import dataset from **cloud storage**
|
|
336
350
|
|
|
337
351
|
```Python
|
|
338
352
|
dataset_data = {
|
|
339
353
|
"name": "Dataset 1",
|
|
340
|
-
"data_source": DataSource.
|
|
354
|
+
"data_source": DataSource.AWS,
|
|
341
355
|
"storage_url": "storage/url",
|
|
342
|
-
"container_name": "azure container name",
|
|
343
356
|
"data_folder": "datafolder/to/vai_anno",
|
|
344
357
|
"type": DatasetType.ANNOTATED_DATA,
|
|
345
358
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
@@ -347,9 +360,8 @@ dataset_data = {
|
|
|
347
360
|
"sequential": False,
|
|
348
361
|
"render_pcd": False,
|
|
349
362
|
"generate_metadata": False,
|
|
350
|
-
"
|
|
351
|
-
"
|
|
352
|
-
"secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
|
|
363
|
+
"access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
364
|
+
"secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
353
365
|
}
|
|
354
366
|
dataset = project.create_dataset(**dataset_data)
|
|
355
367
|
|
|
@@ -360,10 +372,9 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
360
372
|
| Argument name | Type/Options | Default | Description |
|
|
361
373
|
| :--- | :--- | :--- | :--- |
|
|
362
374
|
| name | str | *-- | name of your dataset |
|
|
363
|
-
| data_source | DataSource.
|
|
375
|
+
| data_source | DataSource.AWS | *-- | the datasource of your dataset |
|
|
364
376
|
| storage_url | str | *-- | your cloud storage url |
|
|
365
|
-
|
|
|
366
|
-
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
377
|
+
| data_folder | str | *-- | the relative data folder from the storage_url |
|
|
367
378
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
368
379
|
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
369
380
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
@@ -371,11 +382,10 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
371
382
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
372
383
|
| generate_metadata | bool | False | generate image meta data or not |
|
|
373
384
|
| description | str | None | your dataset description |
|
|
374
|
-
| sas_token | str | None | SAStoken for azure container |
|
|
375
385
|
| access_key_id | str | None | access key id for AWS private s3 bucket |
|
|
376
386
|
| secret_access_key | str | None| secret access key for AWS private s3 bucket |
|
|
377
387
|
|
|
378
|
-
`*--`: required
|
|
388
|
+
`*--`: required for cloud storage
|
|
379
389
|
|
|
380
390
|
* Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
|
|
381
391
|
|
|
@@ -387,15 +397,12 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
387
397
|
dataset_data2 = {
|
|
388
398
|
"name": "dataset-local-upload",
|
|
389
399
|
"data_source": DataSource.LOCAL,
|
|
390
|
-
"storage_url": "",
|
|
391
|
-
"container_name": "",
|
|
392
400
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
393
401
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
394
402
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
395
403
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
396
404
|
"sequential": False,
|
|
397
405
|
"generate_metadata": False,
|
|
398
|
-
"sas_token": ""
|
|
399
406
|
}
|
|
400
407
|
dataset2 = project.create_dataset(**dataset_data2)
|
|
401
408
|
|
|
@@ -3,7 +3,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
|
|
|
3
3
|
Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
|
|
4
4
|
- Create Project with your input ontology and sensors
|
|
5
5
|
- Get Project by project-id
|
|
6
|
-
- Create Dataset from your AWS
|
|
6
|
+
- Create Dataset from your AWS storage or local
|
|
7
7
|
- Get Dataset by dataset-id
|
|
8
8
|
- List models for your selected project-id
|
|
9
9
|
- Get and download your model
|
|
@@ -305,14 +305,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
|
|
|
305
305
|
|
|
306
306
|
### Create Dataset
|
|
307
307
|
|
|
308
|
+
**Required fields by `data_source`:**
|
|
309
|
+
|
|
310
|
+
| `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
|
|
311
|
+
| :--- | :---: | :---: | :---: | :--- |
|
|
312
|
+
| `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
|
|
313
|
+
| `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
|
|
314
|
+
| `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
|
|
315
|
+
|
|
316
|
+
`*--`: required for this `data_source` · `-`: not used (can be omitted)
|
|
317
|
+
|
|
318
|
+
> `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
|
|
319
|
+
>
|
|
320
|
+
> `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
|
|
321
|
+
|
|
308
322
|
#### Use `create_dataset` to import dataset from **cloud storage**
|
|
309
323
|
|
|
310
324
|
```Python
|
|
311
325
|
dataset_data = {
|
|
312
326
|
"name": "Dataset 1",
|
|
313
|
-
"data_source": DataSource.
|
|
327
|
+
"data_source": DataSource.AWS,
|
|
314
328
|
"storage_url": "storage/url",
|
|
315
|
-
"container_name": "azure container name",
|
|
316
329
|
"data_folder": "datafolder/to/vai_anno",
|
|
317
330
|
"type": DatasetType.ANNOTATED_DATA,
|
|
318
331
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
@@ -320,9 +333,8 @@ dataset_data = {
|
|
|
320
333
|
"sequential": False,
|
|
321
334
|
"render_pcd": False,
|
|
322
335
|
"generate_metadata": False,
|
|
323
|
-
"
|
|
324
|
-
"
|
|
325
|
-
"secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
|
|
336
|
+
"access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
337
|
+
"secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
326
338
|
}
|
|
327
339
|
dataset = project.create_dataset(**dataset_data)
|
|
328
340
|
|
|
@@ -333,10 +345,9 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
333
345
|
| Argument name | Type/Options | Default | Description |
|
|
334
346
|
| :--- | :--- | :--- | :--- |
|
|
335
347
|
| name | str | *-- | name of your dataset |
|
|
336
|
-
| data_source | DataSource.
|
|
348
|
+
| data_source | DataSource.AWS | *-- | the datasource of your dataset |
|
|
337
349
|
| storage_url | str | *-- | your cloud storage url |
|
|
338
|
-
|
|
|
339
|
-
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
350
|
+
| data_folder | str | *-- | the relative data folder from the storage_url |
|
|
340
351
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
341
352
|
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
342
353
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
@@ -344,11 +355,10 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
344
355
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
345
356
|
| generate_metadata | bool | False | generate image meta data or not |
|
|
346
357
|
| description | str | None | your dataset description |
|
|
347
|
-
| sas_token | str | None | SAStoken for azure container |
|
|
348
358
|
| access_key_id | str | None | access key id for AWS private s3 bucket |
|
|
349
359
|
| secret_access_key | str | None| secret access key for AWS private s3 bucket |
|
|
350
360
|
|
|
351
|
-
`*--`: required
|
|
361
|
+
`*--`: required for cloud storage
|
|
352
362
|
|
|
353
363
|
* Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
|
|
354
364
|
|
|
@@ -360,15 +370,12 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
360
370
|
dataset_data2 = {
|
|
361
371
|
"name": "dataset-local-upload",
|
|
362
372
|
"data_source": DataSource.LOCAL,
|
|
363
|
-
"storage_url": "",
|
|
364
|
-
"container_name": "",
|
|
365
373
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
366
374
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
367
375
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
368
376
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
369
377
|
"sequential": False,
|
|
370
378
|
"generate_metadata": False,
|
|
371
|
-
"sas_token": ""
|
|
372
379
|
}
|
|
373
380
|
dataset2 = project.create_dataset(**dataset_data2)
|
|
374
381
|
|
|
@@ -402,13 +402,12 @@ class BackendAPI:
|
|
|
402
402
|
project_id: int,
|
|
403
403
|
type: str,
|
|
404
404
|
annotation_format: str,
|
|
405
|
-
storage_url: str,
|
|
406
405
|
data_folder: str,
|
|
406
|
+
storage_url: Optional[str] = None,
|
|
407
407
|
sequential: bool = False,
|
|
408
408
|
generate_metadata: bool = False,
|
|
409
409
|
render_pcd: bool = False,
|
|
410
410
|
container_name: Optional[str] = None,
|
|
411
|
-
sas_token: Optional[str] = None,
|
|
412
411
|
description: Optional[str] = None,
|
|
413
412
|
annotations: Optional[list[str]] = None,
|
|
414
413
|
access_key_id: Optional[str] = None,
|
|
@@ -421,10 +420,7 @@ class BackendAPI:
|
|
|
421
420
|
"name": name,
|
|
422
421
|
"project_id": project_id,
|
|
423
422
|
"data_source": data_source,
|
|
424
|
-
"storage_url": storage_url,
|
|
425
|
-
"container_name": container_name,
|
|
426
423
|
"data_folder": data_folder,
|
|
427
|
-
"sas_token": sas_token,
|
|
428
424
|
"type": type,
|
|
429
425
|
"sequential": sequential,
|
|
430
426
|
"annotation_format": annotation_format,
|
|
@@ -435,6 +431,14 @@ class BackendAPI:
|
|
|
435
431
|
"auto_tagging": [], # FIXME: auto_tagging field is still required by production API.
|
|
436
432
|
}
|
|
437
433
|
|
|
434
|
+
# Only send storage location fields when provided
|
|
435
|
+
for key, value in (
|
|
436
|
+
("storage_url", storage_url),
|
|
437
|
+
("container_name", container_name),
|
|
438
|
+
):
|
|
439
|
+
if value is not None:
|
|
440
|
+
payload_data[key] = value
|
|
441
|
+
|
|
438
442
|
aws_access_key = {secret_access_key, access_key_id}
|
|
439
443
|
if not (all(aws_access_key) or not any(aws_access_key)):
|
|
440
444
|
raise ValueError("Need to assign both secret_access_key and access_key_id")
|
|
@@ -1481,7 +1481,6 @@ of this project OR has been added before"
|
|
|
1481
1481
|
dataset_data.update({"project": project})
|
|
1482
1482
|
return Dataset(**dataset_data, client_alias=client_alias)
|
|
1483
1483
|
|
|
1484
|
-
# TODO: required arguments for different DataSource
|
|
1485
1484
|
@staticmethod
|
|
1486
1485
|
def create_dataset(
|
|
1487
1486
|
name: str,
|
|
@@ -1489,10 +1488,9 @@ of this project OR has been added before"
|
|
|
1489
1488
|
project: Project,
|
|
1490
1489
|
type: DatasetType,
|
|
1491
1490
|
annotation_format: AnnotationFormat,
|
|
1492
|
-
storage_url: str,
|
|
1493
1491
|
data_folder: str,
|
|
1492
|
+
storage_url: Optional[str] = None,
|
|
1494
1493
|
container_name: Optional[str] = None,
|
|
1495
|
-
sas_token: Optional[str] = None,
|
|
1496
1494
|
annotations: Optional[list] = None,
|
|
1497
1495
|
sequential: bool = False,
|
|
1498
1496
|
generate_metadata: bool = False,
|
|
@@ -1519,14 +1517,12 @@ of this project OR has been added before"
|
|
|
1519
1517
|
datasettype (annotation or raw)
|
|
1520
1518
|
annotation_format : AnnotationFormat
|
|
1521
1519
|
format type of annotation
|
|
1522
|
-
storage_url : str
|
|
1523
|
-
storage url for cloud
|
|
1524
1520
|
data_folder : str
|
|
1525
1521
|
data folder of the storage
|
|
1522
|
+
storage_url : Optional[str], optional
|
|
1523
|
+
storage url for cloud storage (e.g. AWS), by default None
|
|
1526
1524
|
container_name : Optional[str], optional
|
|
1527
|
-
|
|
1528
|
-
sas_token : Optional[str], optional
|
|
1529
|
-
SAStoken for Azure, by default None
|
|
1525
|
+
MinIO bucket name, by default None
|
|
1530
1526
|
annotations: list, optional
|
|
1531
1527
|
list of annotation folder name (should be groundtruth or $model_name)
|
|
1532
1528
|
sequential : bool, optional
|
|
@@ -1557,6 +1553,11 @@ of this project OR has been added before"
|
|
|
1557
1553
|
ClientConnectionError
|
|
1558
1554
|
raise exception if there is any error occurs when calling backend APIs.
|
|
1559
1555
|
"""
|
|
1556
|
+
if not data_folder:
|
|
1557
|
+
raise ValueError(
|
|
1558
|
+
"`data_folder` is required to create a dataset and cannot be an empty string"
|
|
1559
|
+
)
|
|
1560
|
+
|
|
1560
1561
|
if annotations is None:
|
|
1561
1562
|
annotations = []
|
|
1562
1563
|
|
|
@@ -1583,7 +1584,6 @@ of this project OR has been added before"
|
|
|
1583
1584
|
storage_url=storage_url,
|
|
1584
1585
|
container_name=container_name,
|
|
1585
1586
|
data_folder=data_folder,
|
|
1586
|
-
sas_token=sas_token,
|
|
1587
1587
|
sequential=sequential,
|
|
1588
1588
|
generate_metadata=generate_metadata,
|
|
1589
1589
|
render_pcd=render_pcd,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from enum import Enum, EnumMeta
|
|
2
|
-
from typing import Any
|
|
2
|
+
from typing import Any
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class BaseEnumMeta(EnumMeta):
|
|
6
|
-
_value_set:
|
|
6
|
+
_value_set: set[Any] | None = None
|
|
7
7
|
|
|
8
8
|
def __contains__(cls, item):
|
|
9
9
|
if cls._value_set is None:
|
|
@@ -16,5 +16,5 @@ class DataverseHost(str, Enum, metaclass=BaseEnumMeta):
|
|
|
16
16
|
DEV2 = "https://dev2.visionai.linkervision.com/dataverse/curation"
|
|
17
17
|
DEV3 = "https://dev3.visionai.linkervision.com/dataverse/curation"
|
|
18
18
|
STAGING = "https://staging.visionai.linkervision.com/dataverse/curation"
|
|
19
|
-
PRODUCTION = "https://visionai.linkervision.
|
|
19
|
+
PRODUCTION = "https://visionai.linkervision.com/dataverse/curation"
|
|
20
20
|
LOCAL = "http://localhost:8000"
|
|
@@ -117,10 +117,9 @@ class DatasetAPISchema(BaseModel):
|
|
|
117
117
|
data_source: DataSource
|
|
118
118
|
type: DatasetType
|
|
119
119
|
annotation_format: AnnotationFormat
|
|
120
|
-
storage_url: str
|
|
121
120
|
data_folder: str
|
|
121
|
+
storage_url: Optional[str] = None
|
|
122
122
|
container_name: Optional[str] = None
|
|
123
|
-
sas_token: Optional[str] = None
|
|
124
123
|
sequential: bool = False
|
|
125
124
|
generate_metadata: bool = False
|
|
126
125
|
render_pcd: Optional[bool] = None
|
|
@@ -369,10 +369,9 @@ class Project(BaseModel):
|
|
|
369
369
|
data_source: DataSource,
|
|
370
370
|
type: DatasetType,
|
|
371
371
|
annotation_format: AnnotationFormat,
|
|
372
|
-
storage_url: str,
|
|
373
372
|
data_folder: str,
|
|
373
|
+
storage_url: Optional[str] = None,
|
|
374
374
|
container_name: Optional[str] = None,
|
|
375
|
-
sas_token: Optional[str] = None,
|
|
376
375
|
annotations: Optional[list] = None,
|
|
377
376
|
sequential: bool = False,
|
|
378
377
|
generate_metadata: bool = False,
|
|
@@ -395,14 +394,12 @@ class Project(BaseModel):
|
|
|
395
394
|
datasettype (annotation or raw)
|
|
396
395
|
annotation_format : AnnotationFormat
|
|
397
396
|
format type of annotation
|
|
398
|
-
storage_url : str
|
|
399
|
-
storage url for cloud
|
|
400
397
|
data_folder : str
|
|
401
398
|
data folder of the storage
|
|
399
|
+
storage_url : Optional[str], optional
|
|
400
|
+
storage url for cloud storage (e.g. AWS), by default None
|
|
402
401
|
container_name : Optional[str], optional
|
|
403
|
-
|
|
404
|
-
sas_token : Optional[str], optional
|
|
405
|
-
SAStoken for Azure, by default None
|
|
402
|
+
MinIO bucket name, by default None
|
|
406
403
|
annotations: list, optional
|
|
407
404
|
list of annotation folder name (should be groundtruth or $model_name)
|
|
408
405
|
sequential : bool, optional
|
|
@@ -447,7 +444,6 @@ class Project(BaseModel):
|
|
|
447
444
|
storage_url=storage_url,
|
|
448
445
|
container_name=container_name,
|
|
449
446
|
data_folder=data_folder,
|
|
450
|
-
sas_token=sas_token,
|
|
451
447
|
annotations=annotations,
|
|
452
448
|
sequential=sequential,
|
|
453
449
|
generate_metadata=generate_metadata,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -30,7 +30,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
|
|
|
30
30
|
Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
|
|
31
31
|
- Create Project with your input ontology and sensors
|
|
32
32
|
- Get Project by project-id
|
|
33
|
-
- Create Dataset from your AWS
|
|
33
|
+
- Create Dataset from your AWS storage or local
|
|
34
34
|
- Get Dataset by dataset-id
|
|
35
35
|
- List models for your selected project-id
|
|
36
36
|
- Get and download your model
|
|
@@ -332,14 +332,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
|
|
|
332
332
|
|
|
333
333
|
### Create Dataset
|
|
334
334
|
|
|
335
|
+
**Required fields by `data_source`:**
|
|
336
|
+
|
|
337
|
+
| `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
|
|
338
|
+
| :--- | :---: | :---: | :---: | :--- |
|
|
339
|
+
| `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
|
|
340
|
+
| `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
|
|
341
|
+
| `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
|
|
342
|
+
|
|
343
|
+
`*--`: required for this `data_source` · `-`: not used (can be omitted)
|
|
344
|
+
|
|
345
|
+
> `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
|
|
346
|
+
>
|
|
347
|
+
> `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
|
|
348
|
+
|
|
335
349
|
#### Use `create_dataset` to import dataset from **cloud storage**
|
|
336
350
|
|
|
337
351
|
```Python
|
|
338
352
|
dataset_data = {
|
|
339
353
|
"name": "Dataset 1",
|
|
340
|
-
"data_source": DataSource.
|
|
354
|
+
"data_source": DataSource.AWS,
|
|
341
355
|
"storage_url": "storage/url",
|
|
342
|
-
"container_name": "azure container name",
|
|
343
356
|
"data_folder": "datafolder/to/vai_anno",
|
|
344
357
|
"type": DatasetType.ANNOTATED_DATA,
|
|
345
358
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
@@ -347,9 +360,8 @@ dataset_data = {
|
|
|
347
360
|
"sequential": False,
|
|
348
361
|
"render_pcd": False,
|
|
349
362
|
"generate_metadata": False,
|
|
350
|
-
"
|
|
351
|
-
"
|
|
352
|
-
"secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
|
|
363
|
+
"access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
364
|
+
"secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
|
|
353
365
|
}
|
|
354
366
|
dataset = project.create_dataset(**dataset_data)
|
|
355
367
|
|
|
@@ -360,10 +372,9 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
360
372
|
| Argument name | Type/Options | Default | Description |
|
|
361
373
|
| :--- | :--- | :--- | :--- |
|
|
362
374
|
| name | str | *-- | name of your dataset |
|
|
363
|
-
| data_source | DataSource.
|
|
375
|
+
| data_source | DataSource.AWS | *-- | the datasource of your dataset |
|
|
364
376
|
| storage_url | str | *-- | your cloud storage url |
|
|
365
|
-
|
|
|
366
|
-
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
377
|
+
| data_folder | str | *-- | the relative data folder from the storage_url |
|
|
367
378
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
368
379
|
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
369
380
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
@@ -371,11 +382,10 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
371
382
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
372
383
|
| generate_metadata | bool | False | generate image meta data or not |
|
|
373
384
|
| description | str | None | your dataset description |
|
|
374
|
-
| sas_token | str | None | SAStoken for azure container |
|
|
375
385
|
| access_key_id | str | None | access key id for AWS private s3 bucket |
|
|
376
386
|
| secret_access_key | str | None| secret access key for AWS private s3 bucket |
|
|
377
387
|
|
|
378
|
-
`*--`: required
|
|
388
|
+
`*--`: required for cloud storage
|
|
379
389
|
|
|
380
390
|
* Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
|
|
381
391
|
|
|
@@ -387,15 +397,12 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
387
397
|
dataset_data2 = {
|
|
388
398
|
"name": "dataset-local-upload",
|
|
389
399
|
"data_source": DataSource.LOCAL,
|
|
390
|
-
"storage_url": "",
|
|
391
|
-
"container_name": "",
|
|
392
400
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
393
401
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
394
402
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
395
403
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
396
404
|
"sequential": False,
|
|
397
405
|
"generate_metadata": False,
|
|
398
|
-
"sas_token": ""
|
|
399
406
|
}
|
|
400
407
|
dataset2 = project.create_dataset(**dataset_data2)
|
|
401
408
|
|
|
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
|
|
|
2
2
|
|
|
3
3
|
AUTHOR = "LinkerVision"
|
|
4
4
|
PACKAGE_NAME = "dataverse-sdk"
|
|
5
|
-
PACKAGE_VERSION = "2.
|
|
5
|
+
PACKAGE_VERSION = "2.6.0"
|
|
6
6
|
DESC = "Dataverse SDK For Python"
|
|
7
7
|
with open("README.md", encoding="utf-8") as fh:
|
|
8
8
|
long_description = fh.read()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|