dataverse-sdk 2.5.4__tar.gz → 2.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/PKG-INFO +22 -15
  2. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/README.md +21 -14
  3. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/backend.py +9 -5
  4. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/client.py +9 -9
  5. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/api.py +1 -2
  6. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/client.py +4 -8
  7. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/common.py +0 -1
  8. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/PKG-INFO +22 -15
  9. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/setup.py +1 -1
  10. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/__init__.py +0 -0
  11. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/__init__.py +0 -0
  12. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/apis/third_party.py +0 -0
  13. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/connections.py +0 -0
  14. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/constants.py +0 -0
  15. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/exceptions/__init__.py +0 -0
  16. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/exceptions/client.py +0 -0
  17. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/__init__.py +0 -0
  18. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/base.py +0 -0
  19. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/coco.py +0 -0
  20. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/constant.py +0 -0
  21. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/exporter.py +0 -0
  22. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/utils.py +0 -0
  23. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/visionai.py +0 -0
  24. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/vqa.py +0 -0
  25. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/export/yolo.py +0 -0
  26. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/__init__.py +0 -0
  27. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/schemas/format.py +0 -0
  28. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/utils/__init__.py +0 -0
  29. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk/utils/utils.py +0 -0
  30. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
  31. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
  32. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/requires.txt +0 -0
  33. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
  34. {dataverse_sdk-2.5.4 → dataverse_sdk-2.6.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.5.4
3
+ Version: 2.6.0
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -30,7 +30,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
30
30
  Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
31
31
  - Create Project with your input ontology and sensors
32
32
  - Get Project by project-id
33
- - Create Dataset from your AWS/Azure storage or local
33
+ - Create Dataset from your AWS storage or local
34
34
  - Get Dataset by dataset-id
35
35
  - List models for your selected project-id
36
36
  - Get and download your model
@@ -332,14 +332,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
332
332
 
333
333
  ### Create Dataset
334
334
 
335
+ **Required fields by `data_source`:**
336
+
337
+ | `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
338
+ | :--- | :---: | :---: | :---: | :--- |
339
+ | `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
340
+ | `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
341
+ | `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
342
+
343
+ `*--`: required for this `data_source` · `-`: not used (can be omitted)
344
+
345
+ > `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
346
+ >
347
+ > `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
348
+
335
349
  #### Use `create_dataset` to import dataset from **cloud storage**
336
350
 
337
351
  ```Python
338
352
  dataset_data = {
339
353
  "name": "Dataset 1",
340
- "data_source": DataSource.Azure/DataSource.AWS,
354
+ "data_source": DataSource.AWS,
341
355
  "storage_url": "storage/url",
342
- "container_name": "azure container name",
343
356
  "data_folder": "datafolder/to/vai_anno",
344
357
  "type": DatasetType.ANNOTATED_DATA,
345
358
  "annotation_format": AnnotationFormat.VISION_AI,
@@ -347,9 +360,8 @@ dataset_data = {
347
360
  "sequential": False,
348
361
  "render_pcd": False,
349
362
  "generate_metadata": False,
350
- "sas_token": "azure sas token", # only for azure storage
351
- "access_key_id" : "aws s3 access key id",# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
352
- "secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
363
+ "access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
364
+ "secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
353
365
  }
354
366
  dataset = project.create_dataset(**dataset_data)
355
367
 
@@ -360,10 +372,9 @@ dataset = project.create_dataset(**dataset_data)
360
372
  | Argument name | Type/Options | Default | Description |
361
373
  | :--- | :--- | :--- | :--- |
362
374
  | name | str | *-- | name of your dataset |
363
- | data_source | DataSource.Azure <br> DataSource.AWS | *-- | the datasource of your dataset |
375
+ | data_source | DataSource.AWS | *-- | the datasource of your dataset |
364
376
  | storage_url | str | *-- | your cloud storage url |
365
- | container_name | str | None | azure container name |
366
- | data_folder | str | *-- | the relative data folder from the storage_url and container |
377
+ | data_folder | str | *-- | the relative data folder from the storage_url |
367
378
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
368
379
  | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
369
380
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
@@ -371,11 +382,10 @@ dataset = project.create_dataset(**dataset_data)
371
382
  | render_pcd | bool | False | render pcd preview image or not |
372
383
  | generate_metadata | bool | False | generate image meta data or not |
373
384
  | description | str | None | your dataset description |
374
- | sas_token | str | None | SAStoken for azure container |
375
385
  | access_key_id | str | None | access key id for AWS private s3 bucket |
376
386
  | secret_access_key | str | None| secret access key for AWS private s3 bucket |
377
387
 
378
- `*--`: required argument without default
388
+ `*--`: required for cloud storage
379
389
 
380
390
  * Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
381
391
 
@@ -387,15 +397,12 @@ dataset = project.create_dataset(**dataset_data)
387
397
  dataset_data2 = {
388
398
  "name": "dataset-local-upload",
389
399
  "data_source": DataSource.LOCAL,
390
- "storage_url": "",
391
- "container_name": "",
392
400
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
393
401
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
394
402
  "annotation_format": AnnotationFormat.VISION_AI,
395
403
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
396
404
  "sequential": False,
397
405
  "generate_metadata": False,
398
- "sas_token": ""
399
406
  }
400
407
  dataset2 = project.create_dataset(**dataset_data2)
401
408
 
@@ -3,7 +3,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
3
3
  Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
4
4
  - Create Project with your input ontology and sensors
5
5
  - Get Project by project-id
6
- - Create Dataset from your AWS/Azure storage or local
6
+ - Create Dataset from your AWS storage or local
7
7
  - Get Dataset by dataset-id
8
8
  - List models for your selected project-id
9
9
  - Get and download your model
@@ -305,14 +305,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
305
305
 
306
306
  ### Create Dataset
307
307
 
308
+ **Required fields by `data_source`:**
309
+
310
+ | `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
311
+ | :--- | :---: | :---: | :---: | :--- |
312
+ | `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
313
+ | `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
314
+ | `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
315
+
316
+ `*--`: required for this `data_source` · `-`: not used (can be omitted)
317
+
318
+ > `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
319
+ >
320
+ > `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
321
+
308
322
  #### Use `create_dataset` to import dataset from **cloud storage**
309
323
 
310
324
  ```Python
311
325
  dataset_data = {
312
326
  "name": "Dataset 1",
313
- "data_source": DataSource.Azure/DataSource.AWS,
327
+ "data_source": DataSource.AWS,
314
328
  "storage_url": "storage/url",
315
- "container_name": "azure container name",
316
329
  "data_folder": "datafolder/to/vai_anno",
317
330
  "type": DatasetType.ANNOTATED_DATA,
318
331
  "annotation_format": AnnotationFormat.VISION_AI,
@@ -320,9 +333,8 @@ dataset_data = {
320
333
  "sequential": False,
321
334
  "render_pcd": False,
322
335
  "generate_metadata": False,
323
- "sas_token": "azure sas token", # only for azure storage
324
- "access_key_id" : "aws s3 access key id",# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
325
- "secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
336
+ "access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
337
+ "secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
326
338
  }
327
339
  dataset = project.create_dataset(**dataset_data)
328
340
 
@@ -333,10 +345,9 @@ dataset = project.create_dataset(**dataset_data)
333
345
  | Argument name | Type/Options | Default | Description |
334
346
  | :--- | :--- | :--- | :--- |
335
347
  | name | str | *-- | name of your dataset |
336
- | data_source | DataSource.Azure <br> DataSource.AWS | *-- | the datasource of your dataset |
348
+ | data_source | DataSource.AWS | *-- | the datasource of your dataset |
337
349
  | storage_url | str | *-- | your cloud storage url |
338
- | container_name | str | None | azure container name |
339
- | data_folder | str | *-- | the relative data folder from the storage_url and container |
350
+ | data_folder | str | *-- | the relative data folder from the storage_url |
340
351
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
341
352
  | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
342
353
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
@@ -344,11 +355,10 @@ dataset = project.create_dataset(**dataset_data)
344
355
  | render_pcd | bool | False | render pcd preview image or not |
345
356
  | generate_metadata | bool | False | generate image meta data or not |
346
357
  | description | str | None | your dataset description |
347
- | sas_token | str | None | SAStoken for azure container |
348
358
  | access_key_id | str | None | access key id for AWS private s3 bucket |
349
359
  | secret_access_key | str | None| secret access key for AWS private s3 bucket |
350
360
 
351
- `*--`: required argument without default
361
+ `*--`: required for cloud storage
352
362
 
353
363
  * Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
354
364
 
@@ -360,15 +370,12 @@ dataset = project.create_dataset(**dataset_data)
360
370
  dataset_data2 = {
361
371
  "name": "dataset-local-upload",
362
372
  "data_source": DataSource.LOCAL,
363
- "storage_url": "",
364
- "container_name": "",
365
373
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
366
374
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
367
375
  "annotation_format": AnnotationFormat.VISION_AI,
368
376
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
369
377
  "sequential": False,
370
378
  "generate_metadata": False,
371
- "sas_token": ""
372
379
  }
373
380
  dataset2 = project.create_dataset(**dataset_data2)
374
381
 
@@ -402,13 +402,12 @@ class BackendAPI:
402
402
  project_id: int,
403
403
  type: str,
404
404
  annotation_format: str,
405
- storage_url: str,
406
405
  data_folder: str,
406
+ storage_url: Optional[str] = None,
407
407
  sequential: bool = False,
408
408
  generate_metadata: bool = False,
409
409
  render_pcd: bool = False,
410
410
  container_name: Optional[str] = None,
411
- sas_token: Optional[str] = None,
412
411
  description: Optional[str] = None,
413
412
  annotations: Optional[list[str]] = None,
414
413
  access_key_id: Optional[str] = None,
@@ -421,10 +420,7 @@ class BackendAPI:
421
420
  "name": name,
422
421
  "project_id": project_id,
423
422
  "data_source": data_source,
424
- "storage_url": storage_url,
425
- "container_name": container_name,
426
423
  "data_folder": data_folder,
427
- "sas_token": sas_token,
428
424
  "type": type,
429
425
  "sequential": sequential,
430
426
  "annotation_format": annotation_format,
@@ -435,6 +431,14 @@ class BackendAPI:
435
431
  "auto_tagging": [], # FIXME: auto_tagging field is still required by production API.
436
432
  }
437
433
 
434
+ # Only send storage location fields when provided
435
+ for key, value in (
436
+ ("storage_url", storage_url),
437
+ ("container_name", container_name),
438
+ ):
439
+ if value is not None:
440
+ payload_data[key] = value
441
+
438
442
  aws_access_key = {secret_access_key, access_key_id}
439
443
  if not (all(aws_access_key) or not any(aws_access_key)):
440
444
  raise ValueError("Need to assign both secret_access_key and access_key_id")
@@ -1481,7 +1481,6 @@ of this project OR has been added before"
1481
1481
  dataset_data.update({"project": project})
1482
1482
  return Dataset(**dataset_data, client_alias=client_alias)
1483
1483
 
1484
- # TODO: required arguments for different DataSource
1485
1484
  @staticmethod
1486
1485
  def create_dataset(
1487
1486
  name: str,
@@ -1489,10 +1488,9 @@ of this project OR has been added before"
1489
1488
  project: Project,
1490
1489
  type: DatasetType,
1491
1490
  annotation_format: AnnotationFormat,
1492
- storage_url: str,
1493
1491
  data_folder: str,
1492
+ storage_url: Optional[str] = None,
1494
1493
  container_name: Optional[str] = None,
1495
- sas_token: Optional[str] = None,
1496
1494
  annotations: Optional[list] = None,
1497
1495
  sequential: bool = False,
1498
1496
  generate_metadata: bool = False,
@@ -1519,14 +1517,12 @@ of this project OR has been added before"
1519
1517
  datasettype (annotation or raw)
1520
1518
  annotation_format : AnnotationFormat
1521
1519
  format type of annotation
1522
- storage_url : str
1523
- storage url for cloud
1524
1520
  data_folder : str
1525
1521
  data folder of the storage
1522
+ storage_url : Optional[str], optional
1523
+ storage url for cloud storage (e.g. AWS), by default None
1526
1524
  container_name : Optional[str], optional
1527
- container name for Azure, by default None
1528
- sas_token : Optional[str], optional
1529
- SAStoken for Azure, by default None
1525
+ MinIO bucket name, by default None
1530
1526
  annotations: list, optional
1531
1527
  list of annotation folder name (should be groundtruth or $model_name)
1532
1528
  sequential : bool, optional
@@ -1557,6 +1553,11 @@ of this project OR has been added before"
1557
1553
  ClientConnectionError
1558
1554
  raise exception if there is any error occurs when calling backend APIs.
1559
1555
  """
1556
+ if not data_folder:
1557
+ raise ValueError(
1558
+ "`data_folder` is required to create a dataset and cannot be an empty string"
1559
+ )
1560
+
1560
1561
  if annotations is None:
1561
1562
  annotations = []
1562
1563
 
@@ -1583,7 +1584,6 @@ of this project OR has been added before"
1583
1584
  storage_url=storage_url,
1584
1585
  container_name=container_name,
1585
1586
  data_folder=data_folder,
1586
- sas_token=sas_token,
1587
1587
  sequential=sequential,
1588
1588
  generate_metadata=generate_metadata,
1589
1589
  render_pcd=render_pcd,
@@ -117,10 +117,9 @@ class DatasetAPISchema(BaseModel):
117
117
  data_source: DataSource
118
118
  type: DatasetType
119
119
  annotation_format: AnnotationFormat
120
- storage_url: str
121
120
  data_folder: str
121
+ storage_url: Optional[str] = None
122
122
  container_name: Optional[str] = None
123
- sas_token: Optional[str] = None
124
123
  sequential: bool = False
125
124
  generate_metadata: bool = False
126
125
  render_pcd: Optional[bool] = None
@@ -369,10 +369,9 @@ class Project(BaseModel):
369
369
  data_source: DataSource,
370
370
  type: DatasetType,
371
371
  annotation_format: AnnotationFormat,
372
- storage_url: str,
373
372
  data_folder: str,
373
+ storage_url: Optional[str] = None,
374
374
  container_name: Optional[str] = None,
375
- sas_token: Optional[str] = None,
376
375
  annotations: Optional[list] = None,
377
376
  sequential: bool = False,
378
377
  generate_metadata: bool = False,
@@ -395,14 +394,12 @@ class Project(BaseModel):
395
394
  datasettype (annotation or raw)
396
395
  annotation_format : AnnotationFormat
397
396
  format type of annotation
398
- storage_url : str
399
- storage url for cloud
400
397
  data_folder : str
401
398
  data folder of the storage
399
+ storage_url : Optional[str], optional
400
+ storage url for cloud storage (e.g. AWS), by default None
402
401
  container_name : Optional[str], optional
403
- container name for Azure, by default None
404
- sas_token : Optional[str], optional
405
- SAStoken for Azure, by default None
402
+ MinIO bucket name, by default None
406
403
  annotations: list, optional
407
404
  list of annotation folder name (should be groundtruth or $model_name)
408
405
  sequential : bool, optional
@@ -447,7 +444,6 @@ class Project(BaseModel):
447
444
  storage_url=storage_url,
448
445
  container_name=container_name,
449
446
  data_folder=data_folder,
450
- sas_token=sas_token,
451
447
  annotations=annotations,
452
448
  sequential=sequential,
453
449
  generate_metadata=generate_metadata,
@@ -55,7 +55,6 @@ class DatasetStatus(str, Enum, metaclass=BaseEnumMeta):
55
55
 
56
56
 
57
57
  class DataSource(str, Enum, metaclass=BaseEnumMeta):
58
- Azure = "azure"
59
58
  AWS = "aws"
60
59
  LOCAL = "local"
61
60
  SDK = "sdk"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.5.4
3
+ Version: 2.6.0
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -30,7 +30,7 @@ Dataverse is a MLOPs platform for assisting in data selection, data visualizatio
30
30
  Use Dataverse-SDK for Python to help you to interact with the Dataverse platform by Python. Currently, the library supports:
31
31
  - Create Project with your input ontology and sensors
32
32
  - Get Project by project-id
33
- - Create Dataset from your AWS/Azure storage or local
33
+ - Create Dataset from your AWS storage or local
34
34
  - Get Dataset by dataset-id
35
35
  - List models for your selected project-id
36
36
  - Get and download your model
@@ -332,14 +332,27 @@ client.update_alias(project_id=123, alias_file_path= "/Users/Downloads/alias.csv
332
332
 
333
333
  ### Create Dataset
334
334
 
335
+ **Required fields by `data_source`:**
336
+
337
+ | `data_source` | `storage_url` | `container_name` | `data_folder` | Notes |
338
+ | :--- | :---: | :---: | :---: | :--- |
339
+ | `DataSource.AWS` | *-- | - | *-- | use `access_key_id` + `secret_access_key` for a private S3 bucket |
340
+ | `DataSource.LOCAL` | - | - | *-- | local folder; SDK uploads files and sends `create_dataset_uuid` for you |
341
+ | `DataSource.SDK` | - | *-- | *-- | offline MinIO import: `container_name` = bucket, `data_folder` = path in bucket; needs Dataverse deployed in offline mode |
342
+
343
+ `*--`: required for this `data_source` · `-`: not used (can be omitted)
344
+
345
+ > `DataSource.DATA_GENERATION` and `DataSource.PRE_IMPORT` require a `data_source_search_body` that the SDK does not currently send, so they are not supported via `create_dataset`.
346
+ >
347
+ > `DataSource.EXISTING_DATASETS` / `EXISTING_DATASLICES` only appear on datasets you read back from the API; they are not inputs for `create_dataset`.
348
+
335
349
  #### Use `create_dataset` to import dataset from **cloud storage**
336
350
 
337
351
  ```Python
338
352
  dataset_data = {
339
353
  "name": "Dataset 1",
340
- "data_source": DataSource.Azure/DataSource.AWS,
354
+ "data_source": DataSource.AWS,
341
355
  "storage_url": "storage/url",
342
- "container_name": "azure container name",
343
356
  "data_folder": "datafolder/to/vai_anno",
344
357
  "type": DatasetType.ANNOTATED_DATA,
345
358
  "annotation_format": AnnotationFormat.VISION_AI,
@@ -347,9 +360,8 @@ dataset_data = {
347
360
  "sequential": False,
348
361
  "render_pcd": False,
349
362
  "generate_metadata": False,
350
- "sas_token": "azure sas token", # only for azure storage
351
- "access_key_id" : "aws s3 access key id",# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
352
- "secret_access_key": "aws s3 secret access key"# only for private s3 bucket, don't need to assign it in case of public s3 bucket or azure data source
363
+ "access_key_id": "aws s3 access key id", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
364
+ "secret_access_key": "aws s3 secret access key", # only for private s3 bucket, don't need to assign it in case of public s3 bucket
353
365
  }
354
366
  dataset = project.create_dataset(**dataset_data)
355
367
 
@@ -360,10 +372,9 @@ dataset = project.create_dataset(**dataset_data)
360
372
  | Argument name | Type/Options | Default | Description |
361
373
  | :--- | :--- | :--- | :--- |
362
374
  | name | str | *-- | name of your dataset |
363
- | data_source | DataSource.Azure <br> DataSource.AWS | *-- | the datasource of your dataset |
375
+ | data_source | DataSource.AWS | *-- | the datasource of your dataset |
364
376
  | storage_url | str | *-- | your cloud storage url |
365
- | container_name | str | None | azure container name |
366
- | data_folder | str | *-- | the relative data folder from the storage_url and container |
377
+ | data_folder | str | *-- | the relative data folder from the storage_url |
367
378
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
368
379
  | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
369
380
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
@@ -371,11 +382,10 @@ dataset = project.create_dataset(**dataset_data)
371
382
  | render_pcd | bool | False | render pcd preview image or not |
372
383
  | generate_metadata | bool | False | generate image meta data or not |
373
384
  | description | str | None | your dataset description |
374
- | sas_token | str | None | SAStoken for azure container |
375
385
  | access_key_id | str | None | access key id for AWS private s3 bucket |
376
386
  | secret_access_key | str | None| secret access key for AWS private s3 bucket |
377
387
 
378
- `*--`: required argument without default
388
+ `*--`: required for cloud storage
379
389
 
380
390
  * Check https://linkervision.gitbook.io/dataverse/data-management/import-dataset for the detail of `Import Dataset`.
381
391
 
@@ -387,15 +397,12 @@ dataset = project.create_dataset(**dataset_data)
387
397
  dataset_data2 = {
388
398
  "name": "dataset-local-upload",
389
399
  "data_source": DataSource.LOCAL,
390
- "storage_url": "",
391
- "container_name": "",
392
400
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
393
401
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
394
402
  "annotation_format": AnnotationFormat.VISION_AI,
395
403
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
396
404
  "sequential": False,
397
405
  "generate_metadata": False,
398
- "sas_token": ""
399
406
  }
400
407
  dataset2 = project.create_dataset(**dataset_data2)
401
408
 
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
2
2
 
3
3
  AUTHOR = "LinkerVision"
4
4
  PACKAGE_NAME = "dataverse-sdk"
5
- PACKAGE_VERSION = "2.5.4"
5
+ PACKAGE_VERSION = "2.6.0"
6
6
  DESC = "Dataverse SDK For Python"
7
7
  with open("README.md", encoding="utf-8") as fh:
8
8
  long_description = fh.read()
File without changes