dataverse-sdk 2.5.1__tar.gz → 2.5.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/PKG-INFO +29 -30
  2. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/README.md +28 -29
  3. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/__init__.py +29 -0
  4. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/client.py +1 -1
  5. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/constants.py +4 -5
  6. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/client.py +60 -0
  7. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/common.py +24 -0
  8. dataverse_sdk-2.5.3/dataverse_sdk/utils/utils.py +339 -0
  9. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/PKG-INFO +29 -30
  10. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/setup.py +1 -1
  11. dataverse_sdk-2.5.1/dataverse_sdk/utils/utils.py +0 -100
  12. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/__init__.py +0 -0
  13. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/backend.py +0 -0
  14. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/third_party.py +0 -0
  15. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/connections.py +0 -0
  16. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/exceptions/__init__.py +0 -0
  17. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/exceptions/client.py +0 -0
  18. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/__init__.py +0 -0
  19. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/base.py +0 -0
  20. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/coco.py +0 -0
  21. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/constant.py +0 -0
  22. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/exporter.py +0 -0
  23. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/utils.py +0 -0
  24. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/visionai.py +0 -0
  25. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/vqa.py +0 -0
  26. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/yolo.py +0 -0
  27. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/__init__.py +0 -0
  28. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/api.py +0 -0
  29. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/format.py +0 -0
  30. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/utils/__init__.py +0 -0
  31. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
  32. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
  33. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/requires.txt +0 -0
  34. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/top_level.txt +0 -0
  35. {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.5.1
3
+ Version: 2.5.3
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -35,7 +35,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
35
35
  - List models for your selected project-id
36
36
  - Get and download your model
37
37
 
38
- [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkernetworks/dataverse-sdk)
38
+ [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
39
39
 
40
40
 
41
41
  ## Getting started
@@ -69,7 +69,7 @@ client2 = DataverseClient(
69
69
  assert client2 is get_connection(client2.alias)
70
70
 
71
71
  client3 = DataverseClient(
72
- host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
72
+ host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
73
73
  )
74
74
  assert client3 is get_connection(client3.alias)
75
75
  ```
@@ -78,7 +78,7 @@ assert client3 is get_connection(client3.alias)
78
78
 
79
79
  | Argument name | Type/Options | Default | Description |
80
80
  | :--- | :--- | :--- | :--- |
81
- | host | str | *-- | the host url of the dataverse site (with curation port) |
81
+ | host | str | *-- | the host url of the dataverse site |
82
82
  | email | str | *-- | the email account of your dataverse workspace |
83
83
  | password | str | *-- | the password of your dataverse workspace |
84
84
  | service_id | str | *-- | The service id of the dataverse you want to connect |
@@ -129,7 +129,7 @@ The `list_projects` method will list all projects of the given sites.
129
129
  ```Python
130
130
  projects = client.list_projects(current_user = True,
131
131
  exclude_sensor_type=SensorType.LIDAR,
132
- image_type= OntologyImageType._2D_BOUNDING_BOX)
132
+ image_type=OntologyImageType._2D_BOUNDING_BOX)
133
133
 
134
134
  ```
135
135
 
@@ -139,7 +139,7 @@ projects = client.list_projects(current_user = True,
139
139
  | :--- | :--- | :--- | :--- |
140
140
  | current_user | bool | True | only show the projects of current user |
141
141
  | exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
142
- | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
142
+ | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
143
143
 
144
144
  <br>
145
145
 
@@ -167,7 +167,7 @@ ontology = Ontology(
167
167
  ],
168
168
  )
169
169
  ```
170
- For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
170
+ For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
171
171
 
172
172
  For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
173
173
 
@@ -213,7 +213,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
213
213
 
214
214
  ### Get Project
215
215
 
216
- The `get_proejct` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
216
+ The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
217
217
 
218
218
  ```Python
219
219
  project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
@@ -364,9 +364,8 @@ dataset = project.create_dataset(**dataset_data)
364
364
  | storage_url | str | *-- | your cloud storage url |
365
365
  | container_name | str | None | azure container name |
366
366
  | data_folder | str | *-- | the relative data folder from the storage_url and container |
367
- | sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
368
367
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
369
- | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE | *-- | the format of your annotation data |
368
+ | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
370
369
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
371
370
  | sequential | bool | False | data is sequential or not |
372
371
  | render_pcd | bool | False | render pcd preview image or not |
@@ -391,7 +390,6 @@ dataset_data2 = {
391
390
  "storage_url": "",
392
391
  "container_name": "",
393
392
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
394
- "sensors": project.sensors,
395
393
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
396
394
  "annotation_format": AnnotationFormat.VISION_AI,
397
395
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
@@ -405,7 +403,7 @@ dataset2 = project.create_dataset(**dataset_data2)
405
403
 
406
404
  Your could also use the script for importing dataset from local
407
405
  ```
408
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
406
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
409
407
  ```
410
408
  <br>
411
409
 
@@ -537,13 +535,13 @@ The `create_vqa_project` method will create project on the connected site with t
537
535
  # 1) Create question class with question and answer type pair
538
536
  question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
539
537
  answer_type="boolean"),
540
- QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yello","green"])
538
+ QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
541
539
  ]
542
540
  ```
543
541
 
544
542
  ```Python
545
543
  # 2) Create your VQA project as below
546
- project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
544
+ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
547
545
  ```
548
546
 
549
547
  * Input arguments for creating project:
@@ -553,7 +551,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
553
551
  | name | str | *-- | name of your project |
554
552
  | sensor_name | str | *-- | the camera sensor name |
555
553
  | ontology_name | str | *-- | the ontology name |
556
- | question_answer| list[QuestionClass] | *-- | your question/answer_type |
554
+ | question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
557
555
  | description | str | None | your project description |
558
556
 
559
557
  `*--`: required argument without default
@@ -572,15 +570,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
572
570
  create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
573
571
  update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
574
572
 
575
- #should provide client_alias if calling from client
576
- client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
577
- create=create_questions,
578
- update=update_questions,
579
- client_alias=client.alias)
580
- #OR
581
- project.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
582
- create=create_questions,
583
- update=update_questions)
573
+ # Through client
574
+ client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
575
+ create=create_questions,
576
+ update=update_questions,
577
+ client_alias=client.alias)
578
+
579
+ # Through project object
580
+ project.edit_vqa_ontology(ontology_name="ontology-new-name",
581
+ create=create_questions,
582
+ update=update_questions)
584
583
  ```
585
584
 
586
585
  ### Get Question List
@@ -598,28 +597,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
598
597
 
599
598
  ### Import Your Local Dataset
600
599
  ```
601
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
600
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
602
601
  ```
603
602
 
604
603
  ### Import VQA Local Dataset
605
604
  ```
606
- python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
605
+ python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
607
606
 
608
607
  ```
609
608
 
610
609
  ### Export Dataslice and download files
611
610
  ```
612
- python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
611
+ python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
613
612
  ```
614
613
 
615
614
  ### Export Large Dataslice and download files
616
615
  ```
617
- python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
616
+ python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
618
617
  ```
619
618
 
620
619
  ### Upload videos to create session tasks
621
620
  ```
622
- python tools/upload_videos_create_session.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
621
+ python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
623
622
  ```
624
623
 
625
624
  - Advanced arguments for video curation (sequential data):
@@ -633,4 +632,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
633
632
 
634
633
  ## Links to language repos
635
634
 
636
- [Python Readme](https://github.com/linkernetworks/dataverse-sdk/tree/master/python/README.md)
635
+ [Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
@@ -8,7 +8,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
8
8
  - List models for your selected project-id
9
9
  - Get and download your model
10
10
 
11
- [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkernetworks/dataverse-sdk)
11
+ [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
12
12
 
13
13
 
14
14
  ## Getting started
@@ -42,7 +42,7 @@ client2 = DataverseClient(
42
42
  assert client2 is get_connection(client2.alias)
43
43
 
44
44
  client3 = DataverseClient(
45
- host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
45
+ host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
46
46
  )
47
47
  assert client3 is get_connection(client3.alias)
48
48
  ```
@@ -51,7 +51,7 @@ assert client3 is get_connection(client3.alias)
51
51
 
52
52
  | Argument name | Type/Options | Default | Description |
53
53
  | :--- | :--- | :--- | :--- |
54
- | host | str | *-- | the host url of the dataverse site (with curation port) |
54
+ | host | str | *-- | the host url of the dataverse site |
55
55
  | email | str | *-- | the email account of your dataverse workspace |
56
56
  | password | str | *-- | the password of your dataverse workspace |
57
57
  | service_id | str | *-- | The service id of the dataverse you want to connect |
@@ -102,7 +102,7 @@ The `list_projects` method will list all projects of the given sites.
102
102
  ```Python
103
103
  projects = client.list_projects(current_user = True,
104
104
  exclude_sensor_type=SensorType.LIDAR,
105
- image_type= OntologyImageType._2D_BOUNDING_BOX)
105
+ image_type=OntologyImageType._2D_BOUNDING_BOX)
106
106
 
107
107
  ```
108
108
 
@@ -112,7 +112,7 @@ projects = client.list_projects(current_user = True,
112
112
  | :--- | :--- | :--- | :--- |
113
113
  | current_user | bool | True | only show the projects of current user |
114
114
  | exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
115
- | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
115
+ | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
116
116
 
117
117
  <br>
118
118
 
@@ -140,7 +140,7 @@ ontology = Ontology(
140
140
  ],
141
141
  )
142
142
  ```
143
- For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
143
+ For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
144
144
 
145
145
  For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
146
146
 
@@ -186,7 +186,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
186
186
 
187
187
  ### Get Project
188
188
 
189
- The `get_proejct` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
189
+ The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
190
190
 
191
191
  ```Python
192
192
  project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
@@ -337,9 +337,8 @@ dataset = project.create_dataset(**dataset_data)
337
337
  | storage_url | str | *-- | your cloud storage url |
338
338
  | container_name | str | None | azure container name |
339
339
  | data_folder | str | *-- | the relative data folder from the storage_url and container |
340
- | sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
341
340
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
342
- | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE | *-- | the format of your annotation data |
341
+ | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
343
342
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
344
343
  | sequential | bool | False | data is sequential or not |
345
344
  | render_pcd | bool | False | render pcd preview image or not |
@@ -364,7 +363,6 @@ dataset_data2 = {
364
363
  "storage_url": "",
365
364
  "container_name": "",
366
365
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
367
- "sensors": project.sensors,
368
366
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
369
367
  "annotation_format": AnnotationFormat.VISION_AI,
370
368
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
@@ -378,7 +376,7 @@ dataset2 = project.create_dataset(**dataset_data2)
378
376
 
379
377
  Your could also use the script for importing dataset from local
380
378
  ```
381
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
379
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
382
380
  ```
383
381
  <br>
384
382
 
@@ -510,13 +508,13 @@ The `create_vqa_project` method will create project on the connected site with t
510
508
  # 1) Create question class with question and answer type pair
511
509
  question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
512
510
  answer_type="boolean"),
513
- QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yello","green"])
511
+ QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
514
512
  ]
515
513
  ```
516
514
 
517
515
  ```Python
518
516
  # 2) Create your VQA project as below
519
- project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
517
+ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
520
518
  ```
521
519
 
522
520
  * Input arguments for creating project:
@@ -526,7 +524,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
526
524
  | name | str | *-- | name of your project |
527
525
  | sensor_name | str | *-- | the camera sensor name |
528
526
  | ontology_name | str | *-- | the ontology name |
529
- | question_answer| list[QuestionClass] | *-- | your question/answer_type |
527
+ | question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
530
528
  | description | str | None | your project description |
531
529
 
532
530
  `*--`: required argument without default
@@ -545,15 +543,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
545
543
  create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
546
544
  update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
547
545
 
548
- #should provide client_alias if calling from client
549
- client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
550
- create=create_questions,
551
- update=update_questions,
552
- client_alias=client.alias)
553
- #OR
554
- project.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
555
- create=create_questions,
556
- update=update_questions)
546
+ # Through client
547
+ client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
548
+ create=create_questions,
549
+ update=update_questions,
550
+ client_alias=client.alias)
551
+
552
+ # Through project object
553
+ project.edit_vqa_ontology(ontology_name="ontology-new-name",
554
+ create=create_questions,
555
+ update=update_questions)
557
556
  ```
558
557
 
559
558
  ### Get Question List
@@ -571,28 +570,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
571
570
 
572
571
  ### Import Your Local Dataset
573
572
  ```
574
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
573
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
575
574
  ```
576
575
 
577
576
  ### Import VQA Local Dataset
578
577
  ```
579
- python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
578
+ python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
580
579
 
581
580
  ```
582
581
 
583
582
  ### Export Dataslice and download files
584
583
  ```
585
- python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
584
+ python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
586
585
  ```
587
586
 
588
587
  ### Export Large Dataslice and download files
589
588
  ```
590
- python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
589
+ python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
591
590
  ```
592
591
 
593
592
  ### Upload videos to create session tasks
594
593
  ```
595
- python tools/upload_videos_create_session.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
594
+ python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
596
595
  ```
597
596
 
598
597
  - Advanced arguments for video curation (sequential data):
@@ -606,4 +605,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
606
605
 
607
606
  ## Links to language repos
608
607
 
609
- [Python Readme](https://github.com/linkernetworks/dataverse-sdk/tree/master/python/README.md)
608
+ [Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
@@ -1,3 +1,6 @@
1
+ import logging
2
+ import re
3
+
1
4
  from . import connections
2
5
  from .client import DataverseClient
3
6
  from .constants import DataverseHost
@@ -45,3 +48,29 @@ __all__ = [
45
48
  "DataSource",
46
49
  "QuestionClass",
47
50
  ]
51
+
52
+
53
+ class _SensitiveDataFilter(logging.Filter):
54
+ """Filter to redact sensitive credentials from logs"""
55
+
56
+ SENSITIVE_PATTERNS = [
57
+ (r"AWSAccessKeyId=[^&\s]*", "AWSAccessKeyId=***"),
58
+ (r"Signature=[^&\s]*", "Signature=***"),
59
+ (r"Expires=\d+", "Expires=***"),
60
+ ]
61
+
62
+ def filter(self, record):
63
+ # Override getMessage to process the final formatted message
64
+ original_getMessage = record.getMessage
65
+
66
+ def patched_getMessage():
67
+ msg = original_getMessage()
68
+ for pattern, replacement in self.SENSITIVE_PATTERNS:
69
+ msg = re.sub(pattern, replacement, msg)
70
+ return msg
71
+
72
+ record.getMessage = patched_getMessage
73
+ return True
74
+
75
+
76
+ logging.getLogger("httpx").addFilter(_SensitiveDataFilter())
@@ -492,7 +492,7 @@ class DataverseClient:
492
492
 
493
493
  Returns
494
494
  -------
495
- list[Projects]
495
+ list[Project]
496
496
  list of project items
497
497
 
498
498
  Raises
@@ -12,10 +12,9 @@ class BaseEnumMeta(EnumMeta):
12
12
 
13
13
 
14
14
  class DataverseHost(str, Enum, metaclass=BaseEnumMeta):
15
- DEV = "https://dev.visionai.linkervision.ai/dataverse/curation"
16
- DEV2 = "https://dev2.visionai.linkervision.ai/dataverse/curation"
17
- DEV3 = "https://dev3.visionai.linkervision.ai/dataverse/curation"
18
- STAGING = "https://staging.visionai.linkervision.ai/dataverse/curation"
19
- DEMO = "https://demo.visionai.linkervision.ai/dataverse/curation"
15
+ DEV = "https://dev.visionai.linkervision.com/dataverse/curation"
16
+ DEV2 = "https://dev2.visionai.linkervision.com/dataverse/curation"
17
+ DEV3 = "https://dev3.visionai.linkervision.com/dataverse/curation"
18
+ STAGING = "https://staging.visionai.linkervision.com/dataverse/curation"
20
19
  PRODUCTION = "https://visionai.linkervision.ai/dataverse/curation"
21
20
  LOCAL = "http://localhost:8000"
@@ -12,6 +12,8 @@ from .common import (
12
12
  DataSource,
13
13
  OntologyImageType,
14
14
  OntologyPcdType,
15
+ ProjectCreateDatasetConfig,
16
+ SensorCounts,
15
17
  SensorType,
16
18
  )
17
19
 
@@ -305,6 +307,62 @@ class Project(BaseModel):
305
307
  )
306
308
  return convert_record_data
307
309
 
310
+ def _validate_before_create_dataset(
311
+ self,
312
+ annotation_format: AnnotationFormat,
313
+ dataset_type: DatasetType,
314
+ sequential: bool,
315
+ ) -> None:
316
+ from dataverse_sdk.utils.utils import validate_before_create_dataset
317
+
318
+ if self.sensors is None:
319
+ from ..client import DataverseClient
320
+
321
+ project = DataverseClient.get_client_project(
322
+ project_id=self.id, client_alias=self.client_alias
323
+ )
324
+ self.sensors = project.sensors or []
325
+
326
+ sensor_counts = SensorCounts(camera=0, lidar=0)
327
+ for sensor in self.sensors:
328
+ sensor_type: SensorType = getattr(sensor, "type", "")
329
+ if sensor_type == SensorType.CAMERA:
330
+ sensor_counts.camera += 1
331
+ elif sensor_type == SensorType.LIDAR:
332
+ sensor_counts.lidar += 1
333
+
334
+ image_type = None
335
+ pcd_type = None
336
+ has_attribute = False
337
+
338
+ if self.ontology:
339
+ if self.ontology.image_type:
340
+ image_type = self.ontology.image_type
341
+
342
+ if self.ontology.pcd_type:
343
+ pcd_type = self.ontology.pcd_type
344
+
345
+ if self.ontology.classes:
346
+ for cls in self.ontology.classes:
347
+ if cls.attributes:
348
+ has_attribute = True
349
+ break
350
+
351
+ config = ProjectCreateDatasetConfig(
352
+ annotation_format=annotation_format,
353
+ dataset_type=dataset_type,
354
+ sensor_counts=sensor_counts,
355
+ is_sequential=sequential,
356
+ image_type=image_type,
357
+ pcd_type=pcd_type,
358
+ has_attribute=has_attribute,
359
+ )
360
+
361
+ is_valid, error_message = validate_before_create_dataset(config)
362
+
363
+ if not is_valid:
364
+ raise ValueError(error_message)
365
+
308
366
  def create_dataset(
309
367
  self,
310
368
  name: str,
@@ -375,6 +433,8 @@ class Project(BaseModel):
375
433
  """
376
434
  from ..client import DataverseClient
377
435
 
436
+ self._validate_before_create_dataset(annotation_format, type, sequential)
437
+
378
438
  if annotations is None:
379
439
  annotations = []
380
440
 
@@ -1,4 +1,6 @@
1
+ from dataclasses import dataclass
1
2
  from enum import Enum
3
+ from typing import Optional
2
4
 
3
5
  from ..constants import BaseEnumMeta
4
6
 
@@ -37,6 +39,7 @@ class AnnotationFormat(str, Enum, metaclass=BaseEnumMeta):
37
39
  IMAGE = "image"
38
40
  KITTI = "kitti"
39
41
  YOLO = "yolo"
42
+ VIDEO = "video"
40
43
  VLM = "vlm"
41
44
 
42
45
 
@@ -56,3 +59,24 @@ class DataSource(str, Enum, metaclass=BaseEnumMeta):
56
59
  AWS = "aws"
57
60
  LOCAL = "local"
58
61
  SDK = "sdk"
62
+ EXISTING_DATASETS = "existing_datasets"
63
+ EXISTING_DATASLICES = "existing_dataslices"
64
+ DATA_GENERATION = "data_generation"
65
+ PRE_IMPORT = "pre_import"
66
+
67
+
68
+ @dataclass
69
+ class SensorCounts:
70
+ camera: int = 0
71
+ lidar: int = 0
72
+
73
+
74
+ @dataclass
75
+ class ProjectCreateDatasetConfig:
76
+ annotation_format: AnnotationFormat
77
+ dataset_type: DatasetType
78
+ sensor_counts: SensorCounts
79
+ is_sequential: bool
80
+ image_type: Optional[OntologyImageType] = None
81
+ pcd_type: Optional[OntologyPcdType] = None
82
+ has_attribute: bool = False
@@ -0,0 +1,339 @@
1
+ from pathlib import Path
2
+ from typing import Optional, Set
3
+
4
+ import requests
5
+ from tqdm import tqdm
6
+
7
+ from dataverse_sdk.schemas.common import (
8
+ AnnotationFormat,
9
+ DatasetType,
10
+ OntologyImageType,
11
+ OntologyPcdType,
12
+ ProjectCreateDatasetConfig,
13
+ )
14
+
15
+ IMAGE_SUPPORTED_FORMAT = {
16
+ "jpeg",
17
+ "jpg",
18
+ "png",
19
+ "bmp",
20
+ }
21
+ CUBOID_SUPPORTED_FORMAT = {"pcd"}
22
+
23
+
24
+ def get_filepaths(directory: str) -> list[str]:
25
+ SUPPORTED_FORMATS: Set[str] = (
26
+ IMAGE_SUPPORTED_FORMAT | CUBOID_SUPPORTED_FORMAT | {"txt", "json"}
27
+ )
28
+
29
+ directory = Path(directory)
30
+ all_files = []
31
+
32
+ for file_path in directory.rglob("*"):
33
+ if (
34
+ file_path.is_file()
35
+ and file_path.suffix.lower().lstrip(".") in SUPPORTED_FORMATS
36
+ ):
37
+ all_files.append(str(file_path))
38
+
39
+ return all_files
40
+
41
+
42
+ def download_file_from_response(response: requests.models.Response, save_path: str):
43
+ with open(save_path, "wb") as file:
44
+ for chunk in response.iter_content(1024 * 1024):
45
+ if chunk:
46
+ file.write(chunk)
47
+ file.flush()
48
+
49
+
50
+ def download_file_from_url(url: str, save_path: str):
51
+ """Downloads a file from a URL and saves it to the specified path.
52
+
53
+ Parameters
54
+ ----------
55
+ url : str
56
+ The URL of the file to download.
57
+ save_path : str
58
+ The local file path where the file will be saved.
59
+
60
+ """
61
+ try:
62
+ # Send a HTTP request to the URL
63
+ response = requests.get(url, stream=True)
64
+ response.raise_for_status() # Check if the request was successful
65
+
66
+ # Get the total file size from headers (if available)
67
+ total_size = int(response.headers.get("content-length", 0))
68
+
69
+ # Initialize tqdm progress bar
70
+ with (
71
+ open(save_path, "wb") as file,
72
+ tqdm(
73
+ total=total_size, unit="B", unit_scale=True, desc="Downloading"
74
+ ) as progress_bar,
75
+ ):
76
+ # Write the file in chunks to avoid using too much memory
77
+ for chunk in response.iter_content(chunk_size=8192):
78
+ if chunk: # Filter out keep-alive chunks
79
+ file.write(chunk)
80
+ progress_bar.update(len(chunk)) # Update progress bar
81
+
82
+ print(f"\nFile downloaded successfully and saved to: {save_path}")
83
+
84
+ except requests.exceptions.RequestException as e:
85
+ print(f"An error occurred while downloading the file: {e}")
86
+
87
+
88
+ def chunks(lst: list, n: int):
89
+ """Yield successive n-sized chunks from lst."""
90
+ for i in range(0, len(lst), n):
91
+ yield lst[i : i + n]
92
+
93
+
94
+ def float_in_range(min_value: float, max_value: float):
95
+ """
96
+ Returns a validator function for argparse to validate floats within a range.
97
+ """
98
+ import argparse
99
+
100
+ def validator(value):
101
+ f_value = float(value)
102
+ if not (min_value <= f_value <= max_value):
103
+ raise argparse.ArgumentTypeError(
104
+ f"value {f_value} not in range [{min_value}, {max_value}]"
105
+ )
106
+ return f_value
107
+
108
+ return validator
109
+
110
+
111
+ class CreateDatasetValidator:
112
+ @staticmethod
113
+ def is_vision_ai_format_supported(
114
+ config: ProjectCreateDatasetConfig,
115
+ ) -> bool:
116
+ """
117
+ VisionAI format is supported for almost all cases except VQA
118
+ """
119
+ if config.image_type == OntologyImageType.VQA:
120
+ return False
121
+ return True
122
+
123
+ @staticmethod
124
+ def is_kitti_format_supported(
125
+ config: ProjectCreateDatasetConfig,
126
+ ) -> bool:
127
+ """
128
+ KITTI format requirements:
129
+ - Exactly 1 LIDAR sensor
130
+ - 0 or 1 camera (raw: 0 cameras, annotated: 0-1 cameras)
131
+ - Non-sequential only
132
+ - If annotated: cuboid PCD type required
133
+ - If annotated with camera: bbox image type required, no attributes
134
+ """
135
+ if config.sensor_counts.lidar != 1:
136
+ return False
137
+
138
+ camera_count = config.sensor_counts.camera
139
+ if camera_count > 1 or (
140
+ config.dataset_type == DatasetType.RAW_DATA and camera_count > 0
141
+ ):
142
+ return False
143
+
144
+ if config.is_sequential:
145
+ return False
146
+
147
+ if config.dataset_type == DatasetType.ANNOTATED_DATA:
148
+ if config.pcd_type != OntologyPcdType.CUBOID:
149
+ return False
150
+ if (
151
+ camera_count == 1
152
+ and config.image_type != OntologyImageType._2D_BOUNDING_BOX
153
+ ):
154
+ return False
155
+ if config.has_attribute:
156
+ return False
157
+
158
+ return True
159
+
160
+ @staticmethod
161
+ def is_coco_format_supported(
162
+ config: ProjectCreateDatasetConfig,
163
+ ) -> bool:
164
+ """
165
+ COCO format requirements:
166
+ - Annotated data only
167
+ - Exactly 1 camera, 0 LIDARs
168
+ - Bounding box image type
169
+ - No attributes
170
+ - Non-sequential only
171
+ """
172
+ if config.dataset_type == DatasetType.RAW_DATA:
173
+ return False
174
+
175
+ if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
176
+ return False
177
+
178
+ if config.image_type != OntologyImageType._2D_BOUNDING_BOX:
179
+ return False
180
+
181
+ if config.has_attribute:
182
+ return False
183
+
184
+ if config.is_sequential:
185
+ return False
186
+
187
+ return True
188
+
189
+ @staticmethod
190
+ def is_image_format_supported(
191
+ config: ProjectCreateDatasetConfig,
192
+ ) -> bool:
193
+ """
194
+ Image format requirements:
195
+ - Raw data only
196
+ - Exactly 1 camera, 0 LIDARs
197
+ - Non-sequential only
198
+ """
199
+ if config.dataset_type == DatasetType.ANNOTATED_DATA:
200
+ return False
201
+
202
+ if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
203
+ return False
204
+
205
+ if config.is_sequential:
206
+ return False
207
+
208
+ return True
209
+
210
+ @staticmethod
211
+ def is_yolo_format_supported(
212
+ config: ProjectCreateDatasetConfig,
213
+ ) -> bool:
214
+ """
215
+ YOLO format requirements:
216
+ - Annotated data only
217
+ - Exactly 1 camera, 0 LIDARs
218
+ - Bounding box image type
219
+ - No attributes
220
+ - Non-sequential only
221
+ """
222
+ if config.dataset_type == DatasetType.RAW_DATA:
223
+ return False
224
+
225
+ if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
226
+ return False
227
+
228
+ if config.image_type != OntologyImageType._2D_BOUNDING_BOX:
229
+ return False
230
+
231
+ if config.has_attribute:
232
+ return False
233
+
234
+ if config.is_sequential:
235
+ return False
236
+
237
+ return True
238
+
239
+ @staticmethod
240
+ def is_video_format_supported(
241
+ config: ProjectCreateDatasetConfig,
242
+ ) -> bool:
243
+ """
244
+ Video format requirements:
245
+ - Raw data only
246
+ - Exactly 1 camera, 0 LIDARs
247
+ - Sequential only
248
+ """
249
+ if config.dataset_type != DatasetType.RAW_DATA:
250
+ return False
251
+
252
+ if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
253
+ return False
254
+
255
+ if not config.is_sequential:
256
+ return False
257
+
258
+ return True
259
+
260
+ @staticmethod
261
+ def is_vlm_format_supported(
262
+ config: ProjectCreateDatasetConfig,
263
+ ) -> bool:
264
+ """
265
+ VLM format requirements:
266
+ - Annotated data only
267
+ - Exactly 1 camera, 0 LIDARs
268
+ - VQA image type
269
+ """
270
+ if config.dataset_type == DatasetType.RAW_DATA:
271
+ return False
272
+
273
+ if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
274
+ return False
275
+
276
+ if config.image_type != OntologyImageType.VQA:
277
+ return False
278
+
279
+ return True
280
+
281
+ @classmethod
282
+ def get_supported_formats(
283
+ cls, config: ProjectCreateDatasetConfig
284
+ ) -> list[AnnotationFormat]:
285
+ supported_formats = []
286
+
287
+ if cls.is_vision_ai_format_supported(config):
288
+ supported_formats.append(AnnotationFormat.VISION_AI)
289
+
290
+ if cls.is_kitti_format_supported(config):
291
+ supported_formats.append(AnnotationFormat.KITTI)
292
+
293
+ if cls.is_coco_format_supported(config):
294
+ supported_formats.append(AnnotationFormat.COCO)
295
+
296
+ if cls.is_image_format_supported(config):
297
+ supported_formats.append(AnnotationFormat.IMAGE)
298
+
299
+ if cls.is_yolo_format_supported(config):
300
+ supported_formats.append(AnnotationFormat.YOLO)
301
+
302
+ if cls.is_video_format_supported(config):
303
+ supported_formats.append(AnnotationFormat.VIDEO)
304
+
305
+ if cls.is_vlm_format_supported(config):
306
+ supported_formats.append(AnnotationFormat.VLM)
307
+
308
+ return supported_formats
309
+
310
+
311
+ def validate_before_create_dataset(
312
+ config: ProjectCreateDatasetConfig,
313
+ ) -> tuple[bool, Optional[str]]:
314
+ supported_formats = CreateDatasetValidator.get_supported_formats(config)
315
+
316
+ try:
317
+ requested_format = AnnotationFormat(config.annotation_format)
318
+ except ValueError:
319
+ return False, f"Unknown annotation format: {config.annotation_format}"
320
+
321
+ if requested_format in supported_formats:
322
+ return True, None
323
+
324
+ error_lines = [
325
+ "",
326
+ "❌ The input arguments in create_dataset are not compatible with the project.",
327
+ "",
328
+ f" • Dataset type: {config.dataset_type}",
329
+ f" • Sensors: {config.sensor_counts.camera} camera(s), {config.sensor_counts.lidar} lidar(s)",
330
+ f" • Annotation format: {config.annotation_format}",
331
+ f" • Image type: {config.image_type or 'N/A'}",
332
+ f" • PCD type: {config.pcd_type or 'N/A'}",
333
+ f" • Has attributes: {config.has_attribute}",
334
+ f" • Sequential: {config.is_sequential}",
335
+ "",
336
+ "Please adjust your input arguments and try again. For detailed guidance, please refer to the documentation: https://linkervision.gitbook.io/dataverse/data-management/import-your-dataset#supported-data-formats",
337
+ ]
338
+
339
+ return False, "\n".join(error_lines)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.5.1
3
+ Version: 2.5.3
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -35,7 +35,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
35
35
  - List models for your selected project-id
36
36
  - Get and download your model
37
37
 
38
- [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkernetworks/dataverse-sdk)
38
+ [Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
39
39
 
40
40
 
41
41
  ## Getting started
@@ -69,7 +69,7 @@ client2 = DataverseClient(
69
69
  assert client2 is get_connection(client2.alias)
70
70
 
71
71
  client3 = DataverseClient(
72
- host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
72
+ host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
73
73
  )
74
74
  assert client3 is get_connection(client3.alias)
75
75
  ```
@@ -78,7 +78,7 @@ assert client3 is get_connection(client3.alias)
78
78
 
79
79
  | Argument name | Type/Options | Default | Description |
80
80
  | :--- | :--- | :--- | :--- |
81
- | host | str | *-- | the host url of the dataverse site (with curation port) |
81
+ | host | str | *-- | the host url of the dataverse site |
82
82
  | email | str | *-- | the email account of your dataverse workspace |
83
83
  | password | str | *-- | the password of your dataverse workspace |
84
84
  | service_id | str | *-- | The service id of the dataverse you want to connect |
@@ -129,7 +129,7 @@ The `list_projects` method will list all projects of the given sites.
129
129
  ```Python
130
130
  projects = client.list_projects(current_user = True,
131
131
  exclude_sensor_type=SensorType.LIDAR,
132
- image_type= OntologyImageType._2D_BOUNDING_BOX)
132
+ image_type=OntologyImageType._2D_BOUNDING_BOX)
133
133
 
134
134
  ```
135
135
 
@@ -139,7 +139,7 @@ projects = client.list_projects(current_user = True,
139
139
  | :--- | :--- | :--- | :--- |
140
140
  | current_user | bool | True | only show the projects of current user |
141
141
  | exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
142
- | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
142
+ | image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
143
143
 
144
144
  <br>
145
145
 
@@ -167,7 +167,7 @@ ontology = Ontology(
167
167
  ],
168
168
  )
169
169
  ```
170
- For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
170
+ For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
171
171
 
172
172
  For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
173
173
 
@@ -213,7 +213,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
213
213
 
214
214
  ### Get Project
215
215
 
216
- The `get_proejct` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
216
+ The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
217
217
 
218
218
  ```Python
219
219
  project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
@@ -364,9 +364,8 @@ dataset = project.create_dataset(**dataset_data)
364
364
  | storage_url | str | *-- | your cloud storage url |
365
365
  | container_name | str | None | azure container name |
366
366
  | data_folder | str | *-- | the relative data folder from the storage_url and container |
367
- | sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
368
367
  | type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
369
- | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE | *-- | the format of your annotation data |
368
+ | annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
370
369
  | annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
371
370
  | sequential | bool | False | data is sequential or not |
372
371
  | render_pcd | bool | False | render pcd preview image or not |
@@ -391,7 +390,6 @@ dataset_data2 = {
391
390
  "storage_url": "",
392
391
  "container_name": "",
393
392
  "data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
394
- "sensors": project.sensors,
395
393
  "type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
396
394
  "annotation_format": AnnotationFormat.VISION_AI,
397
395
  "annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
@@ -405,7 +403,7 @@ dataset2 = project.create_dataset(**dataset_data2)
405
403
 
406
404
  Your could also use the script for importing dataset from local
407
405
  ```
408
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
406
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
409
407
  ```
410
408
  <br>
411
409
 
@@ -537,13 +535,13 @@ The `create_vqa_project` method will create project on the connected site with t
537
535
  # 1) Create question class with question and answer type pair
538
536
  question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
539
537
  answer_type="boolean"),
540
- QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yello","green"])
538
+ QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
541
539
  ]
542
540
  ```
543
541
 
544
542
  ```Python
545
543
  # 2) Create your VQA project as below
546
- project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
544
+ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
547
545
  ```
548
546
 
549
547
  * Input arguments for creating project:
@@ -553,7 +551,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
553
551
  | name | str | *-- | name of your project |
554
552
  | sensor_name | str | *-- | the camera sensor name |
555
553
  | ontology_name | str | *-- | the ontology name |
556
- | question_answer| list[QuestionClass] | *-- | your question/answer_type |
554
+ | question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
557
555
  | description | str | None | your project description |
558
556
 
559
557
  `*--`: required argument without default
@@ -572,15 +570,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
572
570
  create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
573
571
  update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
574
572
 
575
- #should provide client_alias if calling from client
576
- client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
577
- create=create_questions,
578
- update=update_questions,
579
- client_alias=client.alias)
580
- #OR
581
- project.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
582
- create=create_questions,
583
- update=update_questions)
573
+ # Through client
574
+ client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
575
+ create=create_questions,
576
+ update=update_questions,
577
+ client_alias=client.alias)
578
+
579
+ # Through project object
580
+ project.edit_vqa_ontology(ontology_name="ontology-new-name",
581
+ create=create_questions,
582
+ update=update_questions)
584
583
  ```
585
584
 
586
585
  ### Get Question List
@@ -598,28 +597,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
598
597
 
599
598
  ### Import Your Local Dataset
600
599
  ```
601
- python tools/import_dataset_from_local.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
600
+ python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
602
601
  ```
603
602
 
604
603
  ### Import VQA Local Dataset
605
604
  ```
606
- python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
605
+ python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
607
606
 
608
607
  ```
609
608
 
610
609
  ### Export Dataslice and download files
611
610
  ```
612
- python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
611
+ python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
613
612
  ```
614
613
 
615
614
  ### Export Large Dataslice and download files
616
615
  ```
617
- python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
616
+ python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
618
617
  ```
619
618
 
620
619
  ### Upload videos to create session tasks
621
620
  ```
622
- python tools/upload_videos_create_session.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
621
+ python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
623
622
  ```
624
623
 
625
624
  - Advanced arguments for video curation (sequential data):
@@ -633,4 +632,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
633
632
 
634
633
  ## Links to language repos
635
634
 
636
- [Python Readme](https://github.com/linkernetworks/dataverse-sdk/tree/master/python/README.md)
635
+ [Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
2
2
 
3
3
  AUTHOR = "LinkerVision"
4
4
  PACKAGE_NAME = "dataverse-sdk"
5
- PACKAGE_VERSION = "2.5.1"
5
+ PACKAGE_VERSION = "2.5.3"
6
6
  DESC = "Dataverse SDK For Python"
7
7
  with open("README.md", encoding="utf-8") as fh:
8
8
  long_description = fh.read()
@@ -1,100 +0,0 @@
1
- from pathlib import Path
2
- from typing import Set
3
-
4
- import requests
5
- from tqdm import tqdm
6
-
7
- IMAGE_SUPPORTED_FORMAT = {
8
- "jpeg",
9
- "jpg",
10
- "png",
11
- "bmp",
12
- }
13
- CUBOID_SUPPORTED_FORMAT = {"pcd"}
14
-
15
-
16
- def get_filepaths(directory: str) -> list[str]:
17
- SUPPORTED_FORMATS: Set[str] = (
18
- IMAGE_SUPPORTED_FORMAT | CUBOID_SUPPORTED_FORMAT | {"txt", "json"}
19
- )
20
-
21
- directory = Path(directory)
22
- all_files = []
23
-
24
- for file_path in directory.rglob("*"):
25
- if (
26
- file_path.is_file()
27
- and file_path.suffix.lower().lstrip(".") in SUPPORTED_FORMATS
28
- ):
29
- all_files.append(str(file_path))
30
-
31
- return all_files
32
-
33
-
34
- def download_file_from_response(response: requests.models.Response, save_path: str):
35
- with open(save_path, "wb") as file:
36
- for chunk in response.iter_content(1024 * 1024):
37
- if chunk:
38
- file.write(chunk)
39
- file.flush()
40
-
41
-
42
- def download_file_from_url(url: str, save_path: str):
43
- """Downloads a file from a URL and saves it to the specified path.
44
-
45
- Parameters
46
- ----------
47
- url : str
48
- The URL of the file to download.
49
- save_path : str
50
- The local file path where the file will be saved.
51
-
52
- """
53
- try:
54
- # Send a HTTP request to the URL
55
- response = requests.get(url, stream=True)
56
- response.raise_for_status() # Check if the request was successful
57
-
58
- # Get the total file size from headers (if available)
59
- total_size = int(response.headers.get("content-length", 0))
60
-
61
- # Initialize tqdm progress bar
62
- with (
63
- open(save_path, "wb") as file,
64
- tqdm(
65
- total=total_size, unit="B", unit_scale=True, desc="Downloading"
66
- ) as progress_bar,
67
- ):
68
- # Write the file in chunks to avoid using too much memory
69
- for chunk in response.iter_content(chunk_size=8192):
70
- if chunk: # Filter out keep-alive chunks
71
- file.write(chunk)
72
- progress_bar.update(len(chunk)) # Update progress bar
73
-
74
- print(f"\nFile downloaded successfully and saved to: {save_path}")
75
-
76
- except requests.exceptions.RequestException as e:
77
- print(f"An error occurred while downloading the file: {e}")
78
-
79
-
80
- def chunks(lst: list, n: int):
81
- """Yield successive n-sized chunks from lst."""
82
- for i in range(0, len(lst), n):
83
- yield lst[i : i + n]
84
-
85
-
86
- def float_in_range(min_value: float, max_value: float):
87
- """
88
- Returns a validator function for argparse to validate floats within a range.
89
- """
90
- import argparse
91
-
92
- def validator(value):
93
- f_value = float(value)
94
- if not (min_value <= f_value <= max_value):
95
- raise argparse.ArgumentTypeError(
96
- f"value {f_value} not in range [{min_value}, {max_value}]"
97
- )
98
- return f_value
99
-
100
- return validator
File without changes