dataverse-sdk 2.5.1__tar.gz → 2.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/PKG-INFO +29 -30
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/README.md +28 -29
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/__init__.py +29 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/client.py +1 -1
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/constants.py +4 -5
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/client.py +60 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/common.py +24 -0
- dataverse_sdk-2.5.3/dataverse_sdk/utils/utils.py +339 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/PKG-INFO +29 -30
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/setup.py +1 -1
- dataverse_sdk-2.5.1/dataverse_sdk/utils/utils.py +0 -100
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/__init__.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/backend.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/apis/third_party.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/connections.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/exceptions/__init__.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/exceptions/client.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/__init__.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/base.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/coco.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/constant.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/exporter.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/utils.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/visionai.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/vqa.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/export/yolo.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/__init__.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/api.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/schemas/format.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk/utils/__init__.py +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/requires.txt +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/dataverse_sdk.egg-info/top_level.txt +0 -0
- {dataverse_sdk-2.5.1 → dataverse_sdk-2.5.3}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.3
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -35,7 +35,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
|
|
|
35
35
|
- List models for your selected project-id
|
|
36
36
|
- Get and download your model
|
|
37
37
|
|
|
38
|
-
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/
|
|
38
|
+
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
## Getting started
|
|
@@ -69,7 +69,7 @@ client2 = DataverseClient(
|
|
|
69
69
|
assert client2 is get_connection(client2.alias)
|
|
70
70
|
|
|
71
71
|
client3 = DataverseClient(
|
|
72
|
-
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
|
|
72
|
+
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
|
|
73
73
|
)
|
|
74
74
|
assert client3 is get_connection(client3.alias)
|
|
75
75
|
```
|
|
@@ -78,7 +78,7 @@ assert client3 is get_connection(client3.alias)
|
|
|
78
78
|
|
|
79
79
|
| Argument name | Type/Options | Default | Description |
|
|
80
80
|
| :--- | :--- | :--- | :--- |
|
|
81
|
-
| host | str | *-- | the host url of the dataverse site
|
|
81
|
+
| host | str | *-- | the host url of the dataverse site |
|
|
82
82
|
| email | str | *-- | the email account of your dataverse workspace |
|
|
83
83
|
| password | str | *-- | the password of your dataverse workspace |
|
|
84
84
|
| service_id | str | *-- | The service id of the dataverse you want to connect |
|
|
@@ -129,7 +129,7 @@ The `list_projects` method will list all projects of the given sites.
|
|
|
129
129
|
```Python
|
|
130
130
|
projects = client.list_projects(current_user = True,
|
|
131
131
|
exclude_sensor_type=SensorType.LIDAR,
|
|
132
|
-
image_type=
|
|
132
|
+
image_type=OntologyImageType._2D_BOUNDING_BOX)
|
|
133
133
|
|
|
134
134
|
```
|
|
135
135
|
|
|
@@ -139,7 +139,7 @@ projects = client.list_projects(current_user = True,
|
|
|
139
139
|
| :--- | :--- | :--- | :--- |
|
|
140
140
|
| current_user | bool | True | only show the projects of current user |
|
|
141
141
|
| exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
|
|
142
|
-
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
|
|
142
|
+
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
|
|
143
143
|
|
|
144
144
|
<br>
|
|
145
145
|
|
|
@@ -167,7 +167,7 @@ ontology = Ontology(
|
|
|
167
167
|
],
|
|
168
168
|
)
|
|
169
169
|
```
|
|
170
|
-
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
|
|
170
|
+
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
|
|
171
171
|
|
|
172
172
|
For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
|
|
173
173
|
|
|
@@ -213,7 +213,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
|
|
|
213
213
|
|
|
214
214
|
### Get Project
|
|
215
215
|
|
|
216
|
-
The `
|
|
216
|
+
The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
|
|
217
217
|
|
|
218
218
|
```Python
|
|
219
219
|
project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
|
|
@@ -364,9 +364,8 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
364
364
|
| storage_url | str | *-- | your cloud storage url |
|
|
365
365
|
| container_name | str | None | azure container name |
|
|
366
366
|
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
367
|
-
| sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
|
|
368
367
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
369
|
-
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE
|
|
368
|
+
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
370
369
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
371
370
|
| sequential | bool | False | data is sequential or not |
|
|
372
371
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
@@ -391,7 +390,6 @@ dataset_data2 = {
|
|
|
391
390
|
"storage_url": "",
|
|
392
391
|
"container_name": "",
|
|
393
392
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
394
|
-
"sensors": project.sensors,
|
|
395
393
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
396
394
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
397
395
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
@@ -405,7 +403,7 @@ dataset2 = project.create_dataset(**dataset_data2)
|
|
|
405
403
|
|
|
406
404
|
Your could also use the script for importing dataset from local
|
|
407
405
|
```
|
|
408
|
-
python tools/import_dataset_from_local.py -host
|
|
406
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
409
407
|
```
|
|
410
408
|
<br>
|
|
411
409
|
|
|
@@ -537,13 +535,13 @@ The `create_vqa_project` method will create project on the connected site with t
|
|
|
537
535
|
# 1) Create question class with question and answer type pair
|
|
538
536
|
question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
|
|
539
537
|
answer_type="boolean"),
|
|
540
|
-
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","
|
|
538
|
+
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
|
|
541
539
|
]
|
|
542
540
|
```
|
|
543
541
|
|
|
544
542
|
```Python
|
|
545
543
|
# 2) Create your VQA project as below
|
|
546
|
-
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
|
|
544
|
+
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
|
|
547
545
|
```
|
|
548
546
|
|
|
549
547
|
* Input arguments for creating project:
|
|
@@ -553,7 +551,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
553
551
|
| name | str | *-- | name of your project |
|
|
554
552
|
| sensor_name | str | *-- | the camera sensor name |
|
|
555
553
|
| ontology_name | str | *-- | the ontology name |
|
|
556
|
-
| question_answer| list[QuestionClass] | *-- | your question/answer_type |
|
|
554
|
+
| question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
|
|
557
555
|
| description | str | None | your project description |
|
|
558
556
|
|
|
559
557
|
`*--`: required argument without default
|
|
@@ -572,15 +570,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
572
570
|
create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
|
|
573
571
|
update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
|
|
574
572
|
|
|
575
|
-
#
|
|
576
|
-
client.edit_vqa_ontology(project_id=24,
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
project
|
|
582
|
-
|
|
583
|
-
|
|
573
|
+
# Through client
|
|
574
|
+
client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
|
|
575
|
+
create=create_questions,
|
|
576
|
+
update=update_questions,
|
|
577
|
+
client_alias=client.alias)
|
|
578
|
+
|
|
579
|
+
# Through project object
|
|
580
|
+
project.edit_vqa_ontology(ontology_name="ontology-new-name",
|
|
581
|
+
create=create_questions,
|
|
582
|
+
update=update_questions)
|
|
584
583
|
```
|
|
585
584
|
|
|
586
585
|
### Get Question List
|
|
@@ -598,28 +597,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
|
|
|
598
597
|
|
|
599
598
|
### Import Your Local Dataset
|
|
600
599
|
```
|
|
601
|
-
python tools/import_dataset_from_local.py -host
|
|
600
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
602
601
|
```
|
|
603
602
|
|
|
604
603
|
### Import VQA Local Dataset
|
|
605
604
|
```
|
|
606
|
-
python tools/import_vqa_dataset.py -host
|
|
605
|
+
python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
|
|
607
606
|
|
|
608
607
|
```
|
|
609
608
|
|
|
610
609
|
### Export Dataslice and download files
|
|
611
610
|
```
|
|
612
|
-
python tools/export_dataslice.py -host
|
|
611
|
+
python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
|
|
613
612
|
```
|
|
614
613
|
|
|
615
614
|
### Export Large Dataslice and download files
|
|
616
615
|
```
|
|
617
|
-
python tools/export_dataslice_large.py -host
|
|
616
|
+
python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
|
|
618
617
|
```
|
|
619
618
|
|
|
620
619
|
### Upload videos to create session tasks
|
|
621
620
|
```
|
|
622
|
-
python tools/upload_videos_create_session.py -host
|
|
621
|
+
python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
|
|
623
622
|
```
|
|
624
623
|
|
|
625
624
|
- Advanced arguments for video curation (sequential data):
|
|
@@ -633,4 +632,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
|
|
|
633
632
|
|
|
634
633
|
## Links to language repos
|
|
635
634
|
|
|
636
|
-
[Python Readme](https://github.com/
|
|
635
|
+
[Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
|
|
@@ -8,7 +8,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
|
|
|
8
8
|
- List models for your selected project-id
|
|
9
9
|
- Get and download your model
|
|
10
10
|
|
|
11
|
-
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/
|
|
11
|
+
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
## Getting started
|
|
@@ -42,7 +42,7 @@ client2 = DataverseClient(
|
|
|
42
42
|
assert client2 is get_connection(client2.alias)
|
|
43
43
|
|
|
44
44
|
client3 = DataverseClient(
|
|
45
|
-
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
|
|
45
|
+
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
|
|
46
46
|
)
|
|
47
47
|
assert client3 is get_connection(client3.alias)
|
|
48
48
|
```
|
|
@@ -51,7 +51,7 @@ assert client3 is get_connection(client3.alias)
|
|
|
51
51
|
|
|
52
52
|
| Argument name | Type/Options | Default | Description |
|
|
53
53
|
| :--- | :--- | :--- | :--- |
|
|
54
|
-
| host | str | *-- | the host url of the dataverse site
|
|
54
|
+
| host | str | *-- | the host url of the dataverse site |
|
|
55
55
|
| email | str | *-- | the email account of your dataverse workspace |
|
|
56
56
|
| password | str | *-- | the password of your dataverse workspace |
|
|
57
57
|
| service_id | str | *-- | The service id of the dataverse you want to connect |
|
|
@@ -102,7 +102,7 @@ The `list_projects` method will list all projects of the given sites.
|
|
|
102
102
|
```Python
|
|
103
103
|
projects = client.list_projects(current_user = True,
|
|
104
104
|
exclude_sensor_type=SensorType.LIDAR,
|
|
105
|
-
image_type=
|
|
105
|
+
image_type=OntologyImageType._2D_BOUNDING_BOX)
|
|
106
106
|
|
|
107
107
|
```
|
|
108
108
|
|
|
@@ -112,7 +112,7 @@ projects = client.list_projects(current_user = True,
|
|
|
112
112
|
| :--- | :--- | :--- | :--- |
|
|
113
113
|
| current_user | bool | True | only show the projects of current user |
|
|
114
114
|
| exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
|
|
115
|
-
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
|
|
115
|
+
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
|
|
116
116
|
|
|
117
117
|
<br>
|
|
118
118
|
|
|
@@ -140,7 +140,7 @@ ontology = Ontology(
|
|
|
140
140
|
],
|
|
141
141
|
)
|
|
142
142
|
```
|
|
143
|
-
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
|
|
143
|
+
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
|
|
144
144
|
|
|
145
145
|
For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
|
|
146
146
|
|
|
@@ -186,7 +186,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
|
|
|
186
186
|
|
|
187
187
|
### Get Project
|
|
188
188
|
|
|
189
|
-
The `
|
|
189
|
+
The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
|
|
190
190
|
|
|
191
191
|
```Python
|
|
192
192
|
project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
|
|
@@ -337,9 +337,8 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
337
337
|
| storage_url | str | *-- | your cloud storage url |
|
|
338
338
|
| container_name | str | None | azure container name |
|
|
339
339
|
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
340
|
-
| sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
|
|
341
340
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
342
|
-
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE
|
|
341
|
+
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
343
342
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
344
343
|
| sequential | bool | False | data is sequential or not |
|
|
345
344
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
@@ -364,7 +363,6 @@ dataset_data2 = {
|
|
|
364
363
|
"storage_url": "",
|
|
365
364
|
"container_name": "",
|
|
366
365
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
367
|
-
"sensors": project.sensors,
|
|
368
366
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
369
367
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
370
368
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
@@ -378,7 +376,7 @@ dataset2 = project.create_dataset(**dataset_data2)
|
|
|
378
376
|
|
|
379
377
|
Your could also use the script for importing dataset from local
|
|
380
378
|
```
|
|
381
|
-
python tools/import_dataset_from_local.py -host
|
|
379
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
382
380
|
```
|
|
383
381
|
<br>
|
|
384
382
|
|
|
@@ -510,13 +508,13 @@ The `create_vqa_project` method will create project on the connected site with t
|
|
|
510
508
|
# 1) Create question class with question and answer type pair
|
|
511
509
|
question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
|
|
512
510
|
answer_type="boolean"),
|
|
513
|
-
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","
|
|
511
|
+
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
|
|
514
512
|
]
|
|
515
513
|
```
|
|
516
514
|
|
|
517
515
|
```Python
|
|
518
516
|
# 2) Create your VQA project as below
|
|
519
|
-
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
|
|
517
|
+
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
|
|
520
518
|
```
|
|
521
519
|
|
|
522
520
|
* Input arguments for creating project:
|
|
@@ -526,7 +524,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
526
524
|
| name | str | *-- | name of your project |
|
|
527
525
|
| sensor_name | str | *-- | the camera sensor name |
|
|
528
526
|
| ontology_name | str | *-- | the ontology name |
|
|
529
|
-
| question_answer| list[QuestionClass] | *-- | your question/answer_type |
|
|
527
|
+
| question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
|
|
530
528
|
| description | str | None | your project description |
|
|
531
529
|
|
|
532
530
|
`*--`: required argument without default
|
|
@@ -545,15 +543,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
545
543
|
create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
|
|
546
544
|
update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
|
|
547
545
|
|
|
548
|
-
#
|
|
549
|
-
client.edit_vqa_ontology(project_id=24,
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
project
|
|
555
|
-
|
|
556
|
-
|
|
546
|
+
# Through client
|
|
547
|
+
client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
|
|
548
|
+
create=create_questions,
|
|
549
|
+
update=update_questions,
|
|
550
|
+
client_alias=client.alias)
|
|
551
|
+
|
|
552
|
+
# Through project object
|
|
553
|
+
project.edit_vqa_ontology(ontology_name="ontology-new-name",
|
|
554
|
+
create=create_questions,
|
|
555
|
+
update=update_questions)
|
|
557
556
|
```
|
|
558
557
|
|
|
559
558
|
### Get Question List
|
|
@@ -571,28 +570,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
|
|
|
571
570
|
|
|
572
571
|
### Import Your Local Dataset
|
|
573
572
|
```
|
|
574
|
-
python tools/import_dataset_from_local.py -host
|
|
573
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
575
574
|
```
|
|
576
575
|
|
|
577
576
|
### Import VQA Local Dataset
|
|
578
577
|
```
|
|
579
|
-
python tools/import_vqa_dataset.py -host
|
|
578
|
+
python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
|
|
580
579
|
|
|
581
580
|
```
|
|
582
581
|
|
|
583
582
|
### Export Dataslice and download files
|
|
584
583
|
```
|
|
585
|
-
python tools/export_dataslice.py -host
|
|
584
|
+
python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
|
|
586
585
|
```
|
|
587
586
|
|
|
588
587
|
### Export Large Dataslice and download files
|
|
589
588
|
```
|
|
590
|
-
python tools/export_dataslice_large.py -host
|
|
589
|
+
python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
|
|
591
590
|
```
|
|
592
591
|
|
|
593
592
|
### Upload videos to create session tasks
|
|
594
593
|
```
|
|
595
|
-
python tools/upload_videos_create_session.py -host
|
|
594
|
+
python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
|
|
596
595
|
```
|
|
597
596
|
|
|
598
597
|
- Advanced arguments for video curation (sequential data):
|
|
@@ -606,4 +605,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
|
|
|
606
605
|
|
|
607
606
|
## Links to language repos
|
|
608
607
|
|
|
609
|
-
[Python Readme](https://github.com/
|
|
608
|
+
[Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import re
|
|
3
|
+
|
|
1
4
|
from . import connections
|
|
2
5
|
from .client import DataverseClient
|
|
3
6
|
from .constants import DataverseHost
|
|
@@ -45,3 +48,29 @@ __all__ = [
|
|
|
45
48
|
"DataSource",
|
|
46
49
|
"QuestionClass",
|
|
47
50
|
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _SensitiveDataFilter(logging.Filter):
|
|
54
|
+
"""Filter to redact sensitive credentials from logs"""
|
|
55
|
+
|
|
56
|
+
SENSITIVE_PATTERNS = [
|
|
57
|
+
(r"AWSAccessKeyId=[^&\s]*", "AWSAccessKeyId=***"),
|
|
58
|
+
(r"Signature=[^&\s]*", "Signature=***"),
|
|
59
|
+
(r"Expires=\d+", "Expires=***"),
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
def filter(self, record):
|
|
63
|
+
# Override getMessage to process the final formatted message
|
|
64
|
+
original_getMessage = record.getMessage
|
|
65
|
+
|
|
66
|
+
def patched_getMessage():
|
|
67
|
+
msg = original_getMessage()
|
|
68
|
+
for pattern, replacement in self.SENSITIVE_PATTERNS:
|
|
69
|
+
msg = re.sub(pattern, replacement, msg)
|
|
70
|
+
return msg
|
|
71
|
+
|
|
72
|
+
record.getMessage = patched_getMessage
|
|
73
|
+
return True
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
logging.getLogger("httpx").addFilter(_SensitiveDataFilter())
|
|
@@ -12,10 +12,9 @@ class BaseEnumMeta(EnumMeta):
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class DataverseHost(str, Enum, metaclass=BaseEnumMeta):
|
|
15
|
-
DEV = "https://dev.visionai.linkervision.
|
|
16
|
-
DEV2 = "https://dev2.visionai.linkervision.
|
|
17
|
-
DEV3 = "https://dev3.visionai.linkervision.
|
|
18
|
-
STAGING = "https://staging.visionai.linkervision.
|
|
19
|
-
DEMO = "https://demo.visionai.linkervision.ai/dataverse/curation"
|
|
15
|
+
DEV = "https://dev.visionai.linkervision.com/dataverse/curation"
|
|
16
|
+
DEV2 = "https://dev2.visionai.linkervision.com/dataverse/curation"
|
|
17
|
+
DEV3 = "https://dev3.visionai.linkervision.com/dataverse/curation"
|
|
18
|
+
STAGING = "https://staging.visionai.linkervision.com/dataverse/curation"
|
|
20
19
|
PRODUCTION = "https://visionai.linkervision.ai/dataverse/curation"
|
|
21
20
|
LOCAL = "http://localhost:8000"
|
|
@@ -12,6 +12,8 @@ from .common import (
|
|
|
12
12
|
DataSource,
|
|
13
13
|
OntologyImageType,
|
|
14
14
|
OntologyPcdType,
|
|
15
|
+
ProjectCreateDatasetConfig,
|
|
16
|
+
SensorCounts,
|
|
15
17
|
SensorType,
|
|
16
18
|
)
|
|
17
19
|
|
|
@@ -305,6 +307,62 @@ class Project(BaseModel):
|
|
|
305
307
|
)
|
|
306
308
|
return convert_record_data
|
|
307
309
|
|
|
310
|
+
def _validate_before_create_dataset(
|
|
311
|
+
self,
|
|
312
|
+
annotation_format: AnnotationFormat,
|
|
313
|
+
dataset_type: DatasetType,
|
|
314
|
+
sequential: bool,
|
|
315
|
+
) -> None:
|
|
316
|
+
from dataverse_sdk.utils.utils import validate_before_create_dataset
|
|
317
|
+
|
|
318
|
+
if self.sensors is None:
|
|
319
|
+
from ..client import DataverseClient
|
|
320
|
+
|
|
321
|
+
project = DataverseClient.get_client_project(
|
|
322
|
+
project_id=self.id, client_alias=self.client_alias
|
|
323
|
+
)
|
|
324
|
+
self.sensors = project.sensors or []
|
|
325
|
+
|
|
326
|
+
sensor_counts = SensorCounts(camera=0, lidar=0)
|
|
327
|
+
for sensor in self.sensors:
|
|
328
|
+
sensor_type: SensorType = getattr(sensor, "type", "")
|
|
329
|
+
if sensor_type == SensorType.CAMERA:
|
|
330
|
+
sensor_counts.camera += 1
|
|
331
|
+
elif sensor_type == SensorType.LIDAR:
|
|
332
|
+
sensor_counts.lidar += 1
|
|
333
|
+
|
|
334
|
+
image_type = None
|
|
335
|
+
pcd_type = None
|
|
336
|
+
has_attribute = False
|
|
337
|
+
|
|
338
|
+
if self.ontology:
|
|
339
|
+
if self.ontology.image_type:
|
|
340
|
+
image_type = self.ontology.image_type
|
|
341
|
+
|
|
342
|
+
if self.ontology.pcd_type:
|
|
343
|
+
pcd_type = self.ontology.pcd_type
|
|
344
|
+
|
|
345
|
+
if self.ontology.classes:
|
|
346
|
+
for cls in self.ontology.classes:
|
|
347
|
+
if cls.attributes:
|
|
348
|
+
has_attribute = True
|
|
349
|
+
break
|
|
350
|
+
|
|
351
|
+
config = ProjectCreateDatasetConfig(
|
|
352
|
+
annotation_format=annotation_format,
|
|
353
|
+
dataset_type=dataset_type,
|
|
354
|
+
sensor_counts=sensor_counts,
|
|
355
|
+
is_sequential=sequential,
|
|
356
|
+
image_type=image_type,
|
|
357
|
+
pcd_type=pcd_type,
|
|
358
|
+
has_attribute=has_attribute,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
is_valid, error_message = validate_before_create_dataset(config)
|
|
362
|
+
|
|
363
|
+
if not is_valid:
|
|
364
|
+
raise ValueError(error_message)
|
|
365
|
+
|
|
308
366
|
def create_dataset(
|
|
309
367
|
self,
|
|
310
368
|
name: str,
|
|
@@ -375,6 +433,8 @@ class Project(BaseModel):
|
|
|
375
433
|
"""
|
|
376
434
|
from ..client import DataverseClient
|
|
377
435
|
|
|
436
|
+
self._validate_before_create_dataset(annotation_format, type, sequential)
|
|
437
|
+
|
|
378
438
|
if annotations is None:
|
|
379
439
|
annotations = []
|
|
380
440
|
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
1
2
|
from enum import Enum
|
|
3
|
+
from typing import Optional
|
|
2
4
|
|
|
3
5
|
from ..constants import BaseEnumMeta
|
|
4
6
|
|
|
@@ -37,6 +39,7 @@ class AnnotationFormat(str, Enum, metaclass=BaseEnumMeta):
|
|
|
37
39
|
IMAGE = "image"
|
|
38
40
|
KITTI = "kitti"
|
|
39
41
|
YOLO = "yolo"
|
|
42
|
+
VIDEO = "video"
|
|
40
43
|
VLM = "vlm"
|
|
41
44
|
|
|
42
45
|
|
|
@@ -56,3 +59,24 @@ class DataSource(str, Enum, metaclass=BaseEnumMeta):
|
|
|
56
59
|
AWS = "aws"
|
|
57
60
|
LOCAL = "local"
|
|
58
61
|
SDK = "sdk"
|
|
62
|
+
EXISTING_DATASETS = "existing_datasets"
|
|
63
|
+
EXISTING_DATASLICES = "existing_dataslices"
|
|
64
|
+
DATA_GENERATION = "data_generation"
|
|
65
|
+
PRE_IMPORT = "pre_import"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class SensorCounts:
|
|
70
|
+
camera: int = 0
|
|
71
|
+
lidar: int = 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class ProjectCreateDatasetConfig:
|
|
76
|
+
annotation_format: AnnotationFormat
|
|
77
|
+
dataset_type: DatasetType
|
|
78
|
+
sensor_counts: SensorCounts
|
|
79
|
+
is_sequential: bool
|
|
80
|
+
image_type: Optional[OntologyImageType] = None
|
|
81
|
+
pcd_type: Optional[OntologyPcdType] = None
|
|
82
|
+
has_attribute: bool = False
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Optional, Set
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
from tqdm import tqdm
|
|
6
|
+
|
|
7
|
+
from dataverse_sdk.schemas.common import (
|
|
8
|
+
AnnotationFormat,
|
|
9
|
+
DatasetType,
|
|
10
|
+
OntologyImageType,
|
|
11
|
+
OntologyPcdType,
|
|
12
|
+
ProjectCreateDatasetConfig,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
IMAGE_SUPPORTED_FORMAT = {
|
|
16
|
+
"jpeg",
|
|
17
|
+
"jpg",
|
|
18
|
+
"png",
|
|
19
|
+
"bmp",
|
|
20
|
+
}
|
|
21
|
+
CUBOID_SUPPORTED_FORMAT = {"pcd"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_filepaths(directory: str) -> list[str]:
|
|
25
|
+
SUPPORTED_FORMATS: Set[str] = (
|
|
26
|
+
IMAGE_SUPPORTED_FORMAT | CUBOID_SUPPORTED_FORMAT | {"txt", "json"}
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
directory = Path(directory)
|
|
30
|
+
all_files = []
|
|
31
|
+
|
|
32
|
+
for file_path in directory.rglob("*"):
|
|
33
|
+
if (
|
|
34
|
+
file_path.is_file()
|
|
35
|
+
and file_path.suffix.lower().lstrip(".") in SUPPORTED_FORMATS
|
|
36
|
+
):
|
|
37
|
+
all_files.append(str(file_path))
|
|
38
|
+
|
|
39
|
+
return all_files
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def download_file_from_response(response: requests.models.Response, save_path: str):
|
|
43
|
+
with open(save_path, "wb") as file:
|
|
44
|
+
for chunk in response.iter_content(1024 * 1024):
|
|
45
|
+
if chunk:
|
|
46
|
+
file.write(chunk)
|
|
47
|
+
file.flush()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def download_file_from_url(url: str, save_path: str):
|
|
51
|
+
"""Downloads a file from a URL and saves it to the specified path.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
url : str
|
|
56
|
+
The URL of the file to download.
|
|
57
|
+
save_path : str
|
|
58
|
+
The local file path where the file will be saved.
|
|
59
|
+
|
|
60
|
+
"""
|
|
61
|
+
try:
|
|
62
|
+
# Send a HTTP request to the URL
|
|
63
|
+
response = requests.get(url, stream=True)
|
|
64
|
+
response.raise_for_status() # Check if the request was successful
|
|
65
|
+
|
|
66
|
+
# Get the total file size from headers (if available)
|
|
67
|
+
total_size = int(response.headers.get("content-length", 0))
|
|
68
|
+
|
|
69
|
+
# Initialize tqdm progress bar
|
|
70
|
+
with (
|
|
71
|
+
open(save_path, "wb") as file,
|
|
72
|
+
tqdm(
|
|
73
|
+
total=total_size, unit="B", unit_scale=True, desc="Downloading"
|
|
74
|
+
) as progress_bar,
|
|
75
|
+
):
|
|
76
|
+
# Write the file in chunks to avoid using too much memory
|
|
77
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
78
|
+
if chunk: # Filter out keep-alive chunks
|
|
79
|
+
file.write(chunk)
|
|
80
|
+
progress_bar.update(len(chunk)) # Update progress bar
|
|
81
|
+
|
|
82
|
+
print(f"\nFile downloaded successfully and saved to: {save_path}")
|
|
83
|
+
|
|
84
|
+
except requests.exceptions.RequestException as e:
|
|
85
|
+
print(f"An error occurred while downloading the file: {e}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def chunks(lst: list, n: int):
|
|
89
|
+
"""Yield successive n-sized chunks from lst."""
|
|
90
|
+
for i in range(0, len(lst), n):
|
|
91
|
+
yield lst[i : i + n]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def float_in_range(min_value: float, max_value: float):
|
|
95
|
+
"""
|
|
96
|
+
Returns a validator function for argparse to validate floats within a range.
|
|
97
|
+
"""
|
|
98
|
+
import argparse
|
|
99
|
+
|
|
100
|
+
def validator(value):
|
|
101
|
+
f_value = float(value)
|
|
102
|
+
if not (min_value <= f_value <= max_value):
|
|
103
|
+
raise argparse.ArgumentTypeError(
|
|
104
|
+
f"value {f_value} not in range [{min_value}, {max_value}]"
|
|
105
|
+
)
|
|
106
|
+
return f_value
|
|
107
|
+
|
|
108
|
+
return validator
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class CreateDatasetValidator:
|
|
112
|
+
@staticmethod
|
|
113
|
+
def is_vision_ai_format_supported(
|
|
114
|
+
config: ProjectCreateDatasetConfig,
|
|
115
|
+
) -> bool:
|
|
116
|
+
"""
|
|
117
|
+
VisionAI format is supported for almost all cases except VQA
|
|
118
|
+
"""
|
|
119
|
+
if config.image_type == OntologyImageType.VQA:
|
|
120
|
+
return False
|
|
121
|
+
return True
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def is_kitti_format_supported(
|
|
125
|
+
config: ProjectCreateDatasetConfig,
|
|
126
|
+
) -> bool:
|
|
127
|
+
"""
|
|
128
|
+
KITTI format requirements:
|
|
129
|
+
- Exactly 1 LIDAR sensor
|
|
130
|
+
- 0 or 1 camera (raw: 0 cameras, annotated: 0-1 cameras)
|
|
131
|
+
- Non-sequential only
|
|
132
|
+
- If annotated: cuboid PCD type required
|
|
133
|
+
- If annotated with camera: bbox image type required, no attributes
|
|
134
|
+
"""
|
|
135
|
+
if config.sensor_counts.lidar != 1:
|
|
136
|
+
return False
|
|
137
|
+
|
|
138
|
+
camera_count = config.sensor_counts.camera
|
|
139
|
+
if camera_count > 1 or (
|
|
140
|
+
config.dataset_type == DatasetType.RAW_DATA and camera_count > 0
|
|
141
|
+
):
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
if config.is_sequential:
|
|
145
|
+
return False
|
|
146
|
+
|
|
147
|
+
if config.dataset_type == DatasetType.ANNOTATED_DATA:
|
|
148
|
+
if config.pcd_type != OntologyPcdType.CUBOID:
|
|
149
|
+
return False
|
|
150
|
+
if (
|
|
151
|
+
camera_count == 1
|
|
152
|
+
and config.image_type != OntologyImageType._2D_BOUNDING_BOX
|
|
153
|
+
):
|
|
154
|
+
return False
|
|
155
|
+
if config.has_attribute:
|
|
156
|
+
return False
|
|
157
|
+
|
|
158
|
+
return True
|
|
159
|
+
|
|
160
|
+
@staticmethod
|
|
161
|
+
def is_coco_format_supported(
|
|
162
|
+
config: ProjectCreateDatasetConfig,
|
|
163
|
+
) -> bool:
|
|
164
|
+
"""
|
|
165
|
+
COCO format requirements:
|
|
166
|
+
- Annotated data only
|
|
167
|
+
- Exactly 1 camera, 0 LIDARs
|
|
168
|
+
- Bounding box image type
|
|
169
|
+
- No attributes
|
|
170
|
+
- Non-sequential only
|
|
171
|
+
"""
|
|
172
|
+
if config.dataset_type == DatasetType.RAW_DATA:
|
|
173
|
+
return False
|
|
174
|
+
|
|
175
|
+
if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
|
|
176
|
+
return False
|
|
177
|
+
|
|
178
|
+
if config.image_type != OntologyImageType._2D_BOUNDING_BOX:
|
|
179
|
+
return False
|
|
180
|
+
|
|
181
|
+
if config.has_attribute:
|
|
182
|
+
return False
|
|
183
|
+
|
|
184
|
+
if config.is_sequential:
|
|
185
|
+
return False
|
|
186
|
+
|
|
187
|
+
return True
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def is_image_format_supported(
|
|
191
|
+
config: ProjectCreateDatasetConfig,
|
|
192
|
+
) -> bool:
|
|
193
|
+
"""
|
|
194
|
+
Image format requirements:
|
|
195
|
+
- Raw data only
|
|
196
|
+
- Exactly 1 camera, 0 LIDARs
|
|
197
|
+
- Non-sequential only
|
|
198
|
+
"""
|
|
199
|
+
if config.dataset_type == DatasetType.ANNOTATED_DATA:
|
|
200
|
+
return False
|
|
201
|
+
|
|
202
|
+
if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
|
|
203
|
+
return False
|
|
204
|
+
|
|
205
|
+
if config.is_sequential:
|
|
206
|
+
return False
|
|
207
|
+
|
|
208
|
+
return True
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
def is_yolo_format_supported(
|
|
212
|
+
config: ProjectCreateDatasetConfig,
|
|
213
|
+
) -> bool:
|
|
214
|
+
"""
|
|
215
|
+
YOLO format requirements:
|
|
216
|
+
- Annotated data only
|
|
217
|
+
- Exactly 1 camera, 0 LIDARs
|
|
218
|
+
- Bounding box image type
|
|
219
|
+
- No attributes
|
|
220
|
+
- Non-sequential only
|
|
221
|
+
"""
|
|
222
|
+
if config.dataset_type == DatasetType.RAW_DATA:
|
|
223
|
+
return False
|
|
224
|
+
|
|
225
|
+
if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
|
|
226
|
+
return False
|
|
227
|
+
|
|
228
|
+
if config.image_type != OntologyImageType._2D_BOUNDING_BOX:
|
|
229
|
+
return False
|
|
230
|
+
|
|
231
|
+
if config.has_attribute:
|
|
232
|
+
return False
|
|
233
|
+
|
|
234
|
+
if config.is_sequential:
|
|
235
|
+
return False
|
|
236
|
+
|
|
237
|
+
return True
|
|
238
|
+
|
|
239
|
+
@staticmethod
|
|
240
|
+
def is_video_format_supported(
|
|
241
|
+
config: ProjectCreateDatasetConfig,
|
|
242
|
+
) -> bool:
|
|
243
|
+
"""
|
|
244
|
+
Video format requirements:
|
|
245
|
+
- Raw data only
|
|
246
|
+
- Exactly 1 camera, 0 LIDARs
|
|
247
|
+
- Sequential only
|
|
248
|
+
"""
|
|
249
|
+
if config.dataset_type != DatasetType.RAW_DATA:
|
|
250
|
+
return False
|
|
251
|
+
|
|
252
|
+
if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
|
|
253
|
+
return False
|
|
254
|
+
|
|
255
|
+
if not config.is_sequential:
|
|
256
|
+
return False
|
|
257
|
+
|
|
258
|
+
return True
|
|
259
|
+
|
|
260
|
+
@staticmethod
|
|
261
|
+
def is_vlm_format_supported(
|
|
262
|
+
config: ProjectCreateDatasetConfig,
|
|
263
|
+
) -> bool:
|
|
264
|
+
"""
|
|
265
|
+
VLM format requirements:
|
|
266
|
+
- Annotated data only
|
|
267
|
+
- Exactly 1 camera, 0 LIDARs
|
|
268
|
+
- VQA image type
|
|
269
|
+
"""
|
|
270
|
+
if config.dataset_type == DatasetType.RAW_DATA:
|
|
271
|
+
return False
|
|
272
|
+
|
|
273
|
+
if config.sensor_counts.camera != 1 or config.sensor_counts.lidar != 0:
|
|
274
|
+
return False
|
|
275
|
+
|
|
276
|
+
if config.image_type != OntologyImageType.VQA:
|
|
277
|
+
return False
|
|
278
|
+
|
|
279
|
+
return True
|
|
280
|
+
|
|
281
|
+
@classmethod
|
|
282
|
+
def get_supported_formats(
|
|
283
|
+
cls, config: ProjectCreateDatasetConfig
|
|
284
|
+
) -> list[AnnotationFormat]:
|
|
285
|
+
supported_formats = []
|
|
286
|
+
|
|
287
|
+
if cls.is_vision_ai_format_supported(config):
|
|
288
|
+
supported_formats.append(AnnotationFormat.VISION_AI)
|
|
289
|
+
|
|
290
|
+
if cls.is_kitti_format_supported(config):
|
|
291
|
+
supported_formats.append(AnnotationFormat.KITTI)
|
|
292
|
+
|
|
293
|
+
if cls.is_coco_format_supported(config):
|
|
294
|
+
supported_formats.append(AnnotationFormat.COCO)
|
|
295
|
+
|
|
296
|
+
if cls.is_image_format_supported(config):
|
|
297
|
+
supported_formats.append(AnnotationFormat.IMAGE)
|
|
298
|
+
|
|
299
|
+
if cls.is_yolo_format_supported(config):
|
|
300
|
+
supported_formats.append(AnnotationFormat.YOLO)
|
|
301
|
+
|
|
302
|
+
if cls.is_video_format_supported(config):
|
|
303
|
+
supported_formats.append(AnnotationFormat.VIDEO)
|
|
304
|
+
|
|
305
|
+
if cls.is_vlm_format_supported(config):
|
|
306
|
+
supported_formats.append(AnnotationFormat.VLM)
|
|
307
|
+
|
|
308
|
+
return supported_formats
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def validate_before_create_dataset(
|
|
312
|
+
config: ProjectCreateDatasetConfig,
|
|
313
|
+
) -> tuple[bool, Optional[str]]:
|
|
314
|
+
supported_formats = CreateDatasetValidator.get_supported_formats(config)
|
|
315
|
+
|
|
316
|
+
try:
|
|
317
|
+
requested_format = AnnotationFormat(config.annotation_format)
|
|
318
|
+
except ValueError:
|
|
319
|
+
return False, f"Unknown annotation format: {config.annotation_format}"
|
|
320
|
+
|
|
321
|
+
if requested_format in supported_formats:
|
|
322
|
+
return True, None
|
|
323
|
+
|
|
324
|
+
error_lines = [
|
|
325
|
+
"",
|
|
326
|
+
"❌ The input arguments in create_dataset are not compatible with the project.",
|
|
327
|
+
"",
|
|
328
|
+
f" • Dataset type: {config.dataset_type}",
|
|
329
|
+
f" • Sensors: {config.sensor_counts.camera} camera(s), {config.sensor_counts.lidar} lidar(s)",
|
|
330
|
+
f" • Annotation format: {config.annotation_format}",
|
|
331
|
+
f" • Image type: {config.image_type or 'N/A'}",
|
|
332
|
+
f" • PCD type: {config.pcd_type or 'N/A'}",
|
|
333
|
+
f" • Has attributes: {config.has_attribute}",
|
|
334
|
+
f" • Sequential: {config.is_sequential}",
|
|
335
|
+
"",
|
|
336
|
+
"Please adjust your input arguments and try again. For detailed guidance, please refer to the documentation: https://linkervision.gitbook.io/dataverse/data-management/import-your-dataset#supported-data-formats",
|
|
337
|
+
]
|
|
338
|
+
|
|
339
|
+
return False, "\n".join(error_lines)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.3
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -35,7 +35,7 @@ Use Dataverse-SDK for Python to help you to interact with the Dataverse platform
|
|
|
35
35
|
- List models for your selected project-id
|
|
36
36
|
- Get and download your model
|
|
37
37
|
|
|
38
|
-
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/
|
|
38
|
+
[Package (PyPi)](https://pypi.org/project/dataverse-sdk/) | [Source code](https://github.com/linkervision/dataverse-sdk)
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
## Getting started
|
|
@@ -69,7 +69,7 @@ client2 = DataverseClient(
|
|
|
69
69
|
assert client2 is get_connection(client2.alias)
|
|
70
70
|
|
|
71
71
|
client3 = DataverseClient(
|
|
72
|
-
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx"
|
|
72
|
+
host=DataverseHost.PRODUCTION.value, email="XXX", password="", service_id="xxxx-xxxx-xx-xxx", access_token="xxx", alias="client3", force = False
|
|
73
73
|
)
|
|
74
74
|
assert client3 is get_connection(client3.alias)
|
|
75
75
|
```
|
|
@@ -78,7 +78,7 @@ assert client3 is get_connection(client3.alias)
|
|
|
78
78
|
|
|
79
79
|
| Argument name | Type/Options | Default | Description |
|
|
80
80
|
| :--- | :--- | :--- | :--- |
|
|
81
|
-
| host | str | *-- | the host url of the dataverse site
|
|
81
|
+
| host | str | *-- | the host url of the dataverse site |
|
|
82
82
|
| email | str | *-- | the email account of your dataverse workspace |
|
|
83
83
|
| password | str | *-- | the password of your dataverse workspace |
|
|
84
84
|
| service_id | str | *-- | The service id of the dataverse you want to connect |
|
|
@@ -129,7 +129,7 @@ The `list_projects` method will list all projects of the given sites.
|
|
|
129
129
|
```Python
|
|
130
130
|
projects = client.list_projects(current_user = True,
|
|
131
131
|
exclude_sensor_type=SensorType.LIDAR,
|
|
132
|
-
image_type=
|
|
132
|
+
image_type=OntologyImageType._2D_BOUNDING_BOX)
|
|
133
133
|
|
|
134
134
|
```
|
|
135
135
|
|
|
@@ -139,7 +139,7 @@ projects = client.list_projects(current_user = True,
|
|
|
139
139
|
| :--- | :--- | :--- | :--- |
|
|
140
140
|
| current_user | bool | True | only show the projects of current user |
|
|
141
141
|
| exclude_sensor_type | SensorType.CAMERA <br> SensorType.LIDAR| None | exclude the projects with the given sensor type |
|
|
142
|
-
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE | None | only include the projects with the given image type |
|
|
142
|
+
| image_type | OntologyImageType._2D_BOUNDING_BOX <br> OntologyImageType.SEMANTIC_SEGMENTATION <br> OntologyImageType.INSTANCE_SEGMENTATION <br> OntologyImageType.CLASSIFICATION <br> OntologyImageType.POINT<br> OntologyImageType.POLYGON <br> OntologyImageType.POLYLINE <br> OntologyImageType.VQA | None | only include the projects with the given image type |
|
|
143
143
|
|
|
144
144
|
<br>
|
|
145
145
|
|
|
@@ -167,7 +167,7 @@ ontology = Ontology(
|
|
|
167
167
|
],
|
|
168
168
|
)
|
|
169
169
|
```
|
|
170
|
-
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE]`.
|
|
170
|
+
For project with camera sensor, there would be only one image_type for one project. You could choose from `[OntologyImageType._2D_BOUNDING_BOX, OntologyImageType.SEMANTIC_SEGMENTATION, OntologyImageType.INSTANCE_SEGMENTATION, OntologyImageType.CLASSIFICATION, OntologyImageType.POINT, OntologyImageType.POLYGON, OntologyImageType.POLYLINE, OntologyImageType.VQA]`.
|
|
171
171
|
|
|
172
172
|
For project with lidar sensor, your should assign `pcd_type = OntologyPcdType.CUBOID` for the ontology.
|
|
173
173
|
|
|
@@ -213,7 +213,7 @@ project = client.create_project(name="Sample project", ontology=ontology, sensor
|
|
|
213
213
|
|
|
214
214
|
### Get Project
|
|
215
215
|
|
|
216
|
-
The `
|
|
216
|
+
The `get_project` method retrieves the project from the connected site. The `project_id` parameter is the unique integer ID of the project, not its "name" property.
|
|
217
217
|
|
|
218
218
|
```Python
|
|
219
219
|
project = client.get_project(project_id= 1, client_alias=client.alias) # if client_alias is not provided, we'll get it from client
|
|
@@ -364,9 +364,8 @@ dataset = project.create_dataset(**dataset_data)
|
|
|
364
364
|
| storage_url | str | *-- | your cloud storage url |
|
|
365
365
|
| container_name | str | None | azure container name |
|
|
366
366
|
| data_folder | str | *-- | the relative data folder from the storage_url and container |
|
|
367
|
-
| sensors | list[Sensor] | *-- | the list of Sensor of your dataset (one or more from project specified sensors) |
|
|
368
367
|
| type | DatasetType.ANNOTATED_DATA <br> DatasetType.RAW_DATA | *-- | your dataset type (annotated or raw data)|
|
|
369
|
-
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE
|
|
368
|
+
| annotation_format | AnnotationFormat.VISION_AI <br> AnnotationFormat.KITTI <br> AnnotationFormat.COCO <br> AnnotationFormat.YOLO <br> AnnotationFormat.IMAGE <br> AnnotationFormat.BDDP <br> AnnotationFormat.VIDEO <br> AnnotationFormat.VLM <br>| *-- | the format of your annotation data |
|
|
370
369
|
| annotations | list[str] | None | list of names for your annotation data folders, such as ["groundtruth"] |
|
|
371
370
|
| sequential | bool | False | data is sequential or not |
|
|
372
371
|
| render_pcd | bool | False | render pcd preview image or not |
|
|
@@ -391,7 +390,6 @@ dataset_data2 = {
|
|
|
391
390
|
"storage_url": "",
|
|
392
391
|
"container_name": "",
|
|
393
392
|
"data_folder": "/YOUR/TARGET/LOCAL/FOLDER",
|
|
394
|
-
"sensors": project.sensors,
|
|
395
393
|
"type": DatasetType.ANNOTATED_DATA, # or DatasetType.RAW_DATA for images
|
|
396
394
|
"annotation_format": AnnotationFormat.VISION_AI,
|
|
397
395
|
"annotations": ["groundtruth"], # remove it when type is DatasetType.RAW_DATA
|
|
@@ -405,7 +403,7 @@ dataset2 = project.create_dataset(**dataset_data2)
|
|
|
405
403
|
|
|
406
404
|
Your could also use the script for importing dataset from local
|
|
407
405
|
```
|
|
408
|
-
python tools/import_dataset_from_local.py -host
|
|
406
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
409
407
|
```
|
|
410
408
|
<br>
|
|
411
409
|
|
|
@@ -537,13 +535,13 @@ The `create_vqa_project` method will create project on the connected site with t
|
|
|
537
535
|
# 1) Create question class with question and answer type pair
|
|
538
536
|
question_answer = [ QuestionClass(class_name="question1", rank=1, question="Is any person found in the picture?",
|
|
539
537
|
answer_type="boolean"),
|
|
540
|
-
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","
|
|
538
|
+
QuestionClass(class_name="question2", rank=2, question="What is the blob color of traffic light?", answer_type="option",answer_options=["red","yellow","green"])
|
|
541
539
|
]
|
|
542
540
|
```
|
|
543
541
|
|
|
544
542
|
```Python
|
|
545
543
|
# 2) Create your VQA project as below
|
|
546
|
-
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology" question_answer=question_answer)
|
|
544
|
+
project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", ontology_name="vqa-ontology", question_answer=question_answer)
|
|
547
545
|
```
|
|
548
546
|
|
|
549
547
|
* Input arguments for creating project:
|
|
@@ -553,7 +551,7 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
553
551
|
| name | str | *-- | name of your project |
|
|
554
552
|
| sensor_name | str | *-- | the camera sensor name |
|
|
555
553
|
| ontology_name | str | *-- | the ontology name |
|
|
556
|
-
| question_answer| list[QuestionClass] | *-- | your question/answer_type |
|
|
554
|
+
| question_answer| list[QuestionClass] | *-- | your question/answer_type. `QuestionClass.answer_type` valid values: `boolean`, `option`, `number`, `text` |
|
|
557
555
|
| description | str | None | your project description |
|
|
558
556
|
|
|
559
557
|
`*--`: required argument without default
|
|
@@ -572,15 +570,16 @@ project = client.create_vqa_project(name="vqa-project", sensor_name="camera1", o
|
|
|
572
570
|
create_questions = [QuestionClass(class_name="question3", rank=3, question="Age?",answer_type="number")]
|
|
573
571
|
update_questions = [{"rank": 2, "question": "What is the blob color of traffic light?(the closet one)", "options":["black"] }]
|
|
574
572
|
|
|
575
|
-
#
|
|
576
|
-
client.edit_vqa_ontology(project_id=24,
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
project
|
|
582
|
-
|
|
583
|
-
|
|
573
|
+
# Through client
|
|
574
|
+
client.edit_vqa_ontology(project_id=24, ontology_name="ontology-new-name",
|
|
575
|
+
create=create_questions,
|
|
576
|
+
update=update_questions,
|
|
577
|
+
client_alias=client.alias)
|
|
578
|
+
|
|
579
|
+
# Through project object
|
|
580
|
+
project.edit_vqa_ontology(ontology_name="ontology-new-name",
|
|
581
|
+
create=create_questions,
|
|
582
|
+
update=update_questions)
|
|
584
583
|
```
|
|
585
584
|
|
|
586
585
|
### Get Question List
|
|
@@ -598,28 +597,28 @@ output = client.get_question_list(project_id=107, output_file_path="./question.j
|
|
|
598
597
|
|
|
599
598
|
### Import Your Local Dataset
|
|
600
599
|
```
|
|
601
|
-
python tools/import_dataset_from_local.py -host
|
|
600
|
+
python tools/import_dataset_from_local.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -name {dataset-name} -type {raw_data OR annotated_data} -anno {image OR vision_ai} --sequential
|
|
602
601
|
```
|
|
603
602
|
|
|
604
603
|
### Import VQA Local Dataset
|
|
605
604
|
```
|
|
606
|
-
python tools/import_vqa_dataset.py -host
|
|
605
|
+
python tools/import_vqa_dataset.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -project {project-id} --folder {/YOUR/TARGET/LOCAL/FOLDER} -type {raw_data OR annotated_data}
|
|
607
606
|
|
|
608
607
|
```
|
|
609
608
|
|
|
610
609
|
### Export Dataslice and download files
|
|
611
610
|
```
|
|
612
|
-
python tools/export_dataslice.py -host
|
|
611
|
+
python tools/export_dataslice.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
|
|
613
612
|
```
|
|
614
613
|
|
|
615
614
|
### Export Large Dataslice and download files
|
|
616
615
|
```
|
|
617
|
-
python tools/export_dataslice_large.py -host
|
|
616
|
+
python tools/export_dataslice_large.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
|
|
618
617
|
```
|
|
619
618
|
|
|
620
619
|
### Upload videos to create session tasks
|
|
621
620
|
```
|
|
622
|
-
python tools/upload_videos_create_session.py -host
|
|
621
|
+
python tools/upload_videos_create_session.py -host {YOUR_HOST} -e {your-account-email} -p {PASSWORD} -s {service-id} -f {/YOUR/VIDEOS/LOCAL/FOLDER} -n {session-name}
|
|
623
622
|
```
|
|
624
623
|
|
|
625
624
|
- Advanced arguments for video curation (sequential data):
|
|
@@ -633,4 +632,4 @@ python tools/upload_videos_create_session.py -host https://visionai.linkervision
|
|
|
633
632
|
|
|
634
633
|
## Links to language repos
|
|
635
634
|
|
|
636
|
-
[Python Readme](https://github.com/
|
|
635
|
+
[Python Readme](https://github.com/linkervision/dataverse-sdk/blob/master/python/README.md)
|
|
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
|
|
|
2
2
|
|
|
3
3
|
AUTHOR = "LinkerVision"
|
|
4
4
|
PACKAGE_NAME = "dataverse-sdk"
|
|
5
|
-
PACKAGE_VERSION = "2.5.
|
|
5
|
+
PACKAGE_VERSION = "2.5.3"
|
|
6
6
|
DESC = "Dataverse SDK For Python"
|
|
7
7
|
with open("README.md", encoding="utf-8") as fh:
|
|
8
8
|
long_description = fh.read()
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
from typing import Set
|
|
3
|
-
|
|
4
|
-
import requests
|
|
5
|
-
from tqdm import tqdm
|
|
6
|
-
|
|
7
|
-
IMAGE_SUPPORTED_FORMAT = {
|
|
8
|
-
"jpeg",
|
|
9
|
-
"jpg",
|
|
10
|
-
"png",
|
|
11
|
-
"bmp",
|
|
12
|
-
}
|
|
13
|
-
CUBOID_SUPPORTED_FORMAT = {"pcd"}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def get_filepaths(directory: str) -> list[str]:
|
|
17
|
-
SUPPORTED_FORMATS: Set[str] = (
|
|
18
|
-
IMAGE_SUPPORTED_FORMAT | CUBOID_SUPPORTED_FORMAT | {"txt", "json"}
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
directory = Path(directory)
|
|
22
|
-
all_files = []
|
|
23
|
-
|
|
24
|
-
for file_path in directory.rglob("*"):
|
|
25
|
-
if (
|
|
26
|
-
file_path.is_file()
|
|
27
|
-
and file_path.suffix.lower().lstrip(".") in SUPPORTED_FORMATS
|
|
28
|
-
):
|
|
29
|
-
all_files.append(str(file_path))
|
|
30
|
-
|
|
31
|
-
return all_files
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def download_file_from_response(response: requests.models.Response, save_path: str):
|
|
35
|
-
with open(save_path, "wb") as file:
|
|
36
|
-
for chunk in response.iter_content(1024 * 1024):
|
|
37
|
-
if chunk:
|
|
38
|
-
file.write(chunk)
|
|
39
|
-
file.flush()
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def download_file_from_url(url: str, save_path: str):
|
|
43
|
-
"""Downloads a file from a URL and saves it to the specified path.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
url : str
|
|
48
|
-
The URL of the file to download.
|
|
49
|
-
save_path : str
|
|
50
|
-
The local file path where the file will be saved.
|
|
51
|
-
|
|
52
|
-
"""
|
|
53
|
-
try:
|
|
54
|
-
# Send a HTTP request to the URL
|
|
55
|
-
response = requests.get(url, stream=True)
|
|
56
|
-
response.raise_for_status() # Check if the request was successful
|
|
57
|
-
|
|
58
|
-
# Get the total file size from headers (if available)
|
|
59
|
-
total_size = int(response.headers.get("content-length", 0))
|
|
60
|
-
|
|
61
|
-
# Initialize tqdm progress bar
|
|
62
|
-
with (
|
|
63
|
-
open(save_path, "wb") as file,
|
|
64
|
-
tqdm(
|
|
65
|
-
total=total_size, unit="B", unit_scale=True, desc="Downloading"
|
|
66
|
-
) as progress_bar,
|
|
67
|
-
):
|
|
68
|
-
# Write the file in chunks to avoid using too much memory
|
|
69
|
-
for chunk in response.iter_content(chunk_size=8192):
|
|
70
|
-
if chunk: # Filter out keep-alive chunks
|
|
71
|
-
file.write(chunk)
|
|
72
|
-
progress_bar.update(len(chunk)) # Update progress bar
|
|
73
|
-
|
|
74
|
-
print(f"\nFile downloaded successfully and saved to: {save_path}")
|
|
75
|
-
|
|
76
|
-
except requests.exceptions.RequestException as e:
|
|
77
|
-
print(f"An error occurred while downloading the file: {e}")
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
def chunks(lst: list, n: int):
|
|
81
|
-
"""Yield successive n-sized chunks from lst."""
|
|
82
|
-
for i in range(0, len(lst), n):
|
|
83
|
-
yield lst[i : i + n]
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def float_in_range(min_value: float, max_value: float):
|
|
87
|
-
"""
|
|
88
|
-
Returns a validator function for argparse to validate floats within a range.
|
|
89
|
-
"""
|
|
90
|
-
import argparse
|
|
91
|
-
|
|
92
|
-
def validator(value):
|
|
93
|
-
f_value = float(value)
|
|
94
|
-
if not (min_value <= f_value <= max_value):
|
|
95
|
-
raise argparse.ArgumentTypeError(
|
|
96
|
-
f"value {f_value} not in range [{min_value}, {max_value}]"
|
|
97
|
-
)
|
|
98
|
-
return f_value
|
|
99
|
-
|
|
100
|
-
return validator
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|