dataverse-sdk 1.7.2__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/PKG-INFO +10 -2
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/README.md +3 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/apis/backend.py +68 -0
- dataverse_sdk-2.1.0/dataverse_sdk/apis/third_party.py +80 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/client.py +18 -14
- dataverse_sdk-2.1.0/dataverse_sdk/export/__init__.py +4 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/base.py +41 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/coco.py +188 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/constant.py +15 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/exporter.py +256 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/utils.py +26 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/visionai.py +768 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/vqa.py +139 -0
- dataverse_sdk-2.1.0/dataverse_sdk/export/yolo.py +161 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/api.py +13 -19
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/client.py +21 -32
- dataverse_sdk-2.1.0/dataverse_sdk/schemas/format.py +8 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/PKG-INFO +10 -2
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/SOURCES.txt +11 -0
- dataverse_sdk-2.1.0/dataverse_sdk.egg-info/requires.txt +10 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/setup.py +13 -2
- dataverse_sdk-1.7.2/dataverse_sdk.egg-info/requires.txt +0 -5
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/__init__.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/apis/__init__.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/connections.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/constants.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/exceptions/__init__.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/exceptions/client.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/__init__.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/common.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/utils/__init__.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/utils/utils.py +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
- {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/setup.cfg +0 -0
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
7
7
|
Classifier: Programming Language :: Python :: 3
|
|
8
8
|
Requires-Python: >=3.10, <4
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: pydantic==
|
|
10
|
+
Requires-Dist: pydantic==2.*
|
|
11
11
|
Requires-Dist: requests
|
|
12
12
|
Requires-Dist: httpx>=0.23.0
|
|
13
13
|
Requires-Dist: tqdm
|
|
14
14
|
Requires-Dist: aiofiles
|
|
15
|
+
Requires-Dist: visionai-data-format==2.0.0
|
|
16
|
+
Requires-Dist: aiohttp
|
|
17
|
+
Requires-Dist: aiohttp_retry
|
|
18
|
+
Requires-Dist: numpy<2,>=1.21
|
|
19
|
+
Requires-Dist: opencv-contrib-python==4.8.1.78
|
|
15
20
|
Dynamic: author
|
|
16
21
|
Dynamic: classifier
|
|
17
22
|
Dynamic: description
|
|
@@ -555,6 +560,9 @@ python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.a
|
|
|
555
560
|
python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
|
|
556
561
|
```
|
|
557
562
|
|
|
563
|
+
### Export Large Dataslice and download files
|
|
564
|
+
|
|
565
|
+
python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
|
|
558
566
|
|
|
559
567
|
## Links to language repos
|
|
560
568
|
|
|
@@ -533,6 +533,9 @@ python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.a
|
|
|
533
533
|
python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
|
|
534
534
|
```
|
|
535
535
|
|
|
536
|
+
### Export Large Dataslice and download files
|
|
537
|
+
|
|
538
|
+
python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
|
|
536
539
|
|
|
537
540
|
## Links to language repos
|
|
538
541
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
+
from collections.abc import AsyncGenerator, AsyncIterator
|
|
4
5
|
from typing import Optional, Union
|
|
5
6
|
from urllib.parse import urlencode
|
|
6
7
|
|
|
@@ -612,3 +613,70 @@ class AsyncBackendAPI:
|
|
|
612
613
|
except Exception as e:
|
|
613
614
|
print(f"Request failed: {str(e)}")
|
|
614
615
|
return None
|
|
616
|
+
|
|
617
|
+
async def get_datarows(
|
|
618
|
+
self, batch_size: int = 20, order_by: str = "id", **kwargs
|
|
619
|
+
) -> AsyncGenerator[list[dict], None, None]:
|
|
620
|
+
if "offset" in kwargs or "limit" in kwargs:
|
|
621
|
+
raise ValueError("Specifying offset or limit directly is not allowed.")
|
|
622
|
+
|
|
623
|
+
kwargs["order_by"] = order_by
|
|
624
|
+
dataslice_set = kwargs.pop("dataslice_set", [])
|
|
625
|
+
query_params = {
|
|
626
|
+
**kwargs,
|
|
627
|
+
"order_by": order_by,
|
|
628
|
+
"dataslice_set": dataslice_set,
|
|
629
|
+
"limit": batch_size,
|
|
630
|
+
}
|
|
631
|
+
query_string = urlencode(query_params, doseq=True)
|
|
632
|
+
|
|
633
|
+
id_gt = 0
|
|
634
|
+
while True:
|
|
635
|
+
url = f"{self.host}/api/datarows/?{query_string}&id__gt={id_gt}"
|
|
636
|
+
resp: dict = await self.async_send_request(
|
|
637
|
+
url=url,
|
|
638
|
+
method="get",
|
|
639
|
+
headers=self.headers,
|
|
640
|
+
)
|
|
641
|
+
json_data = resp
|
|
642
|
+
if json_data["count"] == 0:
|
|
643
|
+
break
|
|
644
|
+
if not json_data["results"]:
|
|
645
|
+
break
|
|
646
|
+
|
|
647
|
+
# Get last datarow id
|
|
648
|
+
datarows = json_data["results"]
|
|
649
|
+
id_gt = datarows[-1]["id"]
|
|
650
|
+
yield datarows
|
|
651
|
+
|
|
652
|
+
async def get_datarows_flat_parent(
|
|
653
|
+
self, batch_size: int = 20, order_by: str = "id", **kwargs
|
|
654
|
+
) -> AsyncIterator[list[dict]]:
|
|
655
|
+
id_gt = 0
|
|
656
|
+
limit = batch_size
|
|
657
|
+
kwargs["order_by"] = order_by
|
|
658
|
+
while True:
|
|
659
|
+
kwargs.update({"id__gt": id_gt, "limit": limit})
|
|
660
|
+
url = f"{self.host}/api/datarows/flat-parent/?{urlencode(kwargs)}"
|
|
661
|
+
resp: dict = await self.async_send_request(
|
|
662
|
+
url=url,
|
|
663
|
+
method="get",
|
|
664
|
+
headers=self.headers,
|
|
665
|
+
)
|
|
666
|
+
json_data: list[dict] = resp
|
|
667
|
+
if not json_data["results"]:
|
|
668
|
+
break
|
|
669
|
+
|
|
670
|
+
# Get last datarow id
|
|
671
|
+
datarows = json_data["results"]
|
|
672
|
+
id_gt = datarows[-1]["id"]
|
|
673
|
+
yield datarows
|
|
674
|
+
|
|
675
|
+
async def get_dataslice(self, dataslice_id: int) -> dict:
|
|
676
|
+
url = f"{self.host}/api/dataslices/{dataslice_id}/"
|
|
677
|
+
resp: dict = await self.async_send_request(
|
|
678
|
+
url=url,
|
|
679
|
+
method="get",
|
|
680
|
+
headers=self.headers,
|
|
681
|
+
)
|
|
682
|
+
return resp
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
from aiohttp.client_reqrep import ClientResponse
|
|
6
|
+
from aiohttp_retry import ExponentialRetry, RetryClient
|
|
7
|
+
from requests.adapters import HTTPAdapter, Retry
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ThirdPartyAPI:
|
|
13
|
+
adapter = HTTPAdapter(
|
|
14
|
+
max_retries=Retry(
|
|
15
|
+
total=10, backoff_factor=5, status_forcelist=[500, 502, 503, 504, 104]
|
|
16
|
+
)
|
|
17
|
+
)
|
|
18
|
+
retry_options: dict = {
|
|
19
|
+
"attempts": 5,
|
|
20
|
+
"start_timeout": 5,
|
|
21
|
+
"max_timeout": 5 * (2**5),
|
|
22
|
+
"statuses": {500, 502, 503, 504},
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def send_request(
|
|
27
|
+
cls,
|
|
28
|
+
url: str,
|
|
29
|
+
method: str,
|
|
30
|
+
timeout: int = 10,
|
|
31
|
+
custom_retry_options: Optional[dict] = None,
|
|
32
|
+
**kwargs,
|
|
33
|
+
):
|
|
34
|
+
adapter = cls.adapter
|
|
35
|
+
if custom_retry_options:
|
|
36
|
+
adapter = HTTPAdapter(max_retries=Retry(**custom_retry_options))
|
|
37
|
+
|
|
38
|
+
with requests.Session() as session:
|
|
39
|
+
session.mount("http://", adapter)
|
|
40
|
+
session.mount("https://", adapter)
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
response = session.request(
|
|
44
|
+
method=method, url=url, timeout=timeout, **kwargs
|
|
45
|
+
)
|
|
46
|
+
except Exception as e:
|
|
47
|
+
logger.error(f"third party request error : {str(e)}")
|
|
48
|
+
raise
|
|
49
|
+
|
|
50
|
+
if not 200 <= response.status_code <= 299:
|
|
51
|
+
raise Exception(
|
|
52
|
+
f"status code: {response.status_code}, response detail: {response.__dict__}"
|
|
53
|
+
)
|
|
54
|
+
return response
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
async def async_send_request(
|
|
58
|
+
cls, url: str, method: str, **kwargs
|
|
59
|
+
) -> tuple[RetryClient, ClientResponse]:
|
|
60
|
+
retry_options = ExponentialRetry(**cls.retry_options)
|
|
61
|
+
retry_client = RetryClient(raise_for_status=False, retry_options=retry_options)
|
|
62
|
+
try:
|
|
63
|
+
response = await retry_client.request(method=method, url=url, **kwargs)
|
|
64
|
+
except Exception as e:
|
|
65
|
+
logger.error(f"third party request error : {e}")
|
|
66
|
+
raise
|
|
67
|
+
|
|
68
|
+
if not 200 <= response.status <= 299:
|
|
69
|
+
raise Exception(
|
|
70
|
+
f"status code: {response.status}, response detail: {response.__dict__}"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
return retry_client, response
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
async def async_download_file(cls, url: str, method: str, **kwargs) -> bytes:
|
|
77
|
+
client, resp = await cls.async_send_request(url, method, **kwargs)
|
|
78
|
+
data = await resp.read()
|
|
79
|
+
await client.close()
|
|
80
|
+
return data
|
|
@@ -209,7 +209,7 @@ class DataverseClient:
|
|
|
209
209
|
raise InvalidProcessError(
|
|
210
210
|
"Could not create VQA project by this function, please use create_vqa_project"
|
|
211
211
|
)
|
|
212
|
-
raw_ontology_data: dict = ontology.
|
|
212
|
+
raw_ontology_data: dict = ontology.model_dump(exclude_none=True)
|
|
213
213
|
classes_data_list: list[dict] = []
|
|
214
214
|
rank = 1
|
|
215
215
|
# remove `id` field in OntologyClass and Attribute
|
|
@@ -225,15 +225,19 @@ class DataverseClient:
|
|
|
225
225
|
classes_data_list.append(cls_)
|
|
226
226
|
raw_ontology_data["ontology_classes_data"] = classes_data_list
|
|
227
227
|
if project_tag is not None:
|
|
228
|
-
raw_project_tag_data: dict = project_tag.
|
|
228
|
+
raw_project_tag_data: dict = project_tag.model_dump(exclude_none=True)
|
|
229
229
|
if tag_attrs := raw_project_tag_data.pop("attributes", None):
|
|
230
230
|
raw_project_tag_data["attribute_data"] = parse_attribute(tag_attrs)
|
|
231
231
|
else:
|
|
232
232
|
raw_project_tag_data = {}
|
|
233
233
|
|
|
234
|
-
ontology_data = OntologyAPISchema(**raw_ontology_data).
|
|
235
|
-
|
|
236
|
-
|
|
234
|
+
ontology_data = OntologyAPISchema(**raw_ontology_data).model_dump(
|
|
235
|
+
exclude_none=True
|
|
236
|
+
)
|
|
237
|
+
project_tag_data = ProjectTagAPISchema(**raw_project_tag_data).model_dump(
|
|
238
|
+
exclude_none=True
|
|
239
|
+
)
|
|
240
|
+
sensor_data = [sensor.model_dump(exclude_none=True) for sensor in sensors]
|
|
237
241
|
|
|
238
242
|
try:
|
|
239
243
|
raw_project_data = ProjectAPISchema(
|
|
@@ -242,7 +246,7 @@ class DataverseClient:
|
|
|
242
246
|
sensor_data=sensor_data,
|
|
243
247
|
project_tag_data=project_tag_data,
|
|
244
248
|
description=description,
|
|
245
|
-
).
|
|
249
|
+
).model_dump(exclude_none=True)
|
|
246
250
|
except ValidationError as e:
|
|
247
251
|
raise APIValidationError(
|
|
248
252
|
f"Something wrong when composing the final project data: {e}"
|
|
@@ -297,7 +301,7 @@ class DataverseClient:
|
|
|
297
301
|
ontology_name=ontology_name,
|
|
298
302
|
question_answer=question_answer,
|
|
299
303
|
description=description,
|
|
300
|
-
).
|
|
304
|
+
).model_dump(exclude_none=True)
|
|
301
305
|
except ValidationError as e:
|
|
302
306
|
raise APIValidationError(
|
|
303
307
|
f"Something wrong when composing the vqa project data: {e}"
|
|
@@ -411,7 +415,7 @@ class DataverseClient:
|
|
|
411
415
|
if ontology_name:
|
|
412
416
|
edit_vqa_data["ontology_name"] = ontology_name
|
|
413
417
|
if create:
|
|
414
|
-
edit_vqa_data["create"] = [q.
|
|
418
|
+
edit_vqa_data["create"] = [q.model_dump(exclude_none=True) for q in create]
|
|
415
419
|
if update:
|
|
416
420
|
question_table = {
|
|
417
421
|
q.rank: {
|
|
@@ -439,7 +443,7 @@ class DataverseClient:
|
|
|
439
443
|
]["attribute_id"]
|
|
440
444
|
update_question_data["options"] = update_question.options
|
|
441
445
|
update_questions.append(
|
|
442
|
-
UpdateQuestionAPISchema(**update_question_data).
|
|
446
|
+
UpdateQuestionAPISchema(**update_question_data).model_dump(
|
|
443
447
|
exclude_none=True
|
|
444
448
|
)
|
|
445
449
|
)
|
|
@@ -902,7 +906,7 @@ of this project OR has been added before"
|
|
|
902
906
|
if project.ontology.image_type == OntologyImageType.VQA:
|
|
903
907
|
raise InvalidProcessError("Could not add project_tag for VQA project")
|
|
904
908
|
|
|
905
|
-
raw_project_tag: dict = project_tag.
|
|
909
|
+
raw_project_tag: dict = project_tag.model_dump(exclude_none=True)
|
|
906
910
|
# new project tag attributes to be creaeted
|
|
907
911
|
new_attribute_data: list = parse_attribute(
|
|
908
912
|
raw_project_tag.get("attributes", [])
|
|
@@ -962,7 +966,7 @@ of this project OR has been added before"
|
|
|
962
966
|
if project.ontology.image_type == OntologyImageType.VQA:
|
|
963
967
|
raise InvalidProcessError("Could not edit project_tag for VQA project")
|
|
964
968
|
|
|
965
|
-
raw_project_tag: dict = project_tag.
|
|
969
|
+
raw_project_tag: dict = project_tag.model_dump(exclude_none=True)
|
|
966
970
|
# old project tag attributes to be extended
|
|
967
971
|
patched_attribute_data: list = parse_attribute(
|
|
968
972
|
raw_project_tag.get("attributes", [])
|
|
@@ -1025,7 +1029,7 @@ of this project OR has been added before"
|
|
|
1025
1029
|
# new ontology classes to be created
|
|
1026
1030
|
new_classes_data = []
|
|
1027
1031
|
for ontology_class in ontology_classes:
|
|
1028
|
-
raw_ontology_class: dict = ontology_class.
|
|
1032
|
+
raw_ontology_class: dict = ontology_class.model_dump(exclude_none=True)
|
|
1029
1033
|
attribute_data: list = parse_attribute(
|
|
1030
1034
|
raw_ontology_class.get("attributes", [])
|
|
1031
1035
|
)
|
|
@@ -1097,7 +1101,7 @@ of this project OR has been added before"
|
|
|
1097
1101
|
# ontology classes to be edited
|
|
1098
1102
|
patched_classes_data = []
|
|
1099
1103
|
for ontology_class in ontology_classes:
|
|
1100
|
-
raw_ontology_class: dict = ontology_class.
|
|
1104
|
+
raw_ontology_class: dict = ontology_class.model_dump(exclude_none=True)
|
|
1101
1105
|
attribute_data: list = parse_attribute(
|
|
1102
1106
|
raw_ontology_class.get("attributes", [])
|
|
1103
1107
|
)
|
|
@@ -1563,7 +1567,7 @@ of this project OR has been added before"
|
|
|
1563
1567
|
access_key_id=access_key_id,
|
|
1564
1568
|
secret_access_key=secret_access_key,
|
|
1565
1569
|
**kwargs,
|
|
1566
|
-
).
|
|
1570
|
+
).model_dump(exclude_none=True)
|
|
1567
1571
|
except ValidationError as e:
|
|
1568
1572
|
raise APIValidationError(
|
|
1569
1573
|
f"Something wrong when composing the final dataset data: {e}"
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from collections.abc import Generator
|
|
3
|
+
from typing import Callable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ExportAnnotationBase(abc.ABC):
|
|
7
|
+
@abc.abstractmethod
|
|
8
|
+
def producer(
|
|
9
|
+
self,
|
|
10
|
+
target_folder: str,
|
|
11
|
+
class_names: list[str],
|
|
12
|
+
sequence_frame_map: dict[int, dict[int, list[int]]],
|
|
13
|
+
datarow_generator_func: Callable[[list], Generator[dict]],
|
|
14
|
+
annotation_name: str,
|
|
15
|
+
*args,
|
|
16
|
+
**kwargs,
|
|
17
|
+
) -> Generator[bytes, str]:
|
|
18
|
+
"""
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
target_folder : str
|
|
22
|
+
_description_
|
|
23
|
+
class_names : list[str]
|
|
24
|
+
list of class name
|
|
25
|
+
sequence_frame_map : dict[int, dict[int, list[int]]]
|
|
26
|
+
the entire tree of sequence, frame and base datarows
|
|
27
|
+
datarow_generator_func : Callable[[list], AsyncGenerator[dict]]
|
|
28
|
+
a function that takes list of id as input which generates corresponding datarows
|
|
29
|
+
annotation_name: str
|
|
30
|
+
an annotation name
|
|
31
|
+
|
|
32
|
+
Returns
|
|
33
|
+
-------
|
|
34
|
+
AsyncGenerator[bytes, str]
|
|
35
|
+
should yield a tuple with the data to upload and the relative path to rootfolder
|
|
36
|
+
relative path: this/is/the/relative/path/to/file.txt
|
|
37
|
+
Raises
|
|
38
|
+
------
|
|
39
|
+
NotImplementedError
|
|
40
|
+
"""
|
|
41
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
from collections.abc import AsyncGenerator
|
|
4
|
+
from typing import Callable
|
|
5
|
+
|
|
6
|
+
import aiohttp
|
|
7
|
+
from tqdm import tqdm
|
|
8
|
+
from visionai_data_format.converters.vai_to_coco import VAItoCOCO
|
|
9
|
+
from visionai_data_format.schemas.coco_schema import COCO, Category
|
|
10
|
+
from visionai_data_format.utils.common import (
|
|
11
|
+
ANNOT_PATH,
|
|
12
|
+
COCO_IMAGE_PATH,
|
|
13
|
+
COCO_LABEL_FILE,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
from .base import ExportAnnotationBase
|
|
17
|
+
from .constant import (
|
|
18
|
+
BATCH_SIZE,
|
|
19
|
+
GROUND_TRUTH_ANNOTATION_NAME,
|
|
20
|
+
GROUNDTRUTH,
|
|
21
|
+
MAX_CONCURRENT_DOWNLOADS,
|
|
22
|
+
ExportFormat,
|
|
23
|
+
)
|
|
24
|
+
from .exporter import Exporter
|
|
25
|
+
from .utils import convert_to_bytes
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@Exporter.register(format=ExportFormat.COCO)
|
|
29
|
+
class ExportCoco(ExportAnnotationBase):
|
|
30
|
+
async def download_batch(
|
|
31
|
+
self,
|
|
32
|
+
session: aiohttp.ClientSession,
|
|
33
|
+
semaphore: asyncio.Semaphore,
|
|
34
|
+
batch_datarows: list[dict],
|
|
35
|
+
) -> list[tuple[bytes, str]]:
|
|
36
|
+
tasks = []
|
|
37
|
+
|
|
38
|
+
for datarow in batch_datarows:
|
|
39
|
+
url = datarow["url"]
|
|
40
|
+
file_path = os.path.join(COCO_IMAGE_PATH, datarow["unique_file_name"])
|
|
41
|
+
|
|
42
|
+
async def download_single(url, file_path):
|
|
43
|
+
async with semaphore:
|
|
44
|
+
try:
|
|
45
|
+
async with session.get(url) as response:
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
img_bytes = await response.read()
|
|
48
|
+
return img_bytes, file_path
|
|
49
|
+
except Exception as e:
|
|
50
|
+
print(f"Error downloading {url}: {e}")
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
tasks.append(download_single(url, file_path))
|
|
54
|
+
|
|
55
|
+
results = await asyncio.gather(*tasks)
|
|
56
|
+
return [r for r in results if r is not None]
|
|
57
|
+
|
|
58
|
+
async def producer(
|
|
59
|
+
self,
|
|
60
|
+
class_names: list[str],
|
|
61
|
+
sequence_frame_map: dict[int, dict[int, list[int]]],
|
|
62
|
+
datarow_generator_func: Callable[[list], AsyncGenerator[dict]],
|
|
63
|
+
target_folder: str,
|
|
64
|
+
annotation_name: str,
|
|
65
|
+
*_,
|
|
66
|
+
**kwargs,
|
|
67
|
+
) -> AsyncGenerator[tuple[bytes, str], None]:
|
|
68
|
+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS)
|
|
69
|
+
async with aiohttp.ClientSession() as session:
|
|
70
|
+
current_batch = []
|
|
71
|
+
datarows = [] # Keep track of all datarows for annotation
|
|
72
|
+
datarow_id_list = []
|
|
73
|
+
total_datarows = sum(len(v) for v in sequence_frame_map.values())
|
|
74
|
+
existing_files = set()
|
|
75
|
+
|
|
76
|
+
with tqdm(
|
|
77
|
+
total=total_datarows, desc="Downloading images", unit="file"
|
|
78
|
+
) as progress_bar:
|
|
79
|
+
for frame_datarow_map in sequence_frame_map.values():
|
|
80
|
+
for datarow_ids in frame_datarow_map.values():
|
|
81
|
+
datarow_id_list.extend(datarow_ids)
|
|
82
|
+
for start_idx in range(0, len(datarow_id_list), BATCH_SIZE):
|
|
83
|
+
async for datarow in datarow_generator_func(
|
|
84
|
+
datarow_id_list[start_idx : start_idx + BATCH_SIZE]
|
|
85
|
+
):
|
|
86
|
+
original_file_name = os.path.basename(datarow["original_url"])
|
|
87
|
+
unique_file_name = Exporter.get_unique_filename(
|
|
88
|
+
self, original_file_name, existing_files
|
|
89
|
+
)
|
|
90
|
+
existing_files.add(unique_file_name)
|
|
91
|
+
datarow["unique_file_name"] = unique_file_name
|
|
92
|
+
current_batch.append(datarow)
|
|
93
|
+
datarows.append(datarow)
|
|
94
|
+
|
|
95
|
+
if len(current_batch) >= BATCH_SIZE:
|
|
96
|
+
results = await self.download_batch(
|
|
97
|
+
session, semaphore, current_batch
|
|
98
|
+
)
|
|
99
|
+
for result in results:
|
|
100
|
+
if result:
|
|
101
|
+
yield result
|
|
102
|
+
progress_bar.update(1)
|
|
103
|
+
current_batch = []
|
|
104
|
+
|
|
105
|
+
if current_batch:
|
|
106
|
+
results = await self.download_batch(
|
|
107
|
+
session, semaphore, current_batch
|
|
108
|
+
)
|
|
109
|
+
for result in results:
|
|
110
|
+
if result:
|
|
111
|
+
yield result
|
|
112
|
+
progress_bar.update(1)
|
|
113
|
+
|
|
114
|
+
annot_bytes = convert_to_bytes(
|
|
115
|
+
convert_annotation(
|
|
116
|
+
datarows=datarows,
|
|
117
|
+
class_names=class_names,
|
|
118
|
+
target_folder=target_folder,
|
|
119
|
+
annotation_name=annotation_name,
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
yield annot_bytes, os.path.join(ANNOT_PATH, COCO_LABEL_FILE)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def convert_annotation(
|
|
126
|
+
datarows: list[dict],
|
|
127
|
+
class_names: list[str],
|
|
128
|
+
target_folder: str,
|
|
129
|
+
annotation_name: str,
|
|
130
|
+
) -> dict:
|
|
131
|
+
image_id_start = 0
|
|
132
|
+
anno_id_start = 0
|
|
133
|
+
images = []
|
|
134
|
+
annotations = []
|
|
135
|
+
category_idx_map = {category: idx for idx, category in enumerate(class_names)}
|
|
136
|
+
for datarow in datarows:
|
|
137
|
+
camera_sensor_name = datarow["sensor_name"]
|
|
138
|
+
url = datarow["url"]
|
|
139
|
+
file_extension = os.path.splitext(url)[-1]
|
|
140
|
+
if annotation_name == GROUNDTRUTH:
|
|
141
|
+
target_visionai: dict = datarow["items"].get(
|
|
142
|
+
GROUND_TRUTH_ANNOTATION_NAME, {}
|
|
143
|
+
)
|
|
144
|
+
else:
|
|
145
|
+
target_visionai: dict = (
|
|
146
|
+
datarow["items"].get("predictions", {}).get(annotation_name, {})
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
(
|
|
150
|
+
category_idx_map,
|
|
151
|
+
image_update,
|
|
152
|
+
anno_update,
|
|
153
|
+
image_id_start,
|
|
154
|
+
anno_id_start,
|
|
155
|
+
_,
|
|
156
|
+
) = VAItoCOCO.convert_single_visionai_to_coco(
|
|
157
|
+
dest_img_folder=os.path.join(target_folder, COCO_IMAGE_PATH),
|
|
158
|
+
visionai_dict={"visionai": target_visionai},
|
|
159
|
+
copy_sensor_data=False,
|
|
160
|
+
source_data_root="",
|
|
161
|
+
uri_root=target_folder,
|
|
162
|
+
camera_sensor_name=camera_sensor_name,
|
|
163
|
+
image_id_start=image_id_start,
|
|
164
|
+
anno_id_start=anno_id_start,
|
|
165
|
+
category_map=category_idx_map,
|
|
166
|
+
n_frame=-1,
|
|
167
|
+
img_extension=file_extension,
|
|
168
|
+
img_width=datarow["image_width"],
|
|
169
|
+
img_height=datarow["image_height"],
|
|
170
|
+
)
|
|
171
|
+
image_update[0].file_name = datarow["unique_file_name"]
|
|
172
|
+
image_update[
|
|
173
|
+
0
|
|
174
|
+
].coco_url = (
|
|
175
|
+
f"{image_update[0].coco_url.rsplit('/',1)[0]}/{datarow['unique_file_name']}"
|
|
176
|
+
)
|
|
177
|
+
images.extend(image_update)
|
|
178
|
+
annotations.extend(anno_update)
|
|
179
|
+
# generate category objects
|
|
180
|
+
categories = [
|
|
181
|
+
Category(
|
|
182
|
+
id=class_id,
|
|
183
|
+
name=class_name,
|
|
184
|
+
)
|
|
185
|
+
for class_name, class_id in category_idx_map.items()
|
|
186
|
+
]
|
|
187
|
+
coco = COCO(categories=categories, images=images, annotations=annotations).dict()
|
|
188
|
+
return coco
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
GROUNDTRUTH = "groundtruth"
|
|
4
|
+
GROUND_TRUTH_ANNOTATION_NAME = "ground_truths"
|
|
5
|
+
|
|
6
|
+
MAX_CONCURRENT_DOWNLOADS = 100
|
|
7
|
+
BATCH_SIZE = 100
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ExportFormat(str, Enum):
|
|
11
|
+
COCO = "coco"
|
|
12
|
+
VISIONAI = "visionai"
|
|
13
|
+
TRAINING_FORMAT = "training_format"
|
|
14
|
+
YOLO = "yolo"
|
|
15
|
+
VLM = "vlm"
|