dataverse-sdk 1.7.2__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/PKG-INFO +10 -2
  2. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/README.md +3 -0
  3. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/apis/backend.py +68 -0
  4. dataverse_sdk-2.1.0/dataverse_sdk/apis/third_party.py +80 -0
  5. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/client.py +18 -14
  6. dataverse_sdk-2.1.0/dataverse_sdk/export/__init__.py +4 -0
  7. dataverse_sdk-2.1.0/dataverse_sdk/export/base.py +41 -0
  8. dataverse_sdk-2.1.0/dataverse_sdk/export/coco.py +188 -0
  9. dataverse_sdk-2.1.0/dataverse_sdk/export/constant.py +15 -0
  10. dataverse_sdk-2.1.0/dataverse_sdk/export/exporter.py +256 -0
  11. dataverse_sdk-2.1.0/dataverse_sdk/export/utils.py +26 -0
  12. dataverse_sdk-2.1.0/dataverse_sdk/export/visionai.py +768 -0
  13. dataverse_sdk-2.1.0/dataverse_sdk/export/vqa.py +139 -0
  14. dataverse_sdk-2.1.0/dataverse_sdk/export/yolo.py +161 -0
  15. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/api.py +13 -19
  16. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/client.py +21 -32
  17. dataverse_sdk-2.1.0/dataverse_sdk/schemas/format.py +8 -0
  18. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/PKG-INFO +10 -2
  19. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/SOURCES.txt +11 -0
  20. dataverse_sdk-2.1.0/dataverse_sdk.egg-info/requires.txt +10 -0
  21. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/setup.py +13 -2
  22. dataverse_sdk-1.7.2/dataverse_sdk.egg-info/requires.txt +0 -5
  23. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/__init__.py +0 -0
  24. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/apis/__init__.py +0 -0
  25. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/connections.py +0 -0
  26. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/constants.py +0 -0
  27. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/exceptions/__init__.py +0 -0
  28. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/exceptions/client.py +0 -0
  29. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/__init__.py +0 -0
  30. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/schemas/common.py +0 -0
  31. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/utils/__init__.py +0 -0
  32. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk/utils/utils.py +0 -0
  33. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
  34. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
  35. {dataverse_sdk-1.7.2 → dataverse_sdk-2.1.0}/setup.cfg +0 -0
@@ -1,17 +1,22 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dataverse-sdk
3
- Version: 1.7.2
3
+ Version: 2.1.0
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
7
7
  Classifier: Programming Language :: Python :: 3
8
8
  Requires-Python: >=3.10, <4
9
9
  Description-Content-Type: text/markdown
10
- Requires-Dist: pydantic==1.*
10
+ Requires-Dist: pydantic==2.*
11
11
  Requires-Dist: requests
12
12
  Requires-Dist: httpx>=0.23.0
13
13
  Requires-Dist: tqdm
14
14
  Requires-Dist: aiofiles
15
+ Requires-Dist: visionai-data-format==2.0.0
16
+ Requires-Dist: aiohttp
17
+ Requires-Dist: aiohttp_retry
18
+ Requires-Dist: numpy<2,>=1.21
19
+ Requires-Dist: opencv-contrib-python==4.8.1.78
15
20
  Dynamic: author
16
21
  Dynamic: classifier
17
22
  Dynamic: description
@@ -555,6 +560,9 @@ python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.a
555
560
  python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
556
561
  ```
557
562
 
563
+ ### Export Large Dataslice and download files
564
+
565
+ python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
558
566
 
559
567
  ## Links to language repos
560
568
 
@@ -533,6 +533,9 @@ python tools/import_vqa_dataset.py -host https://staging.visionai.linkervision.a
533
533
  python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} -f {/YOUR/TARGET/LOCAL/file.zip}
534
534
  ```
535
535
 
536
+ ### Export Large Dataslice and download files
537
+
538
+ python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
536
539
 
537
540
  ## Links to language repos
538
541
 
@@ -1,6 +1,7 @@
1
1
  import inspect
2
2
  import json
3
3
  import logging
4
+ from collections.abc import AsyncGenerator, AsyncIterator
4
5
  from typing import Optional, Union
5
6
  from urllib.parse import urlencode
6
7
 
@@ -612,3 +613,70 @@ class AsyncBackendAPI:
612
613
  except Exception as e:
613
614
  print(f"Request failed: {str(e)}")
614
615
  return None
616
+
617
+ async def get_datarows(
618
+ self, batch_size: int = 20, order_by: str = "id", **kwargs
619
+ ) -> AsyncGenerator[list[dict], None, None]:
620
+ if "offset" in kwargs or "limit" in kwargs:
621
+ raise ValueError("Specifying offset or limit directly is not allowed.")
622
+
623
+ kwargs["order_by"] = order_by
624
+ dataslice_set = kwargs.pop("dataslice_set", [])
625
+ query_params = {
626
+ **kwargs,
627
+ "order_by": order_by,
628
+ "dataslice_set": dataslice_set,
629
+ "limit": batch_size,
630
+ }
631
+ query_string = urlencode(query_params, doseq=True)
632
+
633
+ id_gt = 0
634
+ while True:
635
+ url = f"{self.host}/api/datarows/?{query_string}&id__gt={id_gt}"
636
+ resp: dict = await self.async_send_request(
637
+ url=url,
638
+ method="get",
639
+ headers=self.headers,
640
+ )
641
+ json_data = resp
642
+ if json_data["count"] == 0:
643
+ break
644
+ if not json_data["results"]:
645
+ break
646
+
647
+ # Get last datarow id
648
+ datarows = json_data["results"]
649
+ id_gt = datarows[-1]["id"]
650
+ yield datarows
651
+
652
+ async def get_datarows_flat_parent(
653
+ self, batch_size: int = 20, order_by: str = "id", **kwargs
654
+ ) -> AsyncIterator[list[dict]]:
655
+ id_gt = 0
656
+ limit = batch_size
657
+ kwargs["order_by"] = order_by
658
+ while True:
659
+ kwargs.update({"id__gt": id_gt, "limit": limit})
660
+ url = f"{self.host}/api/datarows/flat-parent/?{urlencode(kwargs)}"
661
+ resp: dict = await self.async_send_request(
662
+ url=url,
663
+ method="get",
664
+ headers=self.headers,
665
+ )
666
+ json_data: list[dict] = resp
667
+ if not json_data["results"]:
668
+ break
669
+
670
+ # Get last datarow id
671
+ datarows = json_data["results"]
672
+ id_gt = datarows[-1]["id"]
673
+ yield datarows
674
+
675
+ async def get_dataslice(self, dataslice_id: int) -> dict:
676
+ url = f"{self.host}/api/dataslices/{dataslice_id}/"
677
+ resp: dict = await self.async_send_request(
678
+ url=url,
679
+ method="get",
680
+ headers=self.headers,
681
+ )
682
+ return resp
@@ -0,0 +1,80 @@
1
+ import logging
2
+ from typing import Optional
3
+
4
+ import requests
5
+ from aiohttp.client_reqrep import ClientResponse
6
+ from aiohttp_retry import ExponentialRetry, RetryClient
7
+ from requests.adapters import HTTPAdapter, Retry
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ class ThirdPartyAPI:
13
+ adapter = HTTPAdapter(
14
+ max_retries=Retry(
15
+ total=10, backoff_factor=5, status_forcelist=[500, 502, 503, 504, 104]
16
+ )
17
+ )
18
+ retry_options: dict = {
19
+ "attempts": 5,
20
+ "start_timeout": 5,
21
+ "max_timeout": 5 * (2**5),
22
+ "statuses": {500, 502, 503, 504},
23
+ }
24
+
25
+ @classmethod
26
+ def send_request(
27
+ cls,
28
+ url: str,
29
+ method: str,
30
+ timeout: int = 10,
31
+ custom_retry_options: Optional[dict] = None,
32
+ **kwargs,
33
+ ):
34
+ adapter = cls.adapter
35
+ if custom_retry_options:
36
+ adapter = HTTPAdapter(max_retries=Retry(**custom_retry_options))
37
+
38
+ with requests.Session() as session:
39
+ session.mount("http://", adapter)
40
+ session.mount("https://", adapter)
41
+
42
+ try:
43
+ response = session.request(
44
+ method=method, url=url, timeout=timeout, **kwargs
45
+ )
46
+ except Exception as e:
47
+ logger.error(f"third party request error : {str(e)}")
48
+ raise
49
+
50
+ if not 200 <= response.status_code <= 299:
51
+ raise Exception(
52
+ f"status code: {response.status_code}, response detail: {response.__dict__}"
53
+ )
54
+ return response
55
+
56
+ @classmethod
57
+ async def async_send_request(
58
+ cls, url: str, method: str, **kwargs
59
+ ) -> tuple[RetryClient, ClientResponse]:
60
+ retry_options = ExponentialRetry(**cls.retry_options)
61
+ retry_client = RetryClient(raise_for_status=False, retry_options=retry_options)
62
+ try:
63
+ response = await retry_client.request(method=method, url=url, **kwargs)
64
+ except Exception as e:
65
+ logger.error(f"third party request error : {e}")
66
+ raise
67
+
68
+ if not 200 <= response.status <= 299:
69
+ raise Exception(
70
+ f"status code: {response.status}, response detail: {response.__dict__}"
71
+ )
72
+
73
+ return retry_client, response
74
+
75
+ @classmethod
76
+ async def async_download_file(cls, url: str, method: str, **kwargs) -> bytes:
77
+ client, resp = await cls.async_send_request(url, method, **kwargs)
78
+ data = await resp.read()
79
+ await client.close()
80
+ return data
@@ -209,7 +209,7 @@ class DataverseClient:
209
209
  raise InvalidProcessError(
210
210
  "Could not create VQA project by this function, please use create_vqa_project"
211
211
  )
212
- raw_ontology_data: dict = ontology.dict(exclude_none=True)
212
+ raw_ontology_data: dict = ontology.model_dump(exclude_none=True)
213
213
  classes_data_list: list[dict] = []
214
214
  rank = 1
215
215
  # remove `id` field in OntologyClass and Attribute
@@ -225,15 +225,19 @@ class DataverseClient:
225
225
  classes_data_list.append(cls_)
226
226
  raw_ontology_data["ontology_classes_data"] = classes_data_list
227
227
  if project_tag is not None:
228
- raw_project_tag_data: dict = project_tag.dict(exclude_none=True)
228
+ raw_project_tag_data: dict = project_tag.model_dump(exclude_none=True)
229
229
  if tag_attrs := raw_project_tag_data.pop("attributes", None):
230
230
  raw_project_tag_data["attribute_data"] = parse_attribute(tag_attrs)
231
231
  else:
232
232
  raw_project_tag_data = {}
233
233
 
234
- ontology_data = OntologyAPISchema(**raw_ontology_data).dict(exclude_none=True)
235
- project_tag_data = ProjectTagAPISchema(**raw_project_tag_data).dict()
236
- sensor_data = [sensor.dict(exclude_none=True) for sensor in sensors]
234
+ ontology_data = OntologyAPISchema(**raw_ontology_data).model_dump(
235
+ exclude_none=True
236
+ )
237
+ project_tag_data = ProjectTagAPISchema(**raw_project_tag_data).model_dump(
238
+ exclude_none=True
239
+ )
240
+ sensor_data = [sensor.model_dump(exclude_none=True) for sensor in sensors]
237
241
 
238
242
  try:
239
243
  raw_project_data = ProjectAPISchema(
@@ -242,7 +246,7 @@ class DataverseClient:
242
246
  sensor_data=sensor_data,
243
247
  project_tag_data=project_tag_data,
244
248
  description=description,
245
- ).dict(exclude_none=True)
249
+ ).model_dump(exclude_none=True)
246
250
  except ValidationError as e:
247
251
  raise APIValidationError(
248
252
  f"Something wrong when composing the final project data: {e}"
@@ -297,7 +301,7 @@ class DataverseClient:
297
301
  ontology_name=ontology_name,
298
302
  question_answer=question_answer,
299
303
  description=description,
300
- ).dict(exclude_none=True)
304
+ ).model_dump(exclude_none=True)
301
305
  except ValidationError as e:
302
306
  raise APIValidationError(
303
307
  f"Something wrong when composing the vqa project data: {e}"
@@ -411,7 +415,7 @@ class DataverseClient:
411
415
  if ontology_name:
412
416
  edit_vqa_data["ontology_name"] = ontology_name
413
417
  if create:
414
- edit_vqa_data["create"] = [q.dict(exclude_none=True) for q in create]
418
+ edit_vqa_data["create"] = [q.model_dump(exclude_none=True) for q in create]
415
419
  if update:
416
420
  question_table = {
417
421
  q.rank: {
@@ -439,7 +443,7 @@ class DataverseClient:
439
443
  ]["attribute_id"]
440
444
  update_question_data["options"] = update_question.options
441
445
  update_questions.append(
442
- UpdateQuestionAPISchema(**update_question_data).dict(
446
+ UpdateQuestionAPISchema(**update_question_data).model_dump(
443
447
  exclude_none=True
444
448
  )
445
449
  )
@@ -902,7 +906,7 @@ of this project OR has been added before"
902
906
  if project.ontology.image_type == OntologyImageType.VQA:
903
907
  raise InvalidProcessError("Could not add project_tag for VQA project")
904
908
 
905
- raw_project_tag: dict = project_tag.dict(exclude_none=True)
909
+ raw_project_tag: dict = project_tag.model_dump(exclude_none=True)
906
910
  # new project tag attributes to be creaeted
907
911
  new_attribute_data: list = parse_attribute(
908
912
  raw_project_tag.get("attributes", [])
@@ -962,7 +966,7 @@ of this project OR has been added before"
962
966
  if project.ontology.image_type == OntologyImageType.VQA:
963
967
  raise InvalidProcessError("Could not edit project_tag for VQA project")
964
968
 
965
- raw_project_tag: dict = project_tag.dict(exclude_none=True)
969
+ raw_project_tag: dict = project_tag.model_dump(exclude_none=True)
966
970
  # old project tag attributes to be extended
967
971
  patched_attribute_data: list = parse_attribute(
968
972
  raw_project_tag.get("attributes", [])
@@ -1025,7 +1029,7 @@ of this project OR has been added before"
1025
1029
  # new ontology classes to be created
1026
1030
  new_classes_data = []
1027
1031
  for ontology_class in ontology_classes:
1028
- raw_ontology_class: dict = ontology_class.dict(exclude_none=True)
1032
+ raw_ontology_class: dict = ontology_class.model_dump(exclude_none=True)
1029
1033
  attribute_data: list = parse_attribute(
1030
1034
  raw_ontology_class.get("attributes", [])
1031
1035
  )
@@ -1097,7 +1101,7 @@ of this project OR has been added before"
1097
1101
  # ontology classes to be edited
1098
1102
  patched_classes_data = []
1099
1103
  for ontology_class in ontology_classes:
1100
- raw_ontology_class: dict = ontology_class.dict(exclude_none=True)
1104
+ raw_ontology_class: dict = ontology_class.model_dump(exclude_none=True)
1101
1105
  attribute_data: list = parse_attribute(
1102
1106
  raw_ontology_class.get("attributes", [])
1103
1107
  )
@@ -1563,7 +1567,7 @@ of this project OR has been added before"
1563
1567
  access_key_id=access_key_id,
1564
1568
  secret_access_key=secret_access_key,
1565
1569
  **kwargs,
1566
- ).dict(exclude_none=True)
1570
+ ).model_dump(exclude_none=True)
1567
1571
  except ValidationError as e:
1568
1572
  raise APIValidationError(
1569
1573
  f"Something wrong when composing the final dataset data: {e}"
@@ -0,0 +1,4 @@
1
+ from .coco import * # noqa
2
+ from .visionai import * # noqa
3
+ from .vqa import * # noqa
4
+ from .yolo import * # noqa
@@ -0,0 +1,41 @@
1
+ import abc
2
+ from collections.abc import Generator
3
+ from typing import Callable
4
+
5
+
6
+ class ExportAnnotationBase(abc.ABC):
7
+ @abc.abstractmethod
8
+ def producer(
9
+ self,
10
+ target_folder: str,
11
+ class_names: list[str],
12
+ sequence_frame_map: dict[int, dict[int, list[int]]],
13
+ datarow_generator_func: Callable[[list], Generator[dict]],
14
+ annotation_name: str,
15
+ *args,
16
+ **kwargs,
17
+ ) -> Generator[bytes, str]:
18
+ """
19
+ Parameters
20
+ ----------
21
+ target_folder : str
22
+ _description_
23
+ class_names : list[str]
24
+ list of class name
25
+ sequence_frame_map : dict[int, dict[int, list[int]]]
26
+ the entire tree of sequence, frame and base datarows
27
+ datarow_generator_func : Callable[[list], AsyncGenerator[dict]]
28
+ a function that takes list of id as input which generates corresponding datarows
29
+ annotation_name: str
30
+ an annotation name
31
+
32
+ Returns
33
+ -------
34
+ AsyncGenerator[bytes, str]
35
+ should yield a tuple with the data to upload and the relative path to rootfolder
36
+ relative path: this/is/the/relative/path/to/file.txt
37
+ Raises
38
+ ------
39
+ NotImplementedError
40
+ """
41
+ raise NotImplementedError
@@ -0,0 +1,188 @@
1
+ import asyncio
2
+ import os
3
+ from collections.abc import AsyncGenerator
4
+ from typing import Callable
5
+
6
+ import aiohttp
7
+ from tqdm import tqdm
8
+ from visionai_data_format.converters.vai_to_coco import VAItoCOCO
9
+ from visionai_data_format.schemas.coco_schema import COCO, Category
10
+ from visionai_data_format.utils.common import (
11
+ ANNOT_PATH,
12
+ COCO_IMAGE_PATH,
13
+ COCO_LABEL_FILE,
14
+ )
15
+
16
+ from .base import ExportAnnotationBase
17
+ from .constant import (
18
+ BATCH_SIZE,
19
+ GROUND_TRUTH_ANNOTATION_NAME,
20
+ GROUNDTRUTH,
21
+ MAX_CONCURRENT_DOWNLOADS,
22
+ ExportFormat,
23
+ )
24
+ from .exporter import Exporter
25
+ from .utils import convert_to_bytes
26
+
27
+
28
+ @Exporter.register(format=ExportFormat.COCO)
29
+ class ExportCoco(ExportAnnotationBase):
30
+ async def download_batch(
31
+ self,
32
+ session: aiohttp.ClientSession,
33
+ semaphore: asyncio.Semaphore,
34
+ batch_datarows: list[dict],
35
+ ) -> list[tuple[bytes, str]]:
36
+ tasks = []
37
+
38
+ for datarow in batch_datarows:
39
+ url = datarow["url"]
40
+ file_path = os.path.join(COCO_IMAGE_PATH, datarow["unique_file_name"])
41
+
42
+ async def download_single(url, file_path):
43
+ async with semaphore:
44
+ try:
45
+ async with session.get(url) as response:
46
+ response.raise_for_status()
47
+ img_bytes = await response.read()
48
+ return img_bytes, file_path
49
+ except Exception as e:
50
+ print(f"Error downloading {url}: {e}")
51
+ return None
52
+
53
+ tasks.append(download_single(url, file_path))
54
+
55
+ results = await asyncio.gather(*tasks)
56
+ return [r for r in results if r is not None]
57
+
58
+ async def producer(
59
+ self,
60
+ class_names: list[str],
61
+ sequence_frame_map: dict[int, dict[int, list[int]]],
62
+ datarow_generator_func: Callable[[list], AsyncGenerator[dict]],
63
+ target_folder: str,
64
+ annotation_name: str,
65
+ *_,
66
+ **kwargs,
67
+ ) -> AsyncGenerator[tuple[bytes, str], None]:
68
+ semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS)
69
+ async with aiohttp.ClientSession() as session:
70
+ current_batch = []
71
+ datarows = [] # Keep track of all datarows for annotation
72
+ datarow_id_list = []
73
+ total_datarows = sum(len(v) for v in sequence_frame_map.values())
74
+ existing_files = set()
75
+
76
+ with tqdm(
77
+ total=total_datarows, desc="Downloading images", unit="file"
78
+ ) as progress_bar:
79
+ for frame_datarow_map in sequence_frame_map.values():
80
+ for datarow_ids in frame_datarow_map.values():
81
+ datarow_id_list.extend(datarow_ids)
82
+ for start_idx in range(0, len(datarow_id_list), BATCH_SIZE):
83
+ async for datarow in datarow_generator_func(
84
+ datarow_id_list[start_idx : start_idx + BATCH_SIZE]
85
+ ):
86
+ original_file_name = os.path.basename(datarow["original_url"])
87
+ unique_file_name = Exporter.get_unique_filename(
88
+ self, original_file_name, existing_files
89
+ )
90
+ existing_files.add(unique_file_name)
91
+ datarow["unique_file_name"] = unique_file_name
92
+ current_batch.append(datarow)
93
+ datarows.append(datarow)
94
+
95
+ if len(current_batch) >= BATCH_SIZE:
96
+ results = await self.download_batch(
97
+ session, semaphore, current_batch
98
+ )
99
+ for result in results:
100
+ if result:
101
+ yield result
102
+ progress_bar.update(1)
103
+ current_batch = []
104
+
105
+ if current_batch:
106
+ results = await self.download_batch(
107
+ session, semaphore, current_batch
108
+ )
109
+ for result in results:
110
+ if result:
111
+ yield result
112
+ progress_bar.update(1)
113
+
114
+ annot_bytes = convert_to_bytes(
115
+ convert_annotation(
116
+ datarows=datarows,
117
+ class_names=class_names,
118
+ target_folder=target_folder,
119
+ annotation_name=annotation_name,
120
+ )
121
+ )
122
+ yield annot_bytes, os.path.join(ANNOT_PATH, COCO_LABEL_FILE)
123
+
124
+
125
+ def convert_annotation(
126
+ datarows: list[dict],
127
+ class_names: list[str],
128
+ target_folder: str,
129
+ annotation_name: str,
130
+ ) -> dict:
131
+ image_id_start = 0
132
+ anno_id_start = 0
133
+ images = []
134
+ annotations = []
135
+ category_idx_map = {category: idx for idx, category in enumerate(class_names)}
136
+ for datarow in datarows:
137
+ camera_sensor_name = datarow["sensor_name"]
138
+ url = datarow["url"]
139
+ file_extension = os.path.splitext(url)[-1]
140
+ if annotation_name == GROUNDTRUTH:
141
+ target_visionai: dict = datarow["items"].get(
142
+ GROUND_TRUTH_ANNOTATION_NAME, {}
143
+ )
144
+ else:
145
+ target_visionai: dict = (
146
+ datarow["items"].get("predictions", {}).get(annotation_name, {})
147
+ )
148
+
149
+ (
150
+ category_idx_map,
151
+ image_update,
152
+ anno_update,
153
+ image_id_start,
154
+ anno_id_start,
155
+ _,
156
+ ) = VAItoCOCO.convert_single_visionai_to_coco(
157
+ dest_img_folder=os.path.join(target_folder, COCO_IMAGE_PATH),
158
+ visionai_dict={"visionai": target_visionai},
159
+ copy_sensor_data=False,
160
+ source_data_root="",
161
+ uri_root=target_folder,
162
+ camera_sensor_name=camera_sensor_name,
163
+ image_id_start=image_id_start,
164
+ anno_id_start=anno_id_start,
165
+ category_map=category_idx_map,
166
+ n_frame=-1,
167
+ img_extension=file_extension,
168
+ img_width=datarow["image_width"],
169
+ img_height=datarow["image_height"],
170
+ )
171
+ image_update[0].file_name = datarow["unique_file_name"]
172
+ image_update[
173
+ 0
174
+ ].coco_url = (
175
+ f"{image_update[0].coco_url.rsplit('/',1)[0]}/{datarow['unique_file_name']}"
176
+ )
177
+ images.extend(image_update)
178
+ annotations.extend(anno_update)
179
+ # generate category objects
180
+ categories = [
181
+ Category(
182
+ id=class_id,
183
+ name=class_name,
184
+ )
185
+ for class_name, class_id in category_idx_map.items()
186
+ ]
187
+ coco = COCO(categories=categories, images=images, annotations=annotations).dict()
188
+ return coco
@@ -0,0 +1,15 @@
1
+ from enum import Enum
2
+
3
+ GROUNDTRUTH = "groundtruth"
4
+ GROUND_TRUTH_ANNOTATION_NAME = "ground_truths"
5
+
6
+ MAX_CONCURRENT_DOWNLOADS = 100
7
+ BATCH_SIZE = 100
8
+
9
+
10
+ class ExportFormat(str, Enum):
11
+ COCO = "coco"
12
+ VISIONAI = "visionai"
13
+ TRAINING_FORMAT = "training_format"
14
+ YOLO = "yolo"
15
+ VLM = "vlm"