dataverse-sdk 2.8.0__tar.gz → 2.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/PKG-INFO +10 -9
  2. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/README.md +8 -5
  3. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/backend.py +19 -0
  4. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/client.py +8 -2
  5. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/coco.py +2 -4
  6. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/constant.py +3 -0
  7. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/exporter.py +59 -5
  8. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/vqa.py +106 -3
  9. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/yolo.py +2 -2
  10. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/client.py +9 -1
  11. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/common.py +1 -0
  12. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/utils/utils.py +1 -0
  13. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/PKG-INFO +10 -9
  14. dataverse_sdk-2.9.0/dataverse_sdk.egg-info/requires.txt +8 -0
  15. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/setup.py +2 -4
  16. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/import_vqa_dataset.py +10 -1
  17. dataverse_sdk-2.8.0/dataverse_sdk.egg-info/requires.txt +0 -10
  18. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/MANIFEST.in +0 -0
  19. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/__init__.py +0 -0
  20. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/__init__.py +0 -0
  21. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/third_party.py +0 -0
  22. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/connections.py +0 -0
  23. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/constants.py +0 -0
  24. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/exceptions/__init__.py +0 -0
  25. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/exceptions/client.py +0 -0
  26. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/__init__.py +0 -0
  27. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/base.py +0 -0
  28. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/utils.py +0 -0
  29. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/visionai.py +0 -0
  30. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/__init__.py +0 -0
  31. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/api.py +0 -0
  32. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/format.py +0 -0
  33. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/utils/__init__.py +0 -0
  34. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
  35. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
  36. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
  37. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/setup.cfg +0 -0
  38. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/export_dataslice.py +0 -0
  39. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/export_dataslice_large.py +0 -0
  40. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/import_dataset_from_local.py +0 -0
  41. {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/upload_videos_create_session.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.8.0
3
+ Version: 2.9.0
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -12,11 +12,9 @@ Requires-Dist: requests
12
12
  Requires-Dist: httpx>=0.23.0
13
13
  Requires-Dist: tqdm
14
14
  Requires-Dist: aiofiles
15
- Requires-Dist: visionai-data-format==2.0.0
15
+ Requires-Dist: visionai-data-format>=2.1.0
16
16
  Requires-Dist: aiohttp
17
17
  Requires-Dist: aiohttp_retry
18
- Requires-Dist: numpy<2,>=1.21
19
- Requires-Dist: opencv-contrib-python==4.8.1.78
20
18
  Dynamic: author
21
19
  Dynamic: classifier
22
20
  Dynamic: description
@@ -574,6 +572,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
574
572
 
575
573
  `MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
576
574
 
575
+ Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
576
+
577
577
  #### Filtering by Model Type
578
578
 
579
579
  You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
@@ -603,11 +603,12 @@ models = client.list_models(
603
603
 
604
604
  #### Input Arguments
605
605
 
606
- | Argument name | Type/Options | Default | Description |
607
- | ------------- | ----------------------------------------------------------------- | ------------------- | ------------------------ |
608
- | project_id | int | *-- | The project ID |
609
- | client_alias | str | None | The client alias |
610
- | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
606
+ | Argument name | Type/Options | Default | Description |
607
+ | ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
608
+ | project_id | int | *-- | The project ID |
609
+ | client_alias | str | None | The client alias |
610
+ | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
611
+ | include_disabled | bool | False | Also list disabled models, e.g. failed ones |
611
612
 
612
613
  `*--`: required argument without default
613
614
 
@@ -547,6 +547,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
547
547
 
548
548
  `MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
549
549
 
550
+ Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
551
+
550
552
  #### Filtering by Model Type
551
553
 
552
554
  You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
@@ -576,11 +578,12 @@ models = client.list_models(
576
578
 
577
579
  #### Input Arguments
578
580
 
579
- | Argument name | Type/Options | Default | Description |
580
- | ------------- | ----------------------------------------------------------------- | ------------------- | ------------------------ |
581
- | project_id | int | *-- | The project ID |
582
- | client_alias | str | None | The client alias |
583
- | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
581
+ | Argument name | Type/Options | Default | Description |
582
+ | ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
583
+ | project_id | int | *-- | The project ID |
584
+ | client_alias | str | None | The client alias |
585
+ | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
586
+ | include_disabled | bool | False | Also list disabled models, e.g. failed ones |
584
587
 
585
588
  `*--`: required argument without default
586
589
 
@@ -331,10 +331,13 @@ class BackendAPI:
331
331
  self,
332
332
  project_id: int,
333
333
  type: str = "trained,byom",
334
+ include_disabled: bool = False,
334
335
  **kwargs,
335
336
  ) -> list:
336
337
  kwargs["project"] = project_id
337
338
  kwargs["type__in"] = type
339
+ if include_disabled:
340
+ kwargs["include_disabled"] = "true"
338
341
  resp = self.send_request(
339
342
  url=f"{self.host}/api/ml_models/?{urlencode(kwargs)}",
340
343
  method="get",
@@ -721,6 +724,22 @@ class AsyncBackendAPI:
721
724
  data=payload_data,
722
725
  )
723
726
 
727
+ async def get_dataset(self, dataset_id: int) -> dict:
728
+ try:
729
+ resp = await self.client.get(
730
+ url=f"{self.host}/api/datasets/{dataset_id}/",
731
+ headers=self.headers,
732
+ timeout=30,
733
+ )
734
+ resp.raise_for_status()
735
+ return resp.json()
736
+ except httpx.HTTPStatusError as e:
737
+ print(f"HTTP error: {e.response.status_code} - {e.response.text}")
738
+ return None
739
+ except Exception as e:
740
+ print(f"Request failed: {str(e)}")
741
+ return None
742
+
724
743
  async def get_project(self, project_id: str) -> dict:
725
744
  try:
726
745
  resp = await self.client.get(
@@ -1317,6 +1317,7 @@ of this project OR has been added before"
1317
1317
  list[Literal["trained", "byom", "uploaded"]],
1318
1318
  ]
1319
1319
  ] = ["trained", "byom"],
1320
+ include_disabled: bool = False,
1320
1321
  ) -> list[MLModel]:
1321
1322
  """Get the model list by project id
1322
1323
 
@@ -1329,6 +1330,9 @@ of this project OR has been added before"
1329
1330
  project: Optional["Project"]
1330
1331
  project basemodel, by default None
1331
1332
  type : Optional[Union[Literal["trained", "byom", "uploaded"], list[Literal["trained", "byom", "uploaded"]]]], by default ["trained", "byom"]
1333
+ include_disabled : bool, by default False
1334
+ also list disabled models (`MLModel.enabled` is False), e.g. a custom
1335
+ model whose upload failed. Older servers ignore it and list enabled models only.
1332
1336
 
1333
1337
  Returns
1334
1338
  -------
@@ -1346,7 +1350,9 @@ of this project OR has been added before"
1346
1350
  try:
1347
1351
  if isinstance(type, list):
1348
1352
  type = ",".join(type)
1349
- model_list: list = api.list_ml_models(project_id=project_id, type=type)
1353
+ model_list: list = api.list_ml_models(
1354
+ project_id=project_id, type=type, include_disabled=include_disabled
1355
+ )
1350
1356
  except DataverseExceptionBase:
1351
1357
  logging.exception("Got api error from Dataverse")
1352
1358
  raise
@@ -2481,7 +2487,7 @@ of this project OR has been added before"
2481
2487
  return ["images/", "annotations/labels.json"]
2482
2488
  elif annotation_format == AnnotationFormat.YOLO:
2483
2489
  return ["images/", "labels/", "classes.txt"]
2484
- elif annotation_format == AnnotationFormat.VLM:
2490
+ elif annotation_format in (AnnotationFormat.VLM, AnnotationFormat.LLAVA):
2485
2491
  return ["images/", "annotations/"]
2486
2492
  elif annotation_format in (
2487
2493
  AnnotationFormat.VISION_AI,
@@ -93,7 +93,7 @@ class ExportCoco(ExportAnnotationBase):
93
93
  async for datarow in datarow_generator_func(
94
94
  datarow_id_list[start_idx : start_idx + BATCH_SIZE]
95
95
  ):
96
- original_file_name = os.path.basename(datarow["original_url"])
96
+ original_file_name = Exporter.get_export_file_name(datarow)
97
97
  unique_file_name = Exporter.get_unique_filename(
98
98
  self, original_file_name, existing_files
99
99
  )
@@ -184,9 +184,7 @@ def convert_annotation(
184
184
  image_update[0].file_name = datarow["unique_file_name"]
185
185
  image_update[
186
186
  0
187
- ].coco_url = (
188
- f"{image_update[0].coco_url.rsplit('/',1)[0]}/{datarow['unique_file_name']}"
189
- )
187
+ ].coco_url = f"{image_update[0].coco_url.rsplit('/', 1)[0]}/{datarow['unique_file_name']}"
190
188
  images.extend(image_update)
191
189
  annotations.extend(anno_update)
192
190
  # generate category objects
@@ -6,6 +6,8 @@ GROUND_TRUTH_ANNOTATION_NAME = "ground_truths"
6
6
  MAX_CONCURRENT_DOWNLOADS = 100
7
7
  BATCH_SIZE = 50
8
8
 
9
+ VIDEO_SUPPORTED_FORMAT = {".mp4", ".avi", ".mov", ".mpeg", ".flv", ".wmv"}
10
+
9
11
 
10
12
  class ExportFormat(str, Enum):
11
13
  COCO = "coco"
@@ -13,3 +15,4 @@ class ExportFormat(str, Enum):
13
15
  TRAINING_FORMAT = "training_format"
14
16
  YOLO = "yolo"
15
17
  VLM = "vlm"
18
+ LLAVA = "llava"
@@ -2,14 +2,19 @@ import asyncio
2
2
  import os
3
3
  from collections import defaultdict
4
4
  from collections.abc import AsyncGenerator, Generator
5
- from typing import Callable
5
+ from typing import Callable, Optional
6
+ from urllib.parse import urlparse
6
7
 
7
8
  import aiofiles
8
9
 
9
10
  from ..apis.backend import AsyncBackendAPI
10
- from ..schemas.format import AnnotationFormat
11
11
  from .base import ExportAnnotationBase
12
- from .constant import BATCH_SIZE, MAX_CONCURRENT_DOWNLOADS, ExportFormat
12
+ from .constant import (
13
+ BATCH_SIZE,
14
+ MAX_CONCURRENT_DOWNLOADS,
15
+ VIDEO_SUPPORTED_FORMAT,
16
+ ExportFormat,
17
+ )
13
18
 
14
19
  NONE_SEQUENCE_DATAROW_ID = -1
15
20
 
@@ -62,7 +67,7 @@ class Exporter:
62
67
 
63
68
  project_id = dataslice["project"]["id"]
64
69
  question_id_map = {}
65
- if export_format == AnnotationFormat.VLM:
70
+ if export_format in (ExportFormat.VLM, ExportFormat.LLAVA):
66
71
  project: dict = await self.curation_api.get_project(project_id=project_id)
67
72
  for question in project["ontology"]["classes"]:
68
73
  question_id_map[question["rank"]] = question["extended_class"][
@@ -130,6 +135,7 @@ class Exporter:
130
135
  def register(cls, format: ExportFormat):
131
136
  def _wrap(export_annotation: ExportAnnotationBase):
132
137
  cls.export_annotation_m[format] = export_annotation() # register instance
138
+ return export_annotation
133
139
 
134
140
  return _wrap
135
141
 
@@ -137,13 +143,16 @@ class Exporter:
137
143
  async def _gen(
138
144
  curation_api: AsyncBackendAPI,
139
145
  ) -> Callable[[list[int]], AsyncGenerator[dict]]:
146
+ # dataset_id -> frame rate used to split its videos, shared across calls
147
+ frame_rate_map: dict[int, Optional[float]] = {}
148
+
140
149
  async def f(datarow_id_list: list[int]):
141
150
  datarow_id_set: set[int] = set()
142
151
  for id_chunks in chunks(datarow_id_list, 1000):
143
152
  gen: AsyncGenerator = curation_api.get_datarows(
144
153
  id_set_list=id_chunks,
145
154
  batch_size=BATCH_SIZE,
146
- fields="id,items,vlm_items,url,frame_id,image_width,image_height,sensor_name,original_url,type",
155
+ fields="id,items,vlm_items,url,frame_id,image_width,image_height,sensor_name,original_url,type,dataset",
147
156
  )
148
157
  async for batched_datarow in gen:
149
158
  for datarow in batched_datarow:
@@ -152,10 +161,55 @@ class Exporter:
152
161
  # drop duplicate datarows
153
162
  continue
154
163
  datarow_id_set.add(dr_id)
164
+ await Exporter._attach_video_frame_rate(
165
+ curation_api=curation_api,
166
+ datarow=datarow,
167
+ frame_rate_map=frame_rate_map,
168
+ )
155
169
  yield datarow
156
170
 
157
171
  return f
158
172
 
173
+ @staticmethod
174
+ def is_video_frame(datarow: dict) -> bool:
175
+ original_url: str = datarow.get("original_url") or ""
176
+ return os.path.splitext(original_url)[-1].lower() in VIDEO_SUPPORTED_FORMAT
177
+
178
+ @staticmethod
179
+ async def _attach_video_frame_rate(
180
+ curation_api: AsyncBackendAPI,
181
+ datarow: dict,
182
+ frame_rate_map: dict[int, Optional[float]],
183
+ ) -> None:
184
+ """Attach the frame rate used to split the source video to the video frame datarow as `frame_rate`"""
185
+ if not Exporter.is_video_frame(datarow):
186
+ return
187
+ dataset_id: Optional[int] = datarow.get("dataset")
188
+ if dataset_id is not None and dataset_id not in frame_rate_map:
189
+ dataset: Optional[dict] = await curation_api.get_dataset(
190
+ dataset_id=dataset_id
191
+ )
192
+ frame_rate_map[dataset_id] = (dataset or {}).get("frame_rate")
193
+ datarow["frame_rate"] = frame_rate_map.get(dataset_id)
194
+
195
+ @staticmethod
196
+ def get_export_file_name(datarow: dict) -> str:
197
+ """Return the exported image file name of the datarow
198
+
199
+ Frames split from a video share the video as `original_url`, so they are named
200
+ `[VideoName]_frame[frame_id]_[frame_rate]fps.[ext]` instead, e.g. video1_frame000000000001_10fps.jpg
201
+ """
202
+ original_file_name: str = os.path.basename(datarow["original_url"])
203
+ if not Exporter.is_video_frame(datarow) or datarow.get("frame_id") is None:
204
+ return original_file_name
205
+
206
+ video_name: str = os.path.splitext(original_file_name)[0]
207
+ extension: str = os.path.splitext(urlparse(datarow["url"]).path)[-1]
208
+ file_name = f"{video_name}_frame{int(datarow['frame_id']):012d}"
209
+ if frame_rate := datarow.get("frame_rate"):
210
+ file_name += f"_{float(frame_rate):g}fps"
211
+ return f"{file_name}{extension}"
212
+
159
213
  def get_unique_filename(self, original_file_name, existing_files):
160
214
  base_name, extension = os.path.splitext(original_file_name)
161
215
  counter = 1
@@ -1,5 +1,6 @@
1
1
  import asyncio
2
2
  import json
3
+ import logging
3
4
  import os
4
5
  from collections.abc import AsyncGenerator
5
6
  from typing import Callable
@@ -13,11 +14,30 @@ from .constant import BATCH_SIZE, MAX_CONCURRENT_DOWNLOADS, ExportFormat
13
14
  from .exporter import Exporter
14
15
  from .utils import convert_to_bytes
15
16
 
17
+ logger = logging.getLogger(__name__)
18
+
16
19
  VLM_ANNOTATION_FILE = "vlm_annotation.json"
20
+ LLAVA_ANNOTATION_FILE = "llava_annotation.json"
21
+ LLAVA_IMAGE_TOKEN = "<image>"
17
22
 
18
23
 
19
24
  @Exporter.register(format=ExportFormat.VLM)
20
25
  class ExportVQA(ExportAnnotationBase):
26
+ annotation_file = VLM_ANNOTATION_FILE
27
+
28
+ @staticmethod
29
+ def convert(
30
+ datarows: list[dict],
31
+ annotation_name: str,
32
+ question_id_map: dict,
33
+ written_files: set,
34
+ ) -> str:
35
+ return convert_annotation(
36
+ datarows=datarows,
37
+ annotation_name=annotation_name,
38
+ question_id_map=question_id_map,
39
+ )
40
+
21
41
  async def download_batch(
22
42
  self,
23
43
  session: aiohttp.ClientSession,
@@ -71,6 +91,7 @@ class ExportVQA(ExportAnnotationBase):
71
91
  async with aiohttp.ClientSession() as session:
72
92
  current_batch = []
73
93
  datarows = [] # Keep track of all datarows for annotation
94
+ written_files = set() # Basenames of images actually written to disk
74
95
  datarow_id_list = []
75
96
  total_datarows = sum(len(v) for v in sequence_frame_map.values())
76
97
  existing_files = set()
@@ -84,7 +105,7 @@ class ExportVQA(ExportAnnotationBase):
84
105
  async for datarow in datarow_generator_func(
85
106
  datarow_id_list[start_idx : start_idx + BATCH_SIZE]
86
107
  ):
87
- original_file_name = os.path.basename(datarow["original_url"])
108
+ original_file_name = Exporter.get_export_file_name(datarow)
88
109
  unique_file_name = Exporter.get_unique_filename(
89
110
  self, original_file_name, existing_files
90
111
  )
@@ -99,6 +120,7 @@ class ExportVQA(ExportAnnotationBase):
99
120
  )
100
121
  for result in results:
101
122
  if result:
123
+ written_files.add(os.path.basename(result[1]))
102
124
  yield result
103
125
  progress_bar.update(1)
104
126
  current_batch = []
@@ -109,17 +131,38 @@ class ExportVQA(ExportAnnotationBase):
109
131
  )
110
132
  for result in results:
111
133
  if result:
134
+ written_files.add(os.path.basename(result[1]))
112
135
  yield result
113
136
  progress_bar.update(1)
114
137
 
115
138
  annot_bytes = convert_to_bytes(
116
- convert_annotation(
139
+ self.convert(
117
140
  datarows=datarows,
118
141
  annotation_name=annotation_name,
119
142
  question_id_map=question_id_map,
143
+ written_files=written_files,
120
144
  )
121
145
  )
122
- yield annot_bytes, os.path.join(ANNOT_PATH, VLM_ANNOTATION_FILE)
146
+ yield annot_bytes, os.path.join(ANNOT_PATH, self.annotation_file)
147
+
148
+
149
+ @Exporter.register(format=ExportFormat.LLAVA)
150
+ class ExportLLaVA(ExportVQA):
151
+ annotation_file = LLAVA_ANNOTATION_FILE
152
+
153
+ @staticmethod
154
+ def convert(
155
+ datarows: list[dict],
156
+ annotation_name: str,
157
+ question_id_map: dict,
158
+ written_files: set,
159
+ ) -> str:
160
+ return convert_llava_annotation(
161
+ datarows=datarows,
162
+ annotation_name=annotation_name,
163
+ question_id_map=question_id_map,
164
+ written_files=written_files,
165
+ )
123
166
 
124
167
 
125
168
  def convert_annotation(
@@ -147,3 +190,63 @@ def convert_annotation(
147
190
  image_id_start += 1
148
191
 
149
192
  return json.dumps(annotation_list, ensure_ascii=False)
193
+
194
+
195
+ def convert_vlm_answer(answer) -> str:
196
+ """Cast a VLM answer back to LLaVA text: boolean -> Yes/No, option list -> its values, number -> str."""
197
+ if isinstance(answer, bool):
198
+ return "Yes" if answer else "No"
199
+ if isinstance(answer, list):
200
+ return ", ".join(str(value) for value in answer)
201
+ return str(answer)
202
+
203
+
204
+ def build_llava_turns(
205
+ datarow: dict, annotation_name: str, question_id_map: dict
206
+ ) -> list[dict]:
207
+ """Build LLaVA human/gpt turns from a VLM datarow; the first kept question carries <image>."""
208
+ vlm_items = datarow.get("vlm_items") or {}
209
+ turns = []
210
+ for conversation in vlm_items.get("data", {}).get("conversations", []):
211
+ question_id = conversation["question_id"]
212
+ answer = conversation["answer"].get(annotation_name)
213
+ if answer is None:
214
+ continue
215
+ question = question_id_map.get(question_id)
216
+ if question is None:
217
+ logger.warning(
218
+ "Skip question_id %s of datarow %s: not in project ontology",
219
+ question_id,
220
+ datarow["id"],
221
+ )
222
+ continue
223
+ if not turns:
224
+ question = f"{LLAVA_IMAGE_TOKEN}\n{question}"
225
+ meta = {"question_id": question_id}
226
+ turns.append({"from": "human", "value": question, "meta": meta})
227
+ turns.append({"from": "gpt", "value": convert_vlm_answer(answer), "meta": meta})
228
+ return turns
229
+
230
+
231
+ def convert_llava_annotation(
232
+ datarows: list[dict],
233
+ annotation_name: str,
234
+ question_id_map: dict,
235
+ written_files: set,
236
+ ) -> str:
237
+ """Convert VLM datarows to LLaVA 1.5 items; each turn keeps the VLM question_id in `meta`."""
238
+ llava_items = []
239
+ for datarow in datarows:
240
+ # The image failed to download, so the item has nothing to point at
241
+ if datarow.get("unique_file_name") not in written_files:
242
+ logger.warning("Skip datarow %s: image not exported", datarow["id"])
243
+ continue
244
+ # An unannotated frame is still exported, with empty conversations
245
+ llava_items.append(
246
+ {
247
+ "id": f"{len(llava_items):012d}",
248
+ "image": datarow["unique_file_name"],
249
+ "conversations": build_llava_turns(datarow, annotation_name, question_id_map),
250
+ }
251
+ )
252
+ return json.dumps(llava_items, ensure_ascii=False)
@@ -33,7 +33,7 @@ class ExportYolo(ExportAnnotationBase):
33
33
  results = []
34
34
  for datarow in batch_datarows:
35
35
  url = datarow["url"]
36
- file_name_without_format = datarow["unique_file_name"].split(".")[0]
36
+ file_name_without_format = os.path.splitext(datarow["unique_file_name"])[0]
37
37
  anno_name = f"{file_name_without_format}.txt"
38
38
  file_path = os.path.join("images", datarow["unique_file_name"])
39
39
  anno_path = os.path.join("labels", anno_name)
@@ -100,7 +100,7 @@ class ExportYolo(ExportAnnotationBase):
100
100
  async for datarow in datarow_generator_func(
101
101
  datarow_id_list[start_idx : start_idx + BATCH_SIZE]
102
102
  ):
103
- original_file_name = os.path.basename(datarow["original_url"])
103
+ original_file_name = Exporter.get_export_file_name(datarow)
104
104
  unique_file_name = Exporter.get_unique_filename(
105
105
  self, original_file_name, existing_files
106
106
  )
@@ -286,11 +286,16 @@ class Project(BaseModel):
286
286
  list[Literal["trained", "byom", "uploaded"]],
287
287
  ]
288
288
  ] = ["trained", "byom"],
289
+ include_disabled: bool = False,
289
290
  ) -> list:
290
291
  from ..client import DataverseClient
291
292
 
292
293
  model_list: list = DataverseClient.list_models(
293
- project_id=self.id, project=self, client_alias=self.client_alias, type=type
294
+ project_id=self.id,
295
+ project=self,
296
+ client_alias=self.client_alias,
297
+ type=type,
298
+ include_disabled=include_disabled,
294
299
  )
295
300
  return model_list
296
301
 
@@ -645,6 +650,9 @@ class MLModel(BaseModel):
645
650
  configuration: dict = {}
646
651
  # Typed as str, not MLModelStatus: an unknown status must not fail the whole parse.
647
652
  status: Optional[str] = None
653
+ # get_model's payload has no `enabled`, and the detail endpoint 404s for
654
+ # disabled models, so any model it returns is enabled.
655
+ enabled: bool = True
648
656
  # Typed as str, not ModelStructure: an unknown value must not fail the whole list_models parse.
649
657
  # Comparing against ModelStructure still works, since that enum subclasses str.
650
658
  model_structure: Optional[str] = None
@@ -41,6 +41,7 @@ class AnnotationFormat(str, Enum, metaclass=BaseEnumMeta):
41
41
  YOLO = "yolo"
42
42
  VIDEO = "video"
43
43
  VLM = "vlm"
44
+ LLAVA = "llava"
44
45
 
45
46
 
46
47
  class DatasetType(str, Enum, metaclass=BaseEnumMeta):
@@ -330,6 +330,7 @@ class CreateDatasetValidator:
330
330
 
331
331
  if cls.is_vlm_format_supported(config):
332
332
  supported_formats.append(AnnotationFormat.VLM)
333
+ supported_formats.append(AnnotationFormat.LLAVA)
333
334
 
334
335
  return supported_formats
335
336
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataverse-sdk
3
- Version: 2.8.0
3
+ Version: 2.9.0
4
4
  Summary: Dataverse SDK For Python
5
5
  Home-page:
6
6
  Author: LinkerVision
@@ -12,11 +12,9 @@ Requires-Dist: requests
12
12
  Requires-Dist: httpx>=0.23.0
13
13
  Requires-Dist: tqdm
14
14
  Requires-Dist: aiofiles
15
- Requires-Dist: visionai-data-format==2.0.0
15
+ Requires-Dist: visionai-data-format>=2.1.0
16
16
  Requires-Dist: aiohttp
17
17
  Requires-Dist: aiohttp_retry
18
- Requires-Dist: numpy<2,>=1.21
19
- Requires-Dist: opencv-contrib-python==4.8.1.78
20
18
  Dynamic: author
21
19
  Dynamic: classifier
22
20
  Dynamic: description
@@ -574,6 +572,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
574
572
 
575
573
  `MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
576
574
 
575
+ Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
576
+
577
577
  #### Filtering by Model Type
578
578
 
579
579
  You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
@@ -603,11 +603,12 @@ models = client.list_models(
603
603
 
604
604
  #### Input Arguments
605
605
 
606
- | Argument name | Type/Options | Default | Description |
607
- | ------------- | ----------------------------------------------------------------- | ------------------- | ------------------------ |
608
- | project_id | int | *-- | The project ID |
609
- | client_alias | str | None | The client alias |
610
- | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
606
+ | Argument name | Type/Options | Default | Description |
607
+ | ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
608
+ | project_id | int | *-- | The project ID |
609
+ | client_alias | str | None | The client alias |
610
+ | type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
611
+ | include_disabled | bool | False | Also list disabled models, e.g. failed ones |
611
612
 
612
613
  `*--`: required argument without default
613
614
 
@@ -0,0 +1,8 @@
1
+ pydantic==2.*
2
+ requests
3
+ httpx>=0.23.0
4
+ tqdm
5
+ aiofiles
6
+ visionai-data-format>=2.1.0
7
+ aiohttp
8
+ aiohttp_retry
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
2
2
 
3
3
  AUTHOR = "LinkerVision"
4
4
  PACKAGE_NAME = "dataverse-sdk"
5
- PACKAGE_VERSION = "2.8.0"
5
+ PACKAGE_VERSION = "2.9.0"
6
6
  DESC = "Dataverse SDK For Python"
7
7
  with open("README.md", encoding="utf-8") as fh:
8
8
  long_description = fh.read()
@@ -21,11 +21,9 @@ setup(
21
21
  "httpx>=0.23.0",
22
22
  "tqdm",
23
23
  "aiofiles",
24
- "visionai-data-format==2.0.0",
24
+ "visionai-data-format>=2.1.0",
25
25
  "aiohttp",
26
26
  "aiohttp_retry",
27
- "numpy>=1.21,<2",
28
- "opencv-contrib-python==4.8.1.78",
29
27
  ],
30
28
  long_description=long_description,
31
29
  long_description_content_type="text/markdown",
@@ -114,6 +114,13 @@ def make_parser():
114
114
  type=str,
115
115
  help="the dataset type (annotated_data / raw_data)",
116
116
  )
117
+ parser.add_argument(
118
+ "--format",
119
+ default="vlm",
120
+ choices=["vlm", "llava"],
121
+ type=str,
122
+ help="the annotation format for annotated_data imports (ignored for raw_data)",
123
+ )
117
124
  parser.add_argument(
118
125
  "--sequential",
119
126
  default=False,
@@ -147,7 +154,9 @@ if __name__ == "__main__":
147
154
  if not dataset_name:
148
155
  dataset_name = time.strftime("%Y-%m-%d_%H-%M-%S")
149
156
  if args.dataset_type == DatasetType.ANNOTATED_DATA:
150
- annotation_format = AnnotationFormat.VLM
157
+ annotation_format = (
158
+ AnnotationFormat.LLAVA if args.format == "llava" else AnnotationFormat.VLM
159
+ )
151
160
  else:
152
161
  annotation_format = AnnotationFormat.IMAGE
153
162
  import_vqa_dataset_from_local(
@@ -1,10 +0,0 @@
1
- pydantic==2.*
2
- requests
3
- httpx>=0.23.0
4
- tqdm
5
- aiofiles
6
- visionai-data-format==2.0.0
7
- aiohttp
8
- aiohttp_retry
9
- numpy<2,>=1.21
10
- opencv-contrib-python==4.8.1.78
File without changes
File without changes