dataverse-sdk 2.8.0__tar.gz → 2.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/PKG-INFO +10 -9
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/README.md +8 -5
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/backend.py +19 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/client.py +8 -2
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/coco.py +2 -4
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/constant.py +3 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/exporter.py +59 -5
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/vqa.py +106 -3
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/yolo.py +2 -2
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/client.py +9 -1
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/common.py +1 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/utils/utils.py +1 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/PKG-INFO +10 -9
- dataverse_sdk-2.9.0/dataverse_sdk.egg-info/requires.txt +8 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/setup.py +2 -4
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/import_vqa_dataset.py +10 -1
- dataverse_sdk-2.8.0/dataverse_sdk.egg-info/requires.txt +0 -10
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/MANIFEST.in +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/apis/third_party.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/connections.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/constants.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/exceptions/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/exceptions/client.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/base.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/utils.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/export/visionai.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/api.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/schemas/format.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk/utils/__init__.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/setup.cfg +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/export_dataslice.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/export_dataslice_large.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/import_dataset_from_local.py +0 -0
- {dataverse_sdk-2.8.0 → dataverse_sdk-2.9.0}/tools/upload_videos_create_session.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.9.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -12,11 +12,9 @@ Requires-Dist: requests
|
|
|
12
12
|
Requires-Dist: httpx>=0.23.0
|
|
13
13
|
Requires-Dist: tqdm
|
|
14
14
|
Requires-Dist: aiofiles
|
|
15
|
-
Requires-Dist: visionai-data-format
|
|
15
|
+
Requires-Dist: visionai-data-format>=2.1.0
|
|
16
16
|
Requires-Dist: aiohttp
|
|
17
17
|
Requires-Dist: aiohttp_retry
|
|
18
|
-
Requires-Dist: numpy<2,>=1.21
|
|
19
|
-
Requires-Dist: opencv-contrib-python==4.8.1.78
|
|
20
18
|
Dynamic: author
|
|
21
19
|
Dynamic: classifier
|
|
22
20
|
Dynamic: description
|
|
@@ -574,6 +572,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
|
|
|
574
572
|
|
|
575
573
|
`MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
|
|
576
574
|
|
|
575
|
+
Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
|
|
576
|
+
|
|
577
577
|
#### Filtering by Model Type
|
|
578
578
|
|
|
579
579
|
You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
|
|
@@ -603,11 +603,12 @@ models = client.list_models(
|
|
|
603
603
|
|
|
604
604
|
#### Input Arguments
|
|
605
605
|
|
|
606
|
-
| Argument name
|
|
607
|
-
|
|
|
608
|
-
| project_id
|
|
609
|
-
| client_alias
|
|
610
|
-
| type
|
|
606
|
+
| Argument name | Type/Options | Default | Description |
|
|
607
|
+
| ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
|
|
608
|
+
| project_id | int | *-- | The project ID |
|
|
609
|
+
| client_alias | str | None | The client alias |
|
|
610
|
+
| type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
|
|
611
|
+
| include_disabled | bool | False | Also list disabled models, e.g. failed ones |
|
|
611
612
|
|
|
612
613
|
`*--`: required argument without default
|
|
613
614
|
|
|
@@ -547,6 +547,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
|
|
|
547
547
|
|
|
548
548
|
`MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
|
|
549
549
|
|
|
550
|
+
Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
|
|
551
|
+
|
|
550
552
|
#### Filtering by Model Type
|
|
551
553
|
|
|
552
554
|
You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
|
|
@@ -576,11 +578,12 @@ models = client.list_models(
|
|
|
576
578
|
|
|
577
579
|
#### Input Arguments
|
|
578
580
|
|
|
579
|
-
| Argument name
|
|
580
|
-
|
|
|
581
|
-
| project_id
|
|
582
|
-
| client_alias
|
|
583
|
-
| type
|
|
581
|
+
| Argument name | Type/Options | Default | Description |
|
|
582
|
+
| ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
|
|
583
|
+
| project_id | int | *-- | The project ID |
|
|
584
|
+
| client_alias | str | None | The client alias |
|
|
585
|
+
| type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
|
|
586
|
+
| include_disabled | bool | False | Also list disabled models, e.g. failed ones |
|
|
584
587
|
|
|
585
588
|
`*--`: required argument without default
|
|
586
589
|
|
|
@@ -331,10 +331,13 @@ class BackendAPI:
|
|
|
331
331
|
self,
|
|
332
332
|
project_id: int,
|
|
333
333
|
type: str = "trained,byom",
|
|
334
|
+
include_disabled: bool = False,
|
|
334
335
|
**kwargs,
|
|
335
336
|
) -> list:
|
|
336
337
|
kwargs["project"] = project_id
|
|
337
338
|
kwargs["type__in"] = type
|
|
339
|
+
if include_disabled:
|
|
340
|
+
kwargs["include_disabled"] = "true"
|
|
338
341
|
resp = self.send_request(
|
|
339
342
|
url=f"{self.host}/api/ml_models/?{urlencode(kwargs)}",
|
|
340
343
|
method="get",
|
|
@@ -721,6 +724,22 @@ class AsyncBackendAPI:
|
|
|
721
724
|
data=payload_data,
|
|
722
725
|
)
|
|
723
726
|
|
|
727
|
+
async def get_dataset(self, dataset_id: int) -> dict:
|
|
728
|
+
try:
|
|
729
|
+
resp = await self.client.get(
|
|
730
|
+
url=f"{self.host}/api/datasets/{dataset_id}/",
|
|
731
|
+
headers=self.headers,
|
|
732
|
+
timeout=30,
|
|
733
|
+
)
|
|
734
|
+
resp.raise_for_status()
|
|
735
|
+
return resp.json()
|
|
736
|
+
except httpx.HTTPStatusError as e:
|
|
737
|
+
print(f"HTTP error: {e.response.status_code} - {e.response.text}")
|
|
738
|
+
return None
|
|
739
|
+
except Exception as e:
|
|
740
|
+
print(f"Request failed: {str(e)}")
|
|
741
|
+
return None
|
|
742
|
+
|
|
724
743
|
async def get_project(self, project_id: str) -> dict:
|
|
725
744
|
try:
|
|
726
745
|
resp = await self.client.get(
|
|
@@ -1317,6 +1317,7 @@ of this project OR has been added before"
|
|
|
1317
1317
|
list[Literal["trained", "byom", "uploaded"]],
|
|
1318
1318
|
]
|
|
1319
1319
|
] = ["trained", "byom"],
|
|
1320
|
+
include_disabled: bool = False,
|
|
1320
1321
|
) -> list[MLModel]:
|
|
1321
1322
|
"""Get the model list by project id
|
|
1322
1323
|
|
|
@@ -1329,6 +1330,9 @@ of this project OR has been added before"
|
|
|
1329
1330
|
project: Optional["Project"]
|
|
1330
1331
|
project basemodel, by default None
|
|
1331
1332
|
type : Optional[Union[Literal["trained", "byom", "uploaded"], list[Literal["trained", "byom", "uploaded"]]]], by default ["trained", "byom"]
|
|
1333
|
+
include_disabled : bool, by default False
|
|
1334
|
+
also list disabled models (`MLModel.enabled` is False), e.g. a custom
|
|
1335
|
+
model whose upload failed. Older servers ignore it and list enabled models only.
|
|
1332
1336
|
|
|
1333
1337
|
Returns
|
|
1334
1338
|
-------
|
|
@@ -1346,7 +1350,9 @@ of this project OR has been added before"
|
|
|
1346
1350
|
try:
|
|
1347
1351
|
if isinstance(type, list):
|
|
1348
1352
|
type = ",".join(type)
|
|
1349
|
-
model_list: list = api.list_ml_models(
|
|
1353
|
+
model_list: list = api.list_ml_models(
|
|
1354
|
+
project_id=project_id, type=type, include_disabled=include_disabled
|
|
1355
|
+
)
|
|
1350
1356
|
except DataverseExceptionBase:
|
|
1351
1357
|
logging.exception("Got api error from Dataverse")
|
|
1352
1358
|
raise
|
|
@@ -2481,7 +2487,7 @@ of this project OR has been added before"
|
|
|
2481
2487
|
return ["images/", "annotations/labels.json"]
|
|
2482
2488
|
elif annotation_format == AnnotationFormat.YOLO:
|
|
2483
2489
|
return ["images/", "labels/", "classes.txt"]
|
|
2484
|
-
elif annotation_format
|
|
2490
|
+
elif annotation_format in (AnnotationFormat.VLM, AnnotationFormat.LLAVA):
|
|
2485
2491
|
return ["images/", "annotations/"]
|
|
2486
2492
|
elif annotation_format in (
|
|
2487
2493
|
AnnotationFormat.VISION_AI,
|
|
@@ -93,7 +93,7 @@ class ExportCoco(ExportAnnotationBase):
|
|
|
93
93
|
async for datarow in datarow_generator_func(
|
|
94
94
|
datarow_id_list[start_idx : start_idx + BATCH_SIZE]
|
|
95
95
|
):
|
|
96
|
-
original_file_name =
|
|
96
|
+
original_file_name = Exporter.get_export_file_name(datarow)
|
|
97
97
|
unique_file_name = Exporter.get_unique_filename(
|
|
98
98
|
self, original_file_name, existing_files
|
|
99
99
|
)
|
|
@@ -184,9 +184,7 @@ def convert_annotation(
|
|
|
184
184
|
image_update[0].file_name = datarow["unique_file_name"]
|
|
185
185
|
image_update[
|
|
186
186
|
0
|
|
187
|
-
].coco_url = (
|
|
188
|
-
f"{image_update[0].coco_url.rsplit('/',1)[0]}/{datarow['unique_file_name']}"
|
|
189
|
-
)
|
|
187
|
+
].coco_url = f"{image_update[0].coco_url.rsplit('/', 1)[0]}/{datarow['unique_file_name']}"
|
|
190
188
|
images.extend(image_update)
|
|
191
189
|
annotations.extend(anno_update)
|
|
192
190
|
# generate category objects
|
|
@@ -6,6 +6,8 @@ GROUND_TRUTH_ANNOTATION_NAME = "ground_truths"
|
|
|
6
6
|
MAX_CONCURRENT_DOWNLOADS = 100
|
|
7
7
|
BATCH_SIZE = 50
|
|
8
8
|
|
|
9
|
+
VIDEO_SUPPORTED_FORMAT = {".mp4", ".avi", ".mov", ".mpeg", ".flv", ".wmv"}
|
|
10
|
+
|
|
9
11
|
|
|
10
12
|
class ExportFormat(str, Enum):
|
|
11
13
|
COCO = "coco"
|
|
@@ -13,3 +15,4 @@ class ExportFormat(str, Enum):
|
|
|
13
15
|
TRAINING_FORMAT = "training_format"
|
|
14
16
|
YOLO = "yolo"
|
|
15
17
|
VLM = "vlm"
|
|
18
|
+
LLAVA = "llava"
|
|
@@ -2,14 +2,19 @@ import asyncio
|
|
|
2
2
|
import os
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
from collections.abc import AsyncGenerator, Generator
|
|
5
|
-
from typing import Callable
|
|
5
|
+
from typing import Callable, Optional
|
|
6
|
+
from urllib.parse import urlparse
|
|
6
7
|
|
|
7
8
|
import aiofiles
|
|
8
9
|
|
|
9
10
|
from ..apis.backend import AsyncBackendAPI
|
|
10
|
-
from ..schemas.format import AnnotationFormat
|
|
11
11
|
from .base import ExportAnnotationBase
|
|
12
|
-
from .constant import
|
|
12
|
+
from .constant import (
|
|
13
|
+
BATCH_SIZE,
|
|
14
|
+
MAX_CONCURRENT_DOWNLOADS,
|
|
15
|
+
VIDEO_SUPPORTED_FORMAT,
|
|
16
|
+
ExportFormat,
|
|
17
|
+
)
|
|
13
18
|
|
|
14
19
|
NONE_SEQUENCE_DATAROW_ID = -1
|
|
15
20
|
|
|
@@ -62,7 +67,7 @@ class Exporter:
|
|
|
62
67
|
|
|
63
68
|
project_id = dataslice["project"]["id"]
|
|
64
69
|
question_id_map = {}
|
|
65
|
-
if export_format
|
|
70
|
+
if export_format in (ExportFormat.VLM, ExportFormat.LLAVA):
|
|
66
71
|
project: dict = await self.curation_api.get_project(project_id=project_id)
|
|
67
72
|
for question in project["ontology"]["classes"]:
|
|
68
73
|
question_id_map[question["rank"]] = question["extended_class"][
|
|
@@ -130,6 +135,7 @@ class Exporter:
|
|
|
130
135
|
def register(cls, format: ExportFormat):
|
|
131
136
|
def _wrap(export_annotation: ExportAnnotationBase):
|
|
132
137
|
cls.export_annotation_m[format] = export_annotation() # register instance
|
|
138
|
+
return export_annotation
|
|
133
139
|
|
|
134
140
|
return _wrap
|
|
135
141
|
|
|
@@ -137,13 +143,16 @@ class Exporter:
|
|
|
137
143
|
async def _gen(
|
|
138
144
|
curation_api: AsyncBackendAPI,
|
|
139
145
|
) -> Callable[[list[int]], AsyncGenerator[dict]]:
|
|
146
|
+
# dataset_id -> frame rate used to split its videos, shared across calls
|
|
147
|
+
frame_rate_map: dict[int, Optional[float]] = {}
|
|
148
|
+
|
|
140
149
|
async def f(datarow_id_list: list[int]):
|
|
141
150
|
datarow_id_set: set[int] = set()
|
|
142
151
|
for id_chunks in chunks(datarow_id_list, 1000):
|
|
143
152
|
gen: AsyncGenerator = curation_api.get_datarows(
|
|
144
153
|
id_set_list=id_chunks,
|
|
145
154
|
batch_size=BATCH_SIZE,
|
|
146
|
-
fields="id,items,vlm_items,url,frame_id,image_width,image_height,sensor_name,original_url,type",
|
|
155
|
+
fields="id,items,vlm_items,url,frame_id,image_width,image_height,sensor_name,original_url,type,dataset",
|
|
147
156
|
)
|
|
148
157
|
async for batched_datarow in gen:
|
|
149
158
|
for datarow in batched_datarow:
|
|
@@ -152,10 +161,55 @@ class Exporter:
|
|
|
152
161
|
# drop duplicate datarows
|
|
153
162
|
continue
|
|
154
163
|
datarow_id_set.add(dr_id)
|
|
164
|
+
await Exporter._attach_video_frame_rate(
|
|
165
|
+
curation_api=curation_api,
|
|
166
|
+
datarow=datarow,
|
|
167
|
+
frame_rate_map=frame_rate_map,
|
|
168
|
+
)
|
|
155
169
|
yield datarow
|
|
156
170
|
|
|
157
171
|
return f
|
|
158
172
|
|
|
173
|
+
@staticmethod
|
|
174
|
+
def is_video_frame(datarow: dict) -> bool:
|
|
175
|
+
original_url: str = datarow.get("original_url") or ""
|
|
176
|
+
return os.path.splitext(original_url)[-1].lower() in VIDEO_SUPPORTED_FORMAT
|
|
177
|
+
|
|
178
|
+
@staticmethod
|
|
179
|
+
async def _attach_video_frame_rate(
|
|
180
|
+
curation_api: AsyncBackendAPI,
|
|
181
|
+
datarow: dict,
|
|
182
|
+
frame_rate_map: dict[int, Optional[float]],
|
|
183
|
+
) -> None:
|
|
184
|
+
"""Attach the frame rate used to split the source video to the video frame datarow as `frame_rate`"""
|
|
185
|
+
if not Exporter.is_video_frame(datarow):
|
|
186
|
+
return
|
|
187
|
+
dataset_id: Optional[int] = datarow.get("dataset")
|
|
188
|
+
if dataset_id is not None and dataset_id not in frame_rate_map:
|
|
189
|
+
dataset: Optional[dict] = await curation_api.get_dataset(
|
|
190
|
+
dataset_id=dataset_id
|
|
191
|
+
)
|
|
192
|
+
frame_rate_map[dataset_id] = (dataset or {}).get("frame_rate")
|
|
193
|
+
datarow["frame_rate"] = frame_rate_map.get(dataset_id)
|
|
194
|
+
|
|
195
|
+
@staticmethod
|
|
196
|
+
def get_export_file_name(datarow: dict) -> str:
|
|
197
|
+
"""Return the exported image file name of the datarow
|
|
198
|
+
|
|
199
|
+
Frames split from a video share the video as `original_url`, so they are named
|
|
200
|
+
`[VideoName]_frame[frame_id]_[frame_rate]fps.[ext]` instead, e.g. video1_frame000000000001_10fps.jpg
|
|
201
|
+
"""
|
|
202
|
+
original_file_name: str = os.path.basename(datarow["original_url"])
|
|
203
|
+
if not Exporter.is_video_frame(datarow) or datarow.get("frame_id") is None:
|
|
204
|
+
return original_file_name
|
|
205
|
+
|
|
206
|
+
video_name: str = os.path.splitext(original_file_name)[0]
|
|
207
|
+
extension: str = os.path.splitext(urlparse(datarow["url"]).path)[-1]
|
|
208
|
+
file_name = f"{video_name}_frame{int(datarow['frame_id']):012d}"
|
|
209
|
+
if frame_rate := datarow.get("frame_rate"):
|
|
210
|
+
file_name += f"_{float(frame_rate):g}fps"
|
|
211
|
+
return f"{file_name}{extension}"
|
|
212
|
+
|
|
159
213
|
def get_unique_filename(self, original_file_name, existing_files):
|
|
160
214
|
base_name, extension = os.path.splitext(original_file_name)
|
|
161
215
|
counter = 1
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import json
|
|
3
|
+
import logging
|
|
3
4
|
import os
|
|
4
5
|
from collections.abc import AsyncGenerator
|
|
5
6
|
from typing import Callable
|
|
@@ -13,11 +14,30 @@ from .constant import BATCH_SIZE, MAX_CONCURRENT_DOWNLOADS, ExportFormat
|
|
|
13
14
|
from .exporter import Exporter
|
|
14
15
|
from .utils import convert_to_bytes
|
|
15
16
|
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
16
19
|
VLM_ANNOTATION_FILE = "vlm_annotation.json"
|
|
20
|
+
LLAVA_ANNOTATION_FILE = "llava_annotation.json"
|
|
21
|
+
LLAVA_IMAGE_TOKEN = "<image>"
|
|
17
22
|
|
|
18
23
|
|
|
19
24
|
@Exporter.register(format=ExportFormat.VLM)
|
|
20
25
|
class ExportVQA(ExportAnnotationBase):
|
|
26
|
+
annotation_file = VLM_ANNOTATION_FILE
|
|
27
|
+
|
|
28
|
+
@staticmethod
|
|
29
|
+
def convert(
|
|
30
|
+
datarows: list[dict],
|
|
31
|
+
annotation_name: str,
|
|
32
|
+
question_id_map: dict,
|
|
33
|
+
written_files: set,
|
|
34
|
+
) -> str:
|
|
35
|
+
return convert_annotation(
|
|
36
|
+
datarows=datarows,
|
|
37
|
+
annotation_name=annotation_name,
|
|
38
|
+
question_id_map=question_id_map,
|
|
39
|
+
)
|
|
40
|
+
|
|
21
41
|
async def download_batch(
|
|
22
42
|
self,
|
|
23
43
|
session: aiohttp.ClientSession,
|
|
@@ -71,6 +91,7 @@ class ExportVQA(ExportAnnotationBase):
|
|
|
71
91
|
async with aiohttp.ClientSession() as session:
|
|
72
92
|
current_batch = []
|
|
73
93
|
datarows = [] # Keep track of all datarows for annotation
|
|
94
|
+
written_files = set() # Basenames of images actually written to disk
|
|
74
95
|
datarow_id_list = []
|
|
75
96
|
total_datarows = sum(len(v) for v in sequence_frame_map.values())
|
|
76
97
|
existing_files = set()
|
|
@@ -84,7 +105,7 @@ class ExportVQA(ExportAnnotationBase):
|
|
|
84
105
|
async for datarow in datarow_generator_func(
|
|
85
106
|
datarow_id_list[start_idx : start_idx + BATCH_SIZE]
|
|
86
107
|
):
|
|
87
|
-
original_file_name =
|
|
108
|
+
original_file_name = Exporter.get_export_file_name(datarow)
|
|
88
109
|
unique_file_name = Exporter.get_unique_filename(
|
|
89
110
|
self, original_file_name, existing_files
|
|
90
111
|
)
|
|
@@ -99,6 +120,7 @@ class ExportVQA(ExportAnnotationBase):
|
|
|
99
120
|
)
|
|
100
121
|
for result in results:
|
|
101
122
|
if result:
|
|
123
|
+
written_files.add(os.path.basename(result[1]))
|
|
102
124
|
yield result
|
|
103
125
|
progress_bar.update(1)
|
|
104
126
|
current_batch = []
|
|
@@ -109,17 +131,38 @@ class ExportVQA(ExportAnnotationBase):
|
|
|
109
131
|
)
|
|
110
132
|
for result in results:
|
|
111
133
|
if result:
|
|
134
|
+
written_files.add(os.path.basename(result[1]))
|
|
112
135
|
yield result
|
|
113
136
|
progress_bar.update(1)
|
|
114
137
|
|
|
115
138
|
annot_bytes = convert_to_bytes(
|
|
116
|
-
|
|
139
|
+
self.convert(
|
|
117
140
|
datarows=datarows,
|
|
118
141
|
annotation_name=annotation_name,
|
|
119
142
|
question_id_map=question_id_map,
|
|
143
|
+
written_files=written_files,
|
|
120
144
|
)
|
|
121
145
|
)
|
|
122
|
-
yield annot_bytes, os.path.join(ANNOT_PATH,
|
|
146
|
+
yield annot_bytes, os.path.join(ANNOT_PATH, self.annotation_file)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@Exporter.register(format=ExportFormat.LLAVA)
|
|
150
|
+
class ExportLLaVA(ExportVQA):
|
|
151
|
+
annotation_file = LLAVA_ANNOTATION_FILE
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def convert(
|
|
155
|
+
datarows: list[dict],
|
|
156
|
+
annotation_name: str,
|
|
157
|
+
question_id_map: dict,
|
|
158
|
+
written_files: set,
|
|
159
|
+
) -> str:
|
|
160
|
+
return convert_llava_annotation(
|
|
161
|
+
datarows=datarows,
|
|
162
|
+
annotation_name=annotation_name,
|
|
163
|
+
question_id_map=question_id_map,
|
|
164
|
+
written_files=written_files,
|
|
165
|
+
)
|
|
123
166
|
|
|
124
167
|
|
|
125
168
|
def convert_annotation(
|
|
@@ -147,3 +190,63 @@ def convert_annotation(
|
|
|
147
190
|
image_id_start += 1
|
|
148
191
|
|
|
149
192
|
return json.dumps(annotation_list, ensure_ascii=False)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def convert_vlm_answer(answer) -> str:
|
|
196
|
+
"""Cast a VLM answer back to LLaVA text: boolean -> Yes/No, option list -> its values, number -> str."""
|
|
197
|
+
if isinstance(answer, bool):
|
|
198
|
+
return "Yes" if answer else "No"
|
|
199
|
+
if isinstance(answer, list):
|
|
200
|
+
return ", ".join(str(value) for value in answer)
|
|
201
|
+
return str(answer)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def build_llava_turns(
|
|
205
|
+
datarow: dict, annotation_name: str, question_id_map: dict
|
|
206
|
+
) -> list[dict]:
|
|
207
|
+
"""Build LLaVA human/gpt turns from a VLM datarow; the first kept question carries <image>."""
|
|
208
|
+
vlm_items = datarow.get("vlm_items") or {}
|
|
209
|
+
turns = []
|
|
210
|
+
for conversation in vlm_items.get("data", {}).get("conversations", []):
|
|
211
|
+
question_id = conversation["question_id"]
|
|
212
|
+
answer = conversation["answer"].get(annotation_name)
|
|
213
|
+
if answer is None:
|
|
214
|
+
continue
|
|
215
|
+
question = question_id_map.get(question_id)
|
|
216
|
+
if question is None:
|
|
217
|
+
logger.warning(
|
|
218
|
+
"Skip question_id %s of datarow %s: not in project ontology",
|
|
219
|
+
question_id,
|
|
220
|
+
datarow["id"],
|
|
221
|
+
)
|
|
222
|
+
continue
|
|
223
|
+
if not turns:
|
|
224
|
+
question = f"{LLAVA_IMAGE_TOKEN}\n{question}"
|
|
225
|
+
meta = {"question_id": question_id}
|
|
226
|
+
turns.append({"from": "human", "value": question, "meta": meta})
|
|
227
|
+
turns.append({"from": "gpt", "value": convert_vlm_answer(answer), "meta": meta})
|
|
228
|
+
return turns
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def convert_llava_annotation(
|
|
232
|
+
datarows: list[dict],
|
|
233
|
+
annotation_name: str,
|
|
234
|
+
question_id_map: dict,
|
|
235
|
+
written_files: set,
|
|
236
|
+
) -> str:
|
|
237
|
+
"""Convert VLM datarows to LLaVA 1.5 items; each turn keeps the VLM question_id in `meta`."""
|
|
238
|
+
llava_items = []
|
|
239
|
+
for datarow in datarows:
|
|
240
|
+
# The image failed to download, so the item has nothing to point at
|
|
241
|
+
if datarow.get("unique_file_name") not in written_files:
|
|
242
|
+
logger.warning("Skip datarow %s: image not exported", datarow["id"])
|
|
243
|
+
continue
|
|
244
|
+
# An unannotated frame is still exported, with empty conversations
|
|
245
|
+
llava_items.append(
|
|
246
|
+
{
|
|
247
|
+
"id": f"{len(llava_items):012d}",
|
|
248
|
+
"image": datarow["unique_file_name"],
|
|
249
|
+
"conversations": build_llava_turns(datarow, annotation_name, question_id_map),
|
|
250
|
+
}
|
|
251
|
+
)
|
|
252
|
+
return json.dumps(llava_items, ensure_ascii=False)
|
|
@@ -33,7 +33,7 @@ class ExportYolo(ExportAnnotationBase):
|
|
|
33
33
|
results = []
|
|
34
34
|
for datarow in batch_datarows:
|
|
35
35
|
url = datarow["url"]
|
|
36
|
-
file_name_without_format = datarow["unique_file_name"]
|
|
36
|
+
file_name_without_format = os.path.splitext(datarow["unique_file_name"])[0]
|
|
37
37
|
anno_name = f"{file_name_without_format}.txt"
|
|
38
38
|
file_path = os.path.join("images", datarow["unique_file_name"])
|
|
39
39
|
anno_path = os.path.join("labels", anno_name)
|
|
@@ -100,7 +100,7 @@ class ExportYolo(ExportAnnotationBase):
|
|
|
100
100
|
async for datarow in datarow_generator_func(
|
|
101
101
|
datarow_id_list[start_idx : start_idx + BATCH_SIZE]
|
|
102
102
|
):
|
|
103
|
-
original_file_name =
|
|
103
|
+
original_file_name = Exporter.get_export_file_name(datarow)
|
|
104
104
|
unique_file_name = Exporter.get_unique_filename(
|
|
105
105
|
self, original_file_name, existing_files
|
|
106
106
|
)
|
|
@@ -286,11 +286,16 @@ class Project(BaseModel):
|
|
|
286
286
|
list[Literal["trained", "byom", "uploaded"]],
|
|
287
287
|
]
|
|
288
288
|
] = ["trained", "byom"],
|
|
289
|
+
include_disabled: bool = False,
|
|
289
290
|
) -> list:
|
|
290
291
|
from ..client import DataverseClient
|
|
291
292
|
|
|
292
293
|
model_list: list = DataverseClient.list_models(
|
|
293
|
-
project_id=self.id,
|
|
294
|
+
project_id=self.id,
|
|
295
|
+
project=self,
|
|
296
|
+
client_alias=self.client_alias,
|
|
297
|
+
type=type,
|
|
298
|
+
include_disabled=include_disabled,
|
|
294
299
|
)
|
|
295
300
|
return model_list
|
|
296
301
|
|
|
@@ -645,6 +650,9 @@ class MLModel(BaseModel):
|
|
|
645
650
|
configuration: dict = {}
|
|
646
651
|
# Typed as str, not MLModelStatus: an unknown status must not fail the whole parse.
|
|
647
652
|
status: Optional[str] = None
|
|
653
|
+
# get_model's payload has no `enabled`, and the detail endpoint 404s for
|
|
654
|
+
# disabled models, so any model it returns is enabled.
|
|
655
|
+
enabled: bool = True
|
|
648
656
|
# Typed as str, not ModelStructure: an unknown value must not fail the whole list_models parse.
|
|
649
657
|
# Comparing against ModelStructure still works, since that enum subclasses str.
|
|
650
658
|
model_structure: Optional[str] = None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.9.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -12,11 +12,9 @@ Requires-Dist: requests
|
|
|
12
12
|
Requires-Dist: httpx>=0.23.0
|
|
13
13
|
Requires-Dist: tqdm
|
|
14
14
|
Requires-Dist: aiofiles
|
|
15
|
-
Requires-Dist: visionai-data-format
|
|
15
|
+
Requires-Dist: visionai-data-format>=2.1.0
|
|
16
16
|
Requires-Dist: aiohttp
|
|
17
17
|
Requires-Dist: aiohttp_retry
|
|
18
|
-
Requires-Dist: numpy<2,>=1.21
|
|
19
|
-
Requires-Dist: opencv-contrib-python==4.8.1.78
|
|
20
18
|
Dynamic: author
|
|
21
19
|
Dynamic: classifier
|
|
22
20
|
Dynamic: description
|
|
@@ -574,6 +572,8 @@ ready = [model for model in models if model.status == MLModelStatus.READY]
|
|
|
574
572
|
|
|
575
573
|
`MLModelStatus`: `PROCESSING`, `READY`, `DELETING`.
|
|
576
574
|
|
|
575
|
+
Disabled models (`model.enabled` is `False`, e.g. a custom model whose upload failed) are listed only with `include_disabled=True`.
|
|
576
|
+
|
|
577
577
|
#### Filtering by Model Type
|
|
578
578
|
|
|
579
579
|
You can filter models by type using strings or lists of strings. The SDK supports multiple model types:
|
|
@@ -603,11 +603,12 @@ models = client.list_models(
|
|
|
603
603
|
|
|
604
604
|
#### Input Arguments
|
|
605
605
|
|
|
606
|
-
| Argument name
|
|
607
|
-
|
|
|
608
|
-
| project_id
|
|
609
|
-
| client_alias
|
|
610
|
-
| type
|
|
606
|
+
| Argument name | Type/Options | Default | Description |
|
|
607
|
+
| ---------------- | ----------------------------------------------------------------- | ------------------- | -------------------------------------------- |
|
|
608
|
+
| project_id | int | *-- | The project ID |
|
|
609
|
+
| client_alias | str | None | The client alias |
|
|
610
|
+
| type | "trained", "byom", "uploaded", list["trained", "byom", "uploaded] | ["trained", "byom"] | Model types to filter by |
|
|
611
|
+
| include_disabled | bool | False | Also list disabled models, e.g. failed ones |
|
|
611
612
|
|
|
612
613
|
`*--`: required argument without default
|
|
613
614
|
|
|
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
|
|
|
2
2
|
|
|
3
3
|
AUTHOR = "LinkerVision"
|
|
4
4
|
PACKAGE_NAME = "dataverse-sdk"
|
|
5
|
-
PACKAGE_VERSION = "2.
|
|
5
|
+
PACKAGE_VERSION = "2.9.0"
|
|
6
6
|
DESC = "Dataverse SDK For Python"
|
|
7
7
|
with open("README.md", encoding="utf-8") as fh:
|
|
8
8
|
long_description = fh.read()
|
|
@@ -21,11 +21,9 @@ setup(
|
|
|
21
21
|
"httpx>=0.23.0",
|
|
22
22
|
"tqdm",
|
|
23
23
|
"aiofiles",
|
|
24
|
-
"visionai-data-format
|
|
24
|
+
"visionai-data-format>=2.1.0",
|
|
25
25
|
"aiohttp",
|
|
26
26
|
"aiohttp_retry",
|
|
27
|
-
"numpy>=1.21,<2",
|
|
28
|
-
"opencv-contrib-python==4.8.1.78",
|
|
29
27
|
],
|
|
30
28
|
long_description=long_description,
|
|
31
29
|
long_description_content_type="text/markdown",
|
|
@@ -114,6 +114,13 @@ def make_parser():
|
|
|
114
114
|
type=str,
|
|
115
115
|
help="the dataset type (annotated_data / raw_data)",
|
|
116
116
|
)
|
|
117
|
+
parser.add_argument(
|
|
118
|
+
"--format",
|
|
119
|
+
default="vlm",
|
|
120
|
+
choices=["vlm", "llava"],
|
|
121
|
+
type=str,
|
|
122
|
+
help="the annotation format for annotated_data imports (ignored for raw_data)",
|
|
123
|
+
)
|
|
117
124
|
parser.add_argument(
|
|
118
125
|
"--sequential",
|
|
119
126
|
default=False,
|
|
@@ -147,7 +154,9 @@ if __name__ == "__main__":
|
|
|
147
154
|
if not dataset_name:
|
|
148
155
|
dataset_name = time.strftime("%Y-%m-%d_%H-%M-%S")
|
|
149
156
|
if args.dataset_type == DatasetType.ANNOTATED_DATA:
|
|
150
|
-
annotation_format =
|
|
157
|
+
annotation_format = (
|
|
158
|
+
AnnotationFormat.LLAVA if args.format == "llava" else AnnotationFormat.VLM
|
|
159
|
+
)
|
|
151
160
|
else:
|
|
152
161
|
annotation_format = AnnotationFormat.IMAGE
|
|
153
162
|
import_vqa_dataset_from_local(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|