dataverse-sdk 2.6.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/PKG-INFO +56 -1
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/README.md +55 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/__init__.py +4 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/apis/backend.py +6 -7
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/client.py +38 -14
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/schemas/api.py +9 -3
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/schemas/client.py +10 -6
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/schemas/common.py +35 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/utils/utils.py +27 -1
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk.egg-info/PKG-INFO +56 -1
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/setup.py +1 -1
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/apis/__init__.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/apis/third_party.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/connections.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/constants.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/exceptions/__init__.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/exceptions/client.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/__init__.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/base.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/coco.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/constant.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/exporter.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/utils.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/visionai.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/vqa.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/export/yolo.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/schemas/__init__.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/schemas/format.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk/utils/__init__.py +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk.egg-info/SOURCES.txt +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk.egg-info/dependency_links.txt +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk.egg-info/requires.txt +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/dataverse_sdk.egg-info/top_level.txt +0 -0
- {dataverse_sdk-2.6.0 → dataverse_sdk-2.7.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -532,6 +532,61 @@ status, label_file_path = model_record.get_label_file(save_path="./labels.txt",
|
|
|
532
532
|
status, onnx_model_path = model_record.get_onnx_model_file(save_path="./model.onnx", timeout=6000)
|
|
533
533
|
```
|
|
534
534
|
|
|
535
|
+
### Get Convert Model File
|
|
536
|
+
|
|
537
|
+
The `get_convert_model_file` method will download one of the stored artifacts of a convert record, selected by the `file_type` argument.
|
|
538
|
+
|
|
539
|
+
```Python
|
|
540
|
+
from dataverse_sdk import ConvertModelFileType
|
|
541
|
+
|
|
542
|
+
# Method 1: Using client
|
|
543
|
+
status, save_path = client.get_convert_model_file(
|
|
544
|
+
convert_record_id=5,
|
|
545
|
+
file_type=ConvertModelFileType.TRITON,
|
|
546
|
+
client_alias=client.alias,
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
# Method 2: Using convert record object
|
|
550
|
+
model_record = model.get_convert_record(convert_record_id=5)
|
|
551
|
+
status, save_path = model_record.get_convert_model_file(file_type=ConvertModelFileType.TRITON)
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
#### Available File Types
|
|
555
|
+
|
|
556
|
+
| Enum Value | String Value | File downloaded | Available when | Default `save_path` |
|
|
557
|
+
| ----------------------------------- | ----------------- | ---------------------------------------------------------------------- | ---------------------- | ------------------------- |
|
|
558
|
+
| `ConvertModelFileType.TRITON` | `"triton"` | Triton bundle (`.zip`) | always | `./triton.zip` |
|
|
559
|
+
| `ConvertModelFileType.MODEL` | `"model"` | Main artifact: `model.onnx` (format=onnx) or `trt.engine` (format=trt) | always | `./converted_model` |
|
|
560
|
+
| `ConvertModelFileType.RAW_ONNX` | `"raw_onnx"` | Intermediate onnx (fp16: `trt.onnx`; D-FINE int8: fp32 `model.onnx`) | format=trt | `./raw.onnx` |
|
|
561
|
+
| `ConvertModelFileType.CALIB_CACHE` | `"calib_cache"` | `trt_int8_calib.cache` (int8 scale table) | D-FINE + int8 only | `./trt_int8_calib.cache` |
|
|
562
|
+
|
|
563
|
+
When `save_path` is omitted, the filename is resolved in this order:
|
|
564
|
+
|
|
565
|
+
1. The name the server supplies in the response's `Content-Disposition` header — this is the artifact's real stored filename (e.g. `ptq.engine`, `triton_model.zip`), so it already reflects the convert format and precision.
|
|
566
|
+
2. The **Default `save_path`** in the table above, used only when the server sends no usable filename.
|
|
567
|
+
|
|
568
|
+
Pass an explicit `save_path` whenever you need a predictable location — it always takes precedence over the server's name.
|
|
569
|
+
|
|
570
|
+
#### Input Arguments
|
|
571
|
+
|
|
572
|
+
| Argument name | Type/Options | Default | Description |
|
|
573
|
+
| ----------------- | -------------------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------- |
|
|
574
|
+
| convert_record_id | int | \*-- | The convert record to download from |
|
|
575
|
+
| file_type | ConvertModelFileType, "triton", "model", "raw_onnx", "calib_cache" | ConvertModelFileType.TRITON | Which artifact to download |
|
|
576
|
+
| save_path | str | None | Local path to write the file; when omitted the server's filename is used (see above) |
|
|
577
|
+
| timeout | int | 3000 | Maximum timeout of the request |
|
|
578
|
+
| permission | str | "" | Sets the `X-Request-Source` header; pass the caller's permission source if required |
|
|
579
|
+
| client | DataverseClient | None | Client instance; if omitted, `client_alias` must be given |
|
|
580
|
+
| client_alias | str | None | Registered client alias; required when `client` is None |
|
|
581
|
+
|
|
582
|
+
`*--`: required argument without default
|
|
583
|
+
|
|
584
|
+
#### Return
|
|
585
|
+
|
|
586
|
+
`tuple[bool, str]` — `(status, save_path)`. `status` is `False` if the download failed, and `save_path` is the path that was written to.
|
|
587
|
+
|
|
588
|
+
<br>
|
|
589
|
+
|
|
535
590
|
|
|
536
591
|
### Create VQA Project
|
|
537
592
|
|
|
@@ -505,6 +505,61 @@ status, label_file_path = model_record.get_label_file(save_path="./labels.txt",
|
|
|
505
505
|
status, onnx_model_path = model_record.get_onnx_model_file(save_path="./model.onnx", timeout=6000)
|
|
506
506
|
```
|
|
507
507
|
|
|
508
|
+
### Get Convert Model File
|
|
509
|
+
|
|
510
|
+
The `get_convert_model_file` method will download one of the stored artifacts of a convert record, selected by the `file_type` argument.
|
|
511
|
+
|
|
512
|
+
```Python
|
|
513
|
+
from dataverse_sdk import ConvertModelFileType
|
|
514
|
+
|
|
515
|
+
# Method 1: Using client
|
|
516
|
+
status, save_path = client.get_convert_model_file(
|
|
517
|
+
convert_record_id=5,
|
|
518
|
+
file_type=ConvertModelFileType.TRITON,
|
|
519
|
+
client_alias=client.alias,
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
# Method 2: Using convert record object
|
|
523
|
+
model_record = model.get_convert_record(convert_record_id=5)
|
|
524
|
+
status, save_path = model_record.get_convert_model_file(file_type=ConvertModelFileType.TRITON)
|
|
525
|
+
```
|
|
526
|
+
|
|
527
|
+
#### Available File Types
|
|
528
|
+
|
|
529
|
+
| Enum Value | String Value | File downloaded | Available when | Default `save_path` |
|
|
530
|
+
| ----------------------------------- | ----------------- | ---------------------------------------------------------------------- | ---------------------- | ------------------------- |
|
|
531
|
+
| `ConvertModelFileType.TRITON` | `"triton"` | Triton bundle (`.zip`) | always | `./triton.zip` |
|
|
532
|
+
| `ConvertModelFileType.MODEL` | `"model"` | Main artifact: `model.onnx` (format=onnx) or `trt.engine` (format=trt) | always | `./converted_model` |
|
|
533
|
+
| `ConvertModelFileType.RAW_ONNX` | `"raw_onnx"` | Intermediate onnx (fp16: `trt.onnx`; D-FINE int8: fp32 `model.onnx`) | format=trt | `./raw.onnx` |
|
|
534
|
+
| `ConvertModelFileType.CALIB_CACHE` | `"calib_cache"` | `trt_int8_calib.cache` (int8 scale table) | D-FINE + int8 only | `./trt_int8_calib.cache` |
|
|
535
|
+
|
|
536
|
+
When `save_path` is omitted, the filename is resolved in this order:
|
|
537
|
+
|
|
538
|
+
1. The name the server supplies in the response's `Content-Disposition` header — this is the artifact's real stored filename (e.g. `ptq.engine`, `triton_model.zip`), so it already reflects the convert format and precision.
|
|
539
|
+
2. The **Default `save_path`** in the table above, used only when the server sends no usable filename.
|
|
540
|
+
|
|
541
|
+
Pass an explicit `save_path` whenever you need a predictable location — it always takes precedence over the server's name.
|
|
542
|
+
|
|
543
|
+
#### Input Arguments
|
|
544
|
+
|
|
545
|
+
| Argument name | Type/Options | Default | Description |
|
|
546
|
+
| ----------------- | -------------------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------- |
|
|
547
|
+
| convert_record_id | int | \*-- | The convert record to download from |
|
|
548
|
+
| file_type | ConvertModelFileType, "triton", "model", "raw_onnx", "calib_cache" | ConvertModelFileType.TRITON | Which artifact to download |
|
|
549
|
+
| save_path | str | None | Local path to write the file; when omitted the server's filename is used (see above) |
|
|
550
|
+
| timeout | int | 3000 | Maximum timeout of the request |
|
|
551
|
+
| permission | str | "" | Sets the `X-Request-Source` header; pass the caller's permission source if required |
|
|
552
|
+
| client | DataverseClient | None | Client instance; if omitted, `client_alias` must be given |
|
|
553
|
+
| client_alias | str | None | Registered client alias; required when `client` is None |
|
|
554
|
+
|
|
555
|
+
`*--`: required argument without default
|
|
556
|
+
|
|
557
|
+
#### Return
|
|
558
|
+
|
|
559
|
+
`tuple[bool, str]` — `(status, save_path)`. `status` is `False` if the download failed, and `save_path` is the path that was written to.
|
|
560
|
+
|
|
561
|
+
<br>
|
|
562
|
+
|
|
508
563
|
|
|
509
564
|
### Create VQA Project
|
|
510
565
|
|
|
@@ -18,9 +18,11 @@ from .schemas.client import (
|
|
|
18
18
|
from .schemas.common import (
|
|
19
19
|
AnnotationFormat,
|
|
20
20
|
AttributeType,
|
|
21
|
+
ConvertModelFileType,
|
|
21
22
|
DatasetStatus,
|
|
22
23
|
DatasetType,
|
|
23
24
|
DataSource,
|
|
25
|
+
ModelStructure,
|
|
24
26
|
OntologyImageType,
|
|
25
27
|
OntologyPcdType,
|
|
26
28
|
SensorType,
|
|
@@ -47,6 +49,8 @@ __all__ = [
|
|
|
47
49
|
"DatasetStatus",
|
|
48
50
|
"DataSource",
|
|
49
51
|
"QuestionClass",
|
|
52
|
+
"ConvertModelFileType",
|
|
53
|
+
"ModelStructure",
|
|
50
54
|
]
|
|
51
55
|
|
|
52
56
|
|
|
@@ -12,6 +12,7 @@ from requests import sessions
|
|
|
12
12
|
from requests.adapters import HTTPAdapter, Retry
|
|
13
13
|
|
|
14
14
|
from ..exceptions.client import DataverseExceptionBase
|
|
15
|
+
from ..schemas.common import ConvertModelFileType
|
|
15
16
|
from ..utils.utils import chunks
|
|
16
17
|
|
|
17
18
|
logger = logging.getLogger(__name__)
|
|
@@ -375,15 +376,12 @@ class BackendAPI:
|
|
|
375
376
|
self,
|
|
376
377
|
convert_record_id: int,
|
|
377
378
|
timeout: int = 3000,
|
|
378
|
-
|
|
379
|
-
raw_onnx: bool = False,
|
|
379
|
+
file_type: ConvertModelFileType = ConvertModelFileType.TRITON,
|
|
380
380
|
permission: str = "",
|
|
381
381
|
**kwargs,
|
|
382
382
|
) -> requests.models.Response:
|
|
383
383
|
headers = self.headers.copy()
|
|
384
|
-
kwargs["
|
|
385
|
-
if raw_onnx:
|
|
386
|
-
kwargs["raw_onnx"] = raw_onnx
|
|
384
|
+
kwargs["file_type"] = ConvertModelFileType(file_type).value
|
|
387
385
|
if permission:
|
|
388
386
|
headers["X-Request-Source"] = permission
|
|
389
387
|
resp = self.send_request(
|
|
@@ -428,10 +426,11 @@ class BackendAPI:
|
|
|
428
426
|
"render_pcd": render_pcd,
|
|
429
427
|
"description": description if description else "",
|
|
430
428
|
"annotations": annotations if annotations else [],
|
|
431
|
-
"
|
|
429
|
+
"storage_url": "", # FIXME: storage_url field is still required by production API.
|
|
430
|
+
"container_name": "", # FIXME: container_name field is still required by production API.
|
|
431
|
+
"sas_token": "", # FIXME: sas_token field is still required by production API.
|
|
432
432
|
}
|
|
433
433
|
|
|
434
|
-
# Only send storage location fields when provided
|
|
435
434
|
for key, value in (
|
|
436
435
|
("storage_url", storage_url),
|
|
437
436
|
("container_name", container_name),
|
|
@@ -49,10 +49,19 @@ from .schemas.client import (
|
|
|
49
49
|
Sensor,
|
|
50
50
|
UpdateQuestionClass,
|
|
51
51
|
)
|
|
52
|
-
from .schemas.common import
|
|
52
|
+
from .schemas.common import (
|
|
53
|
+
CONVERT_MODEL_FILE_DEFAULT_SAVE_PATHS,
|
|
54
|
+
AnnotationFormat,
|
|
55
|
+
ConvertModelFileType,
|
|
56
|
+
DatasetType,
|
|
57
|
+
ModelStructure,
|
|
58
|
+
OntologyImageType,
|
|
59
|
+
SensorType,
|
|
60
|
+
)
|
|
53
61
|
from .utils.utils import (
|
|
54
62
|
download_file_from_response,
|
|
55
63
|
download_file_from_url,
|
|
64
|
+
filename_from_response,
|
|
56
65
|
get_filepaths,
|
|
57
66
|
)
|
|
58
67
|
|
|
@@ -1398,9 +1407,8 @@ of this project OR has been added before"
|
|
|
1398
1407
|
@staticmethod
|
|
1399
1408
|
def get_convert_model_file(
|
|
1400
1409
|
convert_record_id: int,
|
|
1401
|
-
save_path: str =
|
|
1402
|
-
|
|
1403
|
-
raw_onnx: bool = False,
|
|
1410
|
+
save_path: Optional[str] = None,
|
|
1411
|
+
file_type: ConvertModelFileType = ConvertModelFileType.TRITON,
|
|
1404
1412
|
timeout: int = 3000,
|
|
1405
1413
|
permission: str = "",
|
|
1406
1414
|
client: Optional["DataverseClient"] = None,
|
|
@@ -1411,10 +1419,13 @@ of this project OR has been added before"
|
|
|
1411
1419
|
Parameters
|
|
1412
1420
|
----------
|
|
1413
1421
|
convert_record_id : int
|
|
1414
|
-
save_path : str, optional
|
|
1415
|
-
local path for saving the model file, by default
|
|
1416
|
-
|
|
1417
|
-
|
|
1422
|
+
save_path : Optional[str], optional
|
|
1423
|
+
local path for saving the model file, by default None, which uses the
|
|
1424
|
+
filename the server reports in Content-Disposition, falling back to
|
|
1425
|
+
CONVERT_MODEL_FILE_DEFAULT_SAVE_PATHS when the server sends none
|
|
1426
|
+
file_type: ConvertModelFileType, default=ConvertModelFileType.TRITON
|
|
1427
|
+
which stored artifact to download: the triton bundle, the main model
|
|
1428
|
+
artifact, the intermediate onnx, or the int8 calibration cache
|
|
1418
1429
|
timeout : int, optional
|
|
1419
1430
|
maximum timeout of the request, by default 3000
|
|
1420
1431
|
client : Optional['DataverseClient'], optional
|
|
@@ -1427,25 +1438,38 @@ of this project OR has been added before"
|
|
|
1427
1438
|
the first item means whether the download success or not
|
|
1428
1439
|
the second item shows the save_path
|
|
1429
1440
|
"""
|
|
1441
|
+
try:
|
|
1442
|
+
file_type = ConvertModelFileType(file_type)
|
|
1443
|
+
except ValueError as e:
|
|
1444
|
+
raise APIValidationError(
|
|
1445
|
+
f"Something wrong when getting the convert model file: {e}"
|
|
1446
|
+
) from e
|
|
1447
|
+
|
|
1448
|
+
target_save_path = save_path or CONVERT_MODEL_FILE_DEFAULT_SAVE_PATHS[file_type]
|
|
1430
1449
|
api, client_alias = DataverseClient._get_api_client(
|
|
1431
1450
|
client=client, client_alias=client_alias
|
|
1432
1451
|
)
|
|
1433
1452
|
try:
|
|
1434
1453
|
resp = api.get_convert_model_file(
|
|
1435
1454
|
convert_record_id=convert_record_id,
|
|
1436
|
-
|
|
1437
|
-
raw_onnx=raw_onnx,
|
|
1455
|
+
file_type=file_type,
|
|
1438
1456
|
timeout=timeout,
|
|
1439
1457
|
permission=permission,
|
|
1440
1458
|
)
|
|
1441
|
-
|
|
1442
|
-
|
|
1459
|
+
# Only consulted when the caller named no path: an explicit save_path
|
|
1460
|
+
# always wins over the server's suggestion.
|
|
1461
|
+
if not save_path:
|
|
1462
|
+
server_filename = filename_from_response(resp)
|
|
1463
|
+
if server_filename:
|
|
1464
|
+
target_save_path = f"./{server_filename}"
|
|
1465
|
+
download_file_from_response(response=resp, save_path=target_save_path)
|
|
1466
|
+
return True, target_save_path
|
|
1443
1467
|
except DataverseExceptionBase:
|
|
1444
1468
|
logging.exception("Got api error from Dataverse")
|
|
1445
1469
|
raise
|
|
1446
1470
|
except Exception:
|
|
1447
1471
|
logging.exception("Failed to get the convert model file")
|
|
1448
|
-
return False,
|
|
1472
|
+
return False, target_save_path
|
|
1449
1473
|
|
|
1450
1474
|
def get_dataset(self, dataset_id: int, client_alias: Optional[str] = None):
|
|
1451
1475
|
"""Get dataset detail and status by id
|
|
@@ -1757,7 +1781,7 @@ of this project OR has been added before"
|
|
|
1757
1781
|
input_classes: list[str],
|
|
1758
1782
|
resolution_width: int,
|
|
1759
1783
|
resolution_height: int,
|
|
1760
|
-
model_structure:
|
|
1784
|
+
model_structure: ModelStructure,
|
|
1761
1785
|
weight_url: str,
|
|
1762
1786
|
client: Optional["DataverseClient"] = None,
|
|
1763
1787
|
client_alias: Optional[str] = None,
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import re
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import Optional, Union
|
|
3
3
|
|
|
4
4
|
from pydantic import BaseModel, ConfigDict, field_validator
|
|
5
5
|
|
|
6
6
|
from .client import AnnotationFormat, DatasetType, DataSource, QuestionClass
|
|
7
|
-
from .common import
|
|
7
|
+
from .common import (
|
|
8
|
+
AttributeType,
|
|
9
|
+
ModelStructure,
|
|
10
|
+
OntologyImageType,
|
|
11
|
+
OntologyPcdType,
|
|
12
|
+
SensorType,
|
|
13
|
+
)
|
|
8
14
|
|
|
9
15
|
|
|
10
16
|
class AttributeOptionAPISchema(BaseModel):
|
|
@@ -139,5 +145,5 @@ class CreateCustomModelAPISchema(BaseModel):
|
|
|
139
145
|
input_classes: list[str]
|
|
140
146
|
resolution_width: int
|
|
141
147
|
resolution_height: int
|
|
142
|
-
model_structure:
|
|
148
|
+
model_structure: ModelStructure
|
|
143
149
|
weight_url: str
|
|
@@ -7,9 +7,11 @@ from pydantic_core.core_schema import ValidationInfo
|
|
|
7
7
|
from .common import (
|
|
8
8
|
AnnotationFormat,
|
|
9
9
|
AttributeType,
|
|
10
|
+
ConvertModelFileType,
|
|
10
11
|
DatasetStatus,
|
|
11
12
|
DatasetType,
|
|
12
13
|
DataSource,
|
|
14
|
+
ModelStructure,
|
|
13
15
|
OntologyImageType,
|
|
14
16
|
OntologyPcdType,
|
|
15
17
|
ProjectCreateDatasetConfig,
|
|
@@ -531,9 +533,8 @@ class ConvertRecord(BaseModel):
|
|
|
531
533
|
|
|
532
534
|
def get_convert_model_file(
|
|
533
535
|
self,
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
save_path: str = "./triton.zip",
|
|
536
|
+
file_type: ConvertModelFileType = ConvertModelFileType.TRITON,
|
|
537
|
+
save_path: Optional[str] = None,
|
|
537
538
|
timeout: int = 3000,
|
|
538
539
|
permission: str = "",
|
|
539
540
|
) -> tuple[bool, str]:
|
|
@@ -542,8 +543,7 @@ class ConvertRecord(BaseModel):
|
|
|
542
543
|
return DataverseClient.get_convert_model_file(
|
|
543
544
|
convert_record_id=self.id,
|
|
544
545
|
save_path=save_path,
|
|
545
|
-
|
|
546
|
-
raw_onnx=raw_onnx,
|
|
546
|
+
file_type=file_type,
|
|
547
547
|
timeout=timeout,
|
|
548
548
|
permission=permission,
|
|
549
549
|
client_alias=self.client_alias,
|
|
@@ -560,6 +560,9 @@ class MLModel(BaseModel):
|
|
|
560
560
|
operation_records: list = []
|
|
561
561
|
triton_model_name: str
|
|
562
562
|
description: Optional[str] = None
|
|
563
|
+
# Typed as str, not ModelStructure: an unknown value must not fail the whole list_models parse.
|
|
564
|
+
# Comparing against ModelStructure still works, since that enum subclasses str.
|
|
565
|
+
model_structure: Optional[str] = None
|
|
563
566
|
|
|
564
567
|
model_config = ConfigDict(extra="allow")
|
|
565
568
|
|
|
@@ -594,6 +597,7 @@ class MLModel(BaseModel):
|
|
|
594
597
|
operation_records=model_data.get("model_records", []),
|
|
595
598
|
updated_at=model_data["updated_at"],
|
|
596
599
|
triton_model_name=model_data["triton_model_name"],
|
|
600
|
+
model_structure=model_data.get("model_structure"),
|
|
597
601
|
client_alias=client_alias,
|
|
598
602
|
)
|
|
599
603
|
|
|
@@ -611,7 +615,7 @@ class MLModel(BaseModel):
|
|
|
611
615
|
input_classes: list[str],
|
|
612
616
|
resolution_width: int,
|
|
613
617
|
resolution_height: int,
|
|
614
|
-
model_structure:
|
|
618
|
+
model_structure: ModelStructure,
|
|
615
619
|
weight_url: str,
|
|
616
620
|
permission: str = "",
|
|
617
621
|
):
|
|
@@ -64,6 +64,41 @@ class DataSource(str, Enum, metaclass=BaseEnumMeta):
|
|
|
64
64
|
PRE_IMPORT = "pre_import"
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
class ConvertModelFileType(str, Enum, metaclass=BaseEnumMeta):
|
|
68
|
+
"""Which stored artifact of a convert record to download."""
|
|
69
|
+
|
|
70
|
+
TRITON = "triton"
|
|
71
|
+
MODEL = "model"
|
|
72
|
+
RAW_ONNX = "raw_onnx"
|
|
73
|
+
CALIB_CACHE = "calib_cache"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# Last-resort local filename per artifact, used only when the caller gives no
|
|
77
|
+
# save_path AND the response carries no usable Content-Disposition filename. The
|
|
78
|
+
# server's own name is authoritative: it reflects the convert format and precision
|
|
79
|
+
# (e.g. "ptq.engine"), which a static table cannot. Treat these as a floor to fall
|
|
80
|
+
# back on, not as a description of what the file contains.
|
|
81
|
+
CONVERT_MODEL_FILE_DEFAULT_SAVE_PATHS: dict[ConvertModelFileType, str] = {
|
|
82
|
+
ConvertModelFileType.TRITON: "./triton.zip",
|
|
83
|
+
ConvertModelFileType.MODEL: "./converted_model",
|
|
84
|
+
ConvertModelFileType.RAW_ONNX: "./raw.onnx",
|
|
85
|
+
ConvertModelFileType.CALIB_CACHE: "./trt_int8_calib.cache",
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class ModelStructure(str, Enum, metaclass=BaseEnumMeta):
|
|
90
|
+
"""Architecture to build uploaded custom model weights into."""
|
|
91
|
+
|
|
92
|
+
YOLOV9_C = "yolov9-c"
|
|
93
|
+
YOLOV9_E = "yolov9-e"
|
|
94
|
+
YOLOV9_S = "yolov9-s"
|
|
95
|
+
DFINE_N = "dfine-n"
|
|
96
|
+
DFINE_S = "dfine-s"
|
|
97
|
+
DFINE_M = "dfine-m"
|
|
98
|
+
DFINE_L = "dfine-l"
|
|
99
|
+
DFINE_X = "dfine-x"
|
|
100
|
+
|
|
101
|
+
|
|
67
102
|
@dataclass
|
|
68
103
|
class SensorCounts:
|
|
69
104
|
camera: int = 0
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from
|
|
1
|
+
from email.message import Message
|
|
2
|
+
from pathlib import Path, PurePosixPath
|
|
2
3
|
from typing import Optional, Set
|
|
3
4
|
|
|
4
5
|
import requests
|
|
@@ -47,6 +48,31 @@ def download_file_from_response(response: requests.models.Response, save_path: s
|
|
|
47
48
|
file.flush()
|
|
48
49
|
|
|
49
50
|
|
|
51
|
+
def filename_from_response(response: requests.models.Response) -> Optional[str]:
|
|
52
|
+
"""Filename the server asked us to use, taken from the Content-Disposition header.
|
|
53
|
+
|
|
54
|
+
Returns None when the header is absent, carries no filename, or the filename is
|
|
55
|
+
unusable once reduced to a bare basename. Callers are expected to fall back to
|
|
56
|
+
their own default in that case.
|
|
57
|
+
"""
|
|
58
|
+
header = response.headers.get("Content-Disposition")
|
|
59
|
+
if not header:
|
|
60
|
+
return None
|
|
61
|
+
# Message.get_filename handles the quoting and the RFC 2231 `filename*` form,
|
|
62
|
+
# and reads `inline` dispositions as well as `attachment` ones.
|
|
63
|
+
message = Message()
|
|
64
|
+
message["Content-Disposition"] = header
|
|
65
|
+
filename = message.get_filename()
|
|
66
|
+
if not filename:
|
|
67
|
+
return None
|
|
68
|
+
# The server controls this string and we are about to open() it: keep only the
|
|
69
|
+
# basename so it can never escape the target directory or turn absolute.
|
|
70
|
+
filename = PurePosixPath(filename.replace("\\", "/")).name
|
|
71
|
+
if filename in {"", ".", ".."}:
|
|
72
|
+
return None
|
|
73
|
+
return filename
|
|
74
|
+
|
|
75
|
+
|
|
50
76
|
def download_file_from_url(url: str, save_path: str):
|
|
51
77
|
"""Downloads a file from a URL and saves it to the specified path.
|
|
52
78
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataverse-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Dataverse SDK For Python
|
|
5
5
|
Home-page:
|
|
6
6
|
Author: LinkerVision
|
|
@@ -532,6 +532,61 @@ status, label_file_path = model_record.get_label_file(save_path="./labels.txt",
|
|
|
532
532
|
status, onnx_model_path = model_record.get_onnx_model_file(save_path="./model.onnx", timeout=6000)
|
|
533
533
|
```
|
|
534
534
|
|
|
535
|
+
### Get Convert Model File
|
|
536
|
+
|
|
537
|
+
The `get_convert_model_file` method will download one of the stored artifacts of a convert record, selected by the `file_type` argument.
|
|
538
|
+
|
|
539
|
+
```Python
|
|
540
|
+
from dataverse_sdk import ConvertModelFileType
|
|
541
|
+
|
|
542
|
+
# Method 1: Using client
|
|
543
|
+
status, save_path = client.get_convert_model_file(
|
|
544
|
+
convert_record_id=5,
|
|
545
|
+
file_type=ConvertModelFileType.TRITON,
|
|
546
|
+
client_alias=client.alias,
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
# Method 2: Using convert record object
|
|
550
|
+
model_record = model.get_convert_record(convert_record_id=5)
|
|
551
|
+
status, save_path = model_record.get_convert_model_file(file_type=ConvertModelFileType.TRITON)
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
#### Available File Types
|
|
555
|
+
|
|
556
|
+
| Enum Value | String Value | File downloaded | Available when | Default `save_path` |
|
|
557
|
+
| ----------------------------------- | ----------------- | ---------------------------------------------------------------------- | ---------------------- | ------------------------- |
|
|
558
|
+
| `ConvertModelFileType.TRITON` | `"triton"` | Triton bundle (`.zip`) | always | `./triton.zip` |
|
|
559
|
+
| `ConvertModelFileType.MODEL` | `"model"` | Main artifact: `model.onnx` (format=onnx) or `trt.engine` (format=trt) | always | `./converted_model` |
|
|
560
|
+
| `ConvertModelFileType.RAW_ONNX` | `"raw_onnx"` | Intermediate onnx (fp16: `trt.onnx`; D-FINE int8: fp32 `model.onnx`) | format=trt | `./raw.onnx` |
|
|
561
|
+
| `ConvertModelFileType.CALIB_CACHE` | `"calib_cache"` | `trt_int8_calib.cache` (int8 scale table) | D-FINE + int8 only | `./trt_int8_calib.cache` |
|
|
562
|
+
|
|
563
|
+
When `save_path` is omitted, the filename is resolved in this order:
|
|
564
|
+
|
|
565
|
+
1. The name the server supplies in the response's `Content-Disposition` header — this is the artifact's real stored filename (e.g. `ptq.engine`, `triton_model.zip`), so it already reflects the convert format and precision.
|
|
566
|
+
2. The **Default `save_path`** in the table above, used only when the server sends no usable filename.
|
|
567
|
+
|
|
568
|
+
Pass an explicit `save_path` whenever you need a predictable location — it always takes precedence over the server's name.
|
|
569
|
+
|
|
570
|
+
#### Input Arguments
|
|
571
|
+
|
|
572
|
+
| Argument name | Type/Options | Default | Description |
|
|
573
|
+
| ----------------- | -------------------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------- |
|
|
574
|
+
| convert_record_id | int | \*-- | The convert record to download from |
|
|
575
|
+
| file_type | ConvertModelFileType, "triton", "model", "raw_onnx", "calib_cache" | ConvertModelFileType.TRITON | Which artifact to download |
|
|
576
|
+
| save_path | str | None | Local path to write the file; when omitted the server's filename is used (see above) |
|
|
577
|
+
| timeout | int | 3000 | Maximum timeout of the request |
|
|
578
|
+
| permission | str | "" | Sets the `X-Request-Source` header; pass the caller's permission source if required |
|
|
579
|
+
| client | DataverseClient | None | Client instance; if omitted, `client_alias` must be given |
|
|
580
|
+
| client_alias | str | None | Registered client alias; required when `client` is None |
|
|
581
|
+
|
|
582
|
+
`*--`: required argument without default
|
|
583
|
+
|
|
584
|
+
#### Return
|
|
585
|
+
|
|
586
|
+
`tuple[bool, str]` — `(status, save_path)`. `status` is `False` if the download failed, and `save_path` is the path that was written to.
|
|
587
|
+
|
|
588
|
+
<br>
|
|
589
|
+
|
|
535
590
|
|
|
536
591
|
### Create VQA Project
|
|
537
592
|
|
|
@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
|
|
|
2
2
|
|
|
3
3
|
AUTHOR = "LinkerVision"
|
|
4
4
|
PACKAGE_NAME = "dataverse-sdk"
|
|
5
|
-
PACKAGE_VERSION = "2.
|
|
5
|
+
PACKAGE_VERSION = "2.7.0"
|
|
6
6
|
DESC = "Dataverse SDK For Python"
|
|
7
7
|
with open("README.md", encoding="utf-8") as fh:
|
|
8
8
|
long_description = fh.read()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|