datamint 2.3.2__tar.gz → 2.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datamint might be problematic. Click here for more details.
- {datamint-2.3.2 → datamint-2.3.4}/PKG-INFO +2 -1
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/base_api.py +72 -13
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/client.py +6 -3
- datamint-2.3.4/datamint/api/dto/__init__.py +18 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/annotations_api.py +47 -7
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/projects_api.py +44 -37
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/resources_api.py +83 -30
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/entity_base_api.py +11 -43
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/dto/annotation_dto.py +6 -2
- {datamint-2.3.2 → datamint-2.3.4}/datamint/client_cmd_tools/datamint_upload.py +3 -1
- {datamint-2.3.2 → datamint-2.3.4}/datamint/configs.py +6 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/dataset/base_dataset.py +3 -3
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/__init__.py +4 -2
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/annotation.py +74 -4
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/base_entity.py +47 -6
- datamint-2.3.4/datamint/entities/cache_manager.py +302 -0
- datamint-2.3.4/datamint/entities/datasetinfo.py +129 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/project.py +47 -6
- datamint-2.3.4/datamint/entities/resource.py +257 -0
- datamint-2.3.4/datamint/types.py +17 -0
- {datamint-2.3.2 → datamint-2.3.4}/pyproject.toml +2 -1
- datamint-2.3.2/datamint/api/dto/__init__.py +0 -10
- datamint-2.3.2/datamint/entities/datasetinfo.py +0 -22
- datamint-2.3.2/datamint/entities/resource.py +0 -130
- {datamint-2.3.2 → datamint-2.3.4}/README.md +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/annotationsets_api.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/channels_api.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/datasetsinfo_api.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/models_api.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/api/endpoints/users_api.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/annotation_api_handler.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/api_handler.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/base_api_handler.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/dto/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/exp_api_handler.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/apihandler/root_api_handler.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/client_cmd_tools/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/client_cmd_tools/datamint_config.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/dataset/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/dataset/annotation.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/dataset/dataset.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/channel.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/entities/user.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/examples/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/examples/example_projects.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/exceptions.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/experiment/__init__.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/experiment/_patcher.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/experiment/experiment.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/logging.yaml +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/utils/logging_utils.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/utils/torchmetrics.py +0 -0
- {datamint-2.3.2 → datamint-2.3.4}/datamint/utils/visualization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datamint
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.4
|
|
4
4
|
Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -15,6 +15,7 @@ Requires-Dist: Deprecated (>=1.2.0)
|
|
|
15
15
|
Requires-Dist: aiohttp (>=3.0.0,<4.0.0)
|
|
16
16
|
Requires-Dist: aioresponses (>=0.7.8,<0.8.0) ; extra == "dev"
|
|
17
17
|
Requires-Dist: albumentations (>=2.0.0)
|
|
18
|
+
Requires-Dist: backports-strenum ; python_version < "3.11"
|
|
18
19
|
Requires-Dist: datamintapi (==0.0.*)
|
|
19
20
|
Requires-Dist: httpx
|
|
20
21
|
Requires-Dist: humanize (>=4.0.0,<5.0.0)
|
|
@@ -1,27 +1,28 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import Any, Generator, AsyncGenerator, Sequence
|
|
2
|
+
from typing import Any, Generator, AsyncGenerator, Sequence, TYPE_CHECKING
|
|
3
3
|
import httpx
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
6
|
+
from datamint.types import ImagingData
|
|
6
7
|
import aiohttp
|
|
7
8
|
import json
|
|
8
|
-
import pydicom.dataset
|
|
9
9
|
from PIL import Image
|
|
10
10
|
import cv2
|
|
11
11
|
import nibabel as nib
|
|
12
|
-
from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
|
|
13
12
|
from io import BytesIO
|
|
14
13
|
import gzip
|
|
15
14
|
import contextlib
|
|
16
15
|
import asyncio
|
|
17
|
-
from medimgkit.format_detection import GZIP_MIME_TYPES
|
|
16
|
+
from medimgkit.format_detection import GZIP_MIME_TYPES, DEFAULT_MIME_TYPE, guess_typez, guess_extension
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from datamint.api.client import Api
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
21
23
|
# Generic type for entities
|
|
22
24
|
_PAGE_LIMIT = 5000
|
|
23
25
|
|
|
24
|
-
|
|
25
26
|
@dataclass
|
|
26
27
|
class ApiConfig:
|
|
27
28
|
"""Configuration for API client.
|
|
@@ -37,6 +38,15 @@ class ApiConfig:
|
|
|
37
38
|
timeout: float = 30.0
|
|
38
39
|
max_retries: int = 3
|
|
39
40
|
|
|
41
|
+
@property
|
|
42
|
+
def web_app_url(self) -> str:
|
|
43
|
+
"""Get the base URL for the web application."""
|
|
44
|
+
if self.server_url.startswith('http://localhost:3001'):
|
|
45
|
+
return 'http://localhost:3000'
|
|
46
|
+
if self.server_url.startswith('https://stagingapi.datamint.io'):
|
|
47
|
+
return 'https://staging.datamint.io'
|
|
48
|
+
return 'https://app.datamint.io'
|
|
49
|
+
|
|
40
50
|
|
|
41
51
|
class BaseApi:
|
|
42
52
|
"""Base class for all API endpoint handlers."""
|
|
@@ -53,6 +63,7 @@ class BaseApi:
|
|
|
53
63
|
self.config = config
|
|
54
64
|
self.client = client or self._create_client()
|
|
55
65
|
self.semaphore = asyncio.Semaphore(20)
|
|
66
|
+
self._api_instance: 'Api | None' = None # Injected by Api class
|
|
56
67
|
|
|
57
68
|
def _create_client(self) -> httpx.Client:
|
|
58
69
|
"""Create and configure HTTP client with authentication and timeouts."""
|
|
@@ -110,11 +121,12 @@ class BaseApi:
|
|
|
110
121
|
url = endpoint.lstrip('/') # Remove leading slash for httpx
|
|
111
122
|
|
|
112
123
|
try:
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
124
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
125
|
+
curl_command = self._generate_curl_command({"method": method,
|
|
126
|
+
"url": url,
|
|
127
|
+
"headers": self.client.headers,
|
|
128
|
+
**kwargs}, fail_silently=True)
|
|
129
|
+
logger.debug(f'Equivalent curl command: "{curl_command}"')
|
|
118
130
|
response = self.client.request(method, url, **kwargs)
|
|
119
131
|
response.raise_for_status()
|
|
120
132
|
return response
|
|
@@ -399,10 +411,30 @@ class BaseApi:
|
|
|
399
411
|
|
|
400
412
|
@staticmethod
|
|
401
413
|
def convert_format(bytes_array: bytes,
|
|
402
|
-
mimetype: str,
|
|
414
|
+
mimetype: str | None = None,
|
|
403
415
|
file_path: str | None = None
|
|
404
|
-
) ->
|
|
405
|
-
""" Convert the bytes array to the appropriate format based on the mimetype.
|
|
416
|
+
) -> ImagingData | bytes:
|
|
417
|
+
""" Convert the bytes array to the appropriate format based on the mimetype.
|
|
418
|
+
|
|
419
|
+
Args:
|
|
420
|
+
bytes_array: Raw file content bytes
|
|
421
|
+
mimetype: Optional MIME type of the content
|
|
422
|
+
file_path: deprecated
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
Converted content in appropriate format (pydicom.Dataset, PIL Image, cv2.VideoCapture, ...)
|
|
426
|
+
|
|
427
|
+
Example:
|
|
428
|
+
>>> fpath = 'path/to/file.dcm'
|
|
429
|
+
>>> with open(fpath, 'rb') as f:
|
|
430
|
+
... dicom_bytes = f.read()
|
|
431
|
+
>>> dicom = BaseApi.convert_format(dicom_bytes)
|
|
432
|
+
|
|
433
|
+
"""
|
|
434
|
+
if mimetype is None:
|
|
435
|
+
mimetype, ext = BaseApi._determine_mimetype(bytes_array)
|
|
436
|
+
if mimetype is None:
|
|
437
|
+
raise ValueError("Could not determine mimetype from content.")
|
|
406
438
|
content_io = BytesIO(bytes_array)
|
|
407
439
|
if mimetype.endswith('/dicom'):
|
|
408
440
|
return pydicom.dcmread(content_io)
|
|
@@ -429,3 +461,30 @@ class BaseApi:
|
|
|
429
461
|
return nib.Nifti1Image.from_stream(f)
|
|
430
462
|
|
|
431
463
|
raise ValueError(f"Unsupported mimetype: {mimetype}")
|
|
464
|
+
|
|
465
|
+
@staticmethod
|
|
466
|
+
def _determine_mimetype(content: bytes,
|
|
467
|
+
declared_mimetype: str | None = None) -> tuple[str | None, str | None]:
|
|
468
|
+
"""Infer MIME type and file extension from content and optional declared type.
|
|
469
|
+
|
|
470
|
+
Args:
|
|
471
|
+
content: Raw file content bytes
|
|
472
|
+
declared_mimetype: Optional MIME type declared by the source
|
|
473
|
+
|
|
474
|
+
Returns:
|
|
475
|
+
Tuple of (inferred_mimetype, file_extension)
|
|
476
|
+
"""
|
|
477
|
+
# Determine mimetype from file content
|
|
478
|
+
mimetype_list, ext = guess_typez(content, use_magic=True)
|
|
479
|
+
mimetype = mimetype_list[-1]
|
|
480
|
+
|
|
481
|
+
# get mimetype from resource info if not detected
|
|
482
|
+
if declared_mimetype is not None:
|
|
483
|
+
if mimetype is None:
|
|
484
|
+
mimetype = declared_mimetype
|
|
485
|
+
ext = guess_extension(mimetype)
|
|
486
|
+
elif mimetype == DEFAULT_MIME_TYPE:
|
|
487
|
+
mimetype = declared_mimetype
|
|
488
|
+
ext = guess_extension(mimetype)
|
|
489
|
+
|
|
490
|
+
return mimetype, ext
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
|
-
from .base_api import ApiConfig
|
|
2
|
+
from .base_api import ApiConfig, BaseApi
|
|
3
3
|
from .endpoints import (ProjectsApi, ResourcesApi, AnnotationsApi,
|
|
4
4
|
ChannelsApi, UsersApi, DatasetsInfoApi, ModelsApi,
|
|
5
5
|
AnnotationSetsApi
|
|
@@ -13,7 +13,7 @@ class Api:
|
|
|
13
13
|
DEFAULT_SERVER_URL = 'https://api.datamint.io'
|
|
14
14
|
DATAMINT_API_VENV_NAME = datamint.configs.ENV_VARS[datamint.configs.APIKEY_KEY]
|
|
15
15
|
|
|
16
|
-
_API_MAP = {
|
|
16
|
+
_API_MAP : dict[str, type[BaseApi]] = {
|
|
17
17
|
'projects': ProjectsApi,
|
|
18
18
|
'resources': ResourcesApi,
|
|
19
19
|
'annotations': AnnotationsApi,
|
|
@@ -70,7 +70,10 @@ class Api:
|
|
|
70
70
|
def _get_endpoint(self, name: str):
|
|
71
71
|
if name not in self._endpoints:
|
|
72
72
|
api_class = self._API_MAP[name]
|
|
73
|
-
|
|
73
|
+
endpoint = api_class(self.config, self._client)
|
|
74
|
+
# Inject this API instance into the endpoint so it can inject into entities
|
|
75
|
+
endpoint._api_instance = self
|
|
76
|
+
self._endpoints[name] = endpoint
|
|
74
77
|
return self._endpoints[name]
|
|
75
78
|
|
|
76
79
|
@property
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from datamint.apihandler.dto import annotation_dto
|
|
2
|
+
from datamint.apihandler.dto.annotation_dto import (
|
|
3
|
+
AnnotationType, CreateAnnotationDto,
|
|
4
|
+
Geometry, BoxGeometry, LineGeometry,
|
|
5
|
+
CoordinateSystem
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"annotation_dto",
|
|
10
|
+
"AnnotationType",
|
|
11
|
+
"CreateAnnotationDto",
|
|
12
|
+
"Geometry",
|
|
13
|
+
"BoxGeometry",
|
|
14
|
+
"LineGeometry",
|
|
15
|
+
"CoordinateSystem"
|
|
16
|
+
"LineGeometry",
|
|
17
|
+
"CoordinateSystem"
|
|
18
|
+
]
|
|
@@ -7,7 +7,7 @@ from .models_api import ModelsApi
|
|
|
7
7
|
from datamint.entities.annotation import Annotation
|
|
8
8
|
from datamint.entities.resource import Resource
|
|
9
9
|
from datamint.entities.project import Project
|
|
10
|
-
from datamint.
|
|
10
|
+
from datamint.api.dto import AnnotationType, CreateAnnotationDto, LineGeometry, BoxGeometry, CoordinateSystem, Geometry
|
|
11
11
|
import numpy as np
|
|
12
12
|
import os
|
|
13
13
|
import aiohttp
|
|
@@ -31,15 +31,21 @@ MAX_NUMBER_DISTINCT_COLORS = 2048 # Maximum number of distinct colors in a segm
|
|
|
31
31
|
class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotation]):
|
|
32
32
|
"""API handler for annotation-related endpoints."""
|
|
33
33
|
|
|
34
|
-
def __init__(self,
|
|
34
|
+
def __init__(self,
|
|
35
|
+
config: ApiConfig,
|
|
36
|
+
client: httpx.Client | None = None,
|
|
37
|
+
models_api=None,
|
|
38
|
+
resources_api=None) -> None:
|
|
35
39
|
"""Initialize the annotations API handler.
|
|
36
40
|
|
|
37
41
|
Args:
|
|
38
42
|
config: API configuration containing base URL, API key, etc.
|
|
39
43
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
40
44
|
"""
|
|
45
|
+
from .resources_api import ResourcesApi
|
|
41
46
|
super().__init__(config, Annotation, 'annotations', client)
|
|
42
|
-
self._models_api = ModelsApi(config, client=client)
|
|
47
|
+
self._models_api = ModelsApi(config, client=client) if models_api is None else models_api
|
|
48
|
+
self._resources_api = ResourcesApi(config, client=client, annotations_api=self) if resources_api is None else resources_api
|
|
43
49
|
|
|
44
50
|
def get_list(self,
|
|
45
51
|
resource: str | Resource | None = None,
|
|
@@ -377,7 +383,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
377
383
|
|
|
378
384
|
annotations = [annotation_dto] if isinstance(annotation_dto, CreateAnnotationDto) else annotation_dto
|
|
379
385
|
annotations = [ann.to_dict() if isinstance(ann, CreateAnnotationDto) else ann for ann in annotations]
|
|
380
|
-
resource_id =
|
|
386
|
+
resource_id = self._entid(resource)
|
|
381
387
|
respdata = self._make_request('POST',
|
|
382
388
|
f'{self.endpoint_base}/{resource_id}/annotations',
|
|
383
389
|
json=annotations).json()
|
|
@@ -467,7 +473,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
467
473
|
raise ValueError(f"AI model with name '{ai_model_name}' not found. ")
|
|
468
474
|
raise ValueError(f"AI model with name '{ai_model_name}' not found. " +
|
|
469
475
|
f"Available models: {available_models}")
|
|
470
|
-
model_id = model_id['
|
|
476
|
+
model_id = model_id['name']
|
|
471
477
|
|
|
472
478
|
# Handle NIfTI files specially - upload as single volume
|
|
473
479
|
if isinstance(file_path, str) and (file_path.endswith('.nii') or file_path.endswith('.nii.gz')):
|
|
@@ -780,6 +786,37 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
780
786
|
img_bytes.seek(0)
|
|
781
787
|
yield img_bytes
|
|
782
788
|
|
|
789
|
+
def create_image_classification(self,
|
|
790
|
+
resource: str | Resource,
|
|
791
|
+
identifier: str,
|
|
792
|
+
value: str,
|
|
793
|
+
imported_from: str | None = None,
|
|
794
|
+
model_id: str | None = None,
|
|
795
|
+
) -> str:
|
|
796
|
+
"""
|
|
797
|
+
Create an image-level classification annotation.
|
|
798
|
+
|
|
799
|
+
Args:
|
|
800
|
+
resource: The resource unique id or Resource instance.
|
|
801
|
+
identifier: The annotation identifier/label.
|
|
802
|
+
value: The classification value.
|
|
803
|
+
imported_from: The imported from source value.
|
|
804
|
+
model_id: The model unique id.
|
|
805
|
+
|
|
806
|
+
Returns:
|
|
807
|
+
The id of the created annotation.
|
|
808
|
+
"""
|
|
809
|
+
annotation_dto = CreateAnnotationDto(
|
|
810
|
+
type=AnnotationType.CATEGORY,
|
|
811
|
+
identifier=identifier,
|
|
812
|
+
scope='image',
|
|
813
|
+
value=value,
|
|
814
|
+
imported_from=imported_from,
|
|
815
|
+
model_id=model_id
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
return self.create(resource, annotation_dto)
|
|
819
|
+
|
|
783
820
|
def add_line_annotation(self,
|
|
784
821
|
point1: tuple[int, int] | tuple[float, float, float],
|
|
785
822
|
point2: tuple[int, int] | tuple[float, float, float],
|
|
@@ -903,7 +940,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
903
940
|
|
|
904
941
|
def download_file(self,
|
|
905
942
|
annotation: str | Annotation,
|
|
906
|
-
fpath_out: str |
|
|
943
|
+
fpath_out: str | os.PathLike | None = None) -> bytes:
|
|
907
944
|
"""
|
|
908
945
|
Download the segmentation file for a given resource and annotation.
|
|
909
946
|
|
|
@@ -923,7 +960,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
923
960
|
|
|
924
961
|
resp = self._make_request('GET', f'/annotations/{resource_id}/annotations/{annotation_id}/file')
|
|
925
962
|
if fpath_out:
|
|
926
|
-
with open(
|
|
963
|
+
with open(fpath_out, 'wb') as f:
|
|
927
964
|
f.write(resp.content)
|
|
928
965
|
return resp.content
|
|
929
966
|
|
|
@@ -1028,3 +1065,6 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
1028
1065
|
respdata = resp.json()
|
|
1029
1066
|
if isinstance(respdata, dict) and 'error' in respdata:
|
|
1030
1067
|
raise DatamintException(respdata['error'])
|
|
1068
|
+
|
|
1069
|
+
def _get_resource(self, ann: Annotation) -> Resource:
|
|
1070
|
+
return self._resources_api.get_by_id(ann.resource_id)
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
from typing import Sequence, Literal
|
|
1
|
+
from typing import Sequence, Literal, TYPE_CHECKING
|
|
2
2
|
from ..entity_base_api import ApiConfig, CRUDEntityApi
|
|
3
3
|
from datamint.entities.project import Project
|
|
4
|
-
from datamint.entities.resource import Resource
|
|
5
4
|
import httpx
|
|
5
|
+
from datamint.entities.resource import Resource
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from .resources_api import ResourcesApi
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class ProjectsApi(CRUDEntityApi[Project]):
|
|
@@ -10,14 +12,17 @@ class ProjectsApi(CRUDEntityApi[Project]):
|
|
|
10
12
|
|
|
11
13
|
def __init__(self,
|
|
12
14
|
config: ApiConfig,
|
|
13
|
-
client: httpx.Client | None = None
|
|
15
|
+
client: httpx.Client | None = None,
|
|
16
|
+
resources_api: 'ResourcesApi | None' = None) -> None:
|
|
14
17
|
"""Initialize the projects API handler.
|
|
15
18
|
|
|
16
19
|
Args:
|
|
17
20
|
config: API configuration containing base URL, API key, etc.
|
|
18
21
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
19
22
|
"""
|
|
23
|
+
from .resources_api import ResourcesApi
|
|
20
24
|
super().__init__(config, Project, 'projects', client)
|
|
25
|
+
self.resources_api = resources_api or ResourcesApi(config, client, projects_api=self)
|
|
21
26
|
|
|
22
27
|
def get_project_resources(self, project: Project | str) -> list[Resource]:
|
|
23
28
|
"""Get resources associated with a specific project.
|
|
@@ -30,7 +35,8 @@ class ProjectsApi(CRUDEntityApi[Project]):
|
|
|
30
35
|
"""
|
|
31
36
|
response = self._get_child_entities(project, 'resources')
|
|
32
37
|
resources_data = response.json()
|
|
33
|
-
|
|
38
|
+
resources = [self.resources_api._init_entity_obj(**item) for item in resources_data]
|
|
39
|
+
return resources
|
|
34
40
|
|
|
35
41
|
def create(self,
|
|
36
42
|
name: str,
|
|
@@ -148,47 +154,48 @@ class ProjectsApi(CRUDEntityApi[Project]):
|
|
|
148
154
|
self._make_entity_request('POST', project_id, add_path='resources',
|
|
149
155
|
json={'resource_ids_to_add': resources_ids, 'all_files_selected': False})
|
|
150
156
|
|
|
151
|
-
def download(self, project: str | Project,
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
157
|
+
# def download(self, project: str | Project,
|
|
158
|
+
# outpath: str,
|
|
159
|
+
# all_annotations: bool = False,
|
|
160
|
+
# include_unannotated: bool = False,
|
|
161
|
+
# ) -> None:
|
|
162
|
+
# """Download a project by its id.
|
|
163
|
+
|
|
164
|
+
# Args:
|
|
165
|
+
# project: The project id or Project instance.
|
|
166
|
+
# outpath: The path to save the project zip file.
|
|
167
|
+
# all_annotations: Whether to include all annotations in the downloaded dataset,
|
|
168
|
+
# even those not made by the provided project.
|
|
169
|
+
# include_unannotated: Whether to include unannotated resources in the downloaded dataset.
|
|
170
|
+
# """
|
|
171
|
+
# from tqdm.auto import tqdm
|
|
172
|
+
# params = {'all_annotations': all_annotations}
|
|
173
|
+
# if include_unannotated:
|
|
174
|
+
# params['include_unannotated'] = include_unannotated
|
|
175
|
+
|
|
176
|
+
# project_id = self._entid(project)
|
|
177
|
+
# with self._stream_entity_request('GET', project_id,
|
|
178
|
+
# add_path='annotated_dataset',
|
|
179
|
+
# params=params) as response:
|
|
180
|
+
# total_size = int(response.headers.get('content-length', 0))
|
|
181
|
+
# if total_size == 0:
|
|
182
|
+
# total_size = None
|
|
183
|
+
# with tqdm(total=total_size, unit='B', unit_scale=True) as progress_bar:
|
|
184
|
+
# with open(outpath, 'wb') as file:
|
|
185
|
+
# for data in response.iter_bytes(1024):
|
|
186
|
+
# progress_bar.update(len(data))
|
|
187
|
+
# file.write(data)
|
|
182
188
|
|
|
183
189
|
def set_work_status(self,
|
|
184
|
-
resource: str | Resource,
|
|
185
190
|
project: str | Project,
|
|
191
|
+
resource: str | Resource,
|
|
186
192
|
status: Literal['opened', 'annotated', 'closed']) -> None:
|
|
187
193
|
"""
|
|
188
194
|
Set the status of a resource.
|
|
189
195
|
|
|
190
196
|
Args:
|
|
191
|
-
|
|
197
|
+
project: The project unique id or a project object.
|
|
198
|
+
resource: The resource unique id or a resource object.
|
|
192
199
|
status: The new status to set.
|
|
193
200
|
"""
|
|
194
201
|
resource_id = self._entid(resource)
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from typing import Any, Optional, Sequence, TypeAlias, Literal, IO
|
|
2
2
|
from ..base_api import ApiConfig, BaseApi
|
|
3
|
-
from ..entity_base_api import
|
|
4
|
-
from .annotations_api import AnnotationsApi
|
|
5
|
-
from .projects_api import ProjectsApi
|
|
3
|
+
from ..entity_base_api import CreatableEntityApi, DeletableEntityApi
|
|
6
4
|
from datamint.entities.resource import Resource
|
|
5
|
+
from datamint.entities.project import Project
|
|
7
6
|
from datamint.entities.annotation import Annotation
|
|
8
7
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
9
8
|
import httpx
|
|
@@ -23,10 +22,9 @@ import asyncio
|
|
|
23
22
|
import aiohttp
|
|
24
23
|
from pathlib import Path
|
|
25
24
|
import nest_asyncio # For running asyncio in jupyter notebooks
|
|
26
|
-
import cv2
|
|
27
25
|
from PIL import Image
|
|
28
|
-
from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
|
|
29
26
|
import io
|
|
27
|
+
from datamint.types import ImagingData
|
|
30
28
|
|
|
31
29
|
|
|
32
30
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -54,17 +52,25 @@ def _open_io(file_path: str | Path | IO, mode: str = 'rb') -> IO:
|
|
|
54
52
|
class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
55
53
|
"""API handler for resource-related endpoints."""
|
|
56
54
|
|
|
57
|
-
def __init__(self,
|
|
55
|
+
def __init__(self,
|
|
56
|
+
config: ApiConfig,
|
|
57
|
+
client: Optional[httpx.Client] = None,
|
|
58
|
+
annotations_api=None,
|
|
59
|
+
projects_api=None
|
|
60
|
+
) -> None:
|
|
58
61
|
"""Initialize the resources API handler.
|
|
59
62
|
|
|
60
63
|
Args:
|
|
61
64
|
config: API configuration containing base URL, API key, etc.
|
|
62
65
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
63
66
|
"""
|
|
67
|
+
from .annotations_api import AnnotationsApi
|
|
68
|
+
from .projects_api import ProjectsApi
|
|
64
69
|
super().__init__(config, Resource, 'resources', client)
|
|
65
70
|
nest_asyncio.apply()
|
|
66
|
-
self.annotations_api = AnnotationsApi(
|
|
67
|
-
|
|
71
|
+
self.annotations_api = AnnotationsApi(
|
|
72
|
+
config, client, resources_api=self) if annotations_api is None else annotations_api
|
|
73
|
+
self.projects_api = projects_api or ProjectsApi(config, client, resources_api=self)
|
|
68
74
|
|
|
69
75
|
def get_list(self,
|
|
70
76
|
status: Optional[ResourceStatus] = None,
|
|
@@ -427,8 +433,14 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
427
433
|
)
|
|
428
434
|
return rid
|
|
429
435
|
|
|
430
|
-
|
|
431
|
-
|
|
436
|
+
try:
|
|
437
|
+
tasks = [__upload_single_resource(f, segfiles, metadata_file)
|
|
438
|
+
for f, segfiles, metadata_file in zip(files_path, segmentation_files, metadata_files)]
|
|
439
|
+
except ValueError:
|
|
440
|
+
msg = f"Error preparing upload tasks. Try `assemble_dicom=False`."
|
|
441
|
+
_LOGGER.error(msg)
|
|
442
|
+
_USER_LOGGER.error(msg)
|
|
443
|
+
raise
|
|
432
444
|
return await asyncio.gather(*tasks, return_exceptions=on_error == 'skip')
|
|
433
445
|
|
|
434
446
|
def upload_resources(self,
|
|
@@ -710,21 +722,6 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
710
722
|
# This should not happen with single file uploads, but handle it just in case
|
|
711
723
|
raise DatamintException(f"Unexpected return from upload_resources: {type(result)} | {result}")
|
|
712
724
|
|
|
713
|
-
def _determine_mimetype(self,
|
|
714
|
-
content,
|
|
715
|
-
resource: str | Resource) -> tuple[str | None, str | None]:
|
|
716
|
-
# Determine mimetype from file content
|
|
717
|
-
mimetype_list, ext = guess_typez(content, use_magic=True)
|
|
718
|
-
mimetype = mimetype_list[-1]
|
|
719
|
-
|
|
720
|
-
# get mimetype from resource info if not detected
|
|
721
|
-
if mimetype is None or mimetype == DEFAULT_MIME_TYPE:
|
|
722
|
-
if not isinstance(resource, Resource):
|
|
723
|
-
resource = self.get_by_id(resource)
|
|
724
|
-
mimetype = resource.mimetype or mimetype
|
|
725
|
-
|
|
726
|
-
return mimetype, ext
|
|
727
|
-
|
|
728
725
|
async def _async_download_file(self,
|
|
729
726
|
resource: str | Resource,
|
|
730
727
|
save_path: str | Path,
|
|
@@ -761,8 +758,8 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
761
758
|
f.write(data_bytes)
|
|
762
759
|
|
|
763
760
|
# Determine mimetype from file content
|
|
764
|
-
mimetype, ext =
|
|
765
|
-
|
|
761
|
+
mimetype, ext = BaseApi._determine_mimetype(content=data_bytes,
|
|
762
|
+
declared_mimetype=resource.mimetype if isinstance(resource, Resource) else None)
|
|
766
763
|
|
|
767
764
|
# Generate final path with extension if needed
|
|
768
765
|
if mimetype is not None and mimetype != DEFAULT_MIME_TYPE:
|
|
@@ -850,7 +847,7 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
850
847
|
save_path: Optional[str] = None,
|
|
851
848
|
auto_convert: bool = True,
|
|
852
849
|
add_extension: bool = False
|
|
853
|
-
) ->
|
|
850
|
+
) -> ImagingData | tuple[ImagingData, str] | bytes:
|
|
854
851
|
"""
|
|
855
852
|
Download a resource file.
|
|
856
853
|
|
|
@@ -888,8 +885,8 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
888
885
|
mimetype = None
|
|
889
886
|
ext = None
|
|
890
887
|
if auto_convert or add_extension:
|
|
891
|
-
mimetype, ext =
|
|
892
|
-
|
|
888
|
+
mimetype, ext = BaseApi._determine_mimetype(content=response.content,
|
|
889
|
+
declared_mimetype=resource.mimetype if isinstance(resource, Resource) else None)
|
|
893
890
|
if auto_convert:
|
|
894
891
|
if mimetype is None:
|
|
895
892
|
_LOGGER.warning("Could not determine mimetype. Returning a bytes array.")
|
|
@@ -997,6 +994,12 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
997
994
|
resource: str | Resource,
|
|
998
995
|
tags: Sequence[str],
|
|
999
996
|
):
|
|
997
|
+
"""
|
|
998
|
+
Set tags for a resource, IMPORTANT: This replaces all existing tags.
|
|
999
|
+
Args:
|
|
1000
|
+
resource: The resource unique id or Resource object.
|
|
1001
|
+
tags: The tags to set.
|
|
1002
|
+
"""
|
|
1000
1003
|
data = {'tags': tags}
|
|
1001
1004
|
resource_id = self._entid(resource)
|
|
1002
1005
|
|
|
@@ -1005,3 +1008,53 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
1005
1008
|
add_path='tags',
|
|
1006
1009
|
json=data)
|
|
1007
1010
|
return response
|
|
1011
|
+
|
|
1012
|
+
# def get_projects(self, resource: Resource) -> Sequence[Project]:
|
|
1013
|
+
# """
|
|
1014
|
+
# Get all projects this resource belongs to.
|
|
1015
|
+
|
|
1016
|
+
# Args:
|
|
1017
|
+
# resource: The Resource instance.
|
|
1018
|
+
|
|
1019
|
+
# Returns:
|
|
1020
|
+
# List of Project instances
|
|
1021
|
+
# """
|
|
1022
|
+
# resource._ensure_attr('projects')
|
|
1023
|
+
# proj_ids = [p['id'] for p in resource.projects]
|
|
1024
|
+
# return [proj for proj in self.projects_api.get_all() if proj.id in proj_ids]
|
|
1025
|
+
|
|
1026
|
+
def add_tags(self,
|
|
1027
|
+
resource: str | Resource,
|
|
1028
|
+
tags: Sequence[str],
|
|
1029
|
+
):
|
|
1030
|
+
"""
|
|
1031
|
+
Add tags to a resource, IMPORTANT: This appends to existing tags.
|
|
1032
|
+
Args:
|
|
1033
|
+
resource: The resource unique id or Resource object.
|
|
1034
|
+
tags: The tags to add.
|
|
1035
|
+
"""
|
|
1036
|
+
if isinstance(resource, str):
|
|
1037
|
+
resource = self.get_by_id(resource)
|
|
1038
|
+
old_tags = resource.tags if resource.tags is not None else []
|
|
1039
|
+
return self.set_tags(resource, old_tags + list(tags))
|
|
1040
|
+
|
|
1041
|
+
def bulk_delete(self, entities: Sequence[str | Resource]) -> None:
|
|
1042
|
+
"""Delete multiple entities. Faster than deleting them one by one.
|
|
1043
|
+
|
|
1044
|
+
Args:
|
|
1045
|
+
entities: Sequence of unique identifiers for the entities to delete or the entity instances themselves.
|
|
1046
|
+
|
|
1047
|
+
Raises:
|
|
1048
|
+
httpx.HTTPStatusError: If deletion fails or any entity not found
|
|
1049
|
+
"""
|
|
1050
|
+
from math import ceil
|
|
1051
|
+
|
|
1052
|
+
resources_ids = [self._entid(ent) for ent in entities]
|
|
1053
|
+
if len(resources_ids) == 0:
|
|
1054
|
+
return
|
|
1055
|
+
batch_size = 200
|
|
1056
|
+
for i in range(0, ceil(len(resources_ids)/batch_size)):
|
|
1057
|
+
batch_ids = resources_ids[i*batch_size:(i+1)*batch_size]
|
|
1058
|
+
self._make_request('DELETE',
|
|
1059
|
+
f'{self.endpoint_base}',
|
|
1060
|
+
params={'resource_ids': ','.join(batch_ids)})
|