datamint 2.3.2__py3-none-any.whl → 2.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datamint/api/base_api.py +66 -8
- datamint/api/client.py +6 -3
- datamint/api/dto/__init__.py +10 -2
- datamint/api/endpoints/annotations_api.py +47 -7
- datamint/api/endpoints/projects_api.py +36 -34
- datamint/api/endpoints/resources_api.py +75 -28
- datamint/api/entity_base_api.py +11 -43
- datamint/apihandler/dto/annotation_dto.py +6 -2
- datamint/configs.py +6 -0
- datamint/dataset/base_dataset.py +3 -3
- datamint/entities/__init__.py +4 -2
- datamint/entities/annotation.py +74 -4
- datamint/entities/base_entity.py +47 -6
- datamint/entities/cache_manager.py +302 -0
- datamint/entities/datasetinfo.py +108 -1
- datamint/entities/project.py +47 -6
- datamint/entities/resource.py +146 -19
- datamint/types.py +17 -0
- {datamint-2.3.2.dist-info → datamint-2.3.3.dist-info}/METADATA +2 -1
- {datamint-2.3.2.dist-info → datamint-2.3.3.dist-info}/RECORD +22 -20
- {datamint-2.3.2.dist-info → datamint-2.3.3.dist-info}/WHEEL +0 -0
- {datamint-2.3.2.dist-info → datamint-2.3.3.dist-info}/entry_points.txt +0 -0
datamint/api/base_api.py
CHANGED
|
@@ -1,27 +1,28 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import Any, Generator, AsyncGenerator, Sequence
|
|
2
|
+
from typing import Any, Generator, AsyncGenerator, Sequence, TYPE_CHECKING
|
|
3
3
|
import httpx
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
6
|
+
from datamint.types import ImagingData
|
|
6
7
|
import aiohttp
|
|
7
8
|
import json
|
|
8
|
-
import pydicom.dataset
|
|
9
9
|
from PIL import Image
|
|
10
10
|
import cv2
|
|
11
11
|
import nibabel as nib
|
|
12
|
-
from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
|
|
13
12
|
from io import BytesIO
|
|
14
13
|
import gzip
|
|
15
14
|
import contextlib
|
|
16
15
|
import asyncio
|
|
17
|
-
from medimgkit.format_detection import GZIP_MIME_TYPES
|
|
16
|
+
from medimgkit.format_detection import GZIP_MIME_TYPES, DEFAULT_MIME_TYPE, guess_typez, guess_extension
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from datamint.api.client import Api
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
21
23
|
# Generic type for entities
|
|
22
24
|
_PAGE_LIMIT = 5000
|
|
23
25
|
|
|
24
|
-
|
|
25
26
|
@dataclass
|
|
26
27
|
class ApiConfig:
|
|
27
28
|
"""Configuration for API client.
|
|
@@ -37,6 +38,15 @@ class ApiConfig:
|
|
|
37
38
|
timeout: float = 30.0
|
|
38
39
|
max_retries: int = 3
|
|
39
40
|
|
|
41
|
+
@property
|
|
42
|
+
def web_app_url(self) -> str:
|
|
43
|
+
"""Get the base URL for the web application."""
|
|
44
|
+
if self.server_url.startswith('http://localhost:3001'):
|
|
45
|
+
return 'http://localhost:3000'
|
|
46
|
+
if self.server_url.startswith('https://stagingapi.datamint.io'):
|
|
47
|
+
return 'https://staging.datamint.io'
|
|
48
|
+
return 'https://app.datamint.io'
|
|
49
|
+
|
|
40
50
|
|
|
41
51
|
class BaseApi:
|
|
42
52
|
"""Base class for all API endpoint handlers."""
|
|
@@ -53,6 +63,7 @@ class BaseApi:
|
|
|
53
63
|
self.config = config
|
|
54
64
|
self.client = client or self._create_client()
|
|
55
65
|
self.semaphore = asyncio.Semaphore(20)
|
|
66
|
+
self._api_instance: 'Api | None' = None # Injected by Api class
|
|
56
67
|
|
|
57
68
|
def _create_client(self) -> httpx.Client:
|
|
58
69
|
"""Create and configure HTTP client with authentication and timeouts."""
|
|
@@ -399,10 +410,30 @@ class BaseApi:
|
|
|
399
410
|
|
|
400
411
|
@staticmethod
|
|
401
412
|
def convert_format(bytes_array: bytes,
|
|
402
|
-
mimetype: str,
|
|
413
|
+
mimetype: str | None = None,
|
|
403
414
|
file_path: str | None = None
|
|
404
|
-
) ->
|
|
405
|
-
""" Convert the bytes array to the appropriate format based on the mimetype.
|
|
415
|
+
) -> ImagingData | bytes:
|
|
416
|
+
""" Convert the bytes array to the appropriate format based on the mimetype.
|
|
417
|
+
|
|
418
|
+
Args:
|
|
419
|
+
bytes_array: Raw file content bytes
|
|
420
|
+
mimetype: Optional MIME type of the content
|
|
421
|
+
file_path: deprecated
|
|
422
|
+
|
|
423
|
+
Returns:
|
|
424
|
+
Converted content in appropriate format (pydicom.Dataset, PIL Image, cv2.VideoCapture, ...)
|
|
425
|
+
|
|
426
|
+
Example:
|
|
427
|
+
>>> fpath = 'path/to/file.dcm'
|
|
428
|
+
>>> with open(fpath, 'rb') as f:
|
|
429
|
+
... dicom_bytes = f.read()
|
|
430
|
+
>>> dicom = BaseApi.convert_format(dicom_bytes)
|
|
431
|
+
|
|
432
|
+
"""
|
|
433
|
+
if mimetype is None:
|
|
434
|
+
mimetype, ext = BaseApi._determine_mimetype(bytes_array)
|
|
435
|
+
if mimetype is None:
|
|
436
|
+
raise ValueError("Could not determine mimetype from content.")
|
|
406
437
|
content_io = BytesIO(bytes_array)
|
|
407
438
|
if mimetype.endswith('/dicom'):
|
|
408
439
|
return pydicom.dcmread(content_io)
|
|
@@ -429,3 +460,30 @@ class BaseApi:
|
|
|
429
460
|
return nib.Nifti1Image.from_stream(f)
|
|
430
461
|
|
|
431
462
|
raise ValueError(f"Unsupported mimetype: {mimetype}")
|
|
463
|
+
|
|
464
|
+
@staticmethod
|
|
465
|
+
def _determine_mimetype(content: bytes,
|
|
466
|
+
declared_mimetype: str | None = None) -> tuple[str | None, str | None]:
|
|
467
|
+
"""Infer MIME type and file extension from content and optional declared type.
|
|
468
|
+
|
|
469
|
+
Args:
|
|
470
|
+
content: Raw file content bytes
|
|
471
|
+
declared_mimetype: Optional MIME type declared by the source
|
|
472
|
+
|
|
473
|
+
Returns:
|
|
474
|
+
Tuple of (inferred_mimetype, file_extension)
|
|
475
|
+
"""
|
|
476
|
+
# Determine mimetype from file content
|
|
477
|
+
mimetype_list, ext = guess_typez(content, use_magic=True)
|
|
478
|
+
mimetype = mimetype_list[-1]
|
|
479
|
+
|
|
480
|
+
# get mimetype from resource info if not detected
|
|
481
|
+
if declared_mimetype is not None:
|
|
482
|
+
if mimetype is None:
|
|
483
|
+
mimetype = declared_mimetype
|
|
484
|
+
ext = guess_extension(mimetype)
|
|
485
|
+
elif mimetype == DEFAULT_MIME_TYPE:
|
|
486
|
+
mimetype = declared_mimetype
|
|
487
|
+
ext = guess_extension(mimetype)
|
|
488
|
+
|
|
489
|
+
return mimetype, ext
|
datamint/api/client.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
|
-
from .base_api import ApiConfig
|
|
2
|
+
from .base_api import ApiConfig, BaseApi
|
|
3
3
|
from .endpoints import (ProjectsApi, ResourcesApi, AnnotationsApi,
|
|
4
4
|
ChannelsApi, UsersApi, DatasetsInfoApi, ModelsApi,
|
|
5
5
|
AnnotationSetsApi
|
|
@@ -13,7 +13,7 @@ class Api:
|
|
|
13
13
|
DEFAULT_SERVER_URL = 'https://api.datamint.io'
|
|
14
14
|
DATAMINT_API_VENV_NAME = datamint.configs.ENV_VARS[datamint.configs.APIKEY_KEY]
|
|
15
15
|
|
|
16
|
-
_API_MAP = {
|
|
16
|
+
_API_MAP : dict[str, type[BaseApi]] = {
|
|
17
17
|
'projects': ProjectsApi,
|
|
18
18
|
'resources': ResourcesApi,
|
|
19
19
|
'annotations': AnnotationsApi,
|
|
@@ -70,7 +70,10 @@ class Api:
|
|
|
70
70
|
def _get_endpoint(self, name: str):
|
|
71
71
|
if name not in self._endpoints:
|
|
72
72
|
api_class = self._API_MAP[name]
|
|
73
|
-
|
|
73
|
+
endpoint = api_class(self.config, self._client)
|
|
74
|
+
# Inject this API instance into the endpoint so it can inject into entities
|
|
75
|
+
endpoint._api_instance = self
|
|
76
|
+
self._endpoints[name] = endpoint
|
|
74
77
|
return self._endpoints[name]
|
|
75
78
|
|
|
76
79
|
@property
|
datamint/api/dto/__init__.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from datamint.apihandler.dto import annotation_dto
|
|
2
|
-
from datamint.apihandler.dto.annotation_dto import
|
|
2
|
+
from datamint.apihandler.dto.annotation_dto import (
|
|
3
|
+
AnnotationType, CreateAnnotationDto,
|
|
4
|
+
Geometry, BoxGeometry, LineGeometry,
|
|
5
|
+
CoordinateSystem
|
|
6
|
+
)
|
|
3
7
|
|
|
4
8
|
__all__ = [
|
|
5
9
|
"annotation_dto",
|
|
@@ -7,4 +11,8 @@ __all__ = [
|
|
|
7
11
|
"CreateAnnotationDto",
|
|
8
12
|
"Geometry",
|
|
9
13
|
"BoxGeometry",
|
|
10
|
-
|
|
14
|
+
"LineGeometry",
|
|
15
|
+
"CoordinateSystem"
|
|
16
|
+
"LineGeometry",
|
|
17
|
+
"CoordinateSystem"
|
|
18
|
+
]
|
|
@@ -7,7 +7,7 @@ from .models_api import ModelsApi
|
|
|
7
7
|
from datamint.entities.annotation import Annotation
|
|
8
8
|
from datamint.entities.resource import Resource
|
|
9
9
|
from datamint.entities.project import Project
|
|
10
|
-
from datamint.
|
|
10
|
+
from datamint.api.dto import AnnotationType, CreateAnnotationDto, LineGeometry, BoxGeometry, CoordinateSystem, Geometry
|
|
11
11
|
import numpy as np
|
|
12
12
|
import os
|
|
13
13
|
import aiohttp
|
|
@@ -31,15 +31,21 @@ MAX_NUMBER_DISTINCT_COLORS = 2048 # Maximum number of distinct colors in a segm
|
|
|
31
31
|
class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotation]):
|
|
32
32
|
"""API handler for annotation-related endpoints."""
|
|
33
33
|
|
|
34
|
-
def __init__(self,
|
|
34
|
+
def __init__(self,
|
|
35
|
+
config: ApiConfig,
|
|
36
|
+
client: httpx.Client | None = None,
|
|
37
|
+
models_api=None,
|
|
38
|
+
resources_api=None) -> None:
|
|
35
39
|
"""Initialize the annotations API handler.
|
|
36
40
|
|
|
37
41
|
Args:
|
|
38
42
|
config: API configuration containing base URL, API key, etc.
|
|
39
43
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
40
44
|
"""
|
|
45
|
+
from .resources_api import ResourcesApi
|
|
41
46
|
super().__init__(config, Annotation, 'annotations', client)
|
|
42
|
-
self._models_api = ModelsApi(config, client=client)
|
|
47
|
+
self._models_api = ModelsApi(config, client=client) if models_api is None else models_api
|
|
48
|
+
self._resources_api = ResourcesApi(config, client=client, annotations_api=self) if resources_api is None else resources_api
|
|
43
49
|
|
|
44
50
|
def get_list(self,
|
|
45
51
|
resource: str | Resource | None = None,
|
|
@@ -377,7 +383,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
377
383
|
|
|
378
384
|
annotations = [annotation_dto] if isinstance(annotation_dto, CreateAnnotationDto) else annotation_dto
|
|
379
385
|
annotations = [ann.to_dict() if isinstance(ann, CreateAnnotationDto) else ann for ann in annotations]
|
|
380
|
-
resource_id =
|
|
386
|
+
resource_id = self._entid(resource)
|
|
381
387
|
respdata = self._make_request('POST',
|
|
382
388
|
f'{self.endpoint_base}/{resource_id}/annotations',
|
|
383
389
|
json=annotations).json()
|
|
@@ -467,7 +473,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
467
473
|
raise ValueError(f"AI model with name '{ai_model_name}' not found. ")
|
|
468
474
|
raise ValueError(f"AI model with name '{ai_model_name}' not found. " +
|
|
469
475
|
f"Available models: {available_models}")
|
|
470
|
-
model_id = model_id['
|
|
476
|
+
model_id = model_id['name']
|
|
471
477
|
|
|
472
478
|
# Handle NIfTI files specially - upload as single volume
|
|
473
479
|
if isinstance(file_path, str) and (file_path.endswith('.nii') or file_path.endswith('.nii.gz')):
|
|
@@ -780,6 +786,37 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
780
786
|
img_bytes.seek(0)
|
|
781
787
|
yield img_bytes
|
|
782
788
|
|
|
789
|
+
def create_image_classification(self,
|
|
790
|
+
resource: str | Resource,
|
|
791
|
+
identifier: str,
|
|
792
|
+
value: str,
|
|
793
|
+
imported_from: str | None = None,
|
|
794
|
+
model_id: str | None = None,
|
|
795
|
+
) -> str:
|
|
796
|
+
"""
|
|
797
|
+
Create an image-level classification annotation.
|
|
798
|
+
|
|
799
|
+
Args:
|
|
800
|
+
resource: The resource unique id or Resource instance.
|
|
801
|
+
identifier: The annotation identifier/label.
|
|
802
|
+
value: The classification value.
|
|
803
|
+
imported_from: The imported from source value.
|
|
804
|
+
model_id: The model unique id.
|
|
805
|
+
|
|
806
|
+
Returns:
|
|
807
|
+
The id of the created annotation.
|
|
808
|
+
"""
|
|
809
|
+
annotation_dto = CreateAnnotationDto(
|
|
810
|
+
type=AnnotationType.CATEGORY,
|
|
811
|
+
identifier=identifier,
|
|
812
|
+
scope='image',
|
|
813
|
+
value=value,
|
|
814
|
+
imported_from=imported_from,
|
|
815
|
+
model_id=model_id
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
return self.create(resource, annotation_dto)
|
|
819
|
+
|
|
783
820
|
def add_line_annotation(self,
|
|
784
821
|
point1: tuple[int, int] | tuple[float, float, float],
|
|
785
822
|
point2: tuple[int, int] | tuple[float, float, float],
|
|
@@ -903,7 +940,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
903
940
|
|
|
904
941
|
def download_file(self,
|
|
905
942
|
annotation: str | Annotation,
|
|
906
|
-
fpath_out: str |
|
|
943
|
+
fpath_out: str | os.PathLike | None = None) -> bytes:
|
|
907
944
|
"""
|
|
908
945
|
Download the segmentation file for a given resource and annotation.
|
|
909
946
|
|
|
@@ -923,7 +960,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
923
960
|
|
|
924
961
|
resp = self._make_request('GET', f'/annotations/{resource_id}/annotations/{annotation_id}/file')
|
|
925
962
|
if fpath_out:
|
|
926
|
-
with open(
|
|
963
|
+
with open(fpath_out, 'wb') as f:
|
|
927
964
|
f.write(resp.content)
|
|
928
965
|
return resp.content
|
|
929
966
|
|
|
@@ -1028,3 +1065,6 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
|
|
|
1028
1065
|
respdata = resp.json()
|
|
1029
1066
|
if isinstance(respdata, dict) and 'error' in respdata:
|
|
1030
1067
|
raise DatamintException(respdata['error'])
|
|
1068
|
+
|
|
1069
|
+
def _get_resource(self, ann: Annotation) -> Resource:
|
|
1070
|
+
return self._resources_api.get_by_id(ann.resource_id)
|
|
@@ -30,7 +30,8 @@ class ProjectsApi(CRUDEntityApi[Project]):
|
|
|
30
30
|
"""
|
|
31
31
|
response = self._get_child_entities(project, 'resources')
|
|
32
32
|
resources_data = response.json()
|
|
33
|
-
|
|
33
|
+
resources = [Resource(**item) for item in resources_data]
|
|
34
|
+
return resources
|
|
34
35
|
|
|
35
36
|
def create(self,
|
|
36
37
|
name: str,
|
|
@@ -148,47 +149,48 @@ class ProjectsApi(CRUDEntityApi[Project]):
|
|
|
148
149
|
self._make_entity_request('POST', project_id, add_path='resources',
|
|
149
150
|
json={'resource_ids_to_add': resources_ids, 'all_files_selected': False})
|
|
150
151
|
|
|
151
|
-
def download(self, project: str | Project,
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
152
|
+
# def download(self, project: str | Project,
|
|
153
|
+
# outpath: str,
|
|
154
|
+
# all_annotations: bool = False,
|
|
155
|
+
# include_unannotated: bool = False,
|
|
156
|
+
# ) -> None:
|
|
157
|
+
# """Download a project by its id.
|
|
158
|
+
|
|
159
|
+
# Args:
|
|
160
|
+
# project: The project id or Project instance.
|
|
161
|
+
# outpath: The path to save the project zip file.
|
|
162
|
+
# all_annotations: Whether to include all annotations in the downloaded dataset,
|
|
163
|
+
# even those not made by the provided project.
|
|
164
|
+
# include_unannotated: Whether to include unannotated resources in the downloaded dataset.
|
|
165
|
+
# """
|
|
166
|
+
# from tqdm.auto import tqdm
|
|
167
|
+
# params = {'all_annotations': all_annotations}
|
|
168
|
+
# if include_unannotated:
|
|
169
|
+
# params['include_unannotated'] = include_unannotated
|
|
170
|
+
|
|
171
|
+
# project_id = self._entid(project)
|
|
172
|
+
# with self._stream_entity_request('GET', project_id,
|
|
173
|
+
# add_path='annotated_dataset',
|
|
174
|
+
# params=params) as response:
|
|
175
|
+
# total_size = int(response.headers.get('content-length', 0))
|
|
176
|
+
# if total_size == 0:
|
|
177
|
+
# total_size = None
|
|
178
|
+
# with tqdm(total=total_size, unit='B', unit_scale=True) as progress_bar:
|
|
179
|
+
# with open(outpath, 'wb') as file:
|
|
180
|
+
# for data in response.iter_bytes(1024):
|
|
181
|
+
# progress_bar.update(len(data))
|
|
182
|
+
# file.write(data)
|
|
182
183
|
|
|
183
184
|
def set_work_status(self,
|
|
184
|
-
resource: str | Resource,
|
|
185
185
|
project: str | Project,
|
|
186
|
+
resource: str | Resource,
|
|
186
187
|
status: Literal['opened', 'annotated', 'closed']) -> None:
|
|
187
188
|
"""
|
|
188
189
|
Set the status of a resource.
|
|
189
190
|
|
|
190
191
|
Args:
|
|
191
|
-
|
|
192
|
+
project: The project unique id or a project object.
|
|
193
|
+
resource: The resource unique id or a resource object.
|
|
192
194
|
status: The new status to set.
|
|
193
195
|
"""
|
|
194
196
|
resource_id = self._entid(resource)
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from typing import Any, Optional, Sequence, TypeAlias, Literal, IO
|
|
2
2
|
from ..base_api import ApiConfig, BaseApi
|
|
3
|
-
from ..entity_base_api import
|
|
4
|
-
from .annotations_api import AnnotationsApi
|
|
5
|
-
from .projects_api import ProjectsApi
|
|
3
|
+
from ..entity_base_api import CreatableEntityApi, DeletableEntityApi
|
|
6
4
|
from datamint.entities.resource import Resource
|
|
5
|
+
from datamint.entities.project import Project
|
|
7
6
|
from datamint.entities.annotation import Annotation
|
|
8
7
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
9
8
|
import httpx
|
|
@@ -23,10 +22,9 @@ import asyncio
|
|
|
23
22
|
import aiohttp
|
|
24
23
|
from pathlib import Path
|
|
25
24
|
import nest_asyncio # For running asyncio in jupyter notebooks
|
|
26
|
-
import cv2
|
|
27
25
|
from PIL import Image
|
|
28
|
-
from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
|
|
29
26
|
import io
|
|
27
|
+
from datamint.types import ImagingData
|
|
30
28
|
|
|
31
29
|
|
|
32
30
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -54,17 +52,25 @@ def _open_io(file_path: str | Path | IO, mode: str = 'rb') -> IO:
|
|
|
54
52
|
class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
55
53
|
"""API handler for resource-related endpoints."""
|
|
56
54
|
|
|
57
|
-
def __init__(self,
|
|
55
|
+
def __init__(self,
|
|
56
|
+
config: ApiConfig,
|
|
57
|
+
client: Optional[httpx.Client] = None,
|
|
58
|
+
annotations_api=None,
|
|
59
|
+
projects_api=None
|
|
60
|
+
) -> None:
|
|
58
61
|
"""Initialize the resources API handler.
|
|
59
62
|
|
|
60
63
|
Args:
|
|
61
64
|
config: API configuration containing base URL, API key, etc.
|
|
62
65
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
63
66
|
"""
|
|
67
|
+
from .annotations_api import AnnotationsApi
|
|
68
|
+
from .projects_api import ProjectsApi
|
|
64
69
|
super().__init__(config, Resource, 'resources', client)
|
|
65
70
|
nest_asyncio.apply()
|
|
66
|
-
self.annotations_api = AnnotationsApi(
|
|
67
|
-
|
|
71
|
+
self.annotations_api = AnnotationsApi(
|
|
72
|
+
config, client, resources_api=self) if annotations_api is None else annotations_api
|
|
73
|
+
self.projects_api = ProjectsApi(config, client) if projects_api is None else projects_api
|
|
68
74
|
|
|
69
75
|
def get_list(self,
|
|
70
76
|
status: Optional[ResourceStatus] = None,
|
|
@@ -710,21 +716,6 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
710
716
|
# This should not happen with single file uploads, but handle it just in case
|
|
711
717
|
raise DatamintException(f"Unexpected return from upload_resources: {type(result)} | {result}")
|
|
712
718
|
|
|
713
|
-
def _determine_mimetype(self,
|
|
714
|
-
content,
|
|
715
|
-
resource: str | Resource) -> tuple[str | None, str | None]:
|
|
716
|
-
# Determine mimetype from file content
|
|
717
|
-
mimetype_list, ext = guess_typez(content, use_magic=True)
|
|
718
|
-
mimetype = mimetype_list[-1]
|
|
719
|
-
|
|
720
|
-
# get mimetype from resource info if not detected
|
|
721
|
-
if mimetype is None or mimetype == DEFAULT_MIME_TYPE:
|
|
722
|
-
if not isinstance(resource, Resource):
|
|
723
|
-
resource = self.get_by_id(resource)
|
|
724
|
-
mimetype = resource.mimetype or mimetype
|
|
725
|
-
|
|
726
|
-
return mimetype, ext
|
|
727
|
-
|
|
728
719
|
async def _async_download_file(self,
|
|
729
720
|
resource: str | Resource,
|
|
730
721
|
save_path: str | Path,
|
|
@@ -761,8 +752,8 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
761
752
|
f.write(data_bytes)
|
|
762
753
|
|
|
763
754
|
# Determine mimetype from file content
|
|
764
|
-
mimetype, ext =
|
|
765
|
-
|
|
755
|
+
mimetype, ext = BaseApi._determine_mimetype(content=data_bytes,
|
|
756
|
+
declared_mimetype=resource.mimetype if isinstance(resource, Resource) else None)
|
|
766
757
|
|
|
767
758
|
# Generate final path with extension if needed
|
|
768
759
|
if mimetype is not None and mimetype != DEFAULT_MIME_TYPE:
|
|
@@ -850,7 +841,7 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
850
841
|
save_path: Optional[str] = None,
|
|
851
842
|
auto_convert: bool = True,
|
|
852
843
|
add_extension: bool = False
|
|
853
|
-
) ->
|
|
844
|
+
) -> ImagingData | tuple[ImagingData, str] | bytes:
|
|
854
845
|
"""
|
|
855
846
|
Download a resource file.
|
|
856
847
|
|
|
@@ -888,8 +879,8 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
888
879
|
mimetype = None
|
|
889
880
|
ext = None
|
|
890
881
|
if auto_convert or add_extension:
|
|
891
|
-
mimetype, ext =
|
|
892
|
-
|
|
882
|
+
mimetype, ext = BaseApi._determine_mimetype(content=response.content,
|
|
883
|
+
declared_mimetype=resource.mimetype if isinstance(resource, Resource) else None)
|
|
893
884
|
if auto_convert:
|
|
894
885
|
if mimetype is None:
|
|
895
886
|
_LOGGER.warning("Could not determine mimetype. Returning a bytes array.")
|
|
@@ -997,6 +988,12 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
997
988
|
resource: str | Resource,
|
|
998
989
|
tags: Sequence[str],
|
|
999
990
|
):
|
|
991
|
+
"""
|
|
992
|
+
Set tags for a resource, IMPORTANT: This replaces all existing tags.
|
|
993
|
+
Args:
|
|
994
|
+
resource: The resource unique id or Resource object.
|
|
995
|
+
tags: The tags to set.
|
|
996
|
+
"""
|
|
1000
997
|
data = {'tags': tags}
|
|
1001
998
|
resource_id = self._entid(resource)
|
|
1002
999
|
|
|
@@ -1005,3 +1002,53 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
|
|
|
1005
1002
|
add_path='tags',
|
|
1006
1003
|
json=data)
|
|
1007
1004
|
return response
|
|
1005
|
+
|
|
1006
|
+
# def get_projects(self, resource: Resource) -> Sequence[Project]:
|
|
1007
|
+
# """
|
|
1008
|
+
# Get all projects this resource belongs to.
|
|
1009
|
+
|
|
1010
|
+
# Args:
|
|
1011
|
+
# resource: The Resource instance.
|
|
1012
|
+
|
|
1013
|
+
# Returns:
|
|
1014
|
+
# List of Project instances
|
|
1015
|
+
# """
|
|
1016
|
+
# resource._ensure_attr('projects')
|
|
1017
|
+
# proj_ids = [p['id'] for p in resource.projects]
|
|
1018
|
+
# return [proj for proj in self.projects_api.get_all() if proj.id in proj_ids]
|
|
1019
|
+
|
|
1020
|
+
def add_tags(self,
|
|
1021
|
+
resource: str | Resource,
|
|
1022
|
+
tags: Sequence[str],
|
|
1023
|
+
):
|
|
1024
|
+
"""
|
|
1025
|
+
Add tags to a resource, IMPORTANT: This appends to existing tags.
|
|
1026
|
+
Args:
|
|
1027
|
+
resource: The resource unique id or Resource object.
|
|
1028
|
+
tags: The tags to add.
|
|
1029
|
+
"""
|
|
1030
|
+
if isinstance(resource, str):
|
|
1031
|
+
resource = self.get_by_id(resource)
|
|
1032
|
+
old_tags = resource.tags if resource.tags is not None else []
|
|
1033
|
+
return self.set_tags(resource, old_tags + list(tags))
|
|
1034
|
+
|
|
1035
|
+
def bulk_delete(self, entities: Sequence[str | Resource]) -> None:
|
|
1036
|
+
"""Delete multiple entities. Faster than deleting them one by one.
|
|
1037
|
+
|
|
1038
|
+
Args:
|
|
1039
|
+
entities: Sequence of unique identifiers for the entities to delete or the entity instances themselves.
|
|
1040
|
+
|
|
1041
|
+
Raises:
|
|
1042
|
+
httpx.HTTPStatusError: If deletion fails or any entity not found
|
|
1043
|
+
"""
|
|
1044
|
+
from math import ceil
|
|
1045
|
+
|
|
1046
|
+
resources_ids = [self._entid(ent) for ent in entities]
|
|
1047
|
+
if len(resources_ids) == 0:
|
|
1048
|
+
return
|
|
1049
|
+
batch_size = 200
|
|
1050
|
+
for i in range(0, ceil(len(resources_ids)/batch_size)):
|
|
1051
|
+
batch_ids = resources_ids[i*batch_size:(i+1)*batch_size]
|
|
1052
|
+
self._make_request('DELETE',
|
|
1053
|
+
f'{self.endpoint_base}',
|
|
1054
|
+
params={'resource_ids': ','.join(batch_ids)})
|
datamint/api/entity_base_api.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from typing import Any, TypeVar, Generic, Type, Sequence
|
|
2
2
|
import logging
|
|
3
3
|
import httpx
|
|
4
|
-
from dataclasses import dataclass
|
|
5
4
|
from datamint.entities.base_entity import BaseEntity
|
|
6
5
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
7
6
|
import aiohttp
|
|
@@ -37,9 +36,14 @@ class EntityBaseApi(BaseApi, Generic[T]):
|
|
|
37
36
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
38
37
|
"""
|
|
39
38
|
super().__init__(config, client)
|
|
40
|
-
self.
|
|
39
|
+
self.__entity_class = entity_class
|
|
41
40
|
self.endpoint_base = endpoint_base.strip('/')
|
|
42
41
|
|
|
42
|
+
def _init_entity_obj(self, **kwargs) -> T:
|
|
43
|
+
obj = self.__entity_class(**kwargs)
|
|
44
|
+
obj._api = self
|
|
45
|
+
return obj
|
|
46
|
+
|
|
43
47
|
@staticmethod
|
|
44
48
|
def _entid(entity: BaseEntity | str) -> str:
|
|
45
49
|
return entity if isinstance(entity, str) else entity.id
|
|
@@ -117,7 +121,7 @@ class EntityBaseApi(BaseApi, Generic[T]):
|
|
|
117
121
|
for resp, items in items_gen:
|
|
118
122
|
all_items.extend(items)
|
|
119
123
|
|
|
120
|
-
return [self.
|
|
124
|
+
return [self._init_entity_obj(**item) for item in all_items]
|
|
121
125
|
|
|
122
126
|
def get_all(self, limit: int | None = None) -> Sequence[T]:
|
|
123
127
|
"""Get all entities with optional pagination and filtering.
|
|
@@ -143,7 +147,7 @@ class EntityBaseApi(BaseApi, Generic[T]):
|
|
|
143
147
|
httpx.HTTPStatusError: If the entity is not found or request fails.
|
|
144
148
|
"""
|
|
145
149
|
response = self._make_entity_request('GET', entity_id)
|
|
146
|
-
return self.
|
|
150
|
+
return self._init_entity_obj(**response.json())
|
|
147
151
|
|
|
148
152
|
async def _create_async(self, entity_data: dict[str, Any]) -> str | Sequence[str | dict]:
|
|
149
153
|
"""Create a new entity.
|
|
@@ -177,42 +181,6 @@ class EntityBaseApi(BaseApi, Generic[T]):
|
|
|
177
181
|
add_path=child_entity_name)
|
|
178
182
|
return response
|
|
179
183
|
|
|
180
|
-
# def bulk_create(self, entities_data: list[dict[str, Any]]) -> list[T]:
|
|
181
|
-
# """Create multiple entities in a single request.
|
|
182
|
-
|
|
183
|
-
# Args:
|
|
184
|
-
# entities_data: List of dictionaries containing entity data
|
|
185
|
-
|
|
186
|
-
# Returns:
|
|
187
|
-
# List of created entity instances
|
|
188
|
-
|
|
189
|
-
# Raises:
|
|
190
|
-
# httpx.HTTPStatusError: If bulk creation fails
|
|
191
|
-
# """
|
|
192
|
-
# payload = {'items': entities_data} # Common bulk API format
|
|
193
|
-
# response = self._make_request('POST', f'/{self.endpoint_base}/bulk', json=payload)
|
|
194
|
-
# data = response.json()
|
|
195
|
-
|
|
196
|
-
# # Handle response format - may be direct list or wrapped
|
|
197
|
-
# items = data if isinstance(data, list) else data.get('items', [])
|
|
198
|
-
# return [self.entity_class(**item) for item in items]
|
|
199
|
-
|
|
200
|
-
# def count(self, **params: Any) -> int:
|
|
201
|
-
# """Get the total count of entities matching the given filters.
|
|
202
|
-
|
|
203
|
-
# Args:
|
|
204
|
-
# **params: Query parameters for filtering
|
|
205
|
-
|
|
206
|
-
# Returns:
|
|
207
|
-
# Total count of matching entities
|
|
208
|
-
|
|
209
|
-
# Raises:
|
|
210
|
-
# httpx.HTTPStatusError: If the request fails
|
|
211
|
-
# """
|
|
212
|
-
# response = self._make_request('GET', f'/{self.endpoint_base}/count', params=params)
|
|
213
|
-
# data = response.json()
|
|
214
|
-
# return data.get('count', 0) if isinstance(data, dict) else data
|
|
215
|
-
|
|
216
184
|
|
|
217
185
|
class DeletableEntityApi(EntityBaseApi[T]):
|
|
218
186
|
"""Extension of EntityBaseApi for entities that support soft deletion.
|
|
@@ -221,7 +189,7 @@ class DeletableEntityApi(EntityBaseApi[T]):
|
|
|
221
189
|
retrieval and restoration of such entities.
|
|
222
190
|
"""
|
|
223
191
|
|
|
224
|
-
def delete(self, entity: str |
|
|
192
|
+
def delete(self, entity: str | T) -> None:
|
|
225
193
|
"""Delete an entity.
|
|
226
194
|
|
|
227
195
|
Args:
|
|
@@ -232,7 +200,7 @@ class DeletableEntityApi(EntityBaseApi[T]):
|
|
|
232
200
|
"""
|
|
233
201
|
self._make_entity_request('DELETE', entity)
|
|
234
202
|
|
|
235
|
-
def bulk_delete(self, entities: Sequence[str |
|
|
203
|
+
def bulk_delete(self, entities: Sequence[str | T]) -> None:
|
|
236
204
|
"""Delete multiple entities.
|
|
237
205
|
|
|
238
206
|
Args:
|
|
@@ -264,7 +232,7 @@ class DeletableEntityApi(EntityBaseApi[T]):
|
|
|
264
232
|
httpx.HTTPStatusError: If deletion fails or entity not found
|
|
265
233
|
"""
|
|
266
234
|
async with self._make_entity_request_async('DELETE', entity,
|
|
267
|
-
|
|
235
|
+
session=session) as resp:
|
|
268
236
|
await resp.text() # Consume response to complete request
|
|
269
237
|
|
|
270
238
|
# def get_deleted(self, **kwargs) -> Sequence[T]:
|