datamint 1.3.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datamint might be problematic. Click here for more details.
- {datamint-1.3.0 → datamint-1.4.0}/PKG-INFO +1 -1
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/annotation_api_handler.py +138 -27
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/dto/annotation_dto.py +50 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/root_api_handler.py +44 -5
- {datamint-1.3.0 → datamint-1.4.0}/datamint/client_cmd_tools/datamint_upload.py +116 -7
- {datamint-1.3.0 → datamint-1.4.0}/pyproject.toml +1 -1
- {datamint-1.3.0 → datamint-1.4.0}/README.md +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/__init__.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/api_handler.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/base_api_handler.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/apihandler/exp_api_handler.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/client_cmd_tools/__init__.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/client_cmd_tools/datamint_config.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/configs.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/dataset/__init__.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/dataset/base_dataset.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/dataset/dataset.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/examples/__init__.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/examples/example_projects.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/experiment/__init__.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/experiment/_patcher.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/experiment/experiment.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/logging.yaml +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/utils/dicom_utils.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/utils/io_utils.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/utils/logging_utils.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/utils/torchmetrics.py +0 -0
- {datamint-1.3.0 → datamint-1.4.0}/datamint/utils/visualization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: datamint
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -11,7 +11,7 @@ import asyncio
|
|
|
11
11
|
import aiohttp
|
|
12
12
|
from requests.exceptions import HTTPError
|
|
13
13
|
from deprecated.sphinx import deprecated
|
|
14
|
-
from .dto.annotation_dto import CreateAnnotationDto, LineGeometry, CoordinateSystem, AnnotationType
|
|
14
|
+
from .dto.annotation_dto import CreateAnnotationDto, LineGeometry, BoxGeometry, CoordinateSystem, AnnotationType
|
|
15
15
|
import pydicom
|
|
16
16
|
|
|
17
17
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -353,7 +353,7 @@ class AnnotationAPIHandler(BaseAPIHandler):
|
|
|
353
353
|
author_email: Optional[str] = None,
|
|
354
354
|
model_id: Optional[str] = None,
|
|
355
355
|
project: Optional[str] = None,
|
|
356
|
-
):
|
|
356
|
+
) -> list[str]:
|
|
357
357
|
"""
|
|
358
358
|
Add annotations to a resource.
|
|
359
359
|
|
|
@@ -415,6 +415,66 @@ class AnnotationAPIHandler(BaseAPIHandler):
|
|
|
415
415
|
|
|
416
416
|
resp = self._run_request(request_params)
|
|
417
417
|
self._check_errors_response_json(resp)
|
|
418
|
+
return resp.json()
|
|
419
|
+
|
|
420
|
+
def _create_geometry_annotation(self,
|
|
421
|
+
geometry: LineGeometry | BoxGeometry,
|
|
422
|
+
resource_id: str,
|
|
423
|
+
identifier: str,
|
|
424
|
+
frame_index: int | None = None,
|
|
425
|
+
project: Optional[str] = None,
|
|
426
|
+
worklist_id: Optional[str] = None,
|
|
427
|
+
imported_from: Optional[str] = None,
|
|
428
|
+
author_email: Optional[str] = None,
|
|
429
|
+
model_id: Optional[str] = None) -> list[str]:
|
|
430
|
+
"""
|
|
431
|
+
Common method for creating geometry-based annotations.
|
|
432
|
+
|
|
433
|
+
Args:
|
|
434
|
+
geometry: The geometry object (LineGeometry or BoxGeometry)
|
|
435
|
+
resource_id: The resource unique id
|
|
436
|
+
identifier: The annotation identifier
|
|
437
|
+
frame_index: The frame index of the annotation
|
|
438
|
+
project: The project unique id or name
|
|
439
|
+
worklist_id: The annotation worklist unique id
|
|
440
|
+
imported_from: The imported from source value
|
|
441
|
+
author_email: The email to consider as the author of the annotation
|
|
442
|
+
model_id: The model unique id
|
|
443
|
+
"""
|
|
444
|
+
if project is not None and worklist_id is not None:
|
|
445
|
+
raise ValueError('Only one of project or worklist_id can be provided.')
|
|
446
|
+
|
|
447
|
+
if project is not None:
|
|
448
|
+
proj = self.get_project_by_name(project)
|
|
449
|
+
if 'error' in proj.keys():
|
|
450
|
+
raise DatamintException(f"Project {project} not found.")
|
|
451
|
+
worklist_id = proj['worklist_id']
|
|
452
|
+
|
|
453
|
+
anndto = CreateAnnotationDto(
|
|
454
|
+
type=geometry.type,
|
|
455
|
+
identifier=identifier,
|
|
456
|
+
scope='frame',
|
|
457
|
+
annotation_worklist_id=worklist_id,
|
|
458
|
+
value=None,
|
|
459
|
+
imported_from=imported_from,
|
|
460
|
+
import_author=author_email,
|
|
461
|
+
frame_index=frame_index,
|
|
462
|
+
geometry=geometry,
|
|
463
|
+
model_id=model_id,
|
|
464
|
+
is_model=model_id is not None,
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
json_data = anndto.to_dict()
|
|
468
|
+
|
|
469
|
+
request_params = {
|
|
470
|
+
'method': 'POST',
|
|
471
|
+
'url': f'{self.root_url}/annotations/{resource_id}/annotations',
|
|
472
|
+
'json': [json_data]
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
resp = self._run_request(request_params)
|
|
476
|
+
self._check_errors_response_json(resp)
|
|
477
|
+
return resp.json()
|
|
418
478
|
|
|
419
479
|
def add_line_annotation(self,
|
|
420
480
|
point1: tuple[int, int] | tuple[float, float, float],
|
|
@@ -428,7 +488,7 @@ class AnnotationAPIHandler(BaseAPIHandler):
|
|
|
428
488
|
worklist_id: Optional[str] = None,
|
|
429
489
|
imported_from: Optional[str] = None,
|
|
430
490
|
author_email: Optional[str] = None,
|
|
431
|
-
model_id: Optional[str] = None):
|
|
491
|
+
model_id: Optional[str] = None) -> list[str]:
|
|
432
492
|
"""
|
|
433
493
|
Add a line annotation to a resource.
|
|
434
494
|
|
|
@@ -466,12 +526,6 @@ class AnnotationAPIHandler(BaseAPIHandler):
|
|
|
466
526
|
if project is not None and worklist_id is not None:
|
|
467
527
|
raise ValueError('Only one of project or worklist_id can be provided.')
|
|
468
528
|
|
|
469
|
-
if project is not None:
|
|
470
|
-
proj = self.get_project_by_name(project)
|
|
471
|
-
if 'error' in proj.keys():
|
|
472
|
-
raise DatamintException(f"Project {project} not found.")
|
|
473
|
-
worklist_id = proj['worklist_id']
|
|
474
|
-
|
|
475
529
|
if coords_system == 'pixel':
|
|
476
530
|
if dicom_metadata is None:
|
|
477
531
|
point1 = (point1[0], point1[1], frame_index)
|
|
@@ -486,30 +540,87 @@ class AnnotationAPIHandler(BaseAPIHandler):
|
|
|
486
540
|
else:
|
|
487
541
|
raise ValueError(f"Unknown coordinate system: {coords_system}")
|
|
488
542
|
|
|
489
|
-
|
|
490
|
-
|
|
543
|
+
return self._create_geometry_annotation(
|
|
544
|
+
geometry=geom,
|
|
545
|
+
resource_id=resource_id,
|
|
491
546
|
identifier=identifier,
|
|
492
|
-
scope='frame',
|
|
493
|
-
annotation_worklist_id=worklist_id,
|
|
494
|
-
value=None,
|
|
495
|
-
imported_from=imported_from,
|
|
496
|
-
import_author=author_email,
|
|
497
547
|
frame_index=frame_index,
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
548
|
+
project=project,
|
|
549
|
+
worklist_id=worklist_id,
|
|
550
|
+
imported_from=imported_from,
|
|
551
|
+
author_email=author_email,
|
|
552
|
+
model_id=model_id
|
|
501
553
|
)
|
|
502
554
|
|
|
503
|
-
|
|
555
|
+
def add_box_annotation(self,
|
|
556
|
+
point1: tuple[int, int] | tuple[float, float, float],
|
|
557
|
+
point2: tuple[int, int] | tuple[float, float, float],
|
|
558
|
+
resource_id: str,
|
|
559
|
+
identifier: str,
|
|
560
|
+
frame_index: int | None = None,
|
|
561
|
+
dicom_metadata: pydicom.Dataset | str | None = None,
|
|
562
|
+
coords_system: CoordinateSystem = 'pixel',
|
|
563
|
+
project: Optional[str] = None,
|
|
564
|
+
worklist_id: Optional[str] = None,
|
|
565
|
+
imported_from: Optional[str] = None,
|
|
566
|
+
author_email: Optional[str] = None,
|
|
567
|
+
model_id: Optional[str] = None):
|
|
568
|
+
"""
|
|
569
|
+
Add a box annotation to a resource.
|
|
504
570
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
571
|
+
Args:
|
|
572
|
+
point1: The first corner point of the box. Can be a 2d or 3d point.
|
|
573
|
+
If `coords_system` is 'pixel', it must be a 2d point representing pixel coordinates.
|
|
574
|
+
If `coords_system` is 'patient', it must be a 3d point representing patient coordinates.
|
|
575
|
+
point2: The opposite diagonal corner point of the box. See `point1` for more details.
|
|
576
|
+
resource_id: The resource unique id.
|
|
577
|
+
identifier: The annotation identifier, also known as the annotation's label.
|
|
578
|
+
frame_index: The frame index of the annotation.
|
|
579
|
+
dicom_metadata: The DICOM metadata of the image. If provided, coordinates will be converted
|
|
580
|
+
automatically using the DICOM metadata.
|
|
581
|
+
coords_system: The coordinate system of the points. Can be 'pixel' or 'patient'.
|
|
582
|
+
If 'pixel', points are in pixel coordinates. If 'patient', points are in patient coordinates.
|
|
583
|
+
project: The project unique id or name.
|
|
584
|
+
worklist_id: The annotation worklist unique id. Optional.
|
|
585
|
+
imported_from: The imported from source value.
|
|
586
|
+
author_email: The email to consider as the author of the annotation. If None, uses the API key customer.
|
|
587
|
+
model_id: The model unique id. Optional.
|
|
510
588
|
|
|
511
|
-
|
|
512
|
-
|
|
589
|
+
Example:
|
|
590
|
+
.. code-block:: python
|
|
591
|
+
|
|
592
|
+
res_id = 'aa93813c-cef0-4edd-a45c-85d4a8f1ad0d'
|
|
593
|
+
api.add_box_annotation([10, 10], (50, 40),
|
|
594
|
+
resource_id=res_id,
|
|
595
|
+
identifier='BoundingBox1',
|
|
596
|
+
frame_index=2,
|
|
597
|
+
project='Example Project')
|
|
598
|
+
"""
|
|
599
|
+
if coords_system == 'pixel':
|
|
600
|
+
if dicom_metadata is None:
|
|
601
|
+
point1 = (point1[0], point1[1], frame_index)
|
|
602
|
+
point2 = (point2[0], point2[1], frame_index)
|
|
603
|
+
geom = BoxGeometry(point1, point2)
|
|
604
|
+
else:
|
|
605
|
+
if isinstance(dicom_metadata, str):
|
|
606
|
+
dicom_metadata = pydicom.dcmread(dicom_metadata)
|
|
607
|
+
geom = BoxGeometry.from_dicom(dicom_metadata, point1, point2, slice_index=frame_index)
|
|
608
|
+
elif coords_system == 'patient':
|
|
609
|
+
geom = BoxGeometry(point1, point2)
|
|
610
|
+
else:
|
|
611
|
+
raise ValueError(f"Unknown coordinate system: {coords_system}")
|
|
612
|
+
|
|
613
|
+
return self._create_geometry_annotation(
|
|
614
|
+
geometry=geom,
|
|
615
|
+
resource_id=resource_id,
|
|
616
|
+
identifier=identifier,
|
|
617
|
+
frame_index=frame_index,
|
|
618
|
+
project=project,
|
|
619
|
+
worklist_id=worklist_id,
|
|
620
|
+
imported_from=imported_from,
|
|
621
|
+
author_email=author_email,
|
|
622
|
+
model_id=model_id
|
|
623
|
+
)
|
|
513
624
|
|
|
514
625
|
@deprecated(version='0.12.1', reason='Use :meth:`~get_annotations` instead with `resource_id` parameter.')
|
|
515
626
|
def get_resource_annotations(self,
|
|
@@ -97,6 +97,56 @@ class LineGeometry(Geometry):
|
|
|
97
97
|
return LineGeometry(new_point1, new_point2)
|
|
98
98
|
|
|
99
99
|
|
|
100
|
+
class BoxGeometry(Geometry):
|
|
101
|
+
def __init__(self, point1: tuple[float, float, float],
|
|
102
|
+
point2: tuple[float, float, float]):
|
|
103
|
+
"""
|
|
104
|
+
Create a box geometry from two diagonal corner points.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
point1: First corner point (x, y, z) or (x, y, frame_index)
|
|
108
|
+
point2: Opposite diagonal corner point (x, y, z) or (x, y, frame_index)
|
|
109
|
+
"""
|
|
110
|
+
super().__init__(AnnotationType.SQUARE) # Using SQUARE as the box type
|
|
111
|
+
if isinstance(point1, np.ndarray):
|
|
112
|
+
point1 = point1.tolist()
|
|
113
|
+
if isinstance(point2, np.ndarray):
|
|
114
|
+
point2 = point2.tolist()
|
|
115
|
+
self.point1 = point1
|
|
116
|
+
self.point2 = point2
|
|
117
|
+
|
|
118
|
+
def to_dict(self) -> dict:
|
|
119
|
+
return {
|
|
120
|
+
'points': [self.point1, self.point2],
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def from_dicom(ds: pydicom.Dataset,
|
|
125
|
+
point1: tuple[int, int],
|
|
126
|
+
point2: tuple[int, int],
|
|
127
|
+
slice_index: int | None = None) -> 'BoxGeometry':
|
|
128
|
+
"""
|
|
129
|
+
Create a box geometry from DICOM pixel coordinates.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
ds: DICOM dataset containing spatial metadata
|
|
133
|
+
point1: First corner in pixel coordinates (x, y)
|
|
134
|
+
point2: Opposite corner in pixel coordinates (x, y)
|
|
135
|
+
slice_index: The slice/frame index for 3D positioning
|
|
136
|
+
|
|
137
|
+
Returns:
|
|
138
|
+
BoxGeometry with patient coordinate points
|
|
139
|
+
"""
|
|
140
|
+
pixel_x1, pixel_y1 = point1
|
|
141
|
+
pixel_x2, pixel_y2 = point2
|
|
142
|
+
|
|
143
|
+
new_point1 = pixel_to_patient(ds, pixel_x1, pixel_y1,
|
|
144
|
+
slice_index=slice_index)
|
|
145
|
+
new_point2 = pixel_to_patient(ds, pixel_x2, pixel_y2,
|
|
146
|
+
slice_index=slice_index)
|
|
147
|
+
return BoxGeometry(new_point1, new_point2)
|
|
148
|
+
|
|
149
|
+
|
|
100
150
|
class CreateAnnotationDto:
|
|
101
151
|
def __init__(self,
|
|
102
152
|
type: AnnotationType | str,
|
|
@@ -63,6 +63,7 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
63
63
|
session=None,
|
|
64
64
|
modality: Optional[str] = None,
|
|
65
65
|
publish: bool = False,
|
|
66
|
+
metadata_file: Optional[str] = None,
|
|
66
67
|
) -> str:
|
|
67
68
|
if _is_io_object(file_path):
|
|
68
69
|
name = file_path.name
|
|
@@ -97,6 +98,8 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
97
98
|
is_a_dicom_file = is_dicom(name) or is_dicom(file_path)
|
|
98
99
|
if is_a_dicom_file:
|
|
99
100
|
mimetype = 'application/dicom'
|
|
101
|
+
elif name.endswith('.nii') or name.endswith('.nii.gz'):
|
|
102
|
+
mimetype = 'application/x-nifti'
|
|
100
103
|
|
|
101
104
|
filename = os.path.basename(name)
|
|
102
105
|
_LOGGER.debug(f"File name '{filename}' mimetype: {mimetype}")
|
|
@@ -115,6 +118,25 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
115
118
|
f = _open_io(file_path)
|
|
116
119
|
|
|
117
120
|
try:
|
|
121
|
+
metadata_content = None
|
|
122
|
+
metadata_dict = None
|
|
123
|
+
if metadata_file is not None:
|
|
124
|
+
try:
|
|
125
|
+
with open(metadata_file, 'r') as metadata_f:
|
|
126
|
+
metadata_content = metadata_f.read()
|
|
127
|
+
metadata_dict = json.loads(metadata_content)
|
|
128
|
+
metadata_dict_lower = {k.lower(): v for k, v in metadata_dict.items() if isinstance(k, str)}
|
|
129
|
+
try:
|
|
130
|
+
if modality is None:
|
|
131
|
+
if 'modality' in metadata_dict_lower:
|
|
132
|
+
modality = metadata_dict_lower['modality']
|
|
133
|
+
except Exception as e:
|
|
134
|
+
_LOGGER.debug(f"Failed to extract modality from metadata file {metadata_file}: {e}")
|
|
135
|
+
_LOGGER.debug(f"Metadata dict: {metadata_dict}")
|
|
136
|
+
except Exception as e:
|
|
137
|
+
_LOGGER.warning(f"Failed to read metadata file {metadata_file}: {e}")
|
|
138
|
+
|
|
139
|
+
|
|
118
140
|
form = aiohttp.FormData()
|
|
119
141
|
url = self._get_endpoint_url(RootAPIHandler.ENDPOINT_RESOURCES)
|
|
120
142
|
file_key = 'resource'
|
|
@@ -134,6 +156,14 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
134
156
|
tags = ','.join([l.strip() for l in tags])
|
|
135
157
|
form.add_field('tags', tags)
|
|
136
158
|
|
|
159
|
+
# Add JSON metadata if provided
|
|
160
|
+
if metadata_content is not None:
|
|
161
|
+
try:
|
|
162
|
+
_LOGGER.debug(f"Adding metadata from {metadata_file}")
|
|
163
|
+
form.add_field('metadata', metadata_content, content_type='application/json')
|
|
164
|
+
except Exception as e:
|
|
165
|
+
_LOGGER.warning(f"Failed to read metadata file {metadata_file}: {e}")
|
|
166
|
+
|
|
137
167
|
request_params = {
|
|
138
168
|
'method': 'POST',
|
|
139
169
|
'url': url,
|
|
@@ -170,6 +200,7 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
170
200
|
publish: bool = False,
|
|
171
201
|
segmentation_files: Optional[list[dict]] = None,
|
|
172
202
|
transpose_segmentation: bool = False,
|
|
203
|
+
metadata_files: Optional[list[Optional[str]]] = None,
|
|
173
204
|
) -> list[str]:
|
|
174
205
|
if on_error not in ['raise', 'skip']:
|
|
175
206
|
raise ValueError("on_error must be either 'raise' or 'skip'")
|
|
@@ -177,8 +208,11 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
177
208
|
if segmentation_files is None:
|
|
178
209
|
segmentation_files = _infinite_gen(None)
|
|
179
210
|
|
|
211
|
+
if metadata_files is None:
|
|
212
|
+
metadata_files = _infinite_gen(None)
|
|
213
|
+
|
|
180
214
|
async with aiohttp.ClientSession() as session:
|
|
181
|
-
async def __upload_single_resource(file_path, segfiles: dict):
|
|
215
|
+
async def __upload_single_resource(file_path, segfiles: dict, metadata_file: Optional[str]):
|
|
182
216
|
async with self.semaphore:
|
|
183
217
|
rid = await self._upload_single_resource_async(
|
|
184
218
|
file_path=file_path,
|
|
@@ -191,6 +225,7 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
191
225
|
channel=channel,
|
|
192
226
|
modality=modality,
|
|
193
227
|
publish=publish,
|
|
228
|
+
metadata_file=metadata_file,
|
|
194
229
|
)
|
|
195
230
|
if segfiles is not None:
|
|
196
231
|
fpaths = segfiles['files']
|
|
@@ -208,7 +243,8 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
208
243
|
transpose_segmentation=transpose_segmentation)
|
|
209
244
|
return rid
|
|
210
245
|
|
|
211
|
-
tasks = [__upload_single_resource(f, segfiles
|
|
246
|
+
tasks = [__upload_single_resource(f, segfiles, metadata_file)
|
|
247
|
+
for f, segfiles, metadata_file in zip(files_path, segmentation_files, metadata_files)]
|
|
212
248
|
return await asyncio.gather(*tasks, return_exceptions=on_error == 'skip')
|
|
213
249
|
|
|
214
250
|
def _assemble_dicoms(self, files_path: Sequence[str | IO]) -> tuple[Sequence[str | IO], bool]:
|
|
@@ -248,7 +284,8 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
248
284
|
segmentation_files: Optional[list[Union[list[str], dict]]] = None,
|
|
249
285
|
transpose_segmentation: bool = False,
|
|
250
286
|
modality: Optional[str] = None,
|
|
251
|
-
assemble_dicoms: bool = True
|
|
287
|
+
assemble_dicoms: bool = True,
|
|
288
|
+
metadata_files: Optional[list[Optional[str]]] = None
|
|
252
289
|
) -> list[str | Exception] | str | Exception:
|
|
253
290
|
"""
|
|
254
291
|
Upload resources.
|
|
@@ -274,6 +311,7 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
274
311
|
transpose_segmentation (bool): Whether to transpose the segmentation files or not.
|
|
275
312
|
modality (Optional[str]): The modality of the resources.
|
|
276
313
|
assemble_dicoms (bool): Whether to assemble the dicom files or not based on the SOPInstanceUID and InstanceNumber attributes.
|
|
314
|
+
metadata_files (Optional[list[Optional[str]]]): JSON metadata files to include with each resource.
|
|
277
315
|
|
|
278
316
|
Raises:
|
|
279
317
|
ResourceNotFoundError: If `publish_to` is supplied, and the project does not exists.
|
|
@@ -319,6 +357,7 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
319
357
|
segmentation_files=segmentation_files,
|
|
320
358
|
transpose_segmentation=transpose_segmentation,
|
|
321
359
|
modality=modality,
|
|
360
|
+
metadata_files=metadata_files,
|
|
322
361
|
)
|
|
323
362
|
|
|
324
363
|
resource_ids = loop.run_until_complete(task)
|
|
@@ -690,13 +729,13 @@ class RootAPIHandler(BaseAPIHandler):
|
|
|
690
729
|
'url': url}
|
|
691
730
|
try:
|
|
692
731
|
response = self._run_request(request_params)
|
|
693
|
-
|
|
732
|
+
|
|
694
733
|
# Get mimetype if needed for auto_convert or add_extension
|
|
695
734
|
mimetype = None
|
|
696
735
|
if auto_convert or add_extension:
|
|
697
736
|
resource_info = self.get_resources_by_ids(resource_id)
|
|
698
737
|
mimetype = resource_info['mimetype']
|
|
699
|
-
|
|
738
|
+
|
|
700
739
|
if auto_convert:
|
|
701
740
|
try:
|
|
702
741
|
resource_file = BaseAPIHandler.convert_format(response.content,
|
|
@@ -256,12 +256,86 @@ def _find_segmentation_files(segmentation_root_path: str,
|
|
|
256
256
|
return segmentation_files
|
|
257
257
|
|
|
258
258
|
|
|
259
|
-
def
|
|
259
|
+
def _find_json_metadata(file_path: str | Path) -> Optional[str]:
|
|
260
|
+
"""
|
|
261
|
+
Find a JSON file with the same base name as the given file.
|
|
262
|
+
|
|
263
|
+
Args:
|
|
264
|
+
file_path (str): Path to the main file (e.g., NIFTI file)
|
|
265
|
+
|
|
266
|
+
Returns:
|
|
267
|
+
Optional[str]: Path to the JSON metadata file if found, None otherwise
|
|
268
|
+
"""
|
|
269
|
+
file_path = Path(file_path)
|
|
270
|
+
json_path = file_path.with_suffix('.json')
|
|
271
|
+
|
|
272
|
+
if json_path.exists() and json_path.is_file():
|
|
273
|
+
_LOGGER.debug(f"Found JSON metadata file: {json_path}")
|
|
274
|
+
return str(json_path)
|
|
275
|
+
|
|
276
|
+
return None
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _collect_metadata_files(files_path: list[str], auto_detect_json: bool) -> tuple[list, list[str]]:
|
|
280
|
+
"""
|
|
281
|
+
Collect JSON metadata files for the given files and filter them from main files list.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
files_path (list[str]): List of file paths
|
|
285
|
+
auto_detect_json (bool): Whether to auto-detect JSON metadata files
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
tuple[list[Optional[str]], list[str]]: Tuple of (metadata file paths, filtered files_path)
|
|
289
|
+
- metadata file paths: List of metadata file paths (None if no metadata found)
|
|
290
|
+
- filtered files_path: Original files_path with JSON metadata files removed
|
|
291
|
+
"""
|
|
292
|
+
if not auto_detect_json:
|
|
293
|
+
return [None] * len(files_path), files_path
|
|
294
|
+
|
|
295
|
+
metadata_files = []
|
|
296
|
+
used_json_files = set()
|
|
297
|
+
nifti_extensions = ['.nii', '.nii.gz']
|
|
298
|
+
|
|
299
|
+
for file_path in files_path:
|
|
300
|
+
# Check if this is a NIFTI file
|
|
301
|
+
if any(file_path.endswith(ext) for ext in nifti_extensions):
|
|
302
|
+
json_file = _find_json_metadata(file_path)
|
|
303
|
+
metadata_files.append(json_file)
|
|
304
|
+
if json_file is not None:
|
|
305
|
+
used_json_files.add(json_file)
|
|
306
|
+
else:
|
|
307
|
+
metadata_files.append(None)
|
|
308
|
+
|
|
309
|
+
# Filter out JSON files that are being used as metadata from the main files list
|
|
310
|
+
filtered_files_path = [f for f in files_path if f not in used_json_files]
|
|
311
|
+
|
|
312
|
+
# Update metadata_files to match the filtered list
|
|
313
|
+
if used_json_files:
|
|
314
|
+
_LOGGER.debug(f"Filtering out {len(used_json_files)} JSON metadata files from main upload list")
|
|
315
|
+
filtered_metadata_files = []
|
|
316
|
+
filtered_file_index = 0
|
|
317
|
+
|
|
318
|
+
for original_file in files_path:
|
|
319
|
+
if original_file not in used_json_files:
|
|
320
|
+
filtered_metadata_files.append(metadata_files[files_path.index(original_file)])
|
|
321
|
+
filtered_file_index += 1
|
|
322
|
+
|
|
323
|
+
metadata_files = filtered_metadata_files
|
|
324
|
+
|
|
325
|
+
return metadata_files, filtered_files_path
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _parse_args() -> tuple[Any, list, Optional[list[dict]], Optional[list[str]]]:
|
|
260
329
|
parser = argparse.ArgumentParser(
|
|
261
330
|
description='DatamintAPI command line tool for uploading DICOM files and other resources')
|
|
262
|
-
|
|
263
|
-
|
|
331
|
+
|
|
332
|
+
# Add positional argument for path
|
|
333
|
+
parser.add_argument('path', nargs='?', type=_is_valid_path_argparse, metavar="PATH",
|
|
264
334
|
help='Path to the resource file(s) or a directory')
|
|
335
|
+
|
|
336
|
+
# Keep the --path option for backward compatibility, but make it optional
|
|
337
|
+
parser.add_argument('--path', dest='path_flag', type=_is_valid_path_argparse, metavar="FILE",
|
|
338
|
+
help='Path to the resource file(s) or a directory (alternative to positional argument)')
|
|
265
339
|
parser.add_argument('-r', '--recursive', nargs='?', const=-1, # -1 means infinite
|
|
266
340
|
type=int,
|
|
267
341
|
help='Recurse folders looking for DICOMs. If a number is passed, recurse that number of levels.')
|
|
@@ -302,9 +376,28 @@ def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
|
302
376
|
help='Automatically answer yes to all prompts')
|
|
303
377
|
parser.add_argument('--transpose-segmentation', action='store_true', default=False,
|
|
304
378
|
help='Transpose the segmentation dimensions to match the image dimensions')
|
|
379
|
+
parser.add_argument('--auto-detect-json', action='store_true', default=True,
|
|
380
|
+
help='Automatically detect and include JSON metadata files with the same base name as NIFTI files')
|
|
381
|
+
parser.add_argument('--no-auto-detect-json', dest='auto_detect_json', action='store_false',
|
|
382
|
+
help='Disable automatic detection of JSON metadata files (default behavior)')
|
|
305
383
|
parser.add_argument('--version', action='version', version=f'%(prog)s {datamint_version}')
|
|
306
384
|
parser.add_argument('--verbose', action='store_true', help='Print debug messages', default=False)
|
|
307
385
|
args = parser.parse_args()
|
|
386
|
+
|
|
387
|
+
# Handle path argument priority: positional takes precedence over --path flag
|
|
388
|
+
if args.path is not None and args.path_flag is not None:
|
|
389
|
+
_USER_LOGGER.warning("Both positional path and --path flag provided. Using positional argument.")
|
|
390
|
+
final_path = args.path
|
|
391
|
+
elif args.path is not None:
|
|
392
|
+
final_path = args.path
|
|
393
|
+
elif args.path_flag is not None:
|
|
394
|
+
final_path = args.path_flag
|
|
395
|
+
else:
|
|
396
|
+
parser.error("Path argument is required. Provide it as a positional argument or use --path flag.")
|
|
397
|
+
|
|
398
|
+
# Replace args.path with the final resolved path for consistency
|
|
399
|
+
args.path = final_path
|
|
400
|
+
|
|
308
401
|
if args.verbose:
|
|
309
402
|
# Get the console handler and set to debug
|
|
310
403
|
logging.getLogger().handlers[0].setLevel(logging.DEBUG)
|
|
@@ -319,7 +412,6 @@ def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
|
319
412
|
raise ValueError("--include-extensions and --exclude-extensions are mutually exclusive.")
|
|
320
413
|
|
|
321
414
|
try:
|
|
322
|
-
|
|
323
415
|
if os.path.isfile(args.path):
|
|
324
416
|
file_path = [args.path]
|
|
325
417
|
if args.recursive is not None:
|
|
@@ -337,6 +429,12 @@ def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
|
337
429
|
if len(file_path) == 0:
|
|
338
430
|
raise ValueError(f"No valid file was found in {args.path}")
|
|
339
431
|
|
|
432
|
+
# Collect JSON metadata files and filter them from main files list
|
|
433
|
+
metadata_files, file_path = _collect_metadata_files(file_path, args.auto_detect_json)
|
|
434
|
+
|
|
435
|
+
if len(file_path) == 0:
|
|
436
|
+
raise ValueError(f"No valid non-metadata files found in {args.path}")
|
|
437
|
+
|
|
340
438
|
if args.segmentation_names is not None:
|
|
341
439
|
with open(args.segmentation_names, 'r') as f:
|
|
342
440
|
segmentation_names = yaml.safe_load(f)
|
|
@@ -360,7 +458,7 @@ def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
|
360
458
|
raise ValueError("Cannot use both --tag and --label. Use --tag instead. --label is deprecated.")
|
|
361
459
|
args.tag = args.tag if args.tag is not None else args.label
|
|
362
460
|
|
|
363
|
-
return args, file_path, segmentation_files
|
|
461
|
+
return args, file_path, segmentation_files, metadata_files
|
|
364
462
|
|
|
365
463
|
except Exception as e:
|
|
366
464
|
if args.verbose:
|
|
@@ -371,6 +469,7 @@ def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
|
371
469
|
def print_input_summary(files_path: list[str],
|
|
372
470
|
args,
|
|
373
471
|
segfiles: Optional[list[dict]],
|
|
472
|
+
metadata_files: Optional[list[str]] = None,
|
|
374
473
|
include_extensions=None):
|
|
375
474
|
### Create a summary of the upload ###
|
|
376
475
|
total_files = len(files_path)
|
|
@@ -397,6 +496,7 @@ def print_input_summary(files_path: list[str],
|
|
|
397
496
|
if ext == '':
|
|
398
497
|
ext = 'no extension'
|
|
399
498
|
_USER_LOGGER.info(f"\t{ext}: {count}")
|
|
499
|
+
# Check for multiple extensions
|
|
400
500
|
if len(ext_counts) > 1 and include_extensions is None:
|
|
401
501
|
_USER_LOGGER.warning("Multiple file extensions found!" +
|
|
402
502
|
" Make sure you are uploading the correct files.")
|
|
@@ -419,6 +519,13 @@ def print_input_summary(files_path: list[str],
|
|
|
419
519
|
else:
|
|
420
520
|
_USER_LOGGER.info(msg)
|
|
421
521
|
|
|
522
|
+
if metadata_files is not None:
|
|
523
|
+
num_metadata_files = sum([1 if metadata is not None else 0 for metadata in metadata_files])
|
|
524
|
+
if num_metadata_files > 0:
|
|
525
|
+
msg = f"Number of files with JSON metadata: {num_metadata_files} ({num_metadata_files / total_files:.0%})"
|
|
526
|
+
_USER_LOGGER.info(msg)
|
|
527
|
+
# TODO: Could add validation to ensure JSON metadata files contain valid DICOM metadata structure
|
|
528
|
+
|
|
422
529
|
|
|
423
530
|
def print_results_summary(files_path: list[str],
|
|
424
531
|
results: list[str | Exception]):
|
|
@@ -441,7 +548,7 @@ def main():
|
|
|
441
548
|
load_cmdline_logging_config()
|
|
442
549
|
|
|
443
550
|
try:
|
|
444
|
-
args, files_path, segfiles = _parse_args()
|
|
551
|
+
args, files_path, segfiles, metadata_files = _parse_args()
|
|
445
552
|
except Exception as e:
|
|
446
553
|
_USER_LOGGER.error(f'Error validating arguments. {e}')
|
|
447
554
|
return
|
|
@@ -449,6 +556,7 @@ def main():
|
|
|
449
556
|
print_input_summary(files_path,
|
|
450
557
|
args=args,
|
|
451
558
|
segfiles=segfiles,
|
|
559
|
+
metadata_files=metadata_files,
|
|
452
560
|
include_extensions=args.include_extensions)
|
|
453
561
|
|
|
454
562
|
if not args.yes:
|
|
@@ -471,7 +579,8 @@ def main():
|
|
|
471
579
|
publish=args.publish,
|
|
472
580
|
segmentation_files=segfiles,
|
|
473
581
|
transpose_segmentation=args.transpose_segmentation,
|
|
474
|
-
assemble_dicoms=True
|
|
582
|
+
assemble_dicoms=True,
|
|
583
|
+
metadata_files=metadata_files
|
|
475
584
|
)
|
|
476
585
|
_USER_LOGGER.info('Upload finished!')
|
|
477
586
|
_LOGGER.debug(f"Number of results: {len(results)}")
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "datamint"
|
|
3
3
|
description = "A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows."
|
|
4
|
-
version = "1.
|
|
4
|
+
version = "1.4.0"
|
|
5
5
|
dynamic = ["dependencies"]
|
|
6
6
|
requires-python = ">=3.10"
|
|
7
7
|
readme = "README.md"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|