datamint 1.6.0__tar.gz → 1.6.3__tar.gz

{datamint-1.6.0 → datamint-1.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamint
-Version: 1.6.0
+Version: 1.6.3
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -19,6 +19,7 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
+Requires-Dist: medimgkit
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-1.6.0 → datamint-1.6.3}/datamint/apihandler/annotation_api_handler.py

@@ -10,7 +10,6 @@ import os
 import asyncio
 import aiohttp
 from requests.exceptions import HTTPError
-from deprecated.sphinx import deprecated
 from .dto.annotation_dto import CreateAnnotationDto, LineGeometry, BoxGeometry, CoordinateSystem, AnnotationType
 import pydicom
 import json
@@ -237,7 +236,7 @@ class AnnotationAPIHandler(BaseAPIHandler):
     async def _upload_volume_segmentation_async(self,
                                                 resource_id: str,
                                                 file_path: str | np.ndarray,
-                                                name: dict[int, str] | dict[tuple, str],
+                                                name: str | dict[int, str] | dict[tuple, str] | None,
                                                 imported_from: Optional[str] = None,
                                                 author_email: Optional[str] = None,
                                                 worklist_id: Optional[str] = None,
@@ -263,6 +262,13 @@ class AnnotationAPIHandler(BaseAPIHandler):
         Raises:
             ValueError: If name is not a string or file format is unsupported for volume upload.
         """
+
+        if isinstance(name, str):
+            raise NotImplementedError("`name=string` is not supported yet for volume segmentation.")
+        if isinstance(name, dict):
+            if any(isinstance(k, tuple) for k in name.keys()):
+                raise NotImplementedError("For volume segmentations, `name` must be a dictionary with integer keys only.") 
+                
         # Prepare file for upload
         if isinstance(file_path, str):
             if file_path.endswith('.nii') or file_path.endswith('.nii.gz'):
@@ -275,7 +281,8 @@ class AnnotationAPIHandler(BaseAPIHandler):
                         form.add_field('model_id', model_id)  # Add model_id if provided
                     if worklist_id is not None:
                         form.add_field('annotation_worklist_id', worklist_id)
-                    form.add_field('segmentation_map', json.dumps(name), content_type='application/json')
+                    if name is not None:
+                        form.add_field('segmentation_map', json.dumps(name), content_type='application/json')
 
                     request_params = dict(
                         method='POST',
@@ -449,30 +456,27 @@ class AnnotationAPIHandler(BaseAPIHandler):
         if isinstance(file_path, str) and not os.path.exists(file_path):
             raise FileNotFoundError(f"File {file_path} not found.")
 
-        name = AnnotationAPIHandler.standardize_segmentation_names(name)
-
         # Handle NIfTI files specially - upload as single volume
         if isinstance(file_path, str) and (file_path.endswith('.nii') or file_path.endswith('.nii.gz')):
             _LOGGER.info(f"Uploading NIfTI segmentation file: {file_path}")
             if frame_index is not None:
                 raise ValueError("Do not provide frame_index for NIfTI segmentations.")
             loop = asyncio.get_event_loop()
-            task = self._upload_segmentations_async(
+            task = self._upload_volume_segmentation_async(
                 resource_id=resource_id,
-                frame_index=None,
                 file_path=file_path,
                 name=name,
                 imported_from=imported_from,
                 author_email=author_email,
-                discard_empty_segmentations=False,
                 worklist_id=worklist_id,
                 model_id=model_id,
-                transpose_segmentation=transpose_segmentation,
-                upload_volume=True
+                transpose_segmentation=transpose_segmentation
             )
             return loop.run_until_complete(task)
         # All other file types are converted to multiple PNGs and uploaded frame by frame.
 
+        name = AnnotationAPIHandler.standardize_segmentation_names(name)
+
         to_run = []
         # Generate IOs for the segmentations.
         nframes, fios = AnnotationAPIHandler._generate_segmentations_ios(file_path,

{datamint-1.6.0 → datamint-1.6.3}/datamint/apihandler/dto/annotation_dto.py

@@ -18,7 +18,7 @@ import json
 from typing import Any, TypeAlias, Literal
 import logging
 from enum import Enum
-from datamint.utils.dicom_utils import pixel_to_patient
+from medimgkit.dicom_utils import pixel_to_patient
 import pydicom
 import numpy as np
 

{datamint-1.6.0 → datamint-1.6.3}/datamint/apihandler/root_api_handler.py

@@ -6,8 +6,8 @@ from requests.exceptions import HTTPError
 import logging
 import asyncio
 import aiohttp
-from datamint.utils.dicom_utils import anonymize_dicom, to_bytesio, is_dicom
-from datamint.utils import dicom_utils
+from medimgkit.dicom_utils import anonymize_dicom, to_bytesio, is_dicom
+from medimgkit import dicom_utils
 import pydicom
 from pathlib import Path
 from datetime import date
@@ -447,6 +447,8 @@ class RootAPIHandler(BaseAPIHandler):
                                   for segfiles in segmentation_files]
             
             for segfiles in segmentation_files:
+                if segfiles is None:
+                    continue
                 if 'files' not in segfiles:
                     raise ValueError("segmentation_files must contain a 'files' key with a list of file paths.")
                 if 'names' in segfiles:

{datamint-1.6.0 → datamint-1.6.3}/datamint/client_cmd_tools/datamint_upload.py

@@ -5,7 +5,7 @@ from humanize import naturalsize
 import logging
 from pathlib import Path
 import sys
-from datamint.utils.dicom_utils import is_dicom
+from medimgkit.dicom_utils import is_dicom
 import fnmatch
 from typing import Generator, Optional, Any
 from collections import defaultdict
@@ -15,6 +15,7 @@ from datamint.client_cmd_tools.datamint_config import ask_api_key
 from datamint.utils.logging_utils import load_cmdline_logging_config
 import yaml
 from collections.abc import Iterable
+import pandas as pd
 
 # Create two loggings: one for the user and one for the developer
 _LOGGER = logging.getLogger(__name__)
@@ -23,6 +24,38 @@ _USER_LOGGER = logging.getLogger('user_logger')
 MAX_RECURSION_LIMIT = 1000
 
 
+def _read_segmentation_names(segmentation_names_path: str | Path) -> dict:
+    """
+    Read a segmentation names file (yaml or csv) and return its content as a dictionary.
+    If the file is a YAML file, it should contain two keys: "segmentation_names" and "class_names".
+    If the file is a CSV file, it should contain the following columns:
+    index, r, g, b, ..., name
+    """
+    segmentation_names_path = Path(segmentation_names_path)
+    if segmentation_names_path.suffix in ['.yaml', '.yml']:
+        with open(segmentation_names_path, 'r') as f:
+            metadata = yaml.safe_load(f)
+    elif segmentation_names_path.suffix in ['.csv', '.tsv']:
+        df = pd.read_csv(segmentation_names_path,
+                         header=None,
+                         index_col=0,
+                         sep=None,  # use sep=None to automatically detect the separator
+                         engine='python'
+                         )
+        df = df.rename(columns={1: 'r', 2: 'g', 3: 'b', df.columns[-1]: 'name'})
+        # df = df.set_index(['r', 'g', 'b'])
+        metadata = {'class_names': df['name'].to_dict()}
+    else:
+        raise ValueError(f"Unsupported file format: {segmentation_names_path.suffix}")
+
+    if 'segmentation_names' in metadata:
+        segnames = sorted(metadata['segmentation_names'],
+                          key=lambda x: len(x))
+        metadata['segmentation_names'] = segnames
+
+    return metadata
+
+
 def _is_valid_path_argparse(x):
     """
     argparse type that checks if the path exists
@@ -101,7 +134,6 @@ def walk_to_depth(path: str | Path,
                     continue
                 yield from walk_to_depth(child, depth-1, exclude_pattern)
         else:
-            _LOGGER.debug(f"yielding {child} from {path}")
             yield child
 
 
@@ -157,31 +189,32 @@ def handle_api_key() -> str | None:
 
 def _find_segmentation_files(segmentation_root_path: str,
                              images_files: list[str],
-                             segmentation_metainfo: dict = None
-                             ) -> Optional[list[dict]]:
+                             segmentation_metainfo: dict | None = None
+                             ) -> list[dict]:
     """
     Find the segmentation files that match the images files based on the same folder structure
     """
 
-    if segmentation_root_path is None:
-        return None
-
-    if len(images_files) == 1 and os.path.isfile(images_files[0]) and os.path.isfile(segmentation_root_path):
-        return [{'files': [segmentation_root_path]}]
-
-    segmentation_files = []
-    acceptable_extensions = ['.nii.gz', '.nii', '.png']
-
+    segnames = None
+    classnames = None
     if segmentation_metainfo is not None:
         if 'segmentation_names' in segmentation_metainfo:
             segnames = sorted(segmentation_metainfo['segmentation_names'],
                               key=lambda x: len(x))
-        else:
-            segnames = None
         classnames = segmentation_metainfo.get('class_names', None)
         if classnames is not None:
             _LOGGER.debug(f"Number of class names: {len(classnames)}")
 
+    if len(images_files) == 1 and os.path.isfile(images_files[0]) and os.path.isfile(segmentation_root_path):
+        ret = [{'files': [segmentation_root_path]}]
+        if classnames is not None:
+            ret[0]['names'] = classnames
+        _LOGGER.debug(f"Returning segmentation files: {ret}")
+        return ret
+
+    segmentation_files = []
+    acceptable_extensions = ['.nii.gz', '.nii', '.png']
+
     segmentation_root_path = Path(segmentation_root_path).absolute()
 
     for imgpath in images_files:
@@ -197,7 +230,6 @@ def _find_segmentation_files(segmentation_root_path: str,
         else:
             common_parent = Path(*common_parent)
 
-        _LOGGER.debug(f"_find_segmentation_files::common_parent: {common_parent}")
         path_structure = imgpath_parent.relative_to(common_parent).parts[1:]
 
         # path_structure = imgpath_parent.relative_to(root_path).parts[1:]
@@ -230,24 +262,47 @@ def _find_segmentation_files(segmentation_root_path: str,
             if len(frame_indices) > 0:
                 seginfo['frame_index'] = frame_indices
 
-            if segmentation_metainfo is not None:
-                snames_associated = []
-                for segfile in seg_files:
-                    if segnames is None:
-                        snames_associated.append(classnames)
+            snames_associated = []
+            for segfile in seg_files:
+                # check if there is a metadata file associated, besides json, with the segmentation
+                for ext in ['.yaml', '.yml', '.csv']:
+                    if str(segfile).endswith('nii.gz'):
+                        # has two extensions, so we need to remove both
+                        metadata_file = segfile.with_suffix('').with_suffix(ext)
+                        if not metadata_file.exists():
+                            metadata_file = segfile.with_suffix(ext)
+                    else:
+                        metadata_file = segfile.with_suffix(ext)
+                    if metadata_file.exists():
+                        _LOGGER.debug(f"Found metadata file: {metadata_file}")
+                        try:
+                            new_segmentation_metainfo = _read_segmentation_names(metadata_file)
+                            cur_segnames = new_segmentation_metainfo.get('segmentation_names', segnames)
+                            cur_classnames = new_segmentation_metainfo.get('class_names', classnames)
+                            break
+                        except Exception as e:
+                            _LOGGER.warning(f"Error reading metadata file {metadata_file}: {e}")
+                else:
+                    cur_segnames = segnames
+                    cur_classnames = classnames
+
+                if cur_segnames is None:
+                    _LOGGER.debug(f'adding {cur_classnames}')
+                    snames_associated.append(cur_classnames)
+                else:
+                    for segname in cur_segnames:
+                        if segname in str(segfile):
+                            if cur_classnames is not None:
+                                new_segname = {cid: f'{segname}_{cname}' for cid, cname in cur_classnames.items()}
+                                new_segname.update({'default': segname})
+                            else:
+                                new_segname = segname
+                            snames_associated.append(new_segname)
+                            break
                     else:
-                        for segname in segnames:
-                            if segname in str(segfile):
-                                if classnames is not None:
-                                    new_segname = {cid: f'{segname}_{cname}' for cid, cname in classnames.items()}
-                                    new_segname.update({'default': segname})
-                                else:
-                                    new_segname = segname
-                                snames_associated.append(new_segname)
-                                break
-                        else:
-                            _USER_LOGGER.warning(f"Segmentation file {segname} does not match any segmentation name.")
-                            snames_associated.append(None)
+                        _USER_LOGGER.warning(f"Segmentation file {segfile} does not match any segmentation name.")
+                        snames_associated.append(None)
+            if len(snames_associated) > 0:
                 seginfo['names'] = snames_associated
 
             segmentation_files.append(seginfo)
@@ -268,7 +323,7 @@ def _find_json_metadata(file_path: str | Path) -> Optional[str]:
         Optional[str]: Path to the JSON metadata file if found, None otherwise
     """
     file_path = Path(file_path)
-    
+
     # Handle .nii.gz files specially - need to remove both extensions
     if file_path.name.endswith('.nii.gz'):
         base_name = file_path.name[:-7]  # Remove .nii.gz
@@ -320,7 +375,7 @@ def _collect_metadata_files(files_path: list[str], auto_detect_json: bool) -> tu
     if used_json_files:
         _LOGGER.debug(f"Filtering out {len(used_json_files)} JSON metadata files from main upload list")
         filtered_metadata_files = []
-        
+
         for original_file in files_path:
             if original_file not in used_json_files:
                 original_index = files_path.index(original_file)
@@ -376,8 +431,10 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
                         help='Path to the segmentation file(s) or a directory')
     parser.add_argument('--segmentation_names', type=_is_valid_path_argparse, metavar="FILE",
                         required=False,
-                        help='Path to a yaml file containing the segmentation names.' +
-                        ' The file may contain two keys: "segmentation_names" and "class_names".')
+                        help='Path to a yaml or csv file containing the segmentation names.' +
+                        ' If yaml, the file may contain two keys: "segmentation_names" and "class_names".'
+                        ' If csv, the file should contain the following columns:'
+                        ' index, r, g, b, ..., name')
     parser.add_argument('--yes', action='store_true',
                         help='Automatically answer yes to all prompts')
     parser.add_argument('--transpose-segmentation', action='store_true', default=False,
@@ -407,6 +464,7 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
     if args.verbose:
         # Get the console handler and set to debug
         logging.getLogger().handlers[0].setLevel(logging.DEBUG)
+        logging.getLogger('datamint').handlers[0].setLevel(logging.DEBUG)
         _LOGGER.setLevel(logging.DEBUG)
         _USER_LOGGER.setLevel(logging.DEBUG)
 
@@ -446,15 +504,17 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
             raise ValueError(f"No valid non-metadata files found in {args.path}")
 
         if args.segmentation_names is not None:
-            with open(args.segmentation_names, 'r') as f:
-                segmentation_names = yaml.safe_load(f)
+            segmentation_names = _read_segmentation_names(args.segmentation_names)
         else:
             segmentation_names = None
 
         _LOGGER.debug(f'finding segmentations at {args.segmentation_path}')
-        segmentation_files = _find_segmentation_files(args.segmentation_path,
-                                                      file_path,
-                                                      segmentation_metainfo=segmentation_names)
+        if args.segmentation_path is None:
+            segmentation_files = None
+        else:
+            segmentation_files = _find_segmentation_files(args.segmentation_path,
+                                                          file_path,
+                                                          segmentation_metainfo=segmentation_names)
 
         _LOGGER.info(f"args parsed: {args}")
 

{datamint-1.6.0 → datamint-1.6.3}/datamint/dataset/base_dataset.py

@@ -14,9 +14,9 @@ from torch.utils.data import DataLoader
 import torch
 from torch import Tensor
 from datamint.apihandler.base_api_handler import DatamintException
-from datamint.utils.dicom_utils import is_dicom
+from medimgkit.dicom_utils import is_dicom
 import cv2
-from datamint.utils.io_utils import read_array_normalized
+from medimgkit.io_utils import read_array_normalized
 from datetime import datetime
 
 _LOGGER = logging.getLogger(__name__)

{datamint-1.6.0 → datamint-1.6.3}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "datamint"
 description = "A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows."
-version = "1.6.0"
+version = "1.6.3"
 dynamic = ["dependencies"]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -40,6 +40,7 @@ matplotlib = "*"
 lightning = "*"
 albumentations = ">=2.0.0"
 lazy-loader = ">=0.3.0"
+medimgkit = "*"
 # For compatibility with the datamintapi package
 datamintapi = "0.0.*"
 # Extra dependencies for docs