datamint 1.5.5__py3-none-any.whl → 1.6.2__py3-none-any.whl

datamint/client_cmd_tools/datamint_upload.py

@@ -5,7 +5,7 @@ from humanize import naturalsize
 import logging
 from pathlib import Path
 import sys
-from datamint.utils.dicom_utils import is_dicom
+from medimgkit.dicom_utils import is_dicom
 import fnmatch
 from typing import Generator, Optional, Any
 from collections import defaultdict
@@ -15,6 +15,7 @@ from datamint.client_cmd_tools.datamint_config import ask_api_key
 from datamint.utils.logging_utils import load_cmdline_logging_config
 import yaml
 from collections.abc import Iterable
+import pandas as pd
 
 # Create two loggings: one for the user and one for the developer
 _LOGGER = logging.getLogger(__name__)
@@ -23,6 +24,38 @@ _USER_LOGGER = logging.getLogger('user_logger')
 MAX_RECURSION_LIMIT = 1000
 
 
+def _read_segmentation_names(segmentation_names_path: str | Path) -> dict:
+    """
+    Read a segmentation names file (yaml or csv) and return its content as a dictionary.
+    If the file is a YAML file, it should contain two keys: "segmentation_names" and "class_names".
+    If the file is a CSV file, it should contain the following columns:
+    index, r, g, b, ..., name
+    """
+    segmentation_names_path = Path(segmentation_names_path)
+    if segmentation_names_path.suffix in ['.yaml', '.yml']:
+        with open(segmentation_names_path, 'r') as f:
+            metadata = yaml.safe_load(f)
+    elif segmentation_names_path.suffix in ['.csv', '.tsv']:
+        df = pd.read_csv(segmentation_names_path,
+                         header=None,
+                         index_col=0,
+                         sep=None,  # use sep=None to automatically detect the separator
+                         engine='python'
+                         )
+        df = df.rename(columns={1: 'r', 2: 'g', 3: 'b', df.columns[-1]: 'name'})
+        # df = df.set_index(['r', 'g', 'b'])
+        metadata = {'class_names': df['name'].to_dict()}
+    else:
+        raise ValueError(f"Unsupported file format: {segmentation_names_path.suffix}")
+
+    if 'segmentation_names' in metadata:
+        segnames = sorted(metadata['segmentation_names'],
+                          key=lambda x: len(x))
+        metadata['segmentation_names'] = segnames
+
+    return metadata
+
+
 def _is_valid_path_argparse(x):
     """
     argparse type that checks if the path exists
@@ -101,7 +134,6 @@ def walk_to_depth(path: str | Path,
                     continue
                 yield from walk_to_depth(child, depth-1, exclude_pattern)
         else:
-            _LOGGER.debug(f"yielding {child} from {path}")
             yield child
 
 
@@ -157,31 +189,32 @@ def handle_api_key() -> str | None:
 
 def _find_segmentation_files(segmentation_root_path: str,
                              images_files: list[str],
-                             segmentation_metainfo: dict = None
-                             ) -> Optional[list[dict]]:
+                             segmentation_metainfo: dict | None = None
+                             ) -> list[dict]:
     """
     Find the segmentation files that match the images files based on the same folder structure
     """
 
-    if segmentation_root_path is None:
-        return None
-
-    if len(images_files) == 1 and os.path.isfile(images_files[0]) and os.path.isfile(segmentation_root_path):
-        return [{'files': [segmentation_root_path]}]
-
-    segmentation_files = []
-    acceptable_extensions = ['.nii.gz', '.nii', '.png']
-
+    segnames = None
+    classnames = None
     if segmentation_metainfo is not None:
         if 'segmentation_names' in segmentation_metainfo:
             segnames = sorted(segmentation_metainfo['segmentation_names'],
                               key=lambda x: len(x))
-        else:
-            segnames = None
         classnames = segmentation_metainfo.get('class_names', None)
         if classnames is not None:
             _LOGGER.debug(f"Number of class names: {len(classnames)}")
 
+    if len(images_files) == 1 and os.path.isfile(images_files[0]) and os.path.isfile(segmentation_root_path):
+        ret = [{'files': [segmentation_root_path]}]
+        if classnames is not None:
+            ret[0]['names'] = classnames
+        _LOGGER.debug(f"Returning segmentation files: {ret}")
+        return ret
+
+    segmentation_files = []
+    acceptable_extensions = ['.nii.gz', '.nii', '.png']
+
     segmentation_root_path = Path(segmentation_root_path).absolute()
 
     for imgpath in images_files:
@@ -197,7 +230,6 @@ def _find_segmentation_files(segmentation_root_path: str,
         else:
             common_parent = Path(*common_parent)
 
-        _LOGGER.debug(f"_find_segmentation_files::common_parent: {common_parent}")
         path_structure = imgpath_parent.relative_to(common_parent).parts[1:]
 
         # path_structure = imgpath_parent.relative_to(root_path).parts[1:]
@@ -230,24 +262,47 @@ def _find_segmentation_files(segmentation_root_path: str,
             if len(frame_indices) > 0:
                 seginfo['frame_index'] = frame_indices
 
-            if segmentation_metainfo is not None:
-                snames_associated = []
-                for segfile in seg_files:
-                    if segnames is None:
-                        snames_associated.append(classnames)
+            snames_associated = []
+            for segfile in seg_files:
+                # check if there is a metadata file associated, besides json, with the segmentation
+                for ext in ['.yaml', '.yml', '.csv']:
+                    if str(segfile).endswith('nii.gz'):
+                        # has two extensions, so we need to remove both
+                        metadata_file = segfile.with_suffix('').with_suffix(ext)
+                        if not metadata_file.exists():
+                            metadata_file = segfile.with_suffix(ext)
+                    else:
+                        metadata_file = segfile.with_suffix(ext)
+                    if metadata_file.exists():
+                        _LOGGER.debug(f"Found metadata file: {metadata_file}")
+                        try:
+                            new_segmentation_metainfo = _read_segmentation_names(metadata_file)
+                            cur_segnames = new_segmentation_metainfo.get('segmentation_names', segnames)
+                            cur_classnames = new_segmentation_metainfo.get('class_names', classnames)
+                            break
+                        except Exception as e:
+                            _LOGGER.warning(f"Error reading metadata file {metadata_file}: {e}")
+                else:
+                    cur_segnames = segnames
+                    cur_classnames = classnames
+
+                if cur_segnames is None:
+                    _LOGGER.debug(f'adding {cur_classnames}')
+                    snames_associated.append(cur_classnames)
+                else:
+                    for segname in cur_segnames:
+                        if segname in str(segfile):
+                            if cur_classnames is not None:
+                                new_segname = {cid: f'{segname}_{cname}' for cid, cname in cur_classnames.items()}
+                                new_segname.update({'default': segname})
+                            else:
+                                new_segname = segname
+                            snames_associated.append(new_segname)
+                            break
                     else:
-                        for segname in segnames:
-                            if segname in str(segfile):
-                                if classnames is not None:
-                                    new_segname = {cid: f'{segname}_{cname}' for cid, cname in classnames.items()}
-                                    new_segname.update({'default': segname})
-                                else:
-                                    new_segname = segname
-                                snames_associated.append(new_segname)
-                                break
-                        else:
-                            _USER_LOGGER.warning(f"Segmentation file {segname} does not match any segmentation name.")
-                            snames_associated.append(None)
+                        _USER_LOGGER.warning(f"Segmentation file {segfile} does not match any segmentation name.")
+                        snames_associated.append(None)
+            if len(snames_associated) > 0:
                 seginfo['names'] = snames_associated
 
             segmentation_files.append(seginfo)
@@ -268,7 +323,7 @@ def _find_json_metadata(file_path: str | Path) -> Optional[str]:
         Optional[str]: Path to the JSON metadata file if found, None otherwise
     """
     file_path = Path(file_path)
-    
+
     # Handle .nii.gz files specially - need to remove both extensions
     if file_path.name.endswith('.nii.gz'):
         base_name = file_path.name[:-7]  # Remove .nii.gz
@@ -320,7 +375,7 @@ def _collect_metadata_files(files_path: list[str], auto_detect_json: bool) -> tu
     if used_json_files:
         _LOGGER.debug(f"Filtering out {len(used_json_files)} JSON metadata files from main upload list")
         filtered_metadata_files = []
-        
+
         for original_file in files_path:
             if original_file not in used_json_files:
                 original_index = files_path.index(original_file)
@@ -376,8 +431,10 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
                         help='Path to the segmentation file(s) or a directory')
     parser.add_argument('--segmentation_names', type=_is_valid_path_argparse, metavar="FILE",
                         required=False,
-                        help='Path to a yaml file containing the segmentation names.' +
-                        ' The file may contain two keys: "segmentation_names" and "class_names".')
+                        help='Path to a yaml or csv file containing the segmentation names.' +
+                        ' If yaml, the file may contain two keys: "segmentation_names" and "class_names".'
+                        ' If csv, the file should contain the following columns:'
+                        ' index, r, g, b, ..., name')
     parser.add_argument('--yes', action='store_true',
                         help='Automatically answer yes to all prompts')
     parser.add_argument('--transpose-segmentation', action='store_true', default=False,
@@ -446,15 +503,17 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
             raise ValueError(f"No valid non-metadata files found in {args.path}")
 
         if args.segmentation_names is not None:
-            with open(args.segmentation_names, 'r') as f:
-                segmentation_names = yaml.safe_load(f)
+            segmentation_names = _read_segmentation_names(args.segmentation_names)
         else:
             segmentation_names = None
 
         _LOGGER.debug(f'finding segmentations at {args.segmentation_path}')
-        segmentation_files = _find_segmentation_files(args.segmentation_path,
-                                                      file_path,
-                                                      segmentation_metainfo=segmentation_names)
+        if args.segmentation_path is None:
+            segmentation_files = None
+        else:
+            segmentation_files = _find_segmentation_files(args.segmentation_path,
+                                                          file_path,
+                                                          segmentation_metainfo=segmentation_names)
 
         _LOGGER.info(f"args parsed: {args}")
 

datamint/dataset/base_dataset.py

@@ -14,9 +14,9 @@ from torch.utils.data import DataLoader
 import torch
 from torch import Tensor
 from datamint.apihandler.base_api_handler import DatamintException
-from datamint.utils.dicom_utils import is_dicom
+from medimgkit.dicom_utils import is_dicom
 import cv2
-from datamint.utils.io_utils import read_array_normalized
+from medimgkit.io_utils import read_array_normalized
 from datetime import datetime
 
 _LOGGER = logging.getLogger(__name__)

{datamint-1.5.5.dist-info → datamint-1.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamint
-Version: 1.5.5
+Version: 1.6.2
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -19,6 +19,7 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
+Requires-Dist: medimgkit
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-1.5.5.dist-info → datamint-1.6.2.dist-info}/RECORD

@@ -1,16 +1,16 @@
 datamint/__init__.py,sha256=7rKCCsaa4RBRTIfuHB708rai1xwDHLtkFNFJGKYG5D4,757
-datamint/apihandler/annotation_api_handler.py,sha256=N8WFk-oO84fBKH9t-R1DW5J7hnxQxcz-zxgLuMkNbwA,47766
+datamint/apihandler/annotation_api_handler.py,sha256=ChwaSYjoOAVS7vuyP3-cfpDHaHwk_wXLf8QQaSU_oSM,51893
 datamint/apihandler/api_handler.py,sha256=cdVSddrFCKlF_BJ81LO1aJ0OP49rssjpNEFzJ6Q7YyY,384
 datamint/apihandler/base_api_handler.py,sha256=XSxZEQEkbQpuixGDu_P9jbxUQht3Z3JgxaeiFKPkVDM,11690
-datamint/apihandler/dto/annotation_dto.py,sha256=otCIesoqGBlbSOw4ErqFsXp2HwJsPNUQlkynQh_7pHg,7110
+datamint/apihandler/dto/annotation_dto.py,sha256=qId1RK1VO7dXrvGJ7dqJ31jBQB7Z8yy5x0tLSiMxTB4,7105
 datamint/apihandler/exp_api_handler.py,sha256=hFUgUgBc5rL7odK7gTW3MnrvMY1pVfJUpUdzRNobMQE,6226
-datamint/apihandler/root_api_handler.py,sha256=OIGq6aHX64B94MmAikcFzF0rdekRH4l1S59x2Pa_DJA,51739
+datamint/apihandler/root_api_handler.py,sha256=O8Gn1Gp3w7AYeuT_FbwH413o6P_eAYLoRiW0baGY_b4,51795
 datamint/client_cmd_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamint/client_cmd_tools/datamint_config.py,sha256=md7dnWrbl10lPtXKbmD9yo6onLJsajeG8Vz0ZWH1v4M,8181
-datamint/client_cmd_tools/datamint_upload.py,sha256=VyLL2FgY9ibfbdp4K6HrKt0jgkQH-SVuU71D6e77074,26436
+datamint/client_cmd_tools/datamint_upload.py,sha256=Q_C3HX9EFCdtIqmWY28qcWJmfUZfR1ml_pj0omnrkXw,29396
 datamint/configs.py,sha256=Bdp6NydYwyCJ2dk19_gf_o3M2ZyQOmMHpLi8wEWNHUk,1426
 datamint/dataset/__init__.py,sha256=4PlUKSvVhdfQvvuq8jQXrkdqnot-iTTizM3aM1vgSwg,47
-datamint/dataset/base_dataset.py,sha256=MQZ_wNFex4BKBfb4fAcXV6-fQXFV_zBK1ybWrMm6_pg,39092
+datamint/dataset/base_dataset.py,sha256=bSMuNHUzU7heN0awGemTn3e2zPLhuCsh-qSs_Qt6i9w,39082
 datamint/dataset/dataset.py,sha256=AwS92t5kdmpm9NKFfXFmDmZxEbbPfb_FOMn-FWfu3bE,26590
 datamint/examples/__init__.py,sha256=zcYnd5nLVme9GCTPYH-1JpGo8xXK2WEYvhzcy_2alZc,39
 datamint/examples/example_projects.py,sha256=7Nb_EaIdzJTQa9zopqc-WhTBQWQJSoQZ_KjRS4PB4FI,2931
@@ -18,12 +18,10 @@ datamint/experiment/__init__.py,sha256=5qQOMzoG17DEd1YnTF-vS0qiM-DGdbNh42EUo91CR
 datamint/experiment/_patcher.py,sha256=ZgbezoevAYhJsbiJTvWPALGTcUiMT371xddcTllt3H4,23296
 datamint/experiment/experiment.py,sha256=aHK9dRFdQTi569xgUg1KqlCZLHZpDmSH3g3ndPIZvXw,44546
 datamint/logging.yaml,sha256=a5dsATpul7QHeUHB2TjABFjWaPXBMbO--dgn8GlRqwk,483
-datamint/utils/dicom_utils.py,sha256=sLukP6MB_acx7t868O2HDd_RDEILa97mEe_V9m1EMCY,28991
-datamint/utils/io_utils.py,sha256=lKnUCJEip7W9Xj9wOWsTAA855HnKbjwQON1WjMGqJmM,7374
 datamint/utils/logging_utils.py,sha256=DvoA35ATYG3JTwfXEXYawDyKRfHeCrH0a9czfkmz8kM,1851
 datamint/utils/torchmetrics.py,sha256=lwU0nOtsSWfebyp7dvjlAggaqXtj5ohSEUXOg3L0hJE,2837
 datamint/utils/visualization.py,sha256=yaUVAOHar59VrGUjpAWv5eVvQSfztFG0eP9p5Vt3l-M,4470
-datamint-1.5.5.dist-info/METADATA,sha256=o6BFPA7OS3SSPqflC85pJ_2Q7pETUtoZInY97B2Dxm8,4065
-datamint-1.5.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamint-1.5.5.dist-info/entry_points.txt,sha256=mn5H6jPjO-rY0W0CAZ6Z_KKWhMLvyVaSpoqk77jlTI4,145
-datamint-1.5.5.dist-info/RECORD,,
+datamint-1.6.2.dist-info/METADATA,sha256=Raq2vLoDKmDH63F1DF6IqwIdk5dJDNr6TIvfbYpWUz4,4090
+datamint-1.6.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamint-1.6.2.dist-info/entry_points.txt,sha256=mn5H6jPjO-rY0W0CAZ6Z_KKWhMLvyVaSpoqk77jlTI4,145
+datamint-1.6.2.dist-info/RECORD,,