datamint 1.9.1__tar.gz → 1.9.2__tar.gz

{datamint-1.9.1 → datamint-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamint
-Version: 1.9.1
+Version: 1.9.2
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -19,7 +19,7 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
-Requires-Dist: medimgkit (>=0.4.4)
+Requires-Dist: medimgkit (>=0.5.0)
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-1.9.1 → datamint-1.9.2}/datamint/apihandler/base_api_handler.py

@@ -1,4 +1,4 @@
-from typing import Optional, Literal, Generator, TypeAlias, Dict, Union, List
+from typing import Optional, Literal, Generator, TypeAlias
 import pydicom.dataset
 from requests import Session
 from requests.exceptions import HTTPError
@@ -15,6 +15,7 @@ import nibabel as nib
 from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
 from datamint import configs
 import gzip
+from datamint.exceptions import DatamintException
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -29,13 +30,6 @@ ResourceFields: TypeAlias = Literal['modality', 'created_by', 'published_by', 'p
 _PAGE_LIMIT = 5000
 
 
-class DatamintException(Exception):
-    """
-    Base class for exceptions in this module.
-    """
-    pass
-
-
 class ResourceNotFoundError(DatamintException):
     """
     Exception raised when a resource is not found. 
@@ -196,10 +190,10 @@ class BaseAPIHandler:
                 _LOGGER.error(f"Error in request to {request_args['url']}: {e}")
             if status_code >= 400 and status_code < 500:
                 try:
-                    _LOGGER.error(f"Error response: {response.text}")
+                    _LOGGER.info(f"Error response: {response.text}")
                     error_data = response.json()
                 except Exception as e2:
-                    _LOGGER.error(f"Error parsing the response. {e2}")
+                    _LOGGER.info(f"Error parsing the response. {e2}")
                 else:
                     if isinstance(error_data['message'], str) and ' not found' in error_data['message'].lower():
                         # Will be caught by the caller and properly initialized:

{datamint-1.9.1 → datamint-1.9.2}/datamint/apihandler/root_api_handler.py

@@ -223,7 +223,8 @@ class RootAPIHandler(BaseAPIHandler):
             metadata_files = _infinite_gen(None)
 
         async with aiohttp.ClientSession() as session:
-            async def __upload_single_resource(file_path, segfiles: dict[str, list | dict], metadata_file: str | dict | None):
+            async def __upload_single_resource(file_path, segfiles: dict[str, list | dict],
+                                               metadata_file: str | dict | None):
                 rid = await self._upload_single_resource_async(
                     file_path=file_path,
                     mimetype=mimetype,
@@ -258,30 +259,56 @@ class RootAPIHandler(BaseAPIHandler):
                      for f, segfiles, metadata_file in zip(files_path, segmentation_files, metadata_files)]
             return await asyncio.gather(*tasks, return_exceptions=on_error == 'skip')
 
-    def _assemble_dicoms(self, files_path: Sequence[str | IO]) -> tuple[Sequence[str | IO], bool]:
+    def _assemble_dicoms(self, files_path: Sequence[str | IO]
+                         ) -> tuple[Sequence[str | IO], bool, Sequence[int]]:
+        """
+        Assembles DICOM files into a single file.
+
+        Args:
+            files_path: The paths to the DICOM files to assemble.
+
+        Returns:
+            A tuple containing:
+                - The paths to the assembled DICOM files.
+                - A boolean indicating whether the assembly was successful.
+                - same length as the output assembled DICOMs, mapping assembled DICOM to original DICOMs.
+        """
         dicoms_files_path = []
         other_files_path = []
-        for f in files_path:
+        dicom_original_idxs = []
+        others_original_idxs = []
+        for i, f in enumerate(files_path):
             if is_dicom(f):
                 dicoms_files_path.append(f)
+                dicom_original_idxs.append(i)
             else:
                 other_files_path.append(f)
+                others_original_idxs.append(i)
 
         orig_len = len(dicoms_files_path)
         if orig_len == 0:
             _LOGGER.debug("No DICOM files found to assemble.")
-            return files_path, False
+            return files_path, False, []
         dicoms_files_path = dicom_utils.assemble_dicoms(dicoms_files_path, return_as_IO=True)
 
         new_len = len(dicoms_files_path)
         if new_len != orig_len:
             _LOGGER.info(f"Assembled {new_len} dicom files out of {orig_len} files.")
+            mapping_idx = [None] * len(files_path)
             files_path = itertools.chain(dicoms_files_path, other_files_path)
             assembled = True
+            for orig_idx, value in zip(dicom_original_idxs, dicoms_files_path.inverse_mapping_idx):
+                mapping_idx[orig_idx] = value
+            for i, orig_idx in enumerate(others_original_idxs):
+                mapping_idx[orig_idx] = new_len + i
+            # mapping_idx = [[dicom_original_idxs[i] for i in idxlist]
+            #                for idxlist in dicoms_files_path.mapping_idx]
+            # mapping_idx += [[i] for i in others_original_idxs]
         else:
             assembled = False
+            # mapping_idx = [[i] for i in range(len(files_path))]
 
-        return files_path, assembled
+        return files_path, assembled, mapping_idx
 
     def upload_resource(self,
                         file_path: str | IO | pydicom.dataset.Dataset,
@@ -433,17 +460,17 @@ class RootAPIHandler(BaseAPIHandler):
             # Create filtered lists maintaining index correspondence
             filtered_files = []
             filtered_metadata = []
-            
+
             for i, f in enumerate(files_path):
                 if not is_dicom_report(f):
                     filtered_files.append(f)
                     if metadata is not None:
                         filtered_metadata.append(metadata[i])
-            
+
             files_path = filtered_files
             if metadata is not None:
                 metadata = filtered_metadata
-                
+
             if old_size is not None and old_size != len(files_path):
                 _LOGGER.info(f"Discarded {old_size - len(files_path)} DICOM report files from upload.")
 
@@ -454,9 +481,11 @@ class RootAPIHandler(BaseAPIHandler):
         if metadata is not None and len(metadata) != len(files_path):
             raise ValueError("The number of metadata files must match the number of resources.")
         if assemble_dicoms:
-            files_path, assembled = self._assemble_dicoms(files_path)
+            files_path, assembled, mapping_idx = self._assemble_dicoms(files_path)
             assemble_dicoms = assembled
-        
+        else:
+            mapping_idx = [i for i in range(len(files_path))]
+
         if segmentation_files is not None:
             if assemble_dicoms:
                 raise NotImplementedError("Segmentation files cannot be uploaded when assembling dicoms yet.")
@@ -512,6 +541,10 @@ class RootAPIHandler(BaseAPIHandler):
                 if on_error == 'raise':
                     raise e
 
+        if mapping_idx:
+            _LOGGER.debug(f"Mapping indices for DICOM files: {mapping_idx}")
+            resource_ids = [resource_ids[idx] for idx in mapping_idx]
+
         if is_multiple_resources:
             return resource_ids
         return resource_ids[0]
@@ -652,7 +685,8 @@ class RootAPIHandler(BaseAPIHandler):
             raise e
 
     @staticmethod
-    def __process_files_parameter(file_path: str | IO | Sequence[str | IO] | pydicom.dataset.Dataset) -> tuple[Sequence[str | IO], bool]:
+    def __process_files_parameter(file_path: str | IO | Sequence[str | IO] | pydicom.dataset.Dataset
+                                  ) -> tuple[Sequence[str | IO], bool]:
         """
         Process the file_path parameter to ensure it is a list of file paths or IO objects.
         """
@@ -1123,11 +1157,10 @@ class RootAPIHandler(BaseAPIHandler):
 
         loop = asyncio.get_event_loop()
         loop.run_until_complete(_delete_all_resources_async())
-            
-    
+
     async def _delete_resource_async(self,
-                                   resource_id: str,
-                                   session: aiohttp.ClientSession | None = None) -> None:
+                                     resource_id: str,
+                                     session: aiohttp.ClientSession | None = None) -> None:
         """
         Asynchronously delete a resource by its unique id.
 

{datamint-1.9.1 → datamint-1.9.2}/datamint/client_cmd_tools/datamint_config.py

@@ -125,7 +125,7 @@ def test_connection():
         projects = api.get_projects()
         console.print(f"[success]✅ Connection successful! Found {len(projects)} projects.[/success]")
     except ImportError:
-        console.print("[error]❌ Full API not available. Install with: pip install datamint-python-api[full][/error]")
+        console.print("[error]❌ Full API not available. Install with: pip install datamint[/error]")
     except Exception as e:
         console.print(f"[error]❌ Connection failed: {e}[/error]")
 

{datamint-1.9.1 → datamint-1.9.2}/datamint/client_cmd_tools/datamint_upload.py

@@ -1,3 +1,4 @@
+from datamint.exceptions import DatamintException
 import argparse
 from datamint.apihandler.api_handler import APIHandler
 import os
@@ -16,6 +17,7 @@ from datamint.utils.logging_utils import load_cmdline_logging_config
 import yaml
 from collections.abc import Iterable
 import pandas as pd
+import pydicom.errors
 
 # Create two loggings: one for the user and one for the developer
 _LOGGER = logging.getLogger(__name__)
@@ -24,6 +26,66 @@ _USER_LOGGER = logging.getLogger('user_logger')
 MAX_RECURSION_LIMIT = 1000
 
 
+def _get_minimal_distinguishing_paths(file_paths: list[str]) -> dict[str, str]:
+    """
+    Generate minimal distinguishing paths for files to avoid ambiguity when multiple files have the same name.
+    
+    Args:
+        file_paths: List of file paths
+        
+    Returns:
+        Dictionary mapping full path to minimal distinguishing path
+    """
+    if not file_paths:
+        return {}
+    
+    # Convert to Path objects and get absolute paths
+    paths = [Path(fp).resolve() for fp in file_paths]
+    result = {}
+    
+    # Group files by basename
+    basename_groups = defaultdict(list)
+    for i, path in enumerate(paths):
+        basename_groups[path.name].append((i, path))
+    
+    for basename, path_list in basename_groups.items():
+        if len(path_list) == 1:
+            # Only one file with this name, use just the basename
+            idx, path = path_list[0]
+            result[file_paths[idx]] = basename
+        else:
+            # Multiple files with same name, need to distinguish them
+            path_parts_list = [path.parts for _, path in path_list]
+            
+            # Find the minimum number of parent directories needed to distinguish
+            max_depth_needed = 1
+            for depth in range(1, max(len(parts) for parts in path_parts_list) + 1):
+                # Check if this depth is enough to distinguish all files
+                suffixes = []
+                for parts in path_parts_list:
+                    if depth >= len(parts):
+                        suffixes.append('/'.join(parts))
+                    else:
+                        suffixes.append('/'.join(parts[-depth:]))
+                
+                if len(set(suffixes)) == len(suffixes):
+                    # All suffixes are unique at this depth
+                    max_depth_needed = depth
+                    break
+            
+            # Apply the minimal distinguishing paths
+            for (idx, path), parts in zip(path_list, path_parts_list):
+                if max_depth_needed >= len(parts):
+                    distinguishing_path = '/'.join(parts)
+                else:
+                    distinguishing_path = '/'.join(parts[-max_depth_needed:])
+                result[file_paths[idx]] = distinguishing_path
+    
+    return result
+
+
+
+
 def _read_segmentation_names(segmentation_names_path: str | Path) -> dict:
     """
     Read a segmentation names file (yaml or csv) and return its content as a dictionary.
@@ -124,7 +186,7 @@ def walk_to_depth(path: str | Path,
                   depth: int,
                   exclude_pattern: str | None = None) -> Generator[Path, None, None]:
     path = Path(path)
-    
+
     # Check for DICOMDIR first at current directory level
     dicomdir_path = detect_dicomdir(path)
     if dicomdir_path is not None:
@@ -138,7 +200,7 @@ def walk_to_depth(path: str | Path,
         except Exception as e:
             _USER_LOGGER.warning(f"Failed to parse DICOMDIR at {path}: {e}. Falling back to directory scan.")
             # Continue with regular directory scanning below
-    
+
     # Regular directory scanning
     for child in path.iterdir():
         if _is_system_file(child):
@@ -606,12 +668,15 @@ def print_input_summary(files_path: list[str],
     ext_counts = [(ext, count) for ext, count in ext_dict.items()]
     ext_counts.sort(key=lambda x: x[1], reverse=True)
 
+    # Get distinguishing paths for better display
+    distinguishing_paths = _get_minimal_distinguishing_paths(files_path)
+
     _USER_LOGGER.info(f"Number of files to be uploaded: {total_files}")
-    _USER_LOGGER.info(f"\t{files_path[0]}")
+    _USER_LOGGER.info(f"\t{distinguishing_paths[files_path[0]]}")
     if total_files >= 2:
         if total_files >= 3:
             _USER_LOGGER.info("\t(...)")
-        _USER_LOGGER.info(f"\t{files_path[-1]}")
+        _USER_LOGGER.info(f"\t{distinguishing_paths[files_path[-1]]}")
     _USER_LOGGER.info(f"Total size of the upload: {naturalsize(total_size)}")
     _USER_LOGGER.info(f"Number of files per extension:")
     for ext, count in ext_counts:
@@ -653,17 +718,21 @@ def print_results_summary(files_path: list[str],
                           results: list[str | Exception]) -> int:
     # Check for failed uploads
     failure_files = [f for f, r in zip(files_path, results) if isinstance(r, Exception)]
+    # Get distinguishing paths for better error reporting
+    distinguishing_paths = _get_minimal_distinguishing_paths(files_path)
+    
     _USER_LOGGER.info(f"\nUpload summary:")
     _USER_LOGGER.info(f"\tTotal files: {len(files_path)}")
     _USER_LOGGER.info(f"\tSuccessful uploads: {len(files_path) - len(failure_files)}")
-    _USER_LOGGER.info(f"\tFailed uploads: {len(failure_files)}")
     if len(failure_files) > 0:
-        _USER_LOGGER.warning(f"\tFailed files: {[os.path.basename(f) for f in failure_files]}")
+        _USER_LOGGER.info(f"\t❌ Failed uploads: {len(failure_files)}")
+        _USER_LOGGER.warning(f"\tFailed files: {[distinguishing_paths[f] for f in failure_files]}")
         _USER_LOGGER.warning(f"\nFailures:")
         for f, r in zip(files_path, results):
-            _LOGGER.debug(f"Failure: {f} - {r}")
             if isinstance(r, Exception):
-                _USER_LOGGER.warning(f"\t{os.path.basename(f)}: {r}")
+                _USER_LOGGER.warning(f"\t{distinguishing_paths[f]}: {r}")
+    else:
+        _USER_LOGGER.info(f'✅ All uploads successful!')
     return len(failure_files)
 
 
@@ -691,20 +760,28 @@ def main():
 
     has_a_dicom_file = any(is_dicom(f) for f in files_path)
 
-    api_handler = APIHandler()
-    results = api_handler.upload_resources(channel=args.channel,
-                                           files_path=files_path,
-                                           tags=args.tag,
-                                           on_error='skip',
-                                           anonymize=args.retain_pii == False and has_a_dicom_file,
-                                           anonymize_retain_codes=args.retain_attribute,
-                                           mung_filename=args.mungfilename,
-                                           publish=args.publish,
-                                           segmentation_files=segfiles,
-                                           transpose_segmentation=args.transpose_segmentation,
-                                           assemble_dicoms=True,
-                                           metadata=metadata_files
-                                           )
+    try:
+        api_handler = APIHandler(check_connection=True)
+    except DatamintException as e:
+        _USER_LOGGER.error(f'❌ Connection failed: {e}')
+        return
+    try:
+        results = api_handler.upload_resources(channel=args.channel,
+                                               files_path=files_path,
+                                               tags=args.tag,
+                                               on_error='skip',
+                                               anonymize=args.retain_pii == False and has_a_dicom_file,
+                                               anonymize_retain_codes=args.retain_attribute,
+                                               mung_filename=args.mungfilename,
+                                               publish=args.publish,
+                                               segmentation_files=segfiles,
+                                               transpose_segmentation=args.transpose_segmentation,
+                                               assemble_dicoms=True,
+                                               metadata=metadata_files
+                                               )
+    except pydicom.errors.InvalidDicomError as e:
+        _USER_LOGGER.error(f'❌ Invalid DICOM file: {e}')
+        return
     _USER_LOGGER.info('Upload finished!')
     _LOGGER.debug(f"Number of results: {len(results)}")
 

datamint-1.9.2/datamint/exceptions.py

@@ -0,0 +1,5 @@
+class DatamintException(Exception):
+    """
+    Base class for exceptions in this module.
+    """
+    pass

{datamint-1.9.1 → datamint-1.9.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "datamint"
 description = "A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows."
-version = "1.9.1"
+version = "1.9.2"
 dynamic = ["dependencies"]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -40,7 +40,7 @@ matplotlib = "*"
 lightning = "*"
 albumentations = ">=2.0.0"
 lazy-loader = ">=0.3.0"
-medimgkit = ">=0.4.4"
+medimgkit = ">=0.5.0"
 # For compatibility with the datamintapi package
 datamintapi = "0.0.*"
 # Extra dependencies for docs