datamint 1.7.4__tar.gz → 1.7.6__tar.gz

{datamint-1.7.4 → datamint-1.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamint
-Version: 1.7.4
+Version: 1.7.6
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -19,7 +19,7 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
-Requires-Dist: medimgkit (>=0.2.0)
+Requires-Dist: medimgkit (>=0.2.2)
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-1.7.4 → datamint-1.7.6}/datamint/apihandler/annotation_api_handler.py

@@ -16,6 +16,7 @@ import json
 from deprecated import deprecated
 from pathlib import Path
 from tqdm.auto import tqdm
+from medimgkit.nifti_utils import DEFAULT_NIFTI_MIME
 
 _LOGGER = logging.getLogger(__name__)
 _USER_LOGGER = logging.getLogger('user_logger')
@@ -280,7 +281,7 @@ class AnnotationAPIHandler(BaseAPIHandler):
                 with open(file_path, 'rb') as f:
                     filename = os.path.basename(file_path)
                     form = aiohttp.FormData()
-                    form.add_field('file', f, filename=filename, content_type='application/x-nifti')
+                    form.add_field('file', f, filename=filename, content_type=DEFAULT_NIFTI_MIME)
                     if model_id is not None:
                         form.add_field('model_id', model_id)  # Add model_id if provided
                     if worklist_id is not None:

{datamint-1.7.4 → datamint-1.7.6}/datamint/apihandler/base_api_handler.py

@@ -14,7 +14,7 @@ import cv2
 import nibabel as nib
 from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
 from datamint import configs
-from functools import wraps
+import gzip
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -298,8 +298,15 @@ class BaseAPIHandler:
         elif mimetype == 'application/octet-stream':
             return bytes_array
         elif mimetype.endswith('nifti'):
-            if file_path is None:
-                raise NotImplementedError(f"file_path=None is not implemented yet for {mimetype}.")
-            return nib.load(file_path)
+            try:
+                return nib.Nifti1Image.from_stream(content_io)
+            except Exception as e:
+                if file_path is not None:
+                    return nib.load(file_path)
+                raise e
+        elif mimetype == 'application/gzip':
+            # let's hope it's a .nii.gz
+            with gzip.open(content_io, 'rb') as f:
+                return nib.Nifti1Image.from_stream(f)
 
         raise ValueError(f"Unsupported mimetype: {mimetype}")

{datamint-1.7.4 → datamint-1.7.6}/datamint/apihandler/root_api_handler.py

@@ -6,10 +6,11 @@ from requests.exceptions import HTTPError
 import logging
 import asyncio
 import aiohttp
-from medimgkit.dicom_utils import anonymize_dicom, to_bytesio, is_dicom
-from medimgkit import dicom_utils
+from medimgkit.dicom_utils import anonymize_dicom, to_bytesio, is_dicom, is_dicom_report
+from medimgkit import dicom_utils, standardize_mimetype
 from medimgkit.io_utils import is_io_object
-from medimgkit.format_detection import guess_type, guess_extension
+from medimgkit.format_detection import guess_typez, guess_extension, DEFAULT_MIME_TYPE
+from medimgkit.nifti_utils import DEFAULT_NIFTI_MIME, NIFTI_MIMES
 import pydicom
 from pathlib import Path
 from datetime import date
@@ -87,11 +88,18 @@ class RootAPIHandler(BaseAPIHandler):
 
         is_a_dicom_file = None
         if mimetype is None:
-            mimetype, _ = guess_type(file_path, use_magic=True)
-            if mimetype == 'application/gzip' and name.lower().endswith('nii.gz'):
-                # Special case for gzipped NIfTI files
-                mimetype = 'image/x.nifti'
+            mimetype_list, ext = guess_typez(file_path, use_magic=True)
+            for mime in mimetype_list:
+                if mime in NIFTI_MIMES:
+                    mimetype = DEFAULT_NIFTI_MIME
+                    break
+            else:
+                if ext == '.nii.gz' or name.lower().endswith('nii.gz'):
+                    mimetype = DEFAULT_NIFTI_MIME
+                else:
+                    mimetype = mimetype_list[-1] if mimetype_list else DEFAULT_MIME_TYPE
 
+        mimetype = standardize_mimetype(mimetype)
         filename = os.path.basename(name)
         _LOGGER.debug(f"File name '{filename}' mimetype: {mimetype}")
 
@@ -355,35 +363,6 @@ class RootAPIHandler(BaseAPIHandler):
 
         return result[0]
 
-    @staticmethod
-    def _is_dicom_report(file_path: str | IO) -> bool:
-        """
-        Check if a DICOM file is a report (e.g., Structured Report).
-
-        Args:
-            file_path: Path to the DICOM file or file-like object.
-
-        Returns:
-            bool: True if the DICOM file is a report, False otherwise.
-        """
-        try:
-            if not is_dicom(file_path):
-                return False
-
-            ds = pydicom.dcmread(file_path, stop_before_pixels=True)
-            if hasattr(file_path, 'seek'):
-                file_path.seek(0)
-            modality = getattr(ds, 'Modality', None)
-
-            # Common report modalities
-            # SR=Structured Report, DOC=Document, KO=Key Object, PR=Presentation State
-            report_modalities = {'SR', 'DOC', 'KO', 'PR', 'ESR'}
-
-            return modality in report_modalities
-        except Exception as e:
-            _LOGGER.warning(f"Error checking if DICOM is a report: {e}")
-            return False
-
     def upload_resources(self,
                          files_path: str | IO | Sequence[str | IO] | pydicom.dataset.Dataset,
                          mimetype: Optional[str] = None,
@@ -444,7 +423,7 @@ class RootAPIHandler(BaseAPIHandler):
 
         # Discard DICOM reports
         if discard_dicom_reports:
-            files_path = [f for f in files_path if not RootAPIHandler._is_dicom_report(f)]
+            files_path = [f for f in files_path if not is_dicom_report(f)]
             old_size = len(files_path)
             if old_size is not None and old_size != len(files_path):
                 _LOGGER.info(f"Discarded {old_size - len(files_path)} DICOM report files from upload.")
@@ -866,9 +845,10 @@ class RootAPIHandler(BaseAPIHandler):
 
     async def _async_download_file(self,
                                    resource_id: str,
-                                   save_path: str,
+                                   save_path: str | Path,
                                    session: aiohttp.ClientSession | None = None,
-                                   progress_bar: tqdm | None = None):
+                                   progress_bar: tqdm | None = None,
+                                   add_extension: bool = False) -> str:
         """
         Asynchronously download a file from the server.
 
@@ -877,6 +857,10 @@ class RootAPIHandler(BaseAPIHandler):
             save_path (str): The path to save the file.
             session (aiohttp.ClientSession): The aiohttp session to use for the request.
             progress_bar (tqdm | None): Optional progress bar to update after download completion.
+            add_extension (bool): Whether to add the appropriate file extension based on content type.
+
+        Returns:
+            str: The actual path where the file was saved (important when add_extension=True).
         """
         url = f"{self._get_endpoint_url(RootAPIHandler.ENDPOINT_RESOURCES)}/{resource_id}/file"
         request_params = {
@@ -884,41 +868,90 @@ class RootAPIHandler(BaseAPIHandler):
             'headers': {'accept': 'application/octet-stream'},
             'url': url
         }
+        save_path = str(save_path)  # Ensure save_path is a string for file operations
         try:
             data_bytes = await self._run_request_async(request_params, session, 'content')
-            with open(save_path, 'wb') as f:
-                f.write(data_bytes)
+
+            final_save_path = save_path
+            if add_extension:
+                # Save to temporary file first to determine mimetype from content
+                temp_path = f"{save_path}.tmp"
+                with open(temp_path, 'wb') as f:
+                    f.write(data_bytes)
+
+                # Determine mimetype from file content
+                mimetype_list, ext = guess_typez(temp_path, use_magic=True)
+                mimetype = mimetype_list[-1]
+
+                # get mimetype from resource info if not detected
+                if mimetype is None or mimetype == DEFAULT_MIME_TYPE:
+                    resource_info = self.get_resources_by_ids(resource_id)
+                    mimetype = resource_info.get('mimetype', mimetype)
+
+                # Generate final path with extension if needed
+                if mimetype is not None and mimetype != DEFAULT_MIME_TYPE:
+                    if ext is None:
+                        ext = guess_extension(mimetype)
+                    if ext is not None and not save_path.endswith(ext):
+                        final_save_path = save_path + ext
+
+                # Move file to final location
+                os.rename(temp_path, final_save_path)
+            else:
+                # Standard save without extension detection
+                with open(final_save_path, 'wb') as f:
+                    f.write(data_bytes)
+
             if progress_bar:
                 progress_bar.update(1)
+
+            return final_save_path
+
         except ResourceNotFoundError as e:
             e.set_params('resource', {'resource_id': resource_id})
             raise e
 
     def download_multiple_resources(self,
                                     resource_ids: list[str],
-                                    save_path: list[str] | str
-                                    ) -> None:
+                                    save_path: list[str] | str,
+                                    add_extension: bool = False,
+                                    ) -> list[str]:
         """
         Download multiple resources and save them to the specified paths.
 
         Args:
             resource_ids (list[str]): A list of resource unique ids.
             save_path (list[str] | str): A list of paths to save the files or a directory path.
+            add_extension (bool): Whether to add the appropriate file extension to the save_path based on the content type.
+
+        Returns:
+            list[str]: A list of paths where the files were saved. Important if `add_extension=True`.
         """
+        if isinstance(resource_ids, str):
+            raise ValueError("resource_ids must be a list of strings.")
+
         async def _download_all_async():
             async with aiohttp.ClientSession() as session:
                 tasks = [
-                    self._async_download_file(resource_id, save_path=path, session=session, progress_bar=progress_bar)
+                    self._async_download_file(
+                        resource_id=resource_id,
+                        save_path=path,
+                        session=session,
+                        progress_bar=progress_bar,
+                        add_extension=add_extension
+                    )
                     for resource_id, path in zip(resource_ids, save_path)
                 ]
-                await asyncio.gather(*tasks)
+                return await asyncio.gather(*tasks)
 
         if isinstance(save_path, str):
             save_path = [os.path.join(save_path, r) for r in resource_ids]
 
         with tqdm(total=len(resource_ids), desc="Downloading resources", unit="file") as progress_bar:
             loop = asyncio.get_event_loop()
-            loop.run_until_complete(_download_all_async())
+            final_save_paths = loop.run_until_complete(_download_all_async())
+
+        return final_save_paths
 
     def download_resource_file(self,
                                resource_id: str,
@@ -962,10 +995,14 @@ class RootAPIHandler(BaseAPIHandler):
 
             # Get mimetype if needed for auto_convert or add_extension
             mimetype = None
+            mimetype_list = []
+            ext = None
             if auto_convert or add_extension:
-                resource_info = self.get_resources_by_ids(resource_id)
-                mimetype = resource_info.get('mimetype', guess_type(response.content)[0])
-                
+                mimetype_list, ext = guess_typez(response.content)
+                mimetype = mimetype_list[-1]
+                if mimetype is None or mimetype == DEFAULT_MIME_TYPE:
+                    resource_info = self.get_resources_by_ids(resource_id)
+                    mimetype = resource_info.get('mimetype', None)
 
             if auto_convert:
                 try:
@@ -987,7 +1024,8 @@ class RootAPIHandler(BaseAPIHandler):
 
         if save_path is not None:
             if add_extension and mimetype is not None:
-                ext = guess_extension(mimetype)
+                if ext is None:
+                    ext = guess_extension(mimetype)
                 if ext is not None and not save_path.endswith(ext):
                     save_path += ext
             with open(save_path, 'wb') as f:

{datamint-1.7.4 → datamint-1.7.6}/datamint/dataset/base_dataset.py

@@ -15,10 +15,10 @@ import torch
 from torch import Tensor
 from datamint.apihandler.base_api_handler import DatamintException
 from medimgkit.dicom_utils import is_dicom
-from medimgkit.io_utils import read_array_normalized
+from medimgkit.readers import read_array_normalized
+from medimgkit.format_detection import guess_extension
 from datetime import datetime
 from pathlib import Path
-from mimetypes import guess_extension
 from datamint.dataset.annotation import Annotation
 import cv2
 
@@ -898,8 +898,11 @@ class DatamintBaseDataset:
             new_resources_path = [Path(self.dataset_dir) / r['file'] for r in new_resources]
             new_resources_ids = [r['id'] for r in new_resources]
             _LOGGER.info(f"Downloading {len(new_resources)} new resources...")
-            self.api_handler.download_multiple_resources(new_resources_ids,
-                                                        save_path=new_resources_path)
+            new_res_paths = self.api_handler.download_multiple_resources(new_resources_ids,
+                                                                         save_path=new_resources_path,
+                                                                         add_extension=True)
+            for new_rpath, r in zip(new_res_paths, new_resources):
+                r['file'] = str(Path(new_rpath).relative_to(self.dataset_dir))
             _LOGGER.info(f"Downloaded {len(new_resources)} new resources.")
 
         for r in deleted_resources:
@@ -989,13 +992,13 @@ class DatamintBaseDataset:
         if 'file' in resource and resource['file'] is not None:
             return Path(resource['file'])
         else:
-            ext = guess_extension(resource['mimetype'], strict=False)
-            _LOGGER.debug(f"Guessed extension for resource {resource['id']}|{resource['mimetype']}: {ext}")
-            if ext is None:
-                _LOGGER.warning(f"Could not guess extension for resource {resource['id']}.")
-                ext = ''
-                raise Exception
-            return Path('images', f"{resource['id']}{ext}")
+            # ext = guess_extension(resource['mimetype'])
+            # _LOGGER.debug(f"Guessed extension for resource {resource['id']}|{resource['mimetype']}: {ext}")
+            # if ext is None:
+            #     _LOGGER.warning(f"Could not guess extension for resource {resource['id']}.")
+            #     ext = ''
+            # return Path('images', f"{resource['id']}{ext}")
+            return Path('images', resource['id'])
 
     def _get_annotation_file_path(self, annotation: dict | Annotation) -> Path | None:
         """Get the local file path for an annotation."""

{datamint-1.7.4 → datamint-1.7.6}/datamint/dataset/dataset.py

@@ -478,7 +478,7 @@ class DatamintDataset(DatamintBaseDataset):
 
     def _convert_labels_annotations(self,
                                     annotations: list[Annotation],
-                                    num_frames: int = None) -> dict[str, torch.Tensor]:
+                                    num_frames: int | None = None) -> dict[str, torch.Tensor]:
         """
         Converts the annotations, of the same type and scope, to tensor of shape (num_frames, num_labels)
         for each annotator.

{datamint-1.7.4 → datamint-1.7.6}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "datamint"
 description = "A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows."
-version = "1.7.4"
+version = "1.7.6"
 dynamic = ["dependencies"]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -40,7 +40,7 @@ matplotlib = "*"
 lightning = "*"
 albumentations = ">=2.0.0"
 lazy-loader = ">=0.3.0"
-medimgkit = ">=0.2.0"
+medimgkit = ">=0.2.2"
 # For compatibility with the datamintapi package
 datamintapi = "0.0.*"
 # Extra dependencies for docs