datamint 2.1.4__tar.gz → 2.2.1__tar.gz

{datamint-2.1.4 → datamint-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamint
-Version: 2.1.4
+Version: 2.2.1
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -21,7 +21,7 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
-Requires-Dist: medimgkit (>=0.6.6)
+Requires-Dist: medimgkit (>=0.7.2)
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-2.1.4 → datamint-2.2.1}/datamint/api/endpoints/resources_api.py

@@ -85,8 +85,8 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
 
         Args:
             status: The resource status. Possible values: 'inbox', 'published', 'archived' or None. If None, it will return all resources.
-            from_date : The start date.
-            to_date: The end date.
+            from_date : The start date (inclusive).
+            to_date: The end date (exclusive).
             tags: The tags to filter the resources.
             modality: The modality of the resources.
             mimetype: The mimetype of the resources.
@@ -235,30 +235,26 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
                                             metadata_file: Optional[str | dict] = None,
                                             ) -> str:
         if is_io_object(file_path):
-            name = file_path.name
+            source_filepath = os.path.abspath(os.path.expanduser(file_path.name))
+            filename = os.path.basename(source_filepath)
         else:
-            name = file_path
+            source_filepath = os.path.abspath(os.path.expanduser(file_path))
+            filename = os.path.basename(source_filepath)
 
         if session is not None and not isinstance(session, aiohttp.ClientSession):
             raise ValueError("session must be an aiohttp.ClientSession object.")
 
-        name = os.path.expanduser(os.path.normpath(name))
-        if len(Path(name).parts) == 0:
-            raise ValueError(f"File path '{name}' is not valid.")
-        name = os.path.join(*[x if x != '..' else '_' for x in Path(name).parts])
-
         if mung_filename is not None:
-            file_parts = Path(name).parts
+            file_parts = Path(source_filepath).parts
             if file_parts[0] == os.path.sep:
                 file_parts = file_parts[1:]
             if mung_filename == 'all':
-                new_file_path = '_'.join(file_parts)
+                new_filename = '_'.join(file_parts)
             else:
                 folder_parts = file_parts[:-1]
-                new_file_path = '_'.join([folder_parts[i-1] for i in mung_filename if i <= len(folder_parts)])
-                new_file_path += '_' + file_parts[-1]
-            name = new_file_path
-            _LOGGER.debug(f"New file path: {name}")
+                new_filename = '_'.join([folder_parts[i-1] for i in mung_filename if i <= len(folder_parts)])
+                new_filename += '_' + file_parts[-1]
+            filename = new_filename
 
         is_a_dicom_file = None
         if mimetype is None:
@@ -268,14 +264,12 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
                     mimetype = DEFAULT_NIFTI_MIME
                     break
             else:
-                if ext == '.nii.gz' or name.lower().endswith('nii.gz'):
+                if ext == '.nii.gz' or filename.lower().endswith('nii.gz'):
                     mimetype = DEFAULT_NIFTI_MIME
                 else:
                     mimetype = mimetype_list[-1] if mimetype_list else DEFAULT_MIME_TYPE
 
         mimetype = standardize_mimetype(mimetype)
-        filename = os.path.basename(name)
-        _LOGGER.debug(f"File name '{filename}' mimetype: {mimetype}")
 
         if is_a_dicom_file == True or is_dicom(file_path):
             if tags is None:
@@ -292,7 +286,7 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
             elif lat == 'R':
                 tags.append("right")
             # make the dicom `ds` object a file-like object in order to avoid unnecessary disk writes
-            f = to_bytesio(ds, name)
+            f = to_bytesio(ds, filename)
         else:
             f = _open_io(file_path)
 
@@ -329,7 +323,7 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
             form.add_field('source', 'api')
 
             form.add_field(file_key, f, filename=filename, content_type=mimetype)
-            form.add_field('source_filepath', name)  # full path to the file
+            form.add_field('source_filepath', source_filepath)  # full path to the file
             if mimetype is not None:
                 form.add_field('mimetype', mimetype)
             if channel is not None:
@@ -354,11 +348,11 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
                                                             data=form)
             if 'error' in resp_data:
                 raise DatamintException(resp_data['error'])
-            _LOGGER.debug(f"Response on uploading {name}: {resp_data}")
+            _LOGGER.debug(f"Response on uploading {filename}: {resp_data}")
             return resp_data['id']
         except Exception as e:
-            if 'name' in locals():
-                _LOGGER.error(f"Error uploading {name}: {e}")
+            if 'filename' in locals():
+                _LOGGER.error(f"Error uploading {filename}: {e}")
             else:
                 _LOGGER.error(f"Error uploading {file_path}: {e}")
             raise
@@ -676,7 +670,7 @@ class ResourcesApi(CreatableEntityApi[Resource], DeletableEntityApi[Resource]):
                     channel='study_channel',
                     segmentation_files={
                         'files': ['path/to/segmentation.nii.gz'],
-                        'names': {1: 'Bone', 2: 'Tissue'}
+                        'names': {1: 'Brain', 2: 'Lung'}
                     },
                     metadata={'patient_age': 45, 'modality': 'CT'}
                 )

{datamint-2.1.4 → datamint-2.2.1}/datamint/client_cmd_tools/datamint_upload.py

@@ -492,7 +492,7 @@ def _get_files_from_path(path: str | Path,
     Returns:
         List of file paths as strings
     """
-    path = Path(path)
+    path = Path(path).resolve()
 
     if path.is_file():
         return [str(path)]
@@ -507,7 +507,7 @@ def _get_files_from_path(path: str | Path,
 
         file_paths = walk_to_depth(path, recursive_depth, exclude_pattern)
         filtered_files = filter_files(file_paths, include_extensions, exclude_extensions)
-        return [str(f) for f in filtered_files]
+        return [str(f.resolve()) for f in filtered_files]
 
     except Exception as e:
         _LOGGER.error(f'Error in recursive search: {e}')
@@ -786,7 +786,6 @@ def main():
             _USER_LOGGER.error(f'❌ Connection failed: {e}')
             return
         try:
-            print('>>>', segfiles)
             results = api.resources.upload_resources(channel=args.channel,
                                                      files_path=files_path,
                                                      tags=args.tag,

{datamint-2.1.4 → datamint-2.2.1}/datamint/dataset/base_dataset.py

@@ -1,15 +1,12 @@
 import os
 import requests
-from tqdm.auto import tqdm
 from typing import Optional, Callable, Any, Literal, Sequence
 import logging
 import shutil
 import json
-import yaml
 import pydicom
 from pydicom.dataset import FileDataset
 import numpy as np
-from datamint import configs
 from torch.utils.data import DataLoader
 import torch
 from torch import Tensor

{datamint-2.1.4 → datamint-2.2.1}/datamint/dataset/dataset.py

@@ -1,5 +1,5 @@
 from .base_dataset import DatamintBaseDataset
-from typing import List, Optional, Callable, Any, Dict, Literal, Sequence
+from typing import Optional, Callable, Any, Literal, Sequence
 import torch
 from torch import Tensor
 import os
@@ -120,7 +120,7 @@ class DatamintDataset(DatamintBaseDataset):
 
     def _load_segmentations(self,
                             annotations: Sequence[Annotation],
-                            img_shape) -> tuple[dict[str, list], dict[str, list]]:
+                            img_shape) -> tuple[dict[str, list], dict[str, list], dict[str, Any]]:
         """
         Load segmentations from annotations.
 
@@ -129,12 +129,13 @@ class DatamintDataset(DatamintBaseDataset):
             img_shape: shape of the image (#frames, C, H, W)
 
         Returns:
-            tuple[dict[str, list], dict[str, list]]: a tuple of two dictionaries.
+            tuple[dict[str, list], dict[str, list], dict[str, Any]]: a tuple of two dictionaries and additional metadata.
                 The first dictionary is author -> list of #frames tensors, each tensor has shape (#instances_i, H, W).
                 The second dictionary is author -> list of #frames segmentation labels (tensors).
         """
         segmentations = {}
         seg_labels = {}
+        seg_metainfos = {}
 
         if self.return_frame_by_frame:
             assert len(img_shape) == 3, f"img_shape must have 3 dimensions, got {img_shape}"
@@ -155,11 +156,15 @@ class DatamintDataset(DatamintBaseDataset):
 
             segfilepath = ann.file  # png file
             segfilepath = os.path.join(self.dataset_dir, segfilepath)
-            seg = read_array_normalized(segfilepath)  # (frames, C, H, W)
+            seg, seg_metainfo = read_array_normalized(segfilepath, return_metainfo=True)  # (frames, C, H, W)
             if seg.shape[1] != 1:
                 raise ValueError(f"Segmentation file must have 1 channel, got {seg.shape} in {segfilepath}")
             seg = seg[:, 0, :, :]  # (frames, H, W)
-            
+
+            if seg_metainfo is None:
+                raise Exception
+            seg_metainfos[author] = seg_metainfo
+
             # # FIXME: avoid enforcing resizing the mask
             # seg = (Image.open(segfilepath)
             #        .convert('L')
@@ -217,7 +222,7 @@ class DatamintDataset(DatamintBaseDataset):
                     author_segs[i] = torch.zeros((0, h, w), dtype=torch.bool)
                     author_labels[i] = torch.zeros(0, dtype=torch.int32)
 
-        return segmentations, seg_labels
+        return segmentations, seg_labels, seg_metainfos
 
     def _instanceseg2semanticseg(self,
                                  segmentations: Sequence[Tensor],
@@ -273,19 +278,19 @@ class DatamintDataset(DatamintBaseDataset):
             raise ValueError(f"Unknown semantic_seg_merge_strategy: {self.semantic_seg_merge_strategy}")
         return merged_segs.to(torch.get_default_dtype())
 
-    def _apply_semantic_seg_merge_strategy_union(self, segmentations: Dict[str, torch.Tensor]) -> torch.Tensor:
+    def _apply_semantic_seg_merge_strategy_union(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor:
         new_segmentations = torch.zeros_like(list(segmentations.values())[0])
         for seg in segmentations.values():
             new_segmentations += seg
         return new_segmentations.bool()
 
-    def _apply_semantic_seg_merge_strategy_intersection(self, segmentations: Dict[str, torch.Tensor]) -> torch.Tensor:
+    def _apply_semantic_seg_merge_strategy_intersection(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor:
         new_segmentations = torch.ones_like(list(segmentations.values())[0])
         for seg in segmentations.values():
             new_segmentations += seg
         return new_segmentations.bool()
 
-    def _apply_semantic_seg_merge_strategy_mode(self, segmentations: Dict[str, torch.Tensor]) -> torch.Tensor:
+    def _apply_semantic_seg_merge_strategy_mode(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor:
         new_segmentations = torch.zeros_like(list(segmentations.values())[0])
         for seg in segmentations.values():
             new_segmentations += seg
@@ -428,7 +433,7 @@ class DatamintDataset(DatamintBaseDataset):
 
         try:
             if self.return_segmentations:
-                segmentations, seg_labels = self._load_segmentations(annotations, img.shape)
+                segmentations, seg_labels, seg_metainfos = self._load_segmentations(annotations, img.shape)
                 # seg_labels can be dict[str, list[Tensor]]
                 # apply mask transform
                 if self.mask_transform is not None:
@@ -475,6 +480,7 @@ class DatamintDataset(DatamintBaseDataset):
                 new_item['seg_labels'] = seg_labels
                 # process seg_labels to convert from code to label names
                 new_item['seg_labels_names'] = self._seg_labels_to_names(seg_labels)
+                new_item['seg_metainfo'] = {'file_metainfo': seg_metainfos}
 
         except Exception:
             _LOGGER.error(f'Error in loading/processing segmentations of {metainfo}')

{datamint-2.1.4 → datamint-2.2.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "datamint"
 description = "A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows."
-version = "2.1.4"
+version = "2.2.1"
 dynamic = ["dependencies"]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -40,7 +40,7 @@ matplotlib = "*"
 lightning = "*"
 albumentations = ">=2.0.0"
 lazy-loader = ">=0.3.0"
-medimgkit = ">=0.6.6"
+medimgkit = ">=0.7.2"
 typing_extensions = ">=4.0.0"
 pydantic = ">=2.6.4"
 httpx = "*"