datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

datamint/api/entity_base_api.py

@@ -1,4 +1,5 @@
-from typing import Any, TypeVar, Generic, Type, Sequence
+from typing import Any, Literal, TypeVar, Generic, overload
+from collections.abc import Sequence, AsyncGenerator
 import logging
 import httpx
 from datamint.entities.base_entity import BaseEntity
@@ -7,7 +8,6 @@ import aiohttp
 import asyncio
 from .base_api import ApiConfig, BaseApi
 import contextlib
-from typing import AsyncGenerator
 
 logger = logging.getLogger(__name__)
 T = TypeVar('T', bound=BaseEntity)
@@ -24,9 +24,10 @@ class EntityBaseApi(BaseApi, Generic[T]):
     """
 
     def __init__(self, config: ApiConfig,
-                 entity_class: Type[T],
+                 entity_class: type[T],
                  endpoint_base: str,
-                 client: httpx.Client | None = None) -> None:
+                 client: httpx.Client | None = None
+                 ) -> None:
         """Initialize the entity API handler.
 
         Args:
@@ -53,6 +54,11 @@ class EntityBaseApi(BaseApi, Generic[T]):
                              entity_id: str | BaseEntity,
                              add_path: str = '',
                              **kwargs) -> httpx.Response:
+        """
+        Make an HTTP request for a specific entity by its ID.
+        It is basically a wrapper around :py:meth:`_make_request` that
+        constructs the URL for the entity in this form: `/{endpoint_base}/{entity_id}/{add_path}`
+        """
         try:
             entity_id = self._entid(entity_id)
             add_path = '/'.join(add_path.strip().strip('/').split('/'))
@@ -61,6 +67,10 @@ class EntityBaseApi(BaseApi, Generic[T]):
             if e.response.status_code == 404:
                 raise ResourceNotFoundError(self.endpoint_base, {'id': entity_id}) from e
             raise
+        except ResourceNotFoundError as e:
+            e.resource_type = self.endpoint_base
+            e.params = {'id': entity_id}
+            raise
 
     @contextlib.asynccontextmanager
     async def _make_entity_request_async(self,
@@ -81,6 +91,23 @@ class EntityBaseApi(BaseApi, Generic[T]):
             if e.status == 404:
                 raise ResourceNotFoundError(self.endpoint_base, {'id': entity_id}) from e
             raise
+        except ResourceNotFoundError as e:
+            e.resource_type = self.endpoint_base
+            e.params = {'id': entity_id}
+            raise
+
+    async def _make_entity_request_async_json(self,
+                                              method: str,
+                                              entity_id: str | BaseEntity,
+                                              add_path: str = '',
+                                              session: aiohttp.ClientSession | None = None,
+                                              **kwargs):
+        async with self._make_entity_request_async(method,
+                                                   entity_id,
+                                                   add_path=add_path,
+                                                   session=session,
+                                                   **kwargs) as resp:
+            return await resp.json()
 
     def _stream_entity_request(self,
                                method: str,
@@ -210,7 +237,9 @@ class DeletableEntityApi(EntityBaseApi[T]):
             httpx.HTTPStatusError: If deletion fails or any entity not found
         """
         async def _delete_all_async():
-            async with aiohttp.ClientSession() as session:
+            connector = self._create_aiohttp_connector(force_close=True)
+            timeout = aiohttp.ClientTimeout(total=None, connect=60, sock_read=300)
+            async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
                 tasks = [
                     self._delete_async(entity, session)
                     for entity in entities
@@ -248,7 +277,16 @@ class CreatableEntityApi(EntityBaseApi[T]):
     This class adds methods to handle creation of new entities.
     """
 
-    def _create(self, entity_data: dict[str, Any]) -> str | list[str | dict]:
+    @overload
+    def _create(self, entity_data: dict[str, Any],
+                return_entity: Literal[True] = True) -> T | list[T]: ...
+
+    @overload
+    def _create(self, entity_data: dict[str, Any],
+                return_entity: Literal[False]) -> str | list: ...
+
+    def _create(self, entity_data: dict[str, Any],
+                return_entity: bool = False) -> str | T | list:
         """Create a new entity.
 
         Args:
@@ -263,14 +301,35 @@ class CreatableEntityApi(EntityBaseApi[T]):
         response = self._make_request('POST', f'/{self.endpoint_base}', json=entity_data)
         respdata = response.json()
         if isinstance(respdata, str):
+            if return_entity:
+                return self.get_by_id(respdata)
             return respdata
         if isinstance(respdata, list):
+            if return_entity:
+                logger.warning("Current implementation is slow when returning entities on bulk create."
+                               " Try ``return_entity=False`` for better performance.")
+                return [self.get_by_id(item['id']) if isinstance(item, dict) and 'id' in item else self.get_by_id(item)
+                        for item in respdata]
             return respdata
         if isinstance(respdata, dict):
+            if return_entity:
+                try:
+                    return self._init_entity_obj(**respdata)
+                except:
+                    logger.debug("Failed to init entity obj on create response. Falling back to get_by_id.")
+                    return self.get_by_id(respdata.get('id'))
             return respdata.get('id')
         return respdata
 
-    def create(self, *args, **kwargs) -> str | T:
+    @overload
+    def create(self, *args, return_entity: Literal[True] = True, **kwargs) -> T: ...
+
+    @overload
+    def create(self, *args, return_entity: Literal[False], **kwargs) -> str: ...
+
+    def create(self, *args,
+               return_entity: bool = True,
+               **kwargs) -> str | T:
         raise NotImplementedError("Subclasses must implement the create method with their own custom parameters")
 
 

datamint/apihandler/base_api_handler.py

@@ -30,7 +30,6 @@ ResourceFields: TypeAlias = Literal['modality', 'created_by', 'published_by', 'p
 
 _PAGE_LIMIT = 5000
 
-
 @deprecated(reason="Please use `from datamint import Api` instead.", version="2.0.0")
 class BaseAPIHandler:
     """

datamint/apihandler/dto/annotation_dto.py

@@ -178,6 +178,8 @@ class CreateAnnotationDto:
         if model_id is not None:
             if is_model == False:
                 raise ValueError("model_id==False while self.model_id is provided.")
+            if not isinstance(model_id, str):
+                raise ValueError("model_id must be a string if provided.")
             is_model = True
         self.is_model = is_model
         self.geometry = geometry

datamint/client_cmd_tools/datamint_config.py

@@ -6,7 +6,6 @@ from rich.prompt import Prompt, Confirm
 from rich.console import Console
 import os
 import shutil
-from pathlib import Path
 from rich.table import Table
 
 _LOGGER = logging.getLogger(__name__)

datamint/client_cmd_tools/datamint_upload.py

@@ -574,6 +574,8 @@ def _parse_args() -> tuple[Any, list[str], Optional[list[dict]], Optional[list[s
                         help='Automatically detect and include JSON metadata files with the same base name as NIFTI files')
     parser.add_argument('--no-auto-detect-json', dest='auto_detect_json', action='store_false',
                         help='Disable automatic detection of JSON metadata files (default behavior)')
+    parser.add_argument('--no-assemble-dicoms', dest='assemble_dicoms', action='store_false', default=True,
+                        help='Do not assemble DICOM files into series (default: assemble them)')
     parser.add_argument('--version', action='version', version=f'%(prog)s {datamint_version}')
     parser.add_argument('--verbose', action='store_true', help='Print debug messages', default=False)
     args = parser.parse_args()
@@ -797,7 +799,7 @@ def main():
                                                      publish_to=args.project,
                                                      segmentation_files=segfiles,
                                                      transpose_segmentation=args.transpose_segmentation,
-                                                     assemble_dicoms=True,
+                                                     assemble_dicoms=args.assemble_dicoms,
                                                      metadata=metadata_files,
                                                      progress_bar=True
                                                      )

datamint/configs.py

@@ -1,10 +1,8 @@
 import yaml
 import os
 import logging
-from netrc import netrc
 from platformdirs import PlatformDirs
-from typing import Dict
-from pathlib import Path
+from typing import Any
 
 APIURL_KEY = 'default_api_url'
 APIKEY_KEY = 'api_key'
@@ -14,6 +12,10 @@ ENV_VARS = {
     APIURL_KEY: 'DATAMINT_API_URL'
 }
 
+DEFAULT_VALUES = {
+    APIURL_KEY: 'https://api.datamint.io'
+}
+
 _LOGGER = logging.getLogger(__name__)
 
 DIRS = PlatformDirs(appname='datamintapi')
@@ -23,17 +25,19 @@ try:
 except Exception as e:
     _LOGGER.error(f"Could not determine home directory: {e}")
     DATAMINT_DATA_DIR = None
-    
 
 
 def get_env_var_name(key: str) -> str:
     return ENV_VARS[key]
 
-def read_config() -> Dict:
+
+def read_config() -> dict[str, Any]:
     if os.path.exists(CONFIG_FILE):
         with open(CONFIG_FILE, 'r') as configfile:
-            return yaml.safe_load(configfile)
-    return {}
+            config = yaml.safe_load(configfile)
+        config.update({k: v for k, v in DEFAULT_VALUES.items() if k not in config})
+        return config
+    return DEFAULT_VALUES.copy()
 
 
 def set_value(key: str,

datamint/dataset/base_dataset.py

@@ -307,6 +307,10 @@ class DatamintBaseDataset:
         self.image_lsets, self.image_lcodes = self._get_labels_set(framed=False)
         worklist_id = self.get_info()['worklist_id']
         groups: dict[str, dict] = self.api.annotationsets.get_segmentation_group(worklist_id)['groups']
+        if not groups:
+            self.seglabel_list = []
+            self.seglabel2code = {}
+            return
         # order by 'index' key
         max_index = max([g['index'] for g in groups.values()])
         self.seglabel_list : list[str] = ['UNKNOWN'] * max_index  # 1-based
@@ -928,6 +932,7 @@ class DatamintBaseDataset:
         # Collect all segmentation annotations that need to be downloaded
         segmentations_to_download = []
         segmentation_paths = []
+        segmentation_resource_map = {}  # Maps annotation ID to resource ID for cleanup
 
         # update annotations in resources
         for resource in self.images_metainfo:
@@ -954,6 +959,7 @@ class DatamintBaseDataset:
                     filepath.parent.mkdir(parents=True, exist_ok=True)
                     segmentations_to_download.append(ann)
                     segmentation_paths.append(filepath)
+                    segmentation_resource_map[ann.id] = resource_id
 
             # Process annotation changes
             for ann in annotations_to_remove:
@@ -971,14 +977,28 @@ class DatamintBaseDataset:
                     _LOGGER.error(f"Error deleting annotation file {filepath}: {e}")
 
             # Update resource annotations list - convert to Annotation objects
-            # resource['annotations'] = [Annotation.from_dict(ann) for ann in new_resource_annotations]
             resource['annotations'] = new_resource_annotations
 
         # Batch download all segmentation files
         if segmentations_to_download:
             _LOGGER.info(f"Downloading {len(segmentations_to_download)} segmentation files...")
-            self.api.annotations.download_multiple_files(segmentations_to_download, segmentation_paths)
-            _LOGGER.info(f"Downloaded {len(segmentations_to_download)} segmentation files.")
+            download_results = self.api.annotations.download_multiple_files(
+                segmentations_to_download, segmentation_paths
+            )
+            
+            # Process failed downloads
+            failed_annotations = [result['annotation_id'] for result in download_results if not result['success']]
+            if failed_annotations:
+                _LOGGER.warning(f"Failed to download {len(failed_annotations)} annotations, removing them from metadata")
+                
+                # Remove failed annotations from each resource's annotation list
+                for resource in self.images_metainfo:
+                    resource['annotations'] = [
+                        ann for ann in resource['annotations']
+                        if ann.id not in failed_annotations
+                    ]
+            
+            _LOGGER.info(f"Successfully downloaded {len(segmentations_to_download) - len(failed_annotations)} segmentation files.")
 
         ###################
         # update metadata
@@ -1050,7 +1070,7 @@ class DatamintBaseDataset:
         # delete associated annotations
         for ann in deleted_metainfo.get('annotations', []):
             ann_file = getattr(ann, 'file', None) if hasattr(ann, 'file') else ann.get('file', None)
-            if ann_file is not None:
+            if ann_file is not None and os.path.exists(os.path.join(self.dataset_dir, ann_file)):
                 os.remove(os.path.join(self.dataset_dir, ann_file))
 
     def __add__(self, other):

datamint/dataset/dataset.py

@@ -7,7 +7,7 @@ import numpy as np
 import logging
 from PIL import Image
 import albumentations
-from datamint.entities.annotation import Annotation
+from datamint.entities.annotations.annotation import Annotation
 from medimgkit.readers import read_array_normalized
 
 _LOGGER = logging.getLogger(__name__)

datamint/entities/__init__.py

@@ -1,6 +1,6 @@
 """DataMint entities package."""
 
-from .annotation import Annotation
+from .annotations.annotation import Annotation
 from .base_entity import BaseEntity
 from .channel import Channel, ChannelResourceData
 from .project import Project

datamint/entities/annotations/__init__.py

@@ -0,0 +1,13 @@
+from .image_classification import ImageClassification
+from .image_segmentation import ImageSegmentation
+from .annotation import Annotation
+from .volume_segmentation import VolumeSegmentation
+from datamint.api.dto import AnnotationType # FIXME: move this to this module
+
+__all__ = [
+    "ImageClassification",
+    "ImageSegmentation",
+    "Annotation",
+    "VolumeSegmentation",
+    "AnnotationType",
+]

datamint/entities/{annotation.py → annotations/annotation.py}

@@ -9,16 +9,17 @@ from typing import TYPE_CHECKING, Any
 import logging
 import os
 
-from .base_entity import BaseEntity, MISSING_FIELD
-from .cache_manager import CacheManager
+from ..base_entity import BaseEntity, MISSING_FIELD
+from ..cache_manager import CacheManager
 from pydantic import PrivateAttr
 from datetime import datetime
 from datamint.api.dto import AnnotationType
 from datamint.types import ImagingData
 
+
 if TYPE_CHECKING:
     from datamint.api.endpoints.annotations_api import AnnotationsApi
-    from .resource import Resource
+    from ..resource import Resource
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +34,28 @@ _FIELD_MAPPING = {
 _ANNOTATION_CACHE_KEY = "annotation_data"
 
 
-class Annotation(BaseEntity):
+class AnnotationBase(BaseEntity):
+    """Minimal base class for creating annotations.
+    
+    This class contains only the essential fields needed to create annotations.
+    Use this for creating specific annotation types like ImageClassification.
+    """
+    
+    identifier: str
+    scope: str
+    annotation_type: AnnotationType
+    confiability: float = 1.0
+    
+    def __init__(self, **data):
+        """Initialize the annotation base entity."""
+        super().__init__(**data)
+
+    @property
+    def name(self) -> str:
+        """Get the annotation name (alias for identifier)."""
+        return self.identifier
+
+class Annotation(AnnotationBase):
     """Pydantic Model representing a DataMint annotation.
 
     Attributes:
@@ -67,32 +89,31 @@ class Annotation(BaseEntity):
         values: Optional extra values payload for flexible schemas.
     """
 
-    id: str
+    id: str | None = None
     identifier: str
     scope: str
-    frame_index: int | None
-    annotation_type: AnnotationType
-    text_value: str | None
-    numeric_value: float | int | None
-    units: str | None
-    geometry: list | dict | None
-    created_at: str  # ISO timestamp string
-    created_by: str
-    annotation_worklist_id: str | None
-    status: str
-    approved_at: str | None  # ISO timestamp string
-    approved_by: str | None
-    resource_id: str
-    associated_file: str | None
-    deleted: bool
-    deleted_at: str | None  # ISO timestamp string
-    deleted_by: str | None
-    created_by_model: str | None
-    set_name: str | None
-    resource_filename: str | None
-    resource_modality: str | None
-    annotation_worklist_name: str | None
-    user_info: dict | None
+    frame_index: int | None = None
+    text_value: str | None = None
+    numeric_value: float | int | None = None
+    units: str | None = None
+    geometry: list | dict | None = None
+    created_at: str | None = None  # ISO timestamp string
+    created_by: str | None = None
+    annotation_worklist_id: str | None = None
+    status: str | None = None
+    approved_at: str | None = None  # ISO timestamp string
+    approved_by: str | None = None
+    resource_id: str | None = None
+    associated_file: str | None = None
+    deleted: bool = False
+    deleted_at: str | None = None  # ISO timestamp string
+    deleted_by: str | None = None
+    created_by_model: str | None = None
+    set_name: str | None = None
+    resource_filename: str | None = None
+    resource_modality: str | None = None
+    annotation_worklist_name: str | None = None
+    user_info: dict | None = None
     values: list | None = MISSING_FIELD
     file: str | None = None
 
@@ -101,9 +122,14 @@ class Annotation(BaseEntity):
     def __init__(self, **data):
         """Initialize the annotation entity."""
         super().__init__(**data)
-        self._cache: CacheManager = CacheManager('annotations')
         self._resource: 'Resource | None' = None
 
+    @property
+    def _cache(self) -> CacheManager[bytes]:
+        if not hasattr(self, '__cache'):
+            self.__cache = CacheManager[bytes]('annotations')
+        return self.__cache
+
     @property
     def resource(self) -> 'Resource':
         """Lazily load and cache the associated Resource entity."""
@@ -117,24 +143,37 @@ class Annotation(BaseEntity):
         auto_convert: bool = True,
         use_cache: bool = False,
     ) -> bytes | ImagingData:
+        """Get the file data for this annotation.
+
+        Args:
+            save_path: Optional path to save the file locally. If use_cache is also True,
+                      the file is saved to save_path and cache metadata points to that location
+                      (no duplication - only one file on disk).
+            auto_convert: If True, automatically converts to appropriate format
+            use_cache: If True, uses cached data when available and valid
+
+        Returns:
+            File data (format depends on auto_convert and file type)
+        """
         # Version info for cache validation
         version_info = self._generate_version_info()
 
-        # Try to get from cache
-        img_data = None
-        if use_cache:
-            img_data = self._cache.get(self.id, _ANNOTATION_CACHE_KEY, version_info)
-
-        if img_data is None:
-            # Fetch from server using download_resource_file
-            logger.debug(f"Fetching image data from server for resource {self.id}")
-            img_data = self._api.download_file(
+        # Download callback for the shared caching logic
+        def download_callback(path: str | None) -> bytes:
+            return self._api.download_file(
                 self,
-                fpath_out=save_path
+                fpath_out=path
             )
-            # Cache the data
-            if use_cache:
-                self._cache.set(self.id, _ANNOTATION_CACHE_KEY, img_data, version_info)
+
+        # Use shared caching logic from BaseEntity
+        img_data = self._fetch_and_cache_file_data(
+            cache_manager=self._cache,
+            data_key=_ANNOTATION_CACHE_KEY,
+            version_info=version_info,
+            download_callback=download_callback,
+            save_path=save_path,
+            use_cache=use_cache,
+        )
 
         if auto_convert:
             return self._api.convert_format(img_data)
@@ -190,11 +229,6 @@ class Annotation(BaseEntity):
         """Alias for :attr:`annotation_type`."""
         return self.annotation_type
 
-    @property
-    def name(self) -> str:
-        """Get the annotation name (alias for identifier)."""
-        return self.identifier
-
     @property
     def index(self) -> int | None:
         """Get the frame index (alias for frame_index)."""

datamint/entities/annotations/image_classification.py

@@ -0,0 +1,12 @@
+from .annotation import Annotation
+from datamint.api.dto import AnnotationType
+
+
+class ImageClassification(Annotation):
+    def __init__(self,
+                 name: str,
+                 value: str,
+                 confiability: float = 1.0):
+        super().__init__(identifier=name, text_value=value, scope='image',
+                         confiability=confiability,
+                         annotation_type=AnnotationType.CATEGORY)