datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

datamint/api/endpoints/annotations_api.py

@@ -1,12 +1,11 @@
-from typing import Any, Sequence, Literal, BinaryIO, Generator, IO
+from typing import Literal, BinaryIO, IO, Any, overload
+from collections.abc import Sequence, Generator
 import httpx
 from datetime import date
 import logging
 from ..entity_base_api import ApiConfig, CreatableEntityApi, DeletableEntityApi
-from .models_api import ModelsApi
-from datamint.entities.annotation import Annotation
+from datamint.entities.annotations.annotation import Annotation
 from datamint.entities.resource import Resource
-from datamint.entities.project import Project
 from datamint.api.dto import AnnotationType, CreateAnnotationDto, LineGeometry, BoxGeometry, CoordinateSystem, Geometry
 import numpy as np
 import os
@@ -43,12 +42,16 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
             client: Optional HTTP client instance. If None, a new one will be created.
         """
         from .resources_api import ResourcesApi
+        from .models_api import ModelsApi
+
         super().__init__(config, Annotation, 'annotations', client)
         self._models_api = ModelsApi(config, client=client) if models_api is None else models_api
-        self._resources_api = ResourcesApi(config, client=client, annotations_api=self) if resources_api is None else resources_api
+        self._resources_api = ResourcesApi(
+            config, client=client, annotations_api=self) if resources_api is None else resources_api
 
+    @overload
     def get_list(self,
-                 resource: str | Resource | None = None,
+                 resource: str | Resource | Sequence[str | Resource] | None = None,
                  annotation_type: AnnotationType | str | None = None,
                  annotator_email: str | None = None,
                  date_from: date | None = None,
@@ -57,10 +60,88 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
                  worklist_id: str | None = None,
                  status: Literal['new', 'published'] | None = None,
                  load_ai_segmentations: bool | None = None,
-                 limit: int | None = None
-                 ) -> Sequence[Annotation]:
+                 limit: int | None = None,
+                 group_by_resource: Literal[False] = False
+                 ) -> Sequence[Annotation]: ...
+
+    @overload
+    def get_list(self,
+                 resource: str | Resource | Sequence[str | Resource] | None = None,
+                 annotation_type: AnnotationType | str | None = None,
+                 annotator_email: str | None = None,
+                 date_from: date | None = None,
+                 date_to: date | None = None,
+                 dataset_id: str | None = None,
+                 worklist_id: str | None = None,
+                 status: Literal['new', 'published'] | None = None,
+                 load_ai_segmentations: bool | None = None,
+                 limit: int | None = None,
+                 *,
+                 group_by_resource: Literal[True]
+                 ) -> Sequence[Sequence[Annotation]]: ...
+
+    def get_list(self,
+                 resource: str | Resource | Sequence[str | Resource] | None = None,
+                 annotation_type: AnnotationType | str | None = None,
+                 annotator_email: str | None = None,
+                 date_from: date | None = None,
+                 date_to: date | None = None,
+                 dataset_id: str | None = None,
+                 worklist_id: str | None = None,
+                 status: Literal['new', 'published'] | None = None,
+                 load_ai_segmentations: bool | None = None,
+                 limit: int | None = None,
+                 group_by_resource: bool = False
+                 ) -> Sequence[Annotation] | Sequence[Sequence[Annotation]]:
+        """
+        Retrieve a list of annotations with optional filtering.
+
+        Args:
+            resource: The resource unique id(s) or Resource instance(s). Can be a single resource,
+                a list of resources, or None to retrieve annotations from all resources.
+            annotation_type: Filter by annotation type (e.g., 'segmentation', 'category').
+            annotator_email: Filter by annotator email address.
+            date_from: Filter annotations created on or after this date.
+            date_to: Filter annotations created on or before this date.
+            dataset_id: Filter by dataset unique id.
+            worklist_id: Filter by annotation worklist unique id.
+            status: Filter by annotation status ('new' or 'published').
+            load_ai_segmentations: Whether to load AI-generated segmentations.
+            limit: Maximum number of annotations to return.
+            group_by_resource: If True, return results grouped by resource.
+                For instance, the first index of the returned list will contain all annotations for the first resource.
+
+        Returns:
+            Sequence[Annotation] | Sequence[Sequence[Annotation]]: List of annotations, or list of lists if grouped by resource.
+
+        Example:
+            .. code-block:: python
+
+                # Get all annotations for a single resource
+                annotations = api.annotations.get_list(resource='resource_id')
+
+                # Get annotations with filters
+                annotations = api.annotations.get_list(
+                    resource='resource_id',
+                    annotation_type='segmentation',
+                    status='published'
+                )
+
+                # Get annotations for multiple resources
+                annotations = api.annotations.get_list(
+                    resource=['resource_id_1', 'resource_id_2', 'resource_id_3']
+                )
+        """
+        def group_annotations_by_resource(annotations: Sequence[Annotation],
+                                          resource_ids: Sequence[str]
+                                          ) -> Sequence[Sequence[Annotation]]:
+            resource_annotations_map = {rid: [] for rid in resource_ids}
+            for ann in annotations:
+                resource_annotations_map[ann.resource_id].append(ann)
+            return [resource_annotations_map[rid] for rid in resource_ids]
+
+        # Build search payload according to POST /annotations/search schema
         payload = {
-            'resource_id': resource.id if isinstance(resource, Resource) else resource,
             'annotation_type': annotation_type,
             'annotatorEmail': annotator_email,
             'from': date_from.isoformat() if date_from is not None else None,
@@ -68,12 +149,37 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
             'dataset_id': dataset_id,
             'annotation_worklist_id': worklist_id,
             'status': status,
-            'load_ai_segmentations': load_ai_segmentations
+            'load_ai_segmentations': load_ai_segmentations,
         }
 
-        # remove nones
+        if isinstance(resource, (str, Resource)):
+            resource_id = self._entid(resource)
+            payload['resource_id'] = resource_id
+            resource_ids = None
+        elif resource is not None:
+            resource_ids = [self._entid(res) for res in resource]
+            payload['resource_ids'] = resource_ids
+        else:
+            resource_ids = None
+
+        # Remove None values from payload
         payload = {k: v for k, v in payload.items() if v is not None}
-        return super().get_list(limit=limit, params=payload)
+
+        items_gen = self._make_request_with_pagination('POST',
+                                                       f'{self.endpoint_base}/search',
+                                                       return_field=self.endpoint_base,
+                                                       limit=limit,
+                                                       json=payload)
+
+        all_items = []
+        for _, items in items_gen:
+            all_items.extend(items)
+
+        all_annotations = [self._init_entity_obj(**item) for item in all_items]
+
+        if group_by_resource and resource_ids is not None:
+            return group_annotations_by_resource(all_annotations, resource_ids)
+        return all_annotations
 
     async def _upload_segmentations_async(self,
                                           resource: str | Resource,
@@ -371,14 +477,21 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
                resource: str | Resource,
                annotation_dto: CreateAnnotationDto | Sequence[CreateAnnotationDto]
                ) -> str | Sequence[str]:
-        """Create a new annotation.
+        """Create one or more annotations for a resource.
+
+        .. warning::
+            This is an internal method and should not be used directly by users.
+            Please use specific annotation creation methods like 
+            :py:meth:`create_image_classification` or :py:meth:`upload_segmentations` instead.
 
         Args:
-            resource: The resource unique id or Resource instance.
-            annotation_dto: A CreateAnnotationDto instance or a list of such instances.
+            resource (str | Resource): The resource unique id or Resource instance.
+            annotation_dto (CreateAnnotationDto | Sequence[CreateAnnotationDto]): 
+                A CreateAnnotationDto instance or a list of such instances to be created.
 
         Returns:
-            The id of the created annotation or a list of ids if multiple annotations were created.
+            str | Sequence[str]: The id of the created annotation if a single annotation 
+            was provided, or a list of ids if multiple annotations were created.
         """
 
         annotations = [annotation_dto] if isinstance(annotation_dto, CreateAnnotationDto) else annotation_dto
@@ -396,7 +509,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
 
     def upload_segmentations(self,
                              resource: str | Resource,
-                             file_path: str | np.ndarray,
+                             file_path: str | Path | np.ndarray,
                              name: str | dict[int, str] | dict[tuple, str] | None = None,
                              frame_index: int | list[int] | None = None,
                              imported_from: str | None = None,
@@ -460,6 +573,9 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
         """
         import nest_asyncio
 
+        if isinstance(file_path, Path):
+            file_path = str(file_path)
+
         if isinstance(file_path, str) and not os.path.exists(file_path):
             raise FileNotFoundError(f"File {file_path} not found.")
 
@@ -661,6 +777,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
         if isinstance(file_path, str):
             if file_path.endswith('.nii') or file_path.endswith('.nii.gz'):
                 # Upload NIfTI file directly
+                _LOGGER.debug('uploading segmentation as a volume')
                 with open(file_path, 'rb') as f:
                     filename = os.path.basename(file_path)
                     form = aiohttp.FormData()
@@ -672,9 +789,17 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
                     if name is not None:
                         form.add_field('segmentation_map', json.dumps(name), content_type='application/json')
 
-                    respdata = await self._make_request_async_json('POST',
-                                                                   f'{self.endpoint_base}/{resource_id}/segmentations/file',
-                                                                   data=form)
+                    try:
+                        respdata = await self._make_request_async_json(
+                            'POST',
+                            f'{self.endpoint_base}/{resource_id}/segmentations/file',
+                            data=form
+                        )
+                    except ResourceNotFoundError as e:
+                        e.resource_type = 'resource'
+                        e.params = {'resource_id': resource_id}
+                        raise e
+
                     if 'error' in respdata:
                         raise DatamintException(respdata['error'])
                     return respdata
@@ -815,7 +940,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
             model_id=model_id
         )
 
-        return self.create(resource, annotation_dto)            
+        return self.create(resource, annotation_dto)
 
     def add_line_annotation(self,
                             point1: tuple[int, int] | tuple[float, float, float],
@@ -977,13 +1102,23 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
             save_path (str | Path): The path to save the file.
             session (aiohttp.ClientSession): The aiohttp session to use for the request.
             progress_bar (tqdm | None): Optional progress bar to update after download completion.
+
+        Returns:
+            dict: A dictionary with 'success' (bool) and optional 'error' (str) keys.
         """
         if isinstance(annotation, Annotation):
             annotation_id = annotation.id
             resource_id = annotation.resource_id
         else:
             annotation_id = annotation
-            resource_id = self.get_by_id(annotation_id).resource_id
+            try:
+                resource_id = self.get_by_id(annotation_id).resource_id
+            except Exception as e:
+                error_msg = f"Failed to get resource_id for annotation {annotation_id}: {str(e)}"
+                _LOGGER.error(error_msg)
+                if progress_bar:
+                    progress_bar.update(1)
+                return {'success': False, 'annotation_id': annotation_id, 'error': error_msg}
 
         try:
             async with self._make_request_async('GET',
@@ -994,20 +1129,31 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
                     f.write(data_bytes)
             if progress_bar:
                 progress_bar.update(1)
-        except ResourceNotFoundError as e:
-            e.set_params('annotation', {'annotation_id': annotation_id})
-            raise e
+            return {'success': True, 'annotation_id': annotation_id}
+        except Exception as e:
+            error_msg = f"Failed to download annotation {annotation_id}: {str(e)}"
+            _LOGGER.error(error_msg)
+            if progress_bar:
+                progress_bar.update(1)
+            return {'success': False, 'annotation_id': annotation_id, 'error': error_msg}
 
     def download_multiple_files(self,
                                 annotations: Sequence[str | Annotation],
                                 save_paths: Sequence[str | Path] | str
-                                ) -> None:
+                                ) -> list[dict[str, Any]]:
         """
         Download multiple segmentation files and save them to the specified paths.
 
         Args:
             annotations: A list of annotation unique ids or annotation objects.
             save_paths: A list of paths to save the files or a directory path.
+
+        Returns:
+            List of dictionaries with 'success', 'annotation_id', and optional 'error' keys.
+
+        Note:
+            If any downloads fail, they will be logged but the process will continue.
+            A summary of failed downloads will be logged at the end.
         """
         import nest_asyncio
         nest_asyncio.apply()
@@ -1019,7 +1165,7 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
                         annotation, save_path=path, session=session, progress_bar=progress_bar)
                     for annotation, path in zip(annotations, save_paths)
                 ]
-                await asyncio.gather(*tasks)
+                return await asyncio.gather(*tasks)
 
         if isinstance(save_paths, str):
             save_paths = [os.path.join(save_paths, self._entid(ann))
@@ -1027,7 +1173,19 @@ class AnnotationsApi(CreatableEntityApi[Annotation], DeletableEntityApi[Annotati
 
         with tqdm(total=len(annotations), desc="Downloading segmentations", unit="file") as progress_bar:
             loop = asyncio.get_event_loop()
-            loop.run_until_complete(_download_all_async())
+            results = loop.run_until_complete(_download_all_async())
+
+        # Log summary of failures
+        failures = [r for r in results if not r['success']]
+        if failures:
+            _LOGGER.warning(f"Failed to download {len(failures)} out of {len(annotations)} annotations")
+            _USER_LOGGER.warning(f"Failed to download {len(failures)} out of {len(annotations)} annotations")
+            for failure in failures:
+                _LOGGER.debug(f"  - {failure['annotation_id']}: {failure['error']}")
+        else:
+            _USER_LOGGER.info(f"Successfully downloaded all {len(annotations)} annotations")
+
+        return results
 
     def bulk_download_file(self,
                            annotations: Sequence[str | Annotation],

datamint/api/endpoints/deploy_model_api.py

@@ -0,0 +1,78 @@
+import httpx
+from ..entity_base_api import EntityBaseApi, ApiConfig
+from datamint.entities.deployjob import DeployJob
+
+class DeployModelApi(EntityBaseApi[DeployJob]):
+    """API handler for model deployment endpoints."""
+
+    def __init__(self,
+                 config: ApiConfig,
+                 client: httpx.Client | None = None) -> None:
+        super().__init__(config, DeployJob, 'datamint/api/v1/deploy-model', client)
+
+    def get_by_id(self, entity_id: str) -> DeployJob:
+        """Get deployment job status by ID."""
+        response = self._make_request('GET', f'/{self.endpoint_base}/status/{entity_id}')
+        data = response.json()
+        if 'job_id' in data:
+            data['id'] = data.pop('job_id')
+        return self._init_entity_obj(**data)
+
+    def start(self,
+              model_name: str,
+              model_version: int | None = None,
+              model_alias: str | None = None,
+              image_name: str | None = None,
+              with_gpu: bool = False,
+              convert_to_onnx: bool = False,
+              input_shape: list[int] | None = None) -> DeployJob:
+        """Start a new deployment job."""
+        payload = {
+            "model_name": model_name,
+            "model_version": model_version,
+            "model_alias": model_alias,
+            "image_name": image_name,
+            "with_gpu": with_gpu,
+            "convert_to_onnx": convert_to_onnx,
+            "input_shape": input_shape
+        }
+        # Remove None values
+        payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._make_request('POST', f'/{self.endpoint_base}/start', json=payload)
+        data = response.json()
+        return self.get_by_id(data['job_id'])
+
+    def cancel(self, job: str | DeployJob) -> bool:
+        """Cancel a deployment job."""
+        job_id = self._entid(job)
+        response = self._make_request('POST', f'/{self.endpoint_base}/cancel/{job_id}')
+        return response.json().get('success', False)
+
+    def list_active_jobs(self) -> dict:
+        """List active deployment jobs count."""
+        response = self._make_request('GET', f'/{self.endpoint_base}/jobs')
+        return response.json()
+
+    def list_images(self, model_name: str | None = None) -> list[dict]:
+        """List deployed model images."""
+        params = {}
+        if model_name:
+            params['model_name'] = model_name
+        response = self._make_request('GET', f'/{self.endpoint_base}/images', params=params)
+        return response.json()
+
+    def remove_image(self, model_name: str, tag: str | None = None) -> dict:
+        """Remove a deployed model image."""
+        params = {}
+        if tag:
+            params['tag'] = tag
+        response = self._make_request('DELETE', f'/{self.endpoint_base}/image/{model_name}', params=params)
+        return response.json()
+
+    def image_exists(self, model_name: str, tag: str = "champion") -> bool:
+        """Check if a model image exists."""
+        params = {'tag': tag}
+        response = self._make_request('GET', f'/{self.endpoint_base}/image/{model_name}/exists', params=params)
+        return response.json().get('exists', False)
+

datamint/api/endpoints/models_api.py

@@ -1,3 +1,4 @@
+"""Deprecated: Use MLFlow API instead."""
 from typing import Sequence
 from ..entity_base_api import ApiConfig, BaseApi
 import httpx

datamint/api/endpoints/projects_api.py

@@ -1,8 +1,10 @@
-from typing import Sequence, Literal
+from typing import Sequence, Literal, TYPE_CHECKING, overload
 from ..entity_base_api import ApiConfig, CRUDEntityApi
 from datamint.entities.project import Project
-from datamint.entities.resource import Resource
 import httpx
+from datamint.entities.resource import Resource
+if TYPE_CHECKING:
+    from .resources_api import ResourcesApi
 
 
 class ProjectsApi(CRUDEntityApi[Project]):
@@ -10,14 +12,17 @@ class ProjectsApi(CRUDEntityApi[Project]):
 
     def __init__(self,
                  config: ApiConfig,
-                 client: httpx.Client | None = None) -> None:
+                 client: httpx.Client | None = None,
+                 resources_api: 'ResourcesApi | None' = None) -> None:
         """Initialize the projects API handler.
 
         Args:
             config: API configuration containing base URL, API key, etc.
             client: Optional HTTP client instance. If None, a new one will be created.
         """
+        from .resources_api import ResourcesApi
         super().__init__(config, Project, 'projects', client)
+        self.resources_api = resources_api or ResourcesApi(config, client, projects_api=self)
 
     def get_project_resources(self, project: Project | str) -> list[Resource]:
         """Get resources associated with a specific project.
@@ -30,16 +35,41 @@ class ProjectsApi(CRUDEntityApi[Project]):
         """
         response = self._get_child_entities(project, 'resources')
         resources_data = response.json()
-        resources = [Resource(**item) for item in resources_data]
+        resources = [self.resources_api._init_entity_obj(**item) for item in resources_data]
         return resources
 
+
+    @overload
+    def create(self,
+               name: str,
+               description: str,
+               resources_ids: list[str] | None = None,
+               is_active_learning: bool = False,
+               two_up_display: bool = False,
+               *,
+               return_entity: Literal[True] = True
+               ) -> Project: ...
+
+    @overload
+    def create(self,
+               name: str,
+               description: str,
+               resources_ids: list[str] | None = None,
+               is_active_learning: bool = False,
+               two_up_display: bool = False,
+               *,
+               return_entity: Literal[False]
+               ) -> str: ...
+
     def create(self,
                name: str,
                description: str,
                resources_ids: list[str] | None = None,
                is_active_learning: bool = False,
-               two_up_display: bool = False
-               ) -> str:
+               two_up_display: bool = False,
+               *,
+               return_entity: bool = True
+               ) -> str | Project:
         """Create a new project.
 
         Args:
@@ -48,6 +78,7 @@ class ProjectsApi(CRUDEntityApi[Project]):
             resources_ids: The list of resource ids to be included in the project.
             is_active_learning: Whether the project is an active learning project or not.
             two_up_display: Allow annotators to display multiple resources for annotation.
+            return_entity: Whether to return the created Project instance or just its ID.
 
         Returns:
             The id of the created project.
@@ -67,7 +98,7 @@ class ProjectsApi(CRUDEntityApi[Project]):
                         "require_review": False,
                         'description': description}
 
-        return self._create(project_data)
+        return self._create(project_data, return_entity=return_entity)
 
     def get_all(self, limit: int | None = None) -> Sequence[Project]:
         """Get all projects.