datamint 1.9.2__tar.gz → 2.0.0__tar.gz

{datamint-1.9.2 → datamint-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: datamint
-Version: 1.9.2
+Version: 2.0.0
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -8,6 +8,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: dev
 Provides-Extra: docs
 Requires-Dist: Deprecated (>=1.2.0)
@@ -19,13 +20,14 @@ Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
-Requires-Dist: medimgkit (>=0.5.0)
+Requires-Dist: medimgkit (>=0.6.0)
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy
 Requires-Dist: opencv-python (>=4.0.0)
 Requires-Dist: pandas (>=2.0.0)
 Requires-Dist: platformdirs (>=4.0.0,<5.0.0)
+Requires-Dist: pydantic (>=2.6.4)
 Requires-Dist: pydicom (>=3.0.0,<4.0.0)
 Requires-Dist: pylibjpeg (>=2.0.0,<3.0.0)
 Requires-Dist: pylibjpeg-libjpeg (>=2.0.0,<3.0.0)
@@ -42,6 +44,7 @@ Requires-Dist: sphinx_rtd_theme (>=2.0.0) ; extra == "docs"
 Requires-Dist: torch (>=1.2.0,!=2.3.0)
 Requires-Dist: torchvision (>=0.18.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0)
+Requires-Dist: typing_extensions (>=4.0.0)
 Description-Content-Type: text/markdown
 
 
@@ -91,13 +94,13 @@ import os
 os.environ["DATAMINT_API_KEY"] = "my_api_key"
 ```
 
-### Method 3: APIHandler constructor
+### Method 3: Api constructor
 
-Specify API key in the |APIHandlerClass| constructor:
+Specify API key in the Api constructor:
 
 ```python
-from datamint import APIHandler
-api = APIHandler(api_key='my_api_key')
+from datamint import Api
+api = Api(api_key='my_api_key')
 ```
 
 ## Tutorials
@@ -110,8 +113,9 @@ You can find example notebooks in the `notebooks` folder:
 
 and example scripts in [examples](examples) folder:
 
-- [Running an experiment for classification](examples/experiment_traintest_classifier.py)
-- [Running an experiment for segmentation](examples/experiment_traintest_segmentation.py)
+- [API usage examples](examples/api_usage.ipynb)
+- [Project and entity usage](examples/project_entity_usage.ipynb)
+- [Channels example](examples/channels_example.ipynb)
 
 ## Full documentation
 

{datamint-1.9.2 → datamint-2.0.0}/README.md

@@ -45,13 +45,13 @@ import os
 os.environ["DATAMINT_API_KEY"] = "my_api_key"
 ```
 
-### Method 3: APIHandler constructor
+### Method 3: Api constructor
 
-Specify API key in the |APIHandlerClass| constructor:
+Specify API key in the Api constructor:
 
 ```python
-from datamint import APIHandler
-api = APIHandler(api_key='my_api_key')
+from datamint import Api
+api = Api(api_key='my_api_key')
 ```
 
 ## Tutorials
@@ -64,8 +64,9 @@ You can find example notebooks in the `notebooks` folder:
 
 and example scripts in [examples](examples) folder:
 
-- [Running an experiment for classification](examples/experiment_traintest_classifier.py)
-- [Running an experiment for segmentation](examples/experiment_traintest_segmentation.py)
+- [API usage examples](examples/api_usage.ipynb)
+- [Project and entity usage](examples/project_entity_usage.ipynb)
+- [Channels example](examples/channels_example.ipynb)
 
 ## Full documentation
 

{datamint-1.9.2 → datamint-2.0.0}/datamint/__init__.py

@@ -8,6 +8,7 @@ if TYPE_CHECKING:
     from .dataset.dataset import DatamintDataset as Dataset
     from .apihandler.api_handler import APIHandler
     from .experiment import Experiment
+    from .api.client import Api
 else:
     import lazy_loader as lazy
 
@@ -19,6 +20,7 @@ else:
             "dataset": ['Dataset'],
             "apihandler.api_handler": ["APIHandler"],
             "experiment": ["Experiment"],
+            "api.client": ["Api"],
         },
     )
 

datamint-2.0.0/datamint/api/__init__.py

@@ -0,0 +1,3 @@
+from .client import Api
+
+__all__ = ['Api']

datamint-2.0.0/datamint/api/base_api.py

@@ -0,0 +1,430 @@
+import logging
+from typing import Any, Generator, AsyncGenerator, Sequence
+import httpx
+from dataclasses import dataclass
+from datamint.exceptions import DatamintException, ResourceNotFoundError
+import aiohttp
+import json
+import pydicom.dataset
+from PIL import Image
+import cv2
+import nibabel as nib
+from nibabel.filebasedimages import FileBasedImage as nib_FileBasedImage
+from io import BytesIO
+import gzip
+import contextlib
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+# Generic type for entities
+_PAGE_LIMIT = 5000
+
+
+@dataclass
+class ApiConfig:
+    """Configuration for API client.
+
+    Attributes:
+        server_url: Base URL for the API.
+        api_key: Optional API key for authentication.
+        timeout: Request timeout in seconds.
+        max_retries: Maximum number of retries for requests.
+    """
+    server_url: str
+    api_key: str | None = None
+    timeout: float = 30.0
+    max_retries: int = 3
+
+
+class BaseApi:
+    """Base class for all API endpoint handlers."""
+
+    def __init__(self,
+                 config: ApiConfig,
+                 client: httpx.Client | None = None) -> None:
+        """Initialize the base API handler.
+
+        Args:
+            config: API configuration containing base URL, API key, etc.
+            client: Optional HTTP client instance. If None, a new one will be created.
+        """
+        self.config = config
+        self.client = client or self._create_client()
+        self.semaphore = asyncio.Semaphore(20)
+
+    def _create_client(self) -> httpx.Client:
+        """Create and configure HTTP client with authentication and timeouts."""
+        headers = None
+        if self.config.api_key:
+            headers = {"apikey": self.config.api_key}
+
+        return httpx.Client(
+            base_url=self.config.server_url,
+            headers=headers,
+            timeout=self.config.timeout
+        )
+
+    def _stream_request(self, method: str, endpoint: str, **kwargs):
+        """Make streaming HTTP request with error handling.
+
+        Args:
+            method: HTTP method (GET, POST, PUT, DELETE)
+            endpoint: API endpoint path
+            **kwargs: Additional arguments for the request
+
+        Returns:
+            HTTP response object configured for streaming
+
+        Raises:
+            httpx.HTTPStatusError: If the request fails
+
+        Example:
+            with api._stream_request('GET', '/large-file') as response:
+                for chunk in response.iter_bytes():
+                    process_chunk(chunk)
+        """
+        url = endpoint.lstrip('/')  # Remove leading slash for httpx
+
+        try:
+            return self.client.stream(method, url, **kwargs)
+        except httpx.RequestError as e:
+            logger.error(f"Request error for streaming {method} {endpoint}: {e}")
+            raise
+
+    def _make_request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Make HTTP request with error handling and retries.
+
+        Args:
+            method: HTTP method (GET, POST, PUT, DELETE)
+            endpoint: API endpoint path
+            **kwargs: Additional arguments for the request
+
+        Returns:
+            HTTP response object
+
+        Raises:
+            httpx.HTTPStatusError: If the request fails
+        """
+        url = endpoint.lstrip('/')  # Remove leading slash for httpx
+
+        try:
+            curl_command = self._generate_curl_command({"method": method,
+                                                        "url": url,
+                                                        "headers": self.client.headers,
+                                                        **kwargs}, fail_silently=True)
+            logger.debug(f'Equivalent curl command: "{curl_command}"')
+            response = self.client.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error {e.response.status_code} for {method} {endpoint}: {e.response.text}")
+            raise
+        except httpx.RequestError as e:
+            logger.error(f"Request error for {method} {endpoint}: {e}")
+            raise
+
+    def _generate_curl_command(self,
+                               request_args: dict,
+                               fail_silently: bool = False) -> str:
+        """
+        Generate a curl command for debugging purposes.
+
+        Args:
+            request_args (dict): Request arguments dictionary containing method, url, headers, etc.
+
+        Returns:
+            str: Equivalent curl command
+        """
+        try:
+            method = request_args.get('method', 'GET').upper()
+            url = request_args['url']
+            headers = request_args.get('headers', {})
+            data = request_args.get('json') or request_args.get('data')
+            params = request_args.get('params')
+
+            curl_command = ['curl']
+
+            # Add method if not GET
+            if method != 'GET':
+                curl_command.extend(['-X', method])
+
+            # Add headers
+            for key, value in headers.items():
+                if key.lower() == 'apikey':
+                    value = '<YOUR-API-KEY>'  # Mask API key for security
+                curl_command.extend(['-H', f"'{key}: {value}'"])
+
+            # Add query parameters
+            if params:
+                param_str = '&'.join([f"{k}={v}" for k, v in params.items()])
+                url = f"{url}?{param_str}"
+            # Add URL
+            curl_command.append(f"'{url}'")
+
+            # Add data
+            if data:
+                if isinstance(data, aiohttp.FormData):  # Check if it's aiohttp.FormData
+                    # Handle FormData by extracting fields
+                    form_parts = []
+                    for options, headers, value in data._fields:
+                        # get the name from options
+                        name = options.get('name', 'file')
+                        if hasattr(value, 'read'):  # File-like object
+                            filename = getattr(value, 'name', 'file')
+                            form_parts.extend(['-F', f"'{name}=@{filename}'"])
+                        else:
+                            form_parts.extend(['-F', f"'{name}={value}'"])
+                    curl_command.extend(form_parts)
+                elif isinstance(data, dict):
+                    curl_command.extend(['-d', f"'{json.dumps(data)}'"])
+                else:
+                    curl_command.extend(['-d', f"'{data}'"])
+
+            return ' '.join(curl_command)
+        except Exception as e:
+            if fail_silently:
+                logger.debug(f"Error generating curl command: {e}")
+                return "<error generating curl command>"
+            raise
+
+    @staticmethod
+    def get_status_code(e: httpx.HTTPStatusError | aiohttp.ClientResponseError) -> int:
+        if hasattr(e, 'response') and e.response is not None:
+            # httpx.HTTPStatusError
+            return e.response.status_code
+        if hasattr(e, 'status'):
+            # aiohttp.ClientResponseError
+            return e.status
+        if hasattr(e, 'status_code'):
+            return e.status_code
+        logger.debug(f"Unable to get status code from exception of type {type(e)}")
+        return -1
+
+    @staticmethod
+    def _has_status_code(e: httpx.HTTPError | aiohttp.ClientResponseError,
+                         status_code: int) -> bool:
+        return BaseApi.get_status_code(e) == status_code
+
+    def _check_errors_response(self,
+                               response: httpx.Response | aiohttp.ClientResponse,
+                               url: str):
+        try:
+            response.raise_for_status()
+        except (httpx.HTTPStatusError, aiohttp.ClientResponseError) as e:
+            logger.error(f"HTTP error occurred: {e}")
+            status_code = BaseApi.get_status_code(e)
+            if status_code >= 500 and status_code < 600:
+                logger.error(f"Error in request to {url}: {e}")
+            if status_code >= 400 and status_code < 500:
+                if isinstance(e, aiohttp.ClientResponseError):
+                    # aiohttp.ClientResponse does not have .text or .json() methods directly
+                    error_msg = e.message
+                else:
+                    error_msg = e.response.text
+                logger.info(f"Error response: {error_msg}")
+                if ' not found' in error_msg.lower():
+                    # Will be caught by the caller and properly initialized:
+                    raise ResourceNotFoundError('unknown', {})
+            raise
+
+    @contextlib.asynccontextmanager
+    async def _make_request_async(self,
+                                  method: str,
+                                  endpoint: str,
+                                  session: aiohttp.ClientSession | None = None,
+                                  **kwargs) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """Make asynchronous HTTP request with error handling as an async context manager.
+
+        Args:
+            method: HTTP method (GET, POST, PUT, DELETE)
+            endpoint: API endpoint path
+            session: Optional aiohttp session. If None, a new one will be created.
+            **kwargs: Additional arguments for the request
+
+        Yields:
+            An aiohttp.ClientResponse object.
+
+        Raises:
+            aiohttp.ClientError: If the request fails
+
+        Example:
+            .. code-block:: python
+
+                async with api._make_request_async('GET', '/data') as response:
+                    data = await response.json()
+        """
+
+        if session is None:
+            async with aiohttp.ClientSession() as temp_session:
+                async with self._make_request_async(method, endpoint, temp_session, **kwargs) as resp:
+                    yield resp
+            return
+
+        url = f"{self.config.server_url.rstrip('/')}/{endpoint.lstrip('/')}"
+
+        headers = kwargs.pop('headers', {})
+        if self.config.api_key:
+            headers['apikey'] = self.config.api_key
+
+        timeout = aiohttp.ClientTimeout(total=self.config.timeout)
+
+        response = None
+        curl_cmd = self._generate_curl_command(
+            {"method": method, "url": url, "headers": headers, **kwargs},
+            fail_silently=True
+        )
+        logger.debug(f'Equivalent curl command: "{curl_cmd}"')
+        async with self.semaphore:
+            try:
+                response = await session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    timeout=timeout,
+                    **kwargs
+                )
+                self._check_errors_response(response, url=url)
+                yield response
+            except aiohttp.ClientError as e:
+                logger.error(f"Request error for {method} {endpoint}: {e}")
+                raise
+            finally:
+                if response is not None:
+                    response.release()
+
+    async def _make_request_async_json(self,
+                                       method: str,
+                                       endpoint: str,
+                                       session: aiohttp.ClientSession | None = None,
+                                       **kwargs):
+        """Make asynchronous HTTP request and parse JSON response.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            endpoint: API endpoint path
+            session: Optional aiohttp session. If None, a new one will be created.
+            **kwargs: Additional arguments for the request
+
+        Returns:
+            Parsed JSON response or error information.
+        """
+        async with self._make_request_async(method, endpoint, session=session, **kwargs) as resp:
+            return await resp.json()
+
+    def _make_request_with_pagination(self,
+                                      method: str,
+                                      endpoint: str,
+                                      return_field: str | None = None,
+                                      limit: int | None = None,
+                                      **kwargs
+                                      ) -> Generator[tuple[httpx.Response, list | dict | str], None, None]:
+        """Make paginated HTTP requests, yielding each page of results.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            endpoint: API endpoint path
+            return_field: Optional field name to extract from each item in the response
+            limit: Optional maximum number of items to retrieve
+            **kwargs: Additional arguments for the request (e.g., params, json)
+
+        Yields:
+            Tuples of (HTTP response, items from the current page `response.json()`, for convenience)
+        """
+        offset = 0
+        total_fetched = 0
+        params = dict(kwargs.get('params', {}))
+        # Ensure kwargs carries our params reference so mutations below take effect
+        kwargs['params'] = params
+
+        while True:
+            if limit is not None and total_fetched >= limit:
+                break
+
+            page_limit = _PAGE_LIMIT
+            if limit is not None:
+                remaining = limit - total_fetched
+                page_limit = min(_PAGE_LIMIT, remaining)
+
+            params['offset'] = offset
+            params['limit'] = page_limit
+
+            response = self._make_request(method=method,
+                                          endpoint=endpoint,
+                                          **kwargs)
+            items = self._convert_array_response(response.json(), return_field=return_field)
+
+            if not items:
+                break
+
+            items_to_yield = items
+            if limit is not None:
+                # This ensures we don't yield more than the limit if the API returns more than requested in the last page
+                items_to_yield = items[:limit - total_fetched]
+
+            yield response, items_to_yield
+            total_fetched += len(items_to_yield)
+
+            if len(items) < _PAGE_LIMIT:
+                break
+
+            offset += len(items)
+
+    def _convert_array_response(self,
+                                data: dict | list,
+                                return_field: str | None = None) -> list | dict | str:
+        """Normalize array-like responses into a list when possible.
+
+        Args:
+            data: Parsed JSON response.
+            return_field: Preferred top-level field to extract when present.
+
+        Returns:
+            A list of items when identifiable, otherwise the original data.
+        """
+        if isinstance(data, list):
+            items = data
+        else:
+            if 'data' in data:
+                items = data['data']
+            elif 'items' in data:
+                items = data['items']
+            else:
+                return data
+            if return_field is not None:
+                if 'totalCount' in data and len(items) == 1 and return_field in items[0]:
+                    items = items[0][return_field]
+        return items
+
+    @staticmethod
+    def convert_format(bytes_array: bytes,
+                       mimetype: str,
+                       file_path: str | None = None
+                       ) -> pydicom.dataset.Dataset | Image.Image | cv2.VideoCapture | bytes | nib_FileBasedImage:
+        """ Convert the bytes array to the appropriate format based on the mimetype."""
+        content_io = BytesIO(bytes_array)
+        if mimetype.endswith('/dicom'):
+            return pydicom.dcmread(content_io)
+        elif mimetype.startswith('image/'):
+            return Image.open(content_io)
+        elif mimetype.startswith('video/'):
+            if file_path is None:
+                raise NotImplementedError("file_path=None is not implemented yet for video/* mimetypes.")
+            return cv2.VideoCapture(file_path)
+        elif mimetype == 'application/json':
+            return json.loads(bytes_array)
+        elif mimetype == 'application/octet-stream':
+            return bytes_array
+        elif mimetype.endswith('nifti'):
+            try:
+                return nib.Nifti1Image.from_stream(content_io)
+            except Exception as e:
+                if file_path is not None:
+                    return nib.load(file_path)
+                raise e
+        elif mimetype == 'application/gzip':
+            # let's hope it's a .nii.gz
+            with gzip.open(content_io, 'rb') as f:
+                return nib.Nifti1Image.from_stream(f)
+
+        raise ValueError(f"Unsupported mimetype: {mimetype}")

datamint-2.0.0/datamint/api/client.py

@@ -0,0 +1,91 @@
+from typing import Optional
+import httpx
+from .base_api import ApiConfig
+from .endpoints import ProjectsApi, ResourcesApi, AnnotationsApi, ChannelsApi, UsersApi, DatasetsInfoApi
+import datamint.configs
+from datamint.exceptions import DatamintException
+import asyncio
+
+
+class Api:
+    """Main API client that provides access to all endpoint handlers."""
+    DEFAULT_SERVER_URL = 'https://api.datamint.io'
+    DATAMINT_API_VENV_NAME = datamint.configs.ENV_VARS[datamint.configs.APIKEY_KEY]
+
+    _API_MAP = {
+        'projects': ProjectsApi,
+        'resources': ResourcesApi,
+        'annotations': AnnotationsApi,
+        'channels': ChannelsApi,
+        'users': UsersApi,
+        'datasets': DatasetsInfoApi
+    }
+
+    def __init__(self,
+                 server_url: str | None = None,
+                 api_key: Optional[str] = None,
+                 timeout: float = 60.0, max_retries: int = 2,
+                 check_connection: bool = True) -> None:
+        """Initialize the API client.
+
+        Args:
+            base_url: Base URL for the API
+            api_key: Optional API key for authentication
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts
+            client: Optional HTTP client instance
+        """
+        if server_url is None:
+            server_url = datamint.configs.get_value(datamint.configs.APIURL_KEY)
+            if server_url is None:
+                server_url = Api.DEFAULT_SERVER_URL
+        server_url = server_url.rstrip('/')
+        if api_key is None:
+            api_key = datamint.configs.get_value(datamint.configs.APIKEY_KEY)
+            if api_key is None:
+                msg = f"API key not provided! Use the environment variable " + \
+                    f"{Api.DATAMINT_API_VENV_NAME} or pass it as an argument."
+                raise DatamintException(msg)
+        self.config = ApiConfig(
+            server_url=server_url,
+            api_key=api_key,
+            timeout=timeout,
+            max_retries=max_retries
+        )
+        self._client = None
+        self._endpoints = {}
+        if check_connection:
+            self.check_connection()
+
+    def check_connection(self):
+        try:
+            self.projects.get_list(limit=1)
+        except Exception as e:
+            raise DatamintException("Error connecting to the Datamint API." +
+                                    f" Please check your api_key and/or other configurations. {e}")
+
+    def _get_endpoint(self, name: str):
+        if name not in self._endpoints:
+            api_class = self._API_MAP[name]
+            self._endpoints[name] = api_class(self.config, self._client)
+        return self._endpoints[name]
+
+    @property
+    def projects(self) -> ProjectsApi:
+        return self._get_endpoint('projects')
+    @property
+    def resources(self) -> ResourcesApi:
+        return self._get_endpoint('resources')
+    @property
+    def annotations(self) -> AnnotationsApi:
+        return self._get_endpoint('annotations')
+    @property
+    def channels(self) -> ChannelsApi:
+        return self._get_endpoint('channels')
+    @property
+    def users(self) -> UsersApi:
+        return self._get_endpoint('users')
+    @property
+    def _datasetsinfo(self) -> DatasetsInfoApi:
+        """Internal property to access DatasetsInfoApi."""
+        return self._get_endpoint('datasets')

datamint-2.0.0/datamint/api/dto/__init__.py

@@ -0,0 +1,10 @@
+from datamint.apihandler.dto import annotation_dto
+from datamint.apihandler.dto.annotation_dto import AnnotationType, CreateAnnotationDto, Geometry, BoxGeometry
+
+__all__ = [
+    "annotation_dto",
+    "AnnotationType",
+    "CreateAnnotationDto",
+    "Geometry",
+    "BoxGeometry",
+]

datamint-2.0.0/datamint/api/endpoints/__init__.py

@@ -0,0 +1,17 @@
+"""API endpoint handlers."""
+
+from .annotations_api import AnnotationsApi
+from .channels_api import ChannelsApi
+from .projects_api import ProjectsApi
+from .resources_api import ResourcesApi
+from .users_api import UsersApi
+from .datasetsinfo_api import DatasetsInfoApi
+
+__all__ = [
+    'AnnotationsApi',
+    'ChannelsApi', 
+    'ProjectsApi',
+    'ResourcesApi',
+    'UsersApi',
+    'DatasetsInfoApi'
+]