datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. datamint/__init__.py +1 -3
  2. datamint/api/__init__.py +0 -3
  3. datamint/api/base_api.py +286 -54
  4. datamint/api/client.py +76 -13
  5. datamint/api/endpoints/__init__.py +2 -2
  6. datamint/api/endpoints/annotations_api.py +186 -28
  7. datamint/api/endpoints/deploy_model_api.py +78 -0
  8. datamint/api/endpoints/models_api.py +1 -0
  9. datamint/api/endpoints/projects_api.py +38 -7
  10. datamint/api/endpoints/resources_api.py +227 -100
  11. datamint/api/entity_base_api.py +66 -7
  12. datamint/apihandler/base_api_handler.py +0 -1
  13. datamint/apihandler/dto/annotation_dto.py +2 -0
  14. datamint/client_cmd_tools/datamint_config.py +0 -1
  15. datamint/client_cmd_tools/datamint_upload.py +3 -1
  16. datamint/configs.py +11 -7
  17. datamint/dataset/base_dataset.py +24 -4
  18. datamint/dataset/dataset.py +1 -1
  19. datamint/entities/__init__.py +1 -1
  20. datamint/entities/annotations/__init__.py +13 -0
  21. datamint/entities/{annotation.py → annotations/annotation.py} +81 -47
  22. datamint/entities/annotations/image_classification.py +12 -0
  23. datamint/entities/annotations/image_segmentation.py +252 -0
  24. datamint/entities/annotations/volume_segmentation.py +273 -0
  25. datamint/entities/base_entity.py +100 -6
  26. datamint/entities/cache_manager.py +129 -15
  27. datamint/entities/datasetinfo.py +60 -65
  28. datamint/entities/deployjob.py +18 -0
  29. datamint/entities/project.py +39 -0
  30. datamint/entities/resource.py +310 -46
  31. datamint/lightning/__init__.py +1 -0
  32. datamint/lightning/datamintdatamodule.py +103 -0
  33. datamint/mlflow/__init__.py +65 -0
  34. datamint/mlflow/artifact/__init__.py +1 -0
  35. datamint/mlflow/artifact/datamint_artifacts_repo.py +8 -0
  36. datamint/mlflow/env_utils.py +131 -0
  37. datamint/mlflow/env_vars.py +5 -0
  38. datamint/mlflow/flavors/__init__.py +17 -0
  39. datamint/mlflow/flavors/datamint_flavor.py +150 -0
  40. datamint/mlflow/flavors/model.py +877 -0
  41. datamint/mlflow/lightning/callbacks/__init__.py +1 -0
  42. datamint/mlflow/lightning/callbacks/modelcheckpoint.py +410 -0
  43. datamint/mlflow/models/__init__.py +93 -0
  44. datamint/mlflow/tracking/datamint_store.py +76 -0
  45. datamint/mlflow/tracking/default_experiment.py +27 -0
  46. datamint/mlflow/tracking/fluent.py +91 -0
  47. datamint/utils/env.py +27 -0
  48. datamint/utils/visualization.py +21 -13
  49. datamint-2.9.0.dist-info/METADATA +220 -0
  50. datamint-2.9.0.dist-info/RECORD +73 -0
  51. {datamint-2.3.3.dist-info → datamint-2.9.0.dist-info}/WHEEL +1 -1
  52. datamint-2.9.0.dist-info/entry_points.txt +18 -0
  53. datamint/apihandler/exp_api_handler.py +0 -204
  54. datamint/experiment/__init__.py +0 -1
  55. datamint/experiment/_patcher.py +0 -570
  56. datamint/experiment/experiment.py +0 -1049
  57. datamint-2.3.3.dist-info/METADATA +0 -125
  58. datamint-2.3.3.dist-info/RECORD +0 -54
  59. datamint-2.3.3.dist-info/entry_points.txt +0 -4
datamint/__init__.py CHANGED
@@ -7,19 +7,17 @@ from typing import TYPE_CHECKING
7
7
  if TYPE_CHECKING:
8
8
  from .dataset.dataset import DatamintDataset as Dataset
9
9
  from .apihandler.api_handler import APIHandler
10
- from .experiment import Experiment
11
10
  from .api.client import Api
12
11
  else:
13
12
  import lazy_loader as lazy
14
13
 
15
14
  __getattr__, __dir__, __all__ = lazy.attach(
16
15
  __name__,
17
- submodules=['dataset', "dataset.dataset", "apihandler.api_handler", "experiment"],
16
+ submodules=['dataset', "dataset.dataset", "apihandler.api_handler"],
18
17
  submod_attrs={
19
18
  "dataset.dataset": ["DatamintDataset"],
20
19
  "dataset": ['Dataset'],
21
20
  "apihandler.api_handler": ["APIHandler"],
22
- "experiment": ["Experiment"],
23
21
  "api.client": ["Api"],
24
22
  },
25
23
  )
datamint/api/__init__.py CHANGED
@@ -1,3 +0,0 @@
1
- from .client import Api
2
-
3
- __all__ = ['Api']
datamint/api/base_api.py CHANGED
@@ -1,9 +1,9 @@
1
1
  import logging
2
- from typing import Any, Generator, AsyncGenerator, Sequence, TYPE_CHECKING
2
+ from typing import TYPE_CHECKING
3
+ from collections.abc import Generator, AsyncGenerator
3
4
  import httpx
4
5
  from dataclasses import dataclass
5
6
  from datamint.exceptions import DatamintException, ResourceNotFoundError
6
- from datamint.types import ImagingData
7
7
  import aiohttp
8
8
  import json
9
9
  from PIL import Image
@@ -14,15 +14,17 @@ import gzip
14
14
  import contextlib
15
15
  import asyncio
16
16
  from medimgkit.format_detection import GZIP_MIME_TYPES, DEFAULT_MIME_TYPE, guess_typez, guess_extension
17
+ from datamint.utils.env import ensure_asyncio_loop
17
18
 
18
19
  if TYPE_CHECKING:
19
20
  from datamint.api.client import Api
21
+ from datamint.types import ImagingData
20
22
 
21
23
  logger = logging.getLogger(__name__)
22
24
 
23
- # Generic type for entities
24
25
  _PAGE_LIMIT = 5000
25
26
 
27
+
26
28
  @dataclass
27
29
  class ApiConfig:
28
30
  """Configuration for API client.
@@ -32,18 +34,30 @@ class ApiConfig:
32
34
  api_key: Optional API key for authentication.
33
35
  timeout: Request timeout in seconds.
34
36
  max_retries: Maximum number of retries for requests.
37
+ port: Optional port number for the API server.
38
+ verify_ssl: Whether to verify SSL certificates. Default is True.
39
+ Set to False only in development environments with self-signed certificates.
40
+ Can also be a path to a CA bundle file.
35
41
  """
36
42
  server_url: str
37
43
  api_key: str | None = None
38
44
  timeout: float = 30.0
39
45
  max_retries: int = 3
46
+ port: int | None = None
47
+ verify_ssl: bool | str = True
40
48
 
41
49
  @property
42
50
  def web_app_url(self) -> str:
43
51
  """Get the base URL for the web application."""
44
- if self.server_url.startswith('http://localhost:3001'):
52
+ base_url = self.server_url
53
+
54
+ # Add port to base_url if specified
55
+ if self.port is not None:
56
+ base_url = f"{self.server_url.rstrip('/')}:{self.port}"
57
+
58
+ if base_url.startswith('http://localhost'):
45
59
  return 'http://localhost:3000'
46
- if self.server_url.startswith('https://stagingapi.datamint.io'):
60
+ if base_url.startswith('https://stagingapi.datamint.io'):
47
61
  return 'https://staging.datamint.io'
48
62
  return 'https://app.datamint.io'
49
63
 
@@ -61,22 +75,173 @@ class BaseApi:
61
75
  client: Optional HTTP client instance. If None, a new one will be created.
62
76
  """
63
77
  self.config = config
64
- self.client = client or self._create_client()
78
+ self._owns_client = client is None # Track if we created the client
79
+ self.client = client or BaseApi._create_client(config)
65
80
  self.semaphore = asyncio.Semaphore(20)
66
81
  self._api_instance: 'Api | None' = None # Injected by Api class
82
+ self._aiohttp_connector: aiohttp.TCPConnector | None = None
83
+ self._aiohttp_session: aiohttp.ClientSession | None = None
84
+ ensure_asyncio_loop()
85
+
67
86
 
68
- def _create_client(self) -> httpx.Client:
69
- """Create and configure HTTP client with authentication and timeouts."""
70
- headers = None
71
- if self.config.api_key:
72
- headers = {"apikey": self.config.api_key}
87
+ @staticmethod
88
+ def _create_client(config: ApiConfig) -> httpx.Client:
89
+ """Create and configure HTTP client with authentication and timeouts.
90
+
91
+ The client is designed to be long-lived and reused across multiple requests.
92
+ It maintains connection pooling for improved performance.
93
+ """
94
+ headers = {"apikey": config.api_key, 'Authorization': f"Bearer {config.api_key}"} if config.api_key else None
95
+
96
+ # Add port to base_url if specified
97
+ base_url = config.server_url.rstrip('/').strip()
98
+ if config.port is not None:
99
+ # if the port is already in the URL, replace it
100
+ if ':' in base_url.split('//')[-1]:
101
+ parts = base_url.rsplit(':', 1)
102
+ # confirm parts[1] is numeric
103
+ if parts[1].isdigit():
104
+ base_url = f"{parts[0]}:{config.port}"
105
+ else:
106
+ logger.warning(f"Invalid port detected in server_url: {config.server_url}")
107
+ else:
108
+ base_url = f"{base_url}:{config.port}"
73
109
 
74
110
  return httpx.Client(
75
- base_url=self.config.server_url,
111
+ base_url=base_url,
76
112
  headers=headers,
77
- timeout=self.config.timeout
113
+ timeout=config.timeout,
114
+ verify=config.verify_ssl,
115
+ limits=httpx.Limits(
116
+ max_keepalive_connections=5,
117
+ max_connections=20,
118
+ keepalive_expiry=8
119
+ )
78
120
  )
79
121
 
122
+ def _raise_ssl_error(self, original_error: Exception) -> None:
123
+ """Raise a more helpful SSL certificate error with troubleshooting guidance.
124
+
125
+ Args:
126
+ original_error: The original SSL-related exception
127
+
128
+ Raises:
129
+ DatamintException: With helpful troubleshooting information
130
+ """
131
+ error_msg = (
132
+ f"SSL Certificate verification failed: {original_error}\n\n"
133
+ "This typically happens when Python cannot verify the SSL certificate of the Datamint API.\n\n"
134
+ "Quick fixes:\n"
135
+ "1. Upgrade certifi:\n"
136
+ " pip install --upgrade certifi\n\n"
137
+ "2. Set environment variables:\n"
138
+ " export SSL_CERT_FILE=$(python -m certifi)\n"
139
+ " export REQUESTS_CA_BUNDLE=$(python -m certifi)\n\n"
140
+ "3. Or disable SSL verification (development only):\n"
141
+ " api = Api(verify_ssl=False)\n\n"
142
+ "For more help, see: https://github.com/SonanceAI/datamint-python-api#-ssl-certificate-troubleshooting"
143
+ )
144
+ raise DatamintException(error_msg) from original_error
145
+
146
+ def _create_aiohttp_connector(self, force_close: bool = False) -> aiohttp.TCPConnector:
147
+ """Create aiohttp connector with SSL configuration.
148
+
149
+ Args:
150
+ force_close: Whether to force close connections (disable keep-alive)
151
+
152
+ Returns:
153
+ Configured TCPConnector for aiohttp sessions.
154
+ """
155
+ import ssl
156
+ import certifi
157
+
158
+ limit = 20
159
+ ttl_dns_cache = 300
160
+
161
+ if self.config.verify_ssl is False:
162
+ # Disable SSL verification (not recommended for production)
163
+ return aiohttp.TCPConnector(ssl=False, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
164
+ elif isinstance(self.config.verify_ssl, str):
165
+ # Use custom CA bundle
166
+ ssl_context = ssl.create_default_context(cafile=self.config.verify_ssl)
167
+ return aiohttp.TCPConnector(ssl=ssl_context, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
168
+ else:
169
+ # Use certifi's CA bundle (default behavior)
170
+ ssl_context = ssl.create_default_context(cafile=certifi.where())
171
+ return aiohttp.TCPConnector(ssl=ssl_context, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
172
+
173
+ def _get_aiohttp_session(self) -> aiohttp.ClientSession:
174
+ """Get or create a shared aiohttp session for this API instance.
175
+
176
+ Creating/closing a new TLS connection for each upload can cause intermittent
177
+ connection shutdown timeouts in long-running processes (notably notebooks).
178
+ Reusing a single session keeps connections healthy and avoids excessive churn.
179
+ """
180
+ if self._aiohttp_session is not None and not self._aiohttp_session.closed:
181
+ return self._aiohttp_session
182
+
183
+ # (Re)create connector and session
184
+ self._aiohttp_connector = self._create_aiohttp_connector(force_close=False)
185
+ timeout = aiohttp.ClientTimeout(total=self.config.timeout)
186
+ self._aiohttp_session = aiohttp.ClientSession(connector=self._aiohttp_connector, timeout=timeout)
187
+ return self._aiohttp_session
188
+
189
+ def _close_aiohttp_session(self) -> None:
190
+ """Close the shared aiohttp session if it exists.
191
+
192
+ This is best-effort; in environments with a running loop we rely on
193
+ `nest_asyncio` (enabled via `ensure_asyncio_loop`) to allow nested runs.
194
+ """
195
+ if self._aiohttp_session is None or self._aiohttp_session.closed:
196
+ return
197
+
198
+ ensure_asyncio_loop()
199
+ try:
200
+ loop = asyncio.get_event_loop()
201
+ except RuntimeError:
202
+ loop = asyncio.new_event_loop()
203
+ asyncio.set_event_loop(loop)
204
+
205
+ try:
206
+ loop.run_until_complete(self._aiohttp_session.close())
207
+ except RuntimeError:
208
+ # If we're in an environment where the loop is running and not patched,
209
+ # fall back to scheduling the close.
210
+ try:
211
+ loop.create_task(self._aiohttp_session.close())
212
+ except Exception as e:
213
+ logger.info(f"Unable to schedule aiohttp session close: {e}")
214
+ pass
215
+ finally:
216
+ self._aiohttp_session = None
217
+ self._aiohttp_connector = None
218
+
219
+ def close(self) -> None:
220
+ """Close the HTTP client and release resources.
221
+
222
+ Should be called when the API instance is no longer needed.
223
+ Only closes the client if it was created by this instance.
224
+ """
225
+ # Close shared aiohttp session regardless of httpx client ownership.
226
+ self._close_aiohttp_session()
227
+ if self._owns_client and self.client is not None:
228
+ self.client.close()
229
+
230
+ def __enter__(self):
231
+ """Context manager entry."""
232
+ return self
233
+
234
+ def __exit__(self, exc_type, exc_val, exc_tb):
235
+ """Context manager exit - ensures client is closed."""
236
+ self.close()
237
+
238
+ def __del__(self):
239
+ """Destructor - ensures client is closed when instance is garbage collected."""
240
+ try:
241
+ self.close()
242
+ except Exception:
243
+ pass # Ignore errors during cleanup
244
+
80
245
  def _stream_request(self, method: str, endpoint: str, **kwargs):
81
246
  """Make streaming HTTP request with error handling.
82
247
 
@@ -120,21 +285,15 @@ class BaseApi:
120
285
  """
121
286
  url = endpoint.lstrip('/') # Remove leading slash for httpx
122
287
 
123
- try:
288
+ if logger.isEnabledFor(logging.DEBUG):
124
289
  curl_command = self._generate_curl_command({"method": method,
125
290
  "url": url,
126
291
  "headers": self.client.headers,
127
292
  **kwargs}, fail_silently=True)
128
293
  logger.debug(f'Equivalent curl command: "{curl_command}"')
129
- response = self.client.request(method, url, **kwargs)
130
- response.raise_for_status()
131
- return response
132
- except httpx.HTTPStatusError as e:
133
- logger.error(f"HTTP error {e.response.status_code} for {method} {endpoint}: {e.response.text}")
134
- raise
135
- except httpx.RequestError as e:
136
- logger.error(f"Request error for {method} {endpoint}: {e}")
137
- raise
294
+ response = self.client.request(method, url, **kwargs)
295
+ self._check_errors_response_httpx(response, url=url)
296
+ return response
138
297
 
139
298
  def _generate_curl_command(self,
140
299
  request_args: dict,
@@ -201,7 +360,7 @@ class BaseApi:
201
360
  raise
202
361
 
203
362
  @staticmethod
204
- def get_status_code(e: httpx.HTTPStatusError | aiohttp.ClientResponseError) -> int:
363
+ def get_status_code(e: httpx.HTTPError | aiohttp.ClientError) -> int:
205
364
  if hasattr(e, 'response') and e.response is not None:
206
365
  # httpx.HTTPStatusError
207
366
  return e.response.status_code
@@ -218,27 +377,70 @@ class BaseApi:
218
377
  status_code: int) -> bool:
219
378
  return BaseApi.get_status_code(e) == status_code
220
379
 
221
- def _check_errors_response(self,
222
- response: httpx.Response | aiohttp.ClientResponse,
223
- url: str):
380
+ def _check_errors_response_httpx(self,
381
+ response: httpx.Response,
382
+ url: str):
383
+ response_json = None
224
384
  try:
385
+ try:
386
+ response_json = response.json()
387
+ except Exception:
388
+ pass
225
389
  response.raise_for_status()
226
- except (httpx.HTTPStatusError, aiohttp.ClientResponseError) as e:
227
- logger.error(f"HTTP error occurred: {e}")
390
+ except httpx.ConnectError as e:
391
+ if "CERTIFICATE_VERIFY_FAILED" in str(e) or "certificate verify failed" in str(e).lower():
392
+ self._raise_ssl_error(e)
393
+ raise
394
+ except httpx.HTTPError as e:
395
+ error_msg = f"{getattr(e, 'message', str(e))} | {getattr(response, 'text', '')}"
396
+ if response_json:
397
+ error_msg = f"{error_msg} | {response_json}"
398
+ try:
399
+ e.message = error_msg
400
+ except Exception:
401
+ logger.debug("Unable to set message attribute on exception")
402
+ pass
403
+
404
+ logger.error(f"HTTP error {response.status_code} for {url}: {error_msg}")
405
+ status_code = response.status_code
406
+ if status_code in (400, 404):
407
+ if ' not found' in error_msg.lower() or 'Not Found' in error_msg:
408
+ # Will be caught by the caller and properly initialized:
409
+ raise ResourceNotFoundError('unknown', {})
410
+ raise
411
+ return response_json
412
+
413
+ async def _check_errors_response_aiohttp(self,
414
+ response: aiohttp.ClientResponse,
415
+ url: str):
416
+ response_json = None
417
+ try:
418
+ try:
419
+ response_json = await response.json()
420
+ except Exception:
421
+ logger.debug("Failed to parse JSON from error response")
422
+ pass
423
+ response.raise_for_status()
424
+ except aiohttp.ClientError as e:
425
+ error_msg = str(getattr(e, 'message', e))
426
+ # log the raw response for debugging
228
427
  status_code = BaseApi.get_status_code(e)
229
- if status_code >= 500 and status_code < 600:
230
- logger.error(f"Error in request to {url}: {e}")
231
- if status_code >= 400 and status_code < 500:
232
- if isinstance(e, aiohttp.ClientResponseError):
233
- # aiohttp.ClientResponse does not have .text or .json() methods directly
234
- error_msg = e.message
235
- else:
236
- error_msg = e.response.text
237
- logger.info(f"Error response: {error_msg}")
238
- if ' not found' in error_msg.lower():
428
+ # Try to extract detailed message from JSON response
429
+ if response_json:
430
+ error_msg = f"{error_msg} | {response_json}"
431
+
432
+ logger.error(f"HTTP error {status_code} for {url}: {error_msg}")
433
+ try:
434
+ e.message = error_msg
435
+ except Exception:
436
+ logger.debug("Unable to set message attribute on exception")
437
+ pass
438
+ if status_code in (400, 404):
439
+ if ' not found' in error_msg.lower() or 'Not Found' in error_msg:
239
440
  # Will be caught by the caller and properly initialized:
240
441
  raise ResourceNotFoundError('unknown', {})
241
442
  raise
443
+ return response_json
242
444
 
243
445
  @contextlib.asynccontextmanager
244
446
  async def _make_request_async(self,
@@ -268,9 +470,10 @@ class BaseApi:
268
470
  """
269
471
 
270
472
  if session is None:
271
- async with aiohttp.ClientSession() as temp_session:
272
- async with self._make_request_async(method, endpoint, temp_session, **kwargs) as resp:
273
- yield resp
473
+ session = self._get_aiohttp_session()
474
+
475
+ async with self._make_request_async(method, endpoint, session, **kwargs) as resp:
476
+ yield resp
274
477
  return
275
478
 
276
479
  url = f"{self.config.server_url.rstrip('/')}/{endpoint.lstrip('/')}"
@@ -279,7 +482,13 @@ class BaseApi:
279
482
  if self.config.api_key:
280
483
  headers['apikey'] = self.config.api_key
281
484
 
282
- timeout = aiohttp.ClientTimeout(total=self.config.timeout)
485
+ if 'timeout' in kwargs:
486
+ timeout = kwargs.pop('timeout')
487
+ # Ensure timeout is a ClientTimeout object
488
+ if isinstance(timeout, (int, float)):
489
+ timeout = aiohttp.ClientTimeout(total=timeout)
490
+ else:
491
+ timeout = aiohttp.ClientTimeout(total=self.config.timeout)
283
492
 
284
493
  response = None
285
494
  curl_cmd = self._generate_curl_command(
@@ -296,10 +505,17 @@ class BaseApi:
296
505
  timeout=timeout,
297
506
  **kwargs
298
507
  )
299
- self._check_errors_response(response, url=url)
508
+ data = await self._check_errors_response_aiohttp(response, url=url)
509
+ if data is None:
510
+ data = await response.json()
511
+ logger.debug(f"Successful {method} request to {endpoint}: {data}")
300
512
  yield response
513
+ except aiohttp.ClientConnectorCertificateError as e:
514
+ self._raise_ssl_error(e)
301
515
  except aiohttp.ClientError as e:
302
- logger.error(f"Request error for {method} {endpoint}: {e}")
516
+ # Check for SSL errors in other client errors
517
+ if "certificate" in str(e).lower() or "ssl" in str(e).lower():
518
+ self._raise_ssl_error(e)
303
519
  raise
304
520
  finally:
305
521
  if response is not None:
@@ -345,9 +561,13 @@ class BaseApi:
345
561
  """
346
562
  offset = 0
347
563
  total_fetched = 0
348
- params = dict(kwargs.get('params', {}))
349
- # Ensure kwargs carries our params reference so mutations below take effect
350
- kwargs['params'] = params
564
+
565
+ use_json_pagination = method.upper() == 'POST' and 'json' in kwargs and isinstance(kwargs['json'], dict)
566
+
567
+ if not use_json_pagination:
568
+ params = dict(kwargs.get('params', {}))
569
+ # Ensure kwargs carries our params reference so mutations below take effect
570
+ kwargs['params'] = params
351
571
 
352
572
  while True:
353
573
  if limit is not None and total_fetched >= limit:
@@ -358,8 +578,12 @@ class BaseApi:
358
578
  remaining = limit - total_fetched
359
579
  page_limit = min(_PAGE_LIMIT, remaining)
360
580
 
361
- params['offset'] = offset
362
- params['limit'] = page_limit
581
+ if use_json_pagination:
582
+ kwargs['json']['offset'] = str(offset)
583
+ kwargs['json']['limit'] = str(page_limit)
584
+ else:
585
+ params['offset'] = offset
586
+ params['limit'] = page_limit
363
587
 
364
588
  response = self._make_request(method=method,
365
589
  endpoint=endpoint,
@@ -412,7 +636,7 @@ class BaseApi:
412
636
  def convert_format(bytes_array: bytes,
413
637
  mimetype: str | None = None,
414
638
  file_path: str | None = None
415
- ) -> ImagingData | bytes:
639
+ ) -> 'ImagingData | bytes':
416
640
  """ Convert the bytes array to the appropriate format based on the mimetype.
417
641
 
418
642
  Args:
@@ -430,6 +654,8 @@ class BaseApi:
430
654
  >>> dicom = BaseApi.convert_format(dicom_bytes)
431
655
 
432
656
  """
657
+ import pydicom
658
+
433
659
  if mimetype is None:
434
660
  mimetype, ext = BaseApi._determine_mimetype(bytes_array)
435
661
  if mimetype is None:
@@ -449,15 +675,21 @@ class BaseApi:
449
675
  return bytes_array
450
676
  elif mimetype.endswith('nifti'):
451
677
  try:
452
- return nib.Nifti1Image.from_stream(content_io)
678
+ ndata = nib.Nifti1Image.from_stream(content_io)
679
+ ndata.get_fdata() # force loading before IO is closed
680
+ return ndata
453
681
  except Exception as e:
454
682
  if file_path is not None:
455
- return nib.load(file_path)
683
+ ndata = nib.load(file_path)
684
+ ndata.get_fdata() # force loading before IO is closed
685
+ return ndata
456
686
  raise e
457
687
  elif mimetype in GZIP_MIME_TYPES:
458
688
  # let's hope it's a .nii.gz
459
689
  with gzip.open(content_io, 'rb') as f:
460
- return nib.Nifti1Image.from_stream(f)
690
+ ndata = nib.Nifti1Image.from_stream(f)
691
+ ndata.get_fdata() # force loading before IO is closed
692
+ return ndata
461
693
 
462
694
  raise ValueError(f"Unsupported mimetype: {mimetype}")
463
695
 
datamint/api/client.py CHANGED
@@ -1,19 +1,21 @@
1
- from typing import Optional
2
1
  from .base_api import ApiConfig, BaseApi
3
2
  from .endpoints import (ProjectsApi, ResourcesApi, AnnotationsApi,
4
- ChannelsApi, UsersApi, DatasetsInfoApi, ModelsApi,
5
- AnnotationSetsApi
3
+ ChannelsApi, UsersApi, DatasetsInfoApi,
4
+ AnnotationSetsApi, DeployModelApi
6
5
  )
6
+ from .endpoints.models_api import ModelsApi
7
7
  import datamint.configs
8
8
  from datamint.exceptions import DatamintException
9
+ import logging
9
10
 
11
+ _LOGGER = logging.getLogger(__name__)
10
12
 
11
13
  class Api:
12
14
  """Main API client that provides access to all endpoint handlers."""
13
15
  DEFAULT_SERVER_URL = 'https://api.datamint.io'
14
16
  DATAMINT_API_VENV_NAME = datamint.configs.ENV_VARS[datamint.configs.APIKEY_KEY]
15
17
 
16
- _API_MAP : dict[str, type[BaseApi]] = {
18
+ _API_MAP: dict[str, type[BaseApi]] = {
17
19
  'projects': ProjectsApi,
18
20
  'resources': ResourcesApi,
19
21
  'annotations': AnnotationsApi,
@@ -22,21 +24,26 @@ class Api:
22
24
  'datasets': DatasetsInfoApi,
23
25
  'models': ModelsApi,
24
26
  'annotationsets': AnnotationSetsApi,
27
+ 'deploy': DeployModelApi,
25
28
  }
26
29
 
27
30
  def __init__(self,
28
31
  server_url: str | None = None,
29
- api_key: Optional[str] = None,
32
+ api_key: str | None = None,
30
33
  timeout: float = 60.0, max_retries: int = 2,
31
- check_connection: bool = True) -> None:
34
+ check_connection: bool = True,
35
+ verify_ssl: bool | str = True) -> None:
32
36
  """Initialize the API client.
33
37
 
34
38
  Args:
35
- base_url: Base URL for the API
39
+ server_url: Base URL for the API
36
40
  api_key: Optional API key for authentication
37
41
  timeout: Request timeout in seconds
38
42
  max_retries: Maximum number of retry attempts
39
- client: Optional HTTP client instance
43
+ check_connection: Whether to check connection on initialization
44
+ verify_ssl: Whether to verify SSL certificates. Default is True.
45
+ Set to False only in development environments with self-signed certificates.
46
+ Can also be a path to a CA bundle file for custom certificate verification.
40
47
  """
41
48
  if server_url is None:
42
49
  server_url = datamint.configs.get_value(datamint.configs.APIURL_KEY)
@@ -53,10 +60,28 @@ class Api:
53
60
  server_url=server_url,
54
61
  api_key=api_key,
55
62
  timeout=timeout,
56
- max_retries=max_retries
63
+ max_retries=max_retries,
64
+ verify_ssl=verify_ssl
65
+ )
66
+ self.high_config = ApiConfig(
67
+ server_url=server_url,
68
+ api_key=api_key,
69
+ timeout=timeout*5,
70
+ max_retries=max_retries,
71
+ verify_ssl=verify_ssl,
72
+ )
73
+ self.mlflow_config = ApiConfig(
74
+ server_url=server_url,
75
+ api_key=api_key,
76
+ timeout=timeout,
77
+ max_retries=max_retries,
78
+ port=5000,
79
+ verify_ssl=verify_ssl
57
80
  )
58
81
  self._client = None
59
- self._endpoints = {}
82
+ self._highclient = None
83
+ self._mlclient = None
84
+ self._endpoints: dict[str, BaseApi] = {}
60
85
  if check_connection:
61
86
  self.check_connection()
62
87
 
@@ -65,12 +90,45 @@ class Api:
65
90
  self.projects.get_list(limit=1)
66
91
  except Exception as e:
67
92
  raise DatamintException("Error connecting to the Datamint API." +
68
- f" Please check your api_key and/or other configurations. {e}")
93
+ f" Please check your api_key and/or other configurations.") from e
69
94
 
70
- def _get_endpoint(self, name: str):
95
+ def close(self) -> None:
96
+ """Close underlying HTTP clients and any shared aiohttp sessions.
97
+
98
+ Recommended for notebooks / long-running processes.
99
+ """
100
+ # Close per-endpoint resources (aiohttp sessions, etc.)
101
+ for endpoint in list(self._endpoints.values()):
102
+ try:
103
+ endpoint.close()
104
+ except Exception as e:
105
+ _LOGGER.warning(f"Error closing endpoint {endpoint}: {e}")
106
+ pass
107
+
108
+ # Close shared httpx clients owned by this Api
109
+ for client in (self._client, self._highclient, self._mlclient):
110
+ try:
111
+ if client is not None:
112
+ client.close()
113
+ except Exception as e:
114
+ _LOGGER.info(f"Error closing client {client}: {e}")
115
+
116
+ def _get_endpoint(self, name: str, is_mlflow: bool = False):
117
+ if is_mlflow:
118
+ if self._mlclient is None:
119
+ self._mlclient = BaseApi._create_client(self.mlflow_config)
120
+ client = self._mlclient
121
+ elif name in ['resources', 'annotations']:
122
+ if self._highclient is None:
123
+ self._highclient = BaseApi._create_client(self.high_config)
124
+ client = self._highclient
125
+ else:
126
+ if self._client is None:
127
+ self._client = BaseApi._create_client(self.config)
128
+ client = self._client
71
129
  if name not in self._endpoints:
72
130
  api_class = self._API_MAP[name]
73
- endpoint = api_class(self.config, self._client)
131
+ endpoint = api_class(self.config, client)
74
132
  # Inject this API instance into the endpoint so it can inject into entities
75
133
  endpoint._api_instance = self
76
134
  self._endpoints[name] = endpoint
@@ -108,3 +166,8 @@ class Api:
108
166
  @property
109
167
  def annotationsets(self) -> AnnotationSetsApi:
110
168
  return self._get_endpoint('annotationsets')
169
+
170
+ @property
171
+ def deploy(self) -> DeployModelApi:
172
+ """Access deployment management endpoints."""
173
+ return self._get_endpoint('deploy', is_mlflow=True)
@@ -6,8 +6,8 @@ from .projects_api import ProjectsApi
6
6
  from .resources_api import ResourcesApi
7
7
  from .users_api import UsersApi
8
8
  from .datasetsinfo_api import DatasetsInfoApi
9
- from .models_api import ModelsApi
10
9
  from .annotationsets_api import AnnotationSetsApi
10
+ from .deploy_model_api import DeployModelApi
11
11
 
12
12
  __all__ = [
13
13
  'AnnotationsApi',
@@ -16,6 +16,6 @@ __all__ = [
16
16
  'ResourcesApi',
17
17
  'UsersApi',
18
18
  'DatasetsInfoApi',
19
- 'ModelsApi',
20
19
  'AnnotationSetsApi',
20
+ 'DeployModelApi',
21
21
  ]