datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datamint/__init__.py +1 -3
- datamint/api/__init__.py +0 -3
- datamint/api/base_api.py +286 -54
- datamint/api/client.py +76 -13
- datamint/api/endpoints/__init__.py +2 -2
- datamint/api/endpoints/annotations_api.py +186 -28
- datamint/api/endpoints/deploy_model_api.py +78 -0
- datamint/api/endpoints/models_api.py +1 -0
- datamint/api/endpoints/projects_api.py +38 -7
- datamint/api/endpoints/resources_api.py +227 -100
- datamint/api/entity_base_api.py +66 -7
- datamint/apihandler/base_api_handler.py +0 -1
- datamint/apihandler/dto/annotation_dto.py +2 -0
- datamint/client_cmd_tools/datamint_config.py +0 -1
- datamint/client_cmd_tools/datamint_upload.py +3 -1
- datamint/configs.py +11 -7
- datamint/dataset/base_dataset.py +24 -4
- datamint/dataset/dataset.py +1 -1
- datamint/entities/__init__.py +1 -1
- datamint/entities/annotations/__init__.py +13 -0
- datamint/entities/{annotation.py → annotations/annotation.py} +81 -47
- datamint/entities/annotations/image_classification.py +12 -0
- datamint/entities/annotations/image_segmentation.py +252 -0
- datamint/entities/annotations/volume_segmentation.py +273 -0
- datamint/entities/base_entity.py +100 -6
- datamint/entities/cache_manager.py +129 -15
- datamint/entities/datasetinfo.py +60 -65
- datamint/entities/deployjob.py +18 -0
- datamint/entities/project.py +39 -0
- datamint/entities/resource.py +310 -46
- datamint/lightning/__init__.py +1 -0
- datamint/lightning/datamintdatamodule.py +103 -0
- datamint/mlflow/__init__.py +65 -0
- datamint/mlflow/artifact/__init__.py +1 -0
- datamint/mlflow/artifact/datamint_artifacts_repo.py +8 -0
- datamint/mlflow/env_utils.py +131 -0
- datamint/mlflow/env_vars.py +5 -0
- datamint/mlflow/flavors/__init__.py +17 -0
- datamint/mlflow/flavors/datamint_flavor.py +150 -0
- datamint/mlflow/flavors/model.py +877 -0
- datamint/mlflow/lightning/callbacks/__init__.py +1 -0
- datamint/mlflow/lightning/callbacks/modelcheckpoint.py +410 -0
- datamint/mlflow/models/__init__.py +93 -0
- datamint/mlflow/tracking/datamint_store.py +76 -0
- datamint/mlflow/tracking/default_experiment.py +27 -0
- datamint/mlflow/tracking/fluent.py +91 -0
- datamint/utils/env.py +27 -0
- datamint/utils/visualization.py +21 -13
- datamint-2.9.0.dist-info/METADATA +220 -0
- datamint-2.9.0.dist-info/RECORD +73 -0
- {datamint-2.3.3.dist-info → datamint-2.9.0.dist-info}/WHEEL +1 -1
- datamint-2.9.0.dist-info/entry_points.txt +18 -0
- datamint/apihandler/exp_api_handler.py +0 -204
- datamint/experiment/__init__.py +0 -1
- datamint/experiment/_patcher.py +0 -570
- datamint/experiment/experiment.py +0 -1049
- datamint-2.3.3.dist-info/METADATA +0 -125
- datamint-2.3.3.dist-info/RECORD +0 -54
- datamint-2.3.3.dist-info/entry_points.txt +0 -4
datamint/__init__.py
CHANGED
|
@@ -7,19 +7,17 @@ from typing import TYPE_CHECKING
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
8
|
from .dataset.dataset import DatamintDataset as Dataset
|
|
9
9
|
from .apihandler.api_handler import APIHandler
|
|
10
|
-
from .experiment import Experiment
|
|
11
10
|
from .api.client import Api
|
|
12
11
|
else:
|
|
13
12
|
import lazy_loader as lazy
|
|
14
13
|
|
|
15
14
|
__getattr__, __dir__, __all__ = lazy.attach(
|
|
16
15
|
__name__,
|
|
17
|
-
submodules=['dataset', "dataset.dataset", "apihandler.api_handler"
|
|
16
|
+
submodules=['dataset', "dataset.dataset", "apihandler.api_handler"],
|
|
18
17
|
submod_attrs={
|
|
19
18
|
"dataset.dataset": ["DatamintDataset"],
|
|
20
19
|
"dataset": ['Dataset'],
|
|
21
20
|
"apihandler.api_handler": ["APIHandler"],
|
|
22
|
-
"experiment": ["Experiment"],
|
|
23
21
|
"api.client": ["Api"],
|
|
24
22
|
},
|
|
25
23
|
)
|
datamint/api/__init__.py
CHANGED
datamint/api/base_api.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
from collections.abc import Generator, AsyncGenerator
|
|
3
4
|
import httpx
|
|
4
5
|
from dataclasses import dataclass
|
|
5
6
|
from datamint.exceptions import DatamintException, ResourceNotFoundError
|
|
6
|
-
from datamint.types import ImagingData
|
|
7
7
|
import aiohttp
|
|
8
8
|
import json
|
|
9
9
|
from PIL import Image
|
|
@@ -14,15 +14,17 @@ import gzip
|
|
|
14
14
|
import contextlib
|
|
15
15
|
import asyncio
|
|
16
16
|
from medimgkit.format_detection import GZIP_MIME_TYPES, DEFAULT_MIME_TYPE, guess_typez, guess_extension
|
|
17
|
+
from datamint.utils.env import ensure_asyncio_loop
|
|
17
18
|
|
|
18
19
|
if TYPE_CHECKING:
|
|
19
20
|
from datamint.api.client import Api
|
|
21
|
+
from datamint.types import ImagingData
|
|
20
22
|
|
|
21
23
|
logger = logging.getLogger(__name__)
|
|
22
24
|
|
|
23
|
-
# Generic type for entities
|
|
24
25
|
_PAGE_LIMIT = 5000
|
|
25
26
|
|
|
27
|
+
|
|
26
28
|
@dataclass
|
|
27
29
|
class ApiConfig:
|
|
28
30
|
"""Configuration for API client.
|
|
@@ -32,18 +34,30 @@ class ApiConfig:
|
|
|
32
34
|
api_key: Optional API key for authentication.
|
|
33
35
|
timeout: Request timeout in seconds.
|
|
34
36
|
max_retries: Maximum number of retries for requests.
|
|
37
|
+
port: Optional port number for the API server.
|
|
38
|
+
verify_ssl: Whether to verify SSL certificates. Default is True.
|
|
39
|
+
Set to False only in development environments with self-signed certificates.
|
|
40
|
+
Can also be a path to a CA bundle file.
|
|
35
41
|
"""
|
|
36
42
|
server_url: str
|
|
37
43
|
api_key: str | None = None
|
|
38
44
|
timeout: float = 30.0
|
|
39
45
|
max_retries: int = 3
|
|
46
|
+
port: int | None = None
|
|
47
|
+
verify_ssl: bool | str = True
|
|
40
48
|
|
|
41
49
|
@property
|
|
42
50
|
def web_app_url(self) -> str:
|
|
43
51
|
"""Get the base URL for the web application."""
|
|
44
|
-
|
|
52
|
+
base_url = self.server_url
|
|
53
|
+
|
|
54
|
+
# Add port to base_url if specified
|
|
55
|
+
if self.port is not None:
|
|
56
|
+
base_url = f"{self.server_url.rstrip('/')}:{self.port}"
|
|
57
|
+
|
|
58
|
+
if base_url.startswith('http://localhost'):
|
|
45
59
|
return 'http://localhost:3000'
|
|
46
|
-
if
|
|
60
|
+
if base_url.startswith('https://stagingapi.datamint.io'):
|
|
47
61
|
return 'https://staging.datamint.io'
|
|
48
62
|
return 'https://app.datamint.io'
|
|
49
63
|
|
|
@@ -61,22 +75,173 @@ class BaseApi:
|
|
|
61
75
|
client: Optional HTTP client instance. If None, a new one will be created.
|
|
62
76
|
"""
|
|
63
77
|
self.config = config
|
|
64
|
-
self.
|
|
78
|
+
self._owns_client = client is None # Track if we created the client
|
|
79
|
+
self.client = client or BaseApi._create_client(config)
|
|
65
80
|
self.semaphore = asyncio.Semaphore(20)
|
|
66
81
|
self._api_instance: 'Api | None' = None # Injected by Api class
|
|
82
|
+
self._aiohttp_connector: aiohttp.TCPConnector | None = None
|
|
83
|
+
self._aiohttp_session: aiohttp.ClientSession | None = None
|
|
84
|
+
ensure_asyncio_loop()
|
|
85
|
+
|
|
67
86
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
87
|
+
@staticmethod
|
|
88
|
+
def _create_client(config: ApiConfig) -> httpx.Client:
|
|
89
|
+
"""Create and configure HTTP client with authentication and timeouts.
|
|
90
|
+
|
|
91
|
+
The client is designed to be long-lived and reused across multiple requests.
|
|
92
|
+
It maintains connection pooling for improved performance.
|
|
93
|
+
"""
|
|
94
|
+
headers = {"apikey": config.api_key, 'Authorization': f"Bearer {config.api_key}"} if config.api_key else None
|
|
95
|
+
|
|
96
|
+
# Add port to base_url if specified
|
|
97
|
+
base_url = config.server_url.rstrip('/').strip()
|
|
98
|
+
if config.port is not None:
|
|
99
|
+
# if the port is already in the URL, replace it
|
|
100
|
+
if ':' in base_url.split('//')[-1]:
|
|
101
|
+
parts = base_url.rsplit(':', 1)
|
|
102
|
+
# confirm parts[1] is numeric
|
|
103
|
+
if parts[1].isdigit():
|
|
104
|
+
base_url = f"{parts[0]}:{config.port}"
|
|
105
|
+
else:
|
|
106
|
+
logger.warning(f"Invalid port detected in server_url: {config.server_url}")
|
|
107
|
+
else:
|
|
108
|
+
base_url = f"{base_url}:{config.port}"
|
|
73
109
|
|
|
74
110
|
return httpx.Client(
|
|
75
|
-
base_url=
|
|
111
|
+
base_url=base_url,
|
|
76
112
|
headers=headers,
|
|
77
|
-
timeout=
|
|
113
|
+
timeout=config.timeout,
|
|
114
|
+
verify=config.verify_ssl,
|
|
115
|
+
limits=httpx.Limits(
|
|
116
|
+
max_keepalive_connections=5,
|
|
117
|
+
max_connections=20,
|
|
118
|
+
keepalive_expiry=8
|
|
119
|
+
)
|
|
78
120
|
)
|
|
79
121
|
|
|
122
|
+
def _raise_ssl_error(self, original_error: Exception) -> None:
|
|
123
|
+
"""Raise a more helpful SSL certificate error with troubleshooting guidance.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
original_error: The original SSL-related exception
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
DatamintException: With helpful troubleshooting information
|
|
130
|
+
"""
|
|
131
|
+
error_msg = (
|
|
132
|
+
f"SSL Certificate verification failed: {original_error}\n\n"
|
|
133
|
+
"This typically happens when Python cannot verify the SSL certificate of the Datamint API.\n\n"
|
|
134
|
+
"Quick fixes:\n"
|
|
135
|
+
"1. Upgrade certifi:\n"
|
|
136
|
+
" pip install --upgrade certifi\n\n"
|
|
137
|
+
"2. Set environment variables:\n"
|
|
138
|
+
" export SSL_CERT_FILE=$(python -m certifi)\n"
|
|
139
|
+
" export REQUESTS_CA_BUNDLE=$(python -m certifi)\n\n"
|
|
140
|
+
"3. Or disable SSL verification (development only):\n"
|
|
141
|
+
" api = Api(verify_ssl=False)\n\n"
|
|
142
|
+
"For more help, see: https://github.com/SonanceAI/datamint-python-api#-ssl-certificate-troubleshooting"
|
|
143
|
+
)
|
|
144
|
+
raise DatamintException(error_msg) from original_error
|
|
145
|
+
|
|
146
|
+
def _create_aiohttp_connector(self, force_close: bool = False) -> aiohttp.TCPConnector:
|
|
147
|
+
"""Create aiohttp connector with SSL configuration.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
force_close: Whether to force close connections (disable keep-alive)
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Configured TCPConnector for aiohttp sessions.
|
|
154
|
+
"""
|
|
155
|
+
import ssl
|
|
156
|
+
import certifi
|
|
157
|
+
|
|
158
|
+
limit = 20
|
|
159
|
+
ttl_dns_cache = 300
|
|
160
|
+
|
|
161
|
+
if self.config.verify_ssl is False:
|
|
162
|
+
# Disable SSL verification (not recommended for production)
|
|
163
|
+
return aiohttp.TCPConnector(ssl=False, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
|
|
164
|
+
elif isinstance(self.config.verify_ssl, str):
|
|
165
|
+
# Use custom CA bundle
|
|
166
|
+
ssl_context = ssl.create_default_context(cafile=self.config.verify_ssl)
|
|
167
|
+
return aiohttp.TCPConnector(ssl=ssl_context, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
|
|
168
|
+
else:
|
|
169
|
+
# Use certifi's CA bundle (default behavior)
|
|
170
|
+
ssl_context = ssl.create_default_context(cafile=certifi.where())
|
|
171
|
+
return aiohttp.TCPConnector(ssl=ssl_context, limit=limit, ttl_dns_cache=ttl_dns_cache, force_close=force_close)
|
|
172
|
+
|
|
173
|
+
def _get_aiohttp_session(self) -> aiohttp.ClientSession:
|
|
174
|
+
"""Get or create a shared aiohttp session for this API instance.
|
|
175
|
+
|
|
176
|
+
Creating/closing a new TLS connection for each upload can cause intermittent
|
|
177
|
+
connection shutdown timeouts in long-running processes (notably notebooks).
|
|
178
|
+
Reusing a single session keeps connections healthy and avoids excessive churn.
|
|
179
|
+
"""
|
|
180
|
+
if self._aiohttp_session is not None and not self._aiohttp_session.closed:
|
|
181
|
+
return self._aiohttp_session
|
|
182
|
+
|
|
183
|
+
# (Re)create connector and session
|
|
184
|
+
self._aiohttp_connector = self._create_aiohttp_connector(force_close=False)
|
|
185
|
+
timeout = aiohttp.ClientTimeout(total=self.config.timeout)
|
|
186
|
+
self._aiohttp_session = aiohttp.ClientSession(connector=self._aiohttp_connector, timeout=timeout)
|
|
187
|
+
return self._aiohttp_session
|
|
188
|
+
|
|
189
|
+
def _close_aiohttp_session(self) -> None:
|
|
190
|
+
"""Close the shared aiohttp session if it exists.
|
|
191
|
+
|
|
192
|
+
This is best-effort; in environments with a running loop we rely on
|
|
193
|
+
`nest_asyncio` (enabled via `ensure_asyncio_loop`) to allow nested runs.
|
|
194
|
+
"""
|
|
195
|
+
if self._aiohttp_session is None or self._aiohttp_session.closed:
|
|
196
|
+
return
|
|
197
|
+
|
|
198
|
+
ensure_asyncio_loop()
|
|
199
|
+
try:
|
|
200
|
+
loop = asyncio.get_event_loop()
|
|
201
|
+
except RuntimeError:
|
|
202
|
+
loop = asyncio.new_event_loop()
|
|
203
|
+
asyncio.set_event_loop(loop)
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
loop.run_until_complete(self._aiohttp_session.close())
|
|
207
|
+
except RuntimeError:
|
|
208
|
+
# If we're in an environment where the loop is running and not patched,
|
|
209
|
+
# fall back to scheduling the close.
|
|
210
|
+
try:
|
|
211
|
+
loop.create_task(self._aiohttp_session.close())
|
|
212
|
+
except Exception as e:
|
|
213
|
+
logger.info(f"Unable to schedule aiohttp session close: {e}")
|
|
214
|
+
pass
|
|
215
|
+
finally:
|
|
216
|
+
self._aiohttp_session = None
|
|
217
|
+
self._aiohttp_connector = None
|
|
218
|
+
|
|
219
|
+
def close(self) -> None:
|
|
220
|
+
"""Close the HTTP client and release resources.
|
|
221
|
+
|
|
222
|
+
Should be called when the API instance is no longer needed.
|
|
223
|
+
Only closes the client if it was created by this instance.
|
|
224
|
+
"""
|
|
225
|
+
# Close shared aiohttp session regardless of httpx client ownership.
|
|
226
|
+
self._close_aiohttp_session()
|
|
227
|
+
if self._owns_client and self.client is not None:
|
|
228
|
+
self.client.close()
|
|
229
|
+
|
|
230
|
+
def __enter__(self):
|
|
231
|
+
"""Context manager entry."""
|
|
232
|
+
return self
|
|
233
|
+
|
|
234
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
235
|
+
"""Context manager exit - ensures client is closed."""
|
|
236
|
+
self.close()
|
|
237
|
+
|
|
238
|
+
def __del__(self):
|
|
239
|
+
"""Destructor - ensures client is closed when instance is garbage collected."""
|
|
240
|
+
try:
|
|
241
|
+
self.close()
|
|
242
|
+
except Exception:
|
|
243
|
+
pass # Ignore errors during cleanup
|
|
244
|
+
|
|
80
245
|
def _stream_request(self, method: str, endpoint: str, **kwargs):
|
|
81
246
|
"""Make streaming HTTP request with error handling.
|
|
82
247
|
|
|
@@ -120,21 +285,15 @@ class BaseApi:
|
|
|
120
285
|
"""
|
|
121
286
|
url = endpoint.lstrip('/') # Remove leading slash for httpx
|
|
122
287
|
|
|
123
|
-
|
|
288
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
124
289
|
curl_command = self._generate_curl_command({"method": method,
|
|
125
290
|
"url": url,
|
|
126
291
|
"headers": self.client.headers,
|
|
127
292
|
**kwargs}, fail_silently=True)
|
|
128
293
|
logger.debug(f'Equivalent curl command: "{curl_command}"')
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
except httpx.HTTPStatusError as e:
|
|
133
|
-
logger.error(f"HTTP error {e.response.status_code} for {method} {endpoint}: {e.response.text}")
|
|
134
|
-
raise
|
|
135
|
-
except httpx.RequestError as e:
|
|
136
|
-
logger.error(f"Request error for {method} {endpoint}: {e}")
|
|
137
|
-
raise
|
|
294
|
+
response = self.client.request(method, url, **kwargs)
|
|
295
|
+
self._check_errors_response_httpx(response, url=url)
|
|
296
|
+
return response
|
|
138
297
|
|
|
139
298
|
def _generate_curl_command(self,
|
|
140
299
|
request_args: dict,
|
|
@@ -201,7 +360,7 @@ class BaseApi:
|
|
|
201
360
|
raise
|
|
202
361
|
|
|
203
362
|
@staticmethod
|
|
204
|
-
def get_status_code(e: httpx.
|
|
363
|
+
def get_status_code(e: httpx.HTTPError | aiohttp.ClientError) -> int:
|
|
205
364
|
if hasattr(e, 'response') and e.response is not None:
|
|
206
365
|
# httpx.HTTPStatusError
|
|
207
366
|
return e.response.status_code
|
|
@@ -218,27 +377,70 @@ class BaseApi:
|
|
|
218
377
|
status_code: int) -> bool:
|
|
219
378
|
return BaseApi.get_status_code(e) == status_code
|
|
220
379
|
|
|
221
|
-
def
|
|
222
|
-
|
|
223
|
-
|
|
380
|
+
def _check_errors_response_httpx(self,
|
|
381
|
+
response: httpx.Response,
|
|
382
|
+
url: str):
|
|
383
|
+
response_json = None
|
|
224
384
|
try:
|
|
385
|
+
try:
|
|
386
|
+
response_json = response.json()
|
|
387
|
+
except Exception:
|
|
388
|
+
pass
|
|
225
389
|
response.raise_for_status()
|
|
226
|
-
except
|
|
227
|
-
|
|
390
|
+
except httpx.ConnectError as e:
|
|
391
|
+
if "CERTIFICATE_VERIFY_FAILED" in str(e) or "certificate verify failed" in str(e).lower():
|
|
392
|
+
self._raise_ssl_error(e)
|
|
393
|
+
raise
|
|
394
|
+
except httpx.HTTPError as e:
|
|
395
|
+
error_msg = f"{getattr(e, 'message', str(e))} | {getattr(response, 'text', '')}"
|
|
396
|
+
if response_json:
|
|
397
|
+
error_msg = f"{error_msg} | {response_json}"
|
|
398
|
+
try:
|
|
399
|
+
e.message = error_msg
|
|
400
|
+
except Exception:
|
|
401
|
+
logger.debug("Unable to set message attribute on exception")
|
|
402
|
+
pass
|
|
403
|
+
|
|
404
|
+
logger.error(f"HTTP error {response.status_code} for {url}: {error_msg}")
|
|
405
|
+
status_code = response.status_code
|
|
406
|
+
if status_code in (400, 404):
|
|
407
|
+
if ' not found' in error_msg.lower() or 'Not Found' in error_msg:
|
|
408
|
+
# Will be caught by the caller and properly initialized:
|
|
409
|
+
raise ResourceNotFoundError('unknown', {})
|
|
410
|
+
raise
|
|
411
|
+
return response_json
|
|
412
|
+
|
|
413
|
+
async def _check_errors_response_aiohttp(self,
|
|
414
|
+
response: aiohttp.ClientResponse,
|
|
415
|
+
url: str):
|
|
416
|
+
response_json = None
|
|
417
|
+
try:
|
|
418
|
+
try:
|
|
419
|
+
response_json = await response.json()
|
|
420
|
+
except Exception:
|
|
421
|
+
logger.debug("Failed to parse JSON from error response")
|
|
422
|
+
pass
|
|
423
|
+
response.raise_for_status()
|
|
424
|
+
except aiohttp.ClientError as e:
|
|
425
|
+
error_msg = str(getattr(e, 'message', e))
|
|
426
|
+
# log the raw response for debugging
|
|
228
427
|
status_code = BaseApi.get_status_code(e)
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
logger.
|
|
238
|
-
|
|
428
|
+
# Try to extract detailed message from JSON response
|
|
429
|
+
if response_json:
|
|
430
|
+
error_msg = f"{error_msg} | {response_json}"
|
|
431
|
+
|
|
432
|
+
logger.error(f"HTTP error {status_code} for {url}: {error_msg}")
|
|
433
|
+
try:
|
|
434
|
+
e.message = error_msg
|
|
435
|
+
except Exception:
|
|
436
|
+
logger.debug("Unable to set message attribute on exception")
|
|
437
|
+
pass
|
|
438
|
+
if status_code in (400, 404):
|
|
439
|
+
if ' not found' in error_msg.lower() or 'Not Found' in error_msg:
|
|
239
440
|
# Will be caught by the caller and properly initialized:
|
|
240
441
|
raise ResourceNotFoundError('unknown', {})
|
|
241
442
|
raise
|
|
443
|
+
return response_json
|
|
242
444
|
|
|
243
445
|
@contextlib.asynccontextmanager
|
|
244
446
|
async def _make_request_async(self,
|
|
@@ -268,9 +470,10 @@ class BaseApi:
|
|
|
268
470
|
"""
|
|
269
471
|
|
|
270
472
|
if session is None:
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
473
|
+
session = self._get_aiohttp_session()
|
|
474
|
+
|
|
475
|
+
async with self._make_request_async(method, endpoint, session, **kwargs) as resp:
|
|
476
|
+
yield resp
|
|
274
477
|
return
|
|
275
478
|
|
|
276
479
|
url = f"{self.config.server_url.rstrip('/')}/{endpoint.lstrip('/')}"
|
|
@@ -279,7 +482,13 @@ class BaseApi:
|
|
|
279
482
|
if self.config.api_key:
|
|
280
483
|
headers['apikey'] = self.config.api_key
|
|
281
484
|
|
|
282
|
-
timeout
|
|
485
|
+
if 'timeout' in kwargs:
|
|
486
|
+
timeout = kwargs.pop('timeout')
|
|
487
|
+
# Ensure timeout is a ClientTimeout object
|
|
488
|
+
if isinstance(timeout, (int, float)):
|
|
489
|
+
timeout = aiohttp.ClientTimeout(total=timeout)
|
|
490
|
+
else:
|
|
491
|
+
timeout = aiohttp.ClientTimeout(total=self.config.timeout)
|
|
283
492
|
|
|
284
493
|
response = None
|
|
285
494
|
curl_cmd = self._generate_curl_command(
|
|
@@ -296,10 +505,17 @@ class BaseApi:
|
|
|
296
505
|
timeout=timeout,
|
|
297
506
|
**kwargs
|
|
298
507
|
)
|
|
299
|
-
self.
|
|
508
|
+
data = await self._check_errors_response_aiohttp(response, url=url)
|
|
509
|
+
if data is None:
|
|
510
|
+
data = await response.json()
|
|
511
|
+
logger.debug(f"Successful {method} request to {endpoint}: {data}")
|
|
300
512
|
yield response
|
|
513
|
+
except aiohttp.ClientConnectorCertificateError as e:
|
|
514
|
+
self._raise_ssl_error(e)
|
|
301
515
|
except aiohttp.ClientError as e:
|
|
302
|
-
|
|
516
|
+
# Check for SSL errors in other client errors
|
|
517
|
+
if "certificate" in str(e).lower() or "ssl" in str(e).lower():
|
|
518
|
+
self._raise_ssl_error(e)
|
|
303
519
|
raise
|
|
304
520
|
finally:
|
|
305
521
|
if response is not None:
|
|
@@ -345,9 +561,13 @@ class BaseApi:
|
|
|
345
561
|
"""
|
|
346
562
|
offset = 0
|
|
347
563
|
total_fetched = 0
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
564
|
+
|
|
565
|
+
use_json_pagination = method.upper() == 'POST' and 'json' in kwargs and isinstance(kwargs['json'], dict)
|
|
566
|
+
|
|
567
|
+
if not use_json_pagination:
|
|
568
|
+
params = dict(kwargs.get('params', {}))
|
|
569
|
+
# Ensure kwargs carries our params reference so mutations below take effect
|
|
570
|
+
kwargs['params'] = params
|
|
351
571
|
|
|
352
572
|
while True:
|
|
353
573
|
if limit is not None and total_fetched >= limit:
|
|
@@ -358,8 +578,12 @@ class BaseApi:
|
|
|
358
578
|
remaining = limit - total_fetched
|
|
359
579
|
page_limit = min(_PAGE_LIMIT, remaining)
|
|
360
580
|
|
|
361
|
-
|
|
362
|
-
|
|
581
|
+
if use_json_pagination:
|
|
582
|
+
kwargs['json']['offset'] = str(offset)
|
|
583
|
+
kwargs['json']['limit'] = str(page_limit)
|
|
584
|
+
else:
|
|
585
|
+
params['offset'] = offset
|
|
586
|
+
params['limit'] = page_limit
|
|
363
587
|
|
|
364
588
|
response = self._make_request(method=method,
|
|
365
589
|
endpoint=endpoint,
|
|
@@ -412,7 +636,7 @@ class BaseApi:
|
|
|
412
636
|
def convert_format(bytes_array: bytes,
|
|
413
637
|
mimetype: str | None = None,
|
|
414
638
|
file_path: str | None = None
|
|
415
|
-
) -> ImagingData | bytes:
|
|
639
|
+
) -> 'ImagingData | bytes':
|
|
416
640
|
""" Convert the bytes array to the appropriate format based on the mimetype.
|
|
417
641
|
|
|
418
642
|
Args:
|
|
@@ -430,6 +654,8 @@ class BaseApi:
|
|
|
430
654
|
>>> dicom = BaseApi.convert_format(dicom_bytes)
|
|
431
655
|
|
|
432
656
|
"""
|
|
657
|
+
import pydicom
|
|
658
|
+
|
|
433
659
|
if mimetype is None:
|
|
434
660
|
mimetype, ext = BaseApi._determine_mimetype(bytes_array)
|
|
435
661
|
if mimetype is None:
|
|
@@ -449,15 +675,21 @@ class BaseApi:
|
|
|
449
675
|
return bytes_array
|
|
450
676
|
elif mimetype.endswith('nifti'):
|
|
451
677
|
try:
|
|
452
|
-
|
|
678
|
+
ndata = nib.Nifti1Image.from_stream(content_io)
|
|
679
|
+
ndata.get_fdata() # force loading before IO is closed
|
|
680
|
+
return ndata
|
|
453
681
|
except Exception as e:
|
|
454
682
|
if file_path is not None:
|
|
455
|
-
|
|
683
|
+
ndata = nib.load(file_path)
|
|
684
|
+
ndata.get_fdata() # force loading before IO is closed
|
|
685
|
+
return ndata
|
|
456
686
|
raise e
|
|
457
687
|
elif mimetype in GZIP_MIME_TYPES:
|
|
458
688
|
# let's hope it's a .nii.gz
|
|
459
689
|
with gzip.open(content_io, 'rb') as f:
|
|
460
|
-
|
|
690
|
+
ndata = nib.Nifti1Image.from_stream(f)
|
|
691
|
+
ndata.get_fdata() # force loading before IO is closed
|
|
692
|
+
return ndata
|
|
461
693
|
|
|
462
694
|
raise ValueError(f"Unsupported mimetype: {mimetype}")
|
|
463
695
|
|
datamint/api/client.py
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
1
|
from .base_api import ApiConfig, BaseApi
|
|
3
2
|
from .endpoints import (ProjectsApi, ResourcesApi, AnnotationsApi,
|
|
4
|
-
ChannelsApi, UsersApi, DatasetsInfoApi,
|
|
5
|
-
AnnotationSetsApi
|
|
3
|
+
ChannelsApi, UsersApi, DatasetsInfoApi,
|
|
4
|
+
AnnotationSetsApi, DeployModelApi
|
|
6
5
|
)
|
|
6
|
+
from .endpoints.models_api import ModelsApi
|
|
7
7
|
import datamint.configs
|
|
8
8
|
from datamint.exceptions import DatamintException
|
|
9
|
+
import logging
|
|
9
10
|
|
|
11
|
+
_LOGGER = logging.getLogger(__name__)
|
|
10
12
|
|
|
11
13
|
class Api:
|
|
12
14
|
"""Main API client that provides access to all endpoint handlers."""
|
|
13
15
|
DEFAULT_SERVER_URL = 'https://api.datamint.io'
|
|
14
16
|
DATAMINT_API_VENV_NAME = datamint.configs.ENV_VARS[datamint.configs.APIKEY_KEY]
|
|
15
17
|
|
|
16
|
-
_API_MAP
|
|
18
|
+
_API_MAP: dict[str, type[BaseApi]] = {
|
|
17
19
|
'projects': ProjectsApi,
|
|
18
20
|
'resources': ResourcesApi,
|
|
19
21
|
'annotations': AnnotationsApi,
|
|
@@ -22,21 +24,26 @@ class Api:
|
|
|
22
24
|
'datasets': DatasetsInfoApi,
|
|
23
25
|
'models': ModelsApi,
|
|
24
26
|
'annotationsets': AnnotationSetsApi,
|
|
27
|
+
'deploy': DeployModelApi,
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
def __init__(self,
|
|
28
31
|
server_url: str | None = None,
|
|
29
|
-
api_key:
|
|
32
|
+
api_key: str | None = None,
|
|
30
33
|
timeout: float = 60.0, max_retries: int = 2,
|
|
31
|
-
check_connection: bool = True
|
|
34
|
+
check_connection: bool = True,
|
|
35
|
+
verify_ssl: bool | str = True) -> None:
|
|
32
36
|
"""Initialize the API client.
|
|
33
37
|
|
|
34
38
|
Args:
|
|
35
|
-
|
|
39
|
+
server_url: Base URL for the API
|
|
36
40
|
api_key: Optional API key for authentication
|
|
37
41
|
timeout: Request timeout in seconds
|
|
38
42
|
max_retries: Maximum number of retry attempts
|
|
39
|
-
|
|
43
|
+
check_connection: Whether to check connection on initialization
|
|
44
|
+
verify_ssl: Whether to verify SSL certificates. Default is True.
|
|
45
|
+
Set to False only in development environments with self-signed certificates.
|
|
46
|
+
Can also be a path to a CA bundle file for custom certificate verification.
|
|
40
47
|
"""
|
|
41
48
|
if server_url is None:
|
|
42
49
|
server_url = datamint.configs.get_value(datamint.configs.APIURL_KEY)
|
|
@@ -53,10 +60,28 @@ class Api:
|
|
|
53
60
|
server_url=server_url,
|
|
54
61
|
api_key=api_key,
|
|
55
62
|
timeout=timeout,
|
|
56
|
-
max_retries=max_retries
|
|
63
|
+
max_retries=max_retries,
|
|
64
|
+
verify_ssl=verify_ssl
|
|
65
|
+
)
|
|
66
|
+
self.high_config = ApiConfig(
|
|
67
|
+
server_url=server_url,
|
|
68
|
+
api_key=api_key,
|
|
69
|
+
timeout=timeout*5,
|
|
70
|
+
max_retries=max_retries,
|
|
71
|
+
verify_ssl=verify_ssl,
|
|
72
|
+
)
|
|
73
|
+
self.mlflow_config = ApiConfig(
|
|
74
|
+
server_url=server_url,
|
|
75
|
+
api_key=api_key,
|
|
76
|
+
timeout=timeout,
|
|
77
|
+
max_retries=max_retries,
|
|
78
|
+
port=5000,
|
|
79
|
+
verify_ssl=verify_ssl
|
|
57
80
|
)
|
|
58
81
|
self._client = None
|
|
59
|
-
self.
|
|
82
|
+
self._highclient = None
|
|
83
|
+
self._mlclient = None
|
|
84
|
+
self._endpoints: dict[str, BaseApi] = {}
|
|
60
85
|
if check_connection:
|
|
61
86
|
self.check_connection()
|
|
62
87
|
|
|
@@ -65,12 +90,45 @@ class Api:
|
|
|
65
90
|
self.projects.get_list(limit=1)
|
|
66
91
|
except Exception as e:
|
|
67
92
|
raise DatamintException("Error connecting to the Datamint API." +
|
|
68
|
-
f" Please check your api_key and/or other configurations.
|
|
93
|
+
f" Please check your api_key and/or other configurations.") from e
|
|
69
94
|
|
|
70
|
-
def
|
|
95
|
+
def close(self) -> None:
|
|
96
|
+
"""Close underlying HTTP clients and any shared aiohttp sessions.
|
|
97
|
+
|
|
98
|
+
Recommended for notebooks / long-running processes.
|
|
99
|
+
"""
|
|
100
|
+
# Close per-endpoint resources (aiohttp sessions, etc.)
|
|
101
|
+
for endpoint in list(self._endpoints.values()):
|
|
102
|
+
try:
|
|
103
|
+
endpoint.close()
|
|
104
|
+
except Exception as e:
|
|
105
|
+
_LOGGER.warning(f"Error closing endpoint {endpoint}: {e}")
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
# Close shared httpx clients owned by this Api
|
|
109
|
+
for client in (self._client, self._highclient, self._mlclient):
|
|
110
|
+
try:
|
|
111
|
+
if client is not None:
|
|
112
|
+
client.close()
|
|
113
|
+
except Exception as e:
|
|
114
|
+
_LOGGER.info(f"Error closing client {client}: {e}")
|
|
115
|
+
|
|
116
|
+
def _get_endpoint(self, name: str, is_mlflow: bool = False):
|
|
117
|
+
if is_mlflow:
|
|
118
|
+
if self._mlclient is None:
|
|
119
|
+
self._mlclient = BaseApi._create_client(self.mlflow_config)
|
|
120
|
+
client = self._mlclient
|
|
121
|
+
elif name in ['resources', 'annotations']:
|
|
122
|
+
if self._highclient is None:
|
|
123
|
+
self._highclient = BaseApi._create_client(self.high_config)
|
|
124
|
+
client = self._highclient
|
|
125
|
+
else:
|
|
126
|
+
if self._client is None:
|
|
127
|
+
self._client = BaseApi._create_client(self.config)
|
|
128
|
+
client = self._client
|
|
71
129
|
if name not in self._endpoints:
|
|
72
130
|
api_class = self._API_MAP[name]
|
|
73
|
-
endpoint = api_class(self.config,
|
|
131
|
+
endpoint = api_class(self.config, client)
|
|
74
132
|
# Inject this API instance into the endpoint so it can inject into entities
|
|
75
133
|
endpoint._api_instance = self
|
|
76
134
|
self._endpoints[name] = endpoint
|
|
@@ -108,3 +166,8 @@ class Api:
|
|
|
108
166
|
@property
|
|
109
167
|
def annotationsets(self) -> AnnotationSetsApi:
|
|
110
168
|
return self._get_endpoint('annotationsets')
|
|
169
|
+
|
|
170
|
+
@property
|
|
171
|
+
def deploy(self) -> DeployModelApi:
|
|
172
|
+
"""Access deployment management endpoints."""
|
|
173
|
+
return self._get_endpoint('deploy', is_mlflow=True)
|
|
@@ -6,8 +6,8 @@ from .projects_api import ProjectsApi
|
|
|
6
6
|
from .resources_api import ResourcesApi
|
|
7
7
|
from .users_api import UsersApi
|
|
8
8
|
from .datasetsinfo_api import DatasetsInfoApi
|
|
9
|
-
from .models_api import ModelsApi
|
|
10
9
|
from .annotationsets_api import AnnotationSetsApi
|
|
10
|
+
from .deploy_model_api import DeployModelApi
|
|
11
11
|
|
|
12
12
|
__all__ = [
|
|
13
13
|
'AnnotationsApi',
|
|
@@ -16,6 +16,6 @@ __all__ = [
|
|
|
16
16
|
'ResourcesApi',
|
|
17
17
|
'UsersApi',
|
|
18
18
|
'DatasetsInfoApi',
|
|
19
|
-
'ModelsApi',
|
|
20
19
|
'AnnotationSetsApi',
|
|
20
|
+
'DeployModelApi',
|
|
21
21
|
]
|