datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datamint/__init__.py +1 -3
- datamint/api/__init__.py +0 -3
- datamint/api/base_api.py +286 -54
- datamint/api/client.py +76 -13
- datamint/api/endpoints/__init__.py +2 -2
- datamint/api/endpoints/annotations_api.py +186 -28
- datamint/api/endpoints/deploy_model_api.py +78 -0
- datamint/api/endpoints/models_api.py +1 -0
- datamint/api/endpoints/projects_api.py +38 -7
- datamint/api/endpoints/resources_api.py +227 -100
- datamint/api/entity_base_api.py +66 -7
- datamint/apihandler/base_api_handler.py +0 -1
- datamint/apihandler/dto/annotation_dto.py +2 -0
- datamint/client_cmd_tools/datamint_config.py +0 -1
- datamint/client_cmd_tools/datamint_upload.py +3 -1
- datamint/configs.py +11 -7
- datamint/dataset/base_dataset.py +24 -4
- datamint/dataset/dataset.py +1 -1
- datamint/entities/__init__.py +1 -1
- datamint/entities/annotations/__init__.py +13 -0
- datamint/entities/{annotation.py → annotations/annotation.py} +81 -47
- datamint/entities/annotations/image_classification.py +12 -0
- datamint/entities/annotations/image_segmentation.py +252 -0
- datamint/entities/annotations/volume_segmentation.py +273 -0
- datamint/entities/base_entity.py +100 -6
- datamint/entities/cache_manager.py +129 -15
- datamint/entities/datasetinfo.py +60 -65
- datamint/entities/deployjob.py +18 -0
- datamint/entities/project.py +39 -0
- datamint/entities/resource.py +310 -46
- datamint/lightning/__init__.py +1 -0
- datamint/lightning/datamintdatamodule.py +103 -0
- datamint/mlflow/__init__.py +65 -0
- datamint/mlflow/artifact/__init__.py +1 -0
- datamint/mlflow/artifact/datamint_artifacts_repo.py +8 -0
- datamint/mlflow/env_utils.py +131 -0
- datamint/mlflow/env_vars.py +5 -0
- datamint/mlflow/flavors/__init__.py +17 -0
- datamint/mlflow/flavors/datamint_flavor.py +150 -0
- datamint/mlflow/flavors/model.py +877 -0
- datamint/mlflow/lightning/callbacks/__init__.py +1 -0
- datamint/mlflow/lightning/callbacks/modelcheckpoint.py +410 -0
- datamint/mlflow/models/__init__.py +93 -0
- datamint/mlflow/tracking/datamint_store.py +76 -0
- datamint/mlflow/tracking/default_experiment.py +27 -0
- datamint/mlflow/tracking/fluent.py +91 -0
- datamint/utils/env.py +27 -0
- datamint/utils/visualization.py +21 -13
- datamint-2.9.0.dist-info/METADATA +220 -0
- datamint-2.9.0.dist-info/RECORD +73 -0
- {datamint-2.3.3.dist-info → datamint-2.9.0.dist-info}/WHEEL +1 -1
- datamint-2.9.0.dist-info/entry_points.txt +18 -0
- datamint/apihandler/exp_api_handler.py +0 -204
- datamint/experiment/__init__.py +0 -1
- datamint/experiment/_patcher.py +0 -570
- datamint/experiment/experiment.py +0 -1049
- datamint-2.3.3.dist-info/METADATA +0 -125
- datamint-2.3.3.dist-info/RECORD +0 -54
- datamint-2.3.3.dist-info/entry_points.txt +0 -4
|
@@ -1,204 +0,0 @@
|
|
|
1
|
-
from datamint.apihandler.base_api_handler import BaseAPIHandler
|
|
2
|
-
from typing import Optional, Dict, List, Union, Any
|
|
3
|
-
import json
|
|
4
|
-
import logging
|
|
5
|
-
from io import BytesIO
|
|
6
|
-
|
|
7
|
-
_LOGGER = logging.getLogger(__name__)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class ExperimentAPIHandler(BaseAPIHandler):
|
|
11
|
-
def __init__(self,
|
|
12
|
-
root_url: Optional[str] = None,
|
|
13
|
-
api_key: Optional[str] = None,
|
|
14
|
-
check_connection: bool = True,
|
|
15
|
-
**kwargs):
|
|
16
|
-
super().__init__(root_url=root_url, api_key=api_key, check_connection=check_connection, **kwargs)
|
|
17
|
-
self.exp_url = f"{self.root_url}/experiments"
|
|
18
|
-
|
|
19
|
-
def create_experiment(self,
|
|
20
|
-
dataset_id: str,
|
|
21
|
-
name: str,
|
|
22
|
-
description: str,
|
|
23
|
-
environment: Dict) -> str:
|
|
24
|
-
request_params = {
|
|
25
|
-
'method': 'POST',
|
|
26
|
-
'url': self.exp_url,
|
|
27
|
-
'json': {"dataset_id": dataset_id,
|
|
28
|
-
"name": name,
|
|
29
|
-
"description": description,
|
|
30
|
-
"environment": environment
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
_LOGGER.debug(f"Creating experiment with name {name} and params {json.dumps(request_params)}")
|
|
35
|
-
|
|
36
|
-
response = self._run_request(request_params)
|
|
37
|
-
|
|
38
|
-
return response.json()['id']
|
|
39
|
-
|
|
40
|
-
def get_experiment_by_id(self, exp_id: str) -> Dict:
|
|
41
|
-
request_params = {
|
|
42
|
-
'method': 'GET',
|
|
43
|
-
'url': f"{self.exp_url}/{exp_id}"
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
response = self._run_request(request_params)
|
|
47
|
-
|
|
48
|
-
return response.json()
|
|
49
|
-
|
|
50
|
-
def get_experiments(self) -> List[Dict]:
|
|
51
|
-
request_params = {
|
|
52
|
-
'method': 'GET',
|
|
53
|
-
'url': self.exp_url
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
response = self._run_request(request_params)
|
|
57
|
-
|
|
58
|
-
return response.json()
|
|
59
|
-
|
|
60
|
-
def get_experiment_logs(self, exp_id: str) -> List[Dict]:
|
|
61
|
-
request_params = {
|
|
62
|
-
'method': 'GET',
|
|
63
|
-
'url': f"{self.exp_url}/{exp_id}/log"
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
response = self._run_request(request_params)
|
|
67
|
-
|
|
68
|
-
return response.json()
|
|
69
|
-
|
|
70
|
-
def log_summary(self,
|
|
71
|
-
exp_id: str,
|
|
72
|
-
result_summary: Dict,
|
|
73
|
-
) -> None:
|
|
74
|
-
request_params = {
|
|
75
|
-
'method': 'POST',
|
|
76
|
-
'url': f"{self.exp_url}/{exp_id}/summary",
|
|
77
|
-
'json': {"result_summary": result_summary}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
resp = self._run_request(request_params)
|
|
81
|
-
|
|
82
|
-
def update_experiment(self,
|
|
83
|
-
exp_id: str,
|
|
84
|
-
name: Optional[str] = None,
|
|
85
|
-
description: Optional[str] = None,
|
|
86
|
-
result_summary: Optional[Dict] = None) -> None:
|
|
87
|
-
|
|
88
|
-
# check that at least one of the optional parameters is not None
|
|
89
|
-
if not any([name, description, result_summary]):
|
|
90
|
-
return
|
|
91
|
-
|
|
92
|
-
data = {}
|
|
93
|
-
|
|
94
|
-
if name is not None:
|
|
95
|
-
data['name'] = name
|
|
96
|
-
if description is not None:
|
|
97
|
-
data['description'] = description
|
|
98
|
-
if result_summary is not None:
|
|
99
|
-
data['result_summary'] = result_summary
|
|
100
|
-
|
|
101
|
-
headers = {
|
|
102
|
-
'accept': 'application/json',
|
|
103
|
-
'Content-Type': 'application/json',
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
request_params = {
|
|
107
|
-
'method': 'PATCH',
|
|
108
|
-
'url': f"{self.exp_url}/{exp_id}",
|
|
109
|
-
'json': data,
|
|
110
|
-
'headers': headers
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
resp = self._run_request(request_params)
|
|
114
|
-
|
|
115
|
-
def log_entry(self,
|
|
116
|
-
exp_id: str,
|
|
117
|
-
entry: Dict):
|
|
118
|
-
|
|
119
|
-
if not isinstance(entry, dict):
|
|
120
|
-
raise ValueError(f"Invalid type for entry: {type(entry)}")
|
|
121
|
-
|
|
122
|
-
request_params = {
|
|
123
|
-
'method': 'POST',
|
|
124
|
-
'url': f"{self.exp_url}/{exp_id}/log",
|
|
125
|
-
'json': entry
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
resp = self._run_request(request_params)
|
|
129
|
-
return resp
|
|
130
|
-
|
|
131
|
-
def finish_experiment(self, exp_id: str):
|
|
132
|
-
pass
|
|
133
|
-
# _LOGGER.info(f"Finishing experiment with id {exp_id}")
|
|
134
|
-
# _LOGGER.warning("Finishing experiment not implemented yet")
|
|
135
|
-
# request_params = {
|
|
136
|
-
# 'method': 'POST',
|
|
137
|
-
# 'url': f"{self.exp_url}/{exp_id}/finish"
|
|
138
|
-
# }
|
|
139
|
-
|
|
140
|
-
# resp = self._run_request(request_params)
|
|
141
|
-
|
|
142
|
-
def log_model(self,
|
|
143
|
-
exp_id: str,
|
|
144
|
-
model: Union[Any, str, BytesIO],
|
|
145
|
-
hyper_params: Optional[Dict] = None,
|
|
146
|
-
torch_save_kwargs: Dict = {}) -> Dict:
|
|
147
|
-
import torch
|
|
148
|
-
if isinstance(model, torch.nn.Module):
|
|
149
|
-
f = BytesIO()
|
|
150
|
-
torch.save(model, f, **torch_save_kwargs)
|
|
151
|
-
f.seek(0)
|
|
152
|
-
f.name = None
|
|
153
|
-
elif isinstance(model, str):
|
|
154
|
-
with open(model, 'rb') as f1:
|
|
155
|
-
f = BytesIO(f1.read())
|
|
156
|
-
f.name = None
|
|
157
|
-
elif isinstance(model, BytesIO):
|
|
158
|
-
f = model
|
|
159
|
-
else:
|
|
160
|
-
raise ValueError(f"Invalid type for model: {type(model)}")
|
|
161
|
-
|
|
162
|
-
name = None
|
|
163
|
-
f.name = name
|
|
164
|
-
|
|
165
|
-
try:
|
|
166
|
-
json_data = hyper_params
|
|
167
|
-
json_data['model_name'] = name
|
|
168
|
-
request_params = {
|
|
169
|
-
'method': 'POST',
|
|
170
|
-
'url': f"{self.exp_url}/{exp_id}/model",
|
|
171
|
-
'data': json_data,
|
|
172
|
-
'files': [(None, f)],
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
resp = self._run_request(request_params).json()
|
|
176
|
-
return resp[0]
|
|
177
|
-
finally:
|
|
178
|
-
f.close()
|
|
179
|
-
|
|
180
|
-
def get_experiment_by_name(self, name: str, project: Dict) -> Optional[Dict]:
|
|
181
|
-
"""
|
|
182
|
-
Get the experiment by name of the project.
|
|
183
|
-
|
|
184
|
-
Args:
|
|
185
|
-
name (str): Name of the experiment.
|
|
186
|
-
project (Dict): The project to search for the experiment.
|
|
187
|
-
|
|
188
|
-
Returns:
|
|
189
|
-
Optional[Dict]: The experiment if found, otherwise None.
|
|
190
|
-
"""
|
|
191
|
-
# uses GET /projects/{project_id}/experiments
|
|
192
|
-
|
|
193
|
-
project_id = project['id']
|
|
194
|
-
request_params = {
|
|
195
|
-
'method': 'GET',
|
|
196
|
-
'url': f"{self.root_url}/projects/{project_id}/experiments"
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
response = self._run_request(request_params)
|
|
200
|
-
experiments = response.json()
|
|
201
|
-
for exp in experiments:
|
|
202
|
-
if exp['name'] == name:
|
|
203
|
-
return exp
|
|
204
|
-
return None
|
datamint/experiment/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from .experiment import Experiment
|