digitalhub 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/__init__.py +2 -3
- digitalhub/client/_base/api_builder.py +1 -1
- digitalhub/client/_base/client.py +25 -2
- digitalhub/client/_base/params_builder.py +16 -0
- digitalhub/client/dhcore/api_builder.py +10 -4
- digitalhub/client/dhcore/client.py +30 -398
- digitalhub/client/dhcore/configurator.py +361 -0
- digitalhub/client/dhcore/error_parser.py +107 -0
- digitalhub/client/dhcore/models.py +13 -23
- digitalhub/client/dhcore/params_builder.py +178 -0
- digitalhub/client/dhcore/utils.py +4 -44
- digitalhub/client/local/api_builder.py +13 -18
- digitalhub/client/local/client.py +18 -2
- digitalhub/client/local/enums.py +11 -0
- digitalhub/client/local/params_builder.py +116 -0
- digitalhub/configurator/api.py +31 -0
- digitalhub/configurator/configurator.py +195 -0
- digitalhub/configurator/credentials_store.py +65 -0
- digitalhub/configurator/ini_module.py +74 -0
- digitalhub/entities/_base/_base/entity.py +2 -2
- digitalhub/entities/_base/context/entity.py +4 -4
- digitalhub/entities/_base/entity/builder.py +5 -5
- digitalhub/entities/_base/executable/entity.py +2 -2
- digitalhub/entities/_base/material/entity.py +12 -12
- digitalhub/entities/_base/material/status.py +1 -1
- digitalhub/entities/_base/material/utils.py +2 -2
- digitalhub/entities/_base/unversioned/entity.py +2 -2
- digitalhub/entities/_base/versioned/entity.py +2 -2
- digitalhub/entities/_commons/enums.py +2 -0
- digitalhub/entities/_commons/metrics.py +164 -0
- digitalhub/entities/_commons/types.py +5 -0
- digitalhub/entities/_commons/utils.py +2 -2
- digitalhub/entities/_processors/base.py +527 -0
- digitalhub/entities/{_operations/processor.py → _processors/context.py} +212 -837
- digitalhub/entities/_processors/utils.py +158 -0
- digitalhub/entities/artifact/artifact/spec.py +3 -1
- digitalhub/entities/artifact/crud.py +13 -12
- digitalhub/entities/artifact/utils.py +1 -1
- digitalhub/entities/builders.py +6 -18
- digitalhub/entities/dataitem/_base/entity.py +0 -41
- digitalhub/entities/dataitem/crud.py +27 -15
- digitalhub/entities/dataitem/table/entity.py +49 -35
- digitalhub/entities/dataitem/table/models.py +4 -3
- digitalhub/{utils/data_utils.py → entities/dataitem/table/utils.py} +46 -54
- digitalhub/entities/dataitem/utils.py +58 -10
- digitalhub/entities/function/crud.py +9 -9
- digitalhub/entities/model/_base/entity.py +120 -0
- digitalhub/entities/model/_base/spec.py +6 -17
- digitalhub/entities/model/_base/status.py +10 -0
- digitalhub/entities/model/crud.py +13 -12
- digitalhub/entities/model/huggingface/spec.py +9 -4
- digitalhub/entities/model/mlflow/models.py +2 -2
- digitalhub/entities/model/mlflow/spec.py +7 -7
- digitalhub/entities/model/mlflow/utils.py +44 -5
- digitalhub/entities/project/_base/entity.py +317 -9
- digitalhub/entities/project/_base/spec.py +8 -6
- digitalhub/entities/project/crud.py +12 -11
- digitalhub/entities/run/_base/entity.py +103 -6
- digitalhub/entities/run/_base/spec.py +4 -2
- digitalhub/entities/run/_base/status.py +12 -0
- digitalhub/entities/run/crud.py +8 -8
- digitalhub/entities/secret/_base/entity.py +3 -3
- digitalhub/entities/secret/_base/spec.py +4 -2
- digitalhub/entities/secret/crud.py +11 -9
- digitalhub/entities/task/_base/entity.py +4 -4
- digitalhub/entities/task/_base/models.py +51 -40
- digitalhub/entities/task/_base/spec.py +2 -0
- digitalhub/entities/task/_base/utils.py +2 -2
- digitalhub/entities/task/crud.py +12 -8
- digitalhub/entities/workflow/crud.py +9 -9
- digitalhub/factory/utils.py +9 -9
- digitalhub/readers/{_base → data/_base}/builder.py +1 -1
- digitalhub/readers/{_base → data/_base}/reader.py +16 -4
- digitalhub/readers/{api.py → data/api.py} +2 -2
- digitalhub/readers/{factory.py → data/factory.py} +3 -3
- digitalhub/readers/{pandas → data/pandas}/builder.py +2 -2
- digitalhub/readers/{pandas → data/pandas}/reader.py +110 -30
- digitalhub/readers/query/__init__.py +0 -0
- digitalhub/stores/_base/store.py +59 -69
- digitalhub/stores/api.py +8 -33
- digitalhub/stores/builder.py +44 -161
- digitalhub/stores/local/store.py +106 -89
- digitalhub/stores/remote/store.py +86 -11
- digitalhub/stores/s3/configurator.py +108 -0
- digitalhub/stores/s3/enums.py +17 -0
- digitalhub/stores/s3/models.py +21 -0
- digitalhub/stores/s3/store.py +154 -70
- digitalhub/{utils/s3_utils.py → stores/s3/utils.py} +7 -3
- digitalhub/stores/sql/configurator.py +88 -0
- digitalhub/stores/sql/enums.py +16 -0
- digitalhub/stores/sql/models.py +24 -0
- digitalhub/stores/sql/store.py +106 -85
- digitalhub/{readers/_commons → utils}/enums.py +5 -1
- digitalhub/utils/exceptions.py +6 -0
- digitalhub/utils/file_utils.py +8 -7
- digitalhub/utils/generic_utils.py +28 -15
- digitalhub/utils/git_utils.py +16 -9
- digitalhub/utils/types.py +5 -0
- digitalhub/utils/uri_utils.py +2 -2
- {digitalhub-0.9.1.dist-info → digitalhub-0.10.0.dist-info}/METADATA +25 -31
- {digitalhub-0.9.1.dist-info → digitalhub-0.10.0.dist-info}/RECORD +108 -99
- {digitalhub-0.9.1.dist-info → digitalhub-0.10.0.dist-info}/WHEEL +1 -2
- digitalhub/client/dhcore/env.py +0 -23
- digitalhub/entities/_base/project/entity.py +0 -341
- digitalhub-0.9.1.dist-info/top_level.txt +0 -2
- test/local/CRUD/test_artifacts.py +0 -96
- test/local/CRUD/test_dataitems.py +0 -96
- test/local/CRUD/test_models.py +0 -95
- test/local/imports/test_imports.py +0 -66
- test/local/instances/test_validate.py +0 -55
- test/test_crud_functions.py +0 -109
- test/test_crud_runs.py +0 -86
- test/test_crud_tasks.py +0 -81
- test/testkfp.py +0 -37
- test/testkfp_pipeline.py +0 -22
- /digitalhub/{entities/_base/project → configurator}/__init__.py +0 -0
- /digitalhub/entities/{_operations → _processors}/__init__.py +0 -0
- /digitalhub/readers/{_base → data}/__init__.py +0 -0
- /digitalhub/readers/{_commons → data/_base}/__init__.py +0 -0
- /digitalhub/readers/{pandas → data/pandas}/__init__.py +0 -0
- {digitalhub-0.9.1.dist-info → digitalhub-0.10.0.dist-info/licenses}/LICENSE.txt +0 -0
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
from digitalhub.
|
|
7
|
-
from digitalhub.entities.
|
|
8
|
-
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, Relationship
|
|
7
|
+
from digitalhub.entities._processors.utils import (
|
|
8
|
+
get_context_from_identifier,
|
|
9
|
+
get_context_from_project,
|
|
10
|
+
parse_identifier,
|
|
11
|
+
)
|
|
9
12
|
from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
|
|
10
|
-
from digitalhub.utils.exceptions import
|
|
13
|
+
from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError, EntityNotExistsError
|
|
11
14
|
from digitalhub.utils.io_utils import read_yaml
|
|
15
|
+
from digitalhub.utils.types import SourcesOrListOfSources
|
|
12
16
|
|
|
13
17
|
if typing.TYPE_CHECKING:
|
|
14
|
-
from digitalhub.client._base.client import Client
|
|
15
18
|
from digitalhub.context.context import Context
|
|
16
19
|
from digitalhub.entities._base.context.entity import ContextEntity
|
|
17
20
|
from digitalhub.entities._base.executable.entity import ExecutableEntity
|
|
18
21
|
from digitalhub.entities._base.material.entity import MaterialEntity
|
|
19
|
-
from digitalhub.entities._base.project.entity import ProjectEntity
|
|
20
22
|
from digitalhub.entities._base.unversioned.entity import UnversionedEntity
|
|
21
23
|
|
|
22
24
|
|
|
23
|
-
class
|
|
25
|
+
class ContextEntityOperationsProcessor:
|
|
24
26
|
"""
|
|
25
27
|
Processor for Entity operations.
|
|
26
28
|
|
|
@@ -29,500 +31,6 @@ class OperationsProcessor:
|
|
|
29
31
|
Operations can be CRUD, search, list, etc.
|
|
30
32
|
"""
|
|
31
33
|
|
|
32
|
-
##############################
|
|
33
|
-
# CRUD base entity
|
|
34
|
-
##############################
|
|
35
|
-
|
|
36
|
-
def _create_base_entity(
|
|
37
|
-
self,
|
|
38
|
-
client: Client,
|
|
39
|
-
entity_type: str,
|
|
40
|
-
entity_dict: dict,
|
|
41
|
-
**kwargs,
|
|
42
|
-
) -> dict:
|
|
43
|
-
"""
|
|
44
|
-
Create object in backend.
|
|
45
|
-
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
client : Client
|
|
49
|
-
Client instance.
|
|
50
|
-
entity_type : str
|
|
51
|
-
Entity type.
|
|
52
|
-
entity_dict : dict
|
|
53
|
-
Object instance.
|
|
54
|
-
**kwargs : dict
|
|
55
|
-
Parameters to pass to the API call.
|
|
56
|
-
|
|
57
|
-
Returns
|
|
58
|
-
-------
|
|
59
|
-
dict
|
|
60
|
-
Object instance.
|
|
61
|
-
"""
|
|
62
|
-
api = client.build_api(
|
|
63
|
-
ApiCategories.BASE.value,
|
|
64
|
-
BackendOperations.CREATE.value,
|
|
65
|
-
entity_type=entity_type,
|
|
66
|
-
)
|
|
67
|
-
return client.create_object(api, entity_dict, **kwargs)
|
|
68
|
-
|
|
69
|
-
def create_project_entity(
|
|
70
|
-
self,
|
|
71
|
-
_entity: ProjectEntity | None = None,
|
|
72
|
-
**kwargs,
|
|
73
|
-
) -> ProjectEntity:
|
|
74
|
-
"""
|
|
75
|
-
Create object in backend.
|
|
76
|
-
|
|
77
|
-
Parameters
|
|
78
|
-
----------
|
|
79
|
-
_entity : ProjectEntity
|
|
80
|
-
Object instance.
|
|
81
|
-
**kwargs : dict
|
|
82
|
-
Parameters to pass to entity builder.
|
|
83
|
-
|
|
84
|
-
Returns
|
|
85
|
-
-------
|
|
86
|
-
Project
|
|
87
|
-
Object instance.
|
|
88
|
-
"""
|
|
89
|
-
if _entity is not None:
|
|
90
|
-
client = _entity._client
|
|
91
|
-
obj = _entity
|
|
92
|
-
else:
|
|
93
|
-
client = get_client(kwargs.get("local"), kwargs.pop("config", None))
|
|
94
|
-
obj = build_entity_from_params(**kwargs)
|
|
95
|
-
ent = self._create_base_entity(client, obj.ENTITY_TYPE, obj.to_dict())
|
|
96
|
-
ent["local"] = client.is_local()
|
|
97
|
-
return build_entity_from_dict(ent)
|
|
98
|
-
|
|
99
|
-
def _read_base_entity(
|
|
100
|
-
self,
|
|
101
|
-
client: Client,
|
|
102
|
-
entity_type: str,
|
|
103
|
-
entity_name: str,
|
|
104
|
-
**kwargs,
|
|
105
|
-
) -> dict:
|
|
106
|
-
"""
|
|
107
|
-
Read object from backend.
|
|
108
|
-
|
|
109
|
-
Parameters
|
|
110
|
-
----------
|
|
111
|
-
client : Client
|
|
112
|
-
Client instance.
|
|
113
|
-
entity_type : str
|
|
114
|
-
Entity type.
|
|
115
|
-
entity_name : str
|
|
116
|
-
Entity name.
|
|
117
|
-
**kwargs : dict
|
|
118
|
-
Parameters to pass to the API call.
|
|
119
|
-
|
|
120
|
-
Returns
|
|
121
|
-
-------
|
|
122
|
-
dict
|
|
123
|
-
Object instance.
|
|
124
|
-
"""
|
|
125
|
-
api = client.build_api(
|
|
126
|
-
ApiCategories.BASE.value,
|
|
127
|
-
BackendOperations.READ.value,
|
|
128
|
-
entity_type=entity_type,
|
|
129
|
-
entity_name=entity_name,
|
|
130
|
-
)
|
|
131
|
-
return client.read_object(api, **kwargs)
|
|
132
|
-
|
|
133
|
-
def read_project_entity(
|
|
134
|
-
self,
|
|
135
|
-
entity_type: str,
|
|
136
|
-
entity_name: str,
|
|
137
|
-
**kwargs,
|
|
138
|
-
) -> ProjectEntity:
|
|
139
|
-
"""
|
|
140
|
-
Read object from backend.
|
|
141
|
-
|
|
142
|
-
Parameters
|
|
143
|
-
----------
|
|
144
|
-
entity_type : str
|
|
145
|
-
Entity type.
|
|
146
|
-
entity_name : str
|
|
147
|
-
Entity name.
|
|
148
|
-
**kwargs : dict
|
|
149
|
-
Parameters to pass to entity builder.
|
|
150
|
-
|
|
151
|
-
Returns
|
|
152
|
-
-------
|
|
153
|
-
ProjectEntity
|
|
154
|
-
Object instance.
|
|
155
|
-
"""
|
|
156
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
157
|
-
obj = self._read_base_entity(client, entity_type, entity_name, **kwargs)
|
|
158
|
-
obj["local"] = client.is_local()
|
|
159
|
-
return build_entity_from_dict(obj)
|
|
160
|
-
|
|
161
|
-
def import_project_entity(
|
|
162
|
-
self,
|
|
163
|
-
file: str,
|
|
164
|
-
**kwargs,
|
|
165
|
-
) -> ProjectEntity:
|
|
166
|
-
"""
|
|
167
|
-
Import object from a YAML file and create a new object into the backend.
|
|
168
|
-
|
|
169
|
-
Parameters
|
|
170
|
-
----------
|
|
171
|
-
file : str
|
|
172
|
-
Path to YAML file.
|
|
173
|
-
**kwargs : dict
|
|
174
|
-
Additional keyword arguments.
|
|
175
|
-
|
|
176
|
-
Returns
|
|
177
|
-
-------
|
|
178
|
-
ProjectEntity
|
|
179
|
-
Object instance.
|
|
180
|
-
"""
|
|
181
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
182
|
-
obj: dict = read_yaml(file)
|
|
183
|
-
obj["status"] = {}
|
|
184
|
-
obj["local"] = client.is_local()
|
|
185
|
-
ent: ProjectEntity = build_entity_from_dict(obj)
|
|
186
|
-
|
|
187
|
-
try:
|
|
188
|
-
self._create_base_entity(ent._client, ent.ENTITY_TYPE, ent.to_dict())
|
|
189
|
-
except EntityAlreadyExistsError:
|
|
190
|
-
raise EntityError(f"Entity {ent.name} already exists. If you want to update it, use load instead.")
|
|
191
|
-
|
|
192
|
-
# Import related entities
|
|
193
|
-
ent._import_entities(obj)
|
|
194
|
-
ent.refresh()
|
|
195
|
-
return ent
|
|
196
|
-
|
|
197
|
-
def load_project_entity(
|
|
198
|
-
self,
|
|
199
|
-
file: str,
|
|
200
|
-
**kwargs,
|
|
201
|
-
) -> ProjectEntity:
|
|
202
|
-
"""
|
|
203
|
-
Load object from a YAML file and update an existing object into the backend.
|
|
204
|
-
|
|
205
|
-
Parameters
|
|
206
|
-
----------
|
|
207
|
-
file : str
|
|
208
|
-
Path to YAML file.
|
|
209
|
-
**kwargs : dict
|
|
210
|
-
Additional keyword arguments.
|
|
211
|
-
|
|
212
|
-
Returns
|
|
213
|
-
-------
|
|
214
|
-
ProjectEntity
|
|
215
|
-
Object instance.
|
|
216
|
-
"""
|
|
217
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
218
|
-
obj: dict = read_yaml(file)
|
|
219
|
-
obj["local"] = client.is_local()
|
|
220
|
-
ent: ProjectEntity = build_entity_from_dict(obj)
|
|
221
|
-
|
|
222
|
-
try:
|
|
223
|
-
self._update_base_entity(ent._client, ent.ENTITY_TYPE, ent.name, ent.to_dict())
|
|
224
|
-
except EntityNotExistsError:
|
|
225
|
-
self._create_base_entity(ent._client, ent.ENTITY_TYPE, ent.to_dict())
|
|
226
|
-
|
|
227
|
-
# Load related entities
|
|
228
|
-
ent._load_entities(obj)
|
|
229
|
-
ent.refresh()
|
|
230
|
-
return ent
|
|
231
|
-
|
|
232
|
-
def _list_base_entities(
|
|
233
|
-
self,
|
|
234
|
-
client: Client,
|
|
235
|
-
entity_type: str,
|
|
236
|
-
**kwargs,
|
|
237
|
-
) -> list[dict]:
|
|
238
|
-
"""
|
|
239
|
-
List objects from backend.
|
|
240
|
-
|
|
241
|
-
Parameters
|
|
242
|
-
----------
|
|
243
|
-
client : Client
|
|
244
|
-
Client instance.
|
|
245
|
-
entity_type : str
|
|
246
|
-
Entity type.
|
|
247
|
-
**kwargs : dict
|
|
248
|
-
Parameters to pass to the API call.
|
|
249
|
-
|
|
250
|
-
Returns
|
|
251
|
-
-------
|
|
252
|
-
list[dict]
|
|
253
|
-
List of objects.
|
|
254
|
-
"""
|
|
255
|
-
api = client.build_api(
|
|
256
|
-
ApiCategories.BASE.value,
|
|
257
|
-
BackendOperations.LIST.value,
|
|
258
|
-
entity_type=entity_type,
|
|
259
|
-
)
|
|
260
|
-
return client.list_objects(api, **kwargs)
|
|
261
|
-
|
|
262
|
-
def list_project_entities(
|
|
263
|
-
self,
|
|
264
|
-
entity_type: str,
|
|
265
|
-
**kwargs,
|
|
266
|
-
) -> list[ProjectEntity]:
|
|
267
|
-
"""
|
|
268
|
-
List objects from backend.
|
|
269
|
-
|
|
270
|
-
Parameters
|
|
271
|
-
----------
|
|
272
|
-
entity_type : str
|
|
273
|
-
Entity type.
|
|
274
|
-
**kwargs : dict
|
|
275
|
-
Parameters to pass to API call.
|
|
276
|
-
|
|
277
|
-
Returns
|
|
278
|
-
-------
|
|
279
|
-
list[ProjectEntity]
|
|
280
|
-
List of objects.
|
|
281
|
-
"""
|
|
282
|
-
client = get_client(kwargs.pop("local", False))
|
|
283
|
-
objs = self._list_base_entities(client, entity_type, **kwargs)
|
|
284
|
-
entities = []
|
|
285
|
-
for obj in objs:
|
|
286
|
-
obj["local"] = client.is_local()
|
|
287
|
-
ent = build_entity_from_dict(obj)
|
|
288
|
-
entities.append(ent)
|
|
289
|
-
return entities
|
|
290
|
-
|
|
291
|
-
def _update_base_entity(
|
|
292
|
-
self,
|
|
293
|
-
client: Client,
|
|
294
|
-
entity_type: str,
|
|
295
|
-
entity_name: str,
|
|
296
|
-
entity_dict: dict,
|
|
297
|
-
**kwargs,
|
|
298
|
-
) -> dict:
|
|
299
|
-
"""
|
|
300
|
-
Update object method.
|
|
301
|
-
|
|
302
|
-
Parameters
|
|
303
|
-
----------
|
|
304
|
-
client : Client
|
|
305
|
-
Client instance.
|
|
306
|
-
entity_type : str
|
|
307
|
-
Entity type.
|
|
308
|
-
entity_name : str
|
|
309
|
-
Entity name.
|
|
310
|
-
entity_dict : dict
|
|
311
|
-
Object instance.
|
|
312
|
-
**kwargs : dict
|
|
313
|
-
Parameters to pass to the API call.
|
|
314
|
-
|
|
315
|
-
Returns
|
|
316
|
-
-------
|
|
317
|
-
dict
|
|
318
|
-
Object instance.
|
|
319
|
-
"""
|
|
320
|
-
api = client.build_api(
|
|
321
|
-
ApiCategories.BASE.value,
|
|
322
|
-
BackendOperations.UPDATE.value,
|
|
323
|
-
entity_type=entity_type,
|
|
324
|
-
entity_name=entity_name,
|
|
325
|
-
)
|
|
326
|
-
return client.update_object(api, entity_dict, **kwargs)
|
|
327
|
-
|
|
328
|
-
def update_project_entity(
|
|
329
|
-
self,
|
|
330
|
-
entity_type: str,
|
|
331
|
-
entity_name: str,
|
|
332
|
-
entity_dict: dict,
|
|
333
|
-
**kwargs,
|
|
334
|
-
) -> ProjectEntity:
|
|
335
|
-
"""
|
|
336
|
-
Update object method.
|
|
337
|
-
|
|
338
|
-
Parameters
|
|
339
|
-
----------
|
|
340
|
-
entity_type : str
|
|
341
|
-
Entity type.
|
|
342
|
-
entity_name : str
|
|
343
|
-
Entity name.
|
|
344
|
-
entity_dict : dict
|
|
345
|
-
Object instance.
|
|
346
|
-
**kwargs : dict
|
|
347
|
-
Parameters to pass to entity builder.
|
|
348
|
-
|
|
349
|
-
Returns
|
|
350
|
-
-------
|
|
351
|
-
ProjectEntity
|
|
352
|
-
Object instance.
|
|
353
|
-
"""
|
|
354
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
355
|
-
obj = self._update_base_entity(client, entity_type, entity_name, entity_dict, **kwargs)
|
|
356
|
-
obj["local"] = client.is_local()
|
|
357
|
-
return build_entity_from_dict(obj)
|
|
358
|
-
|
|
359
|
-
def _delete_base_entity(
|
|
360
|
-
self,
|
|
361
|
-
client: Client,
|
|
362
|
-
entity_type: str,
|
|
363
|
-
entity_name: str,
|
|
364
|
-
**kwargs,
|
|
365
|
-
) -> dict:
|
|
366
|
-
"""
|
|
367
|
-
Delete object method.
|
|
368
|
-
|
|
369
|
-
Parameters
|
|
370
|
-
----------
|
|
371
|
-
client : Client
|
|
372
|
-
Client instance.
|
|
373
|
-
entity_type : str
|
|
374
|
-
Entity type.
|
|
375
|
-
entity_name : str
|
|
376
|
-
Entity name.
|
|
377
|
-
**kwargs : dict
|
|
378
|
-
Parameters to pass to the API call.
|
|
379
|
-
|
|
380
|
-
Returns
|
|
381
|
-
-------
|
|
382
|
-
dict
|
|
383
|
-
Response from backend.
|
|
384
|
-
"""
|
|
385
|
-
api = client.build_api(
|
|
386
|
-
ApiCategories.BASE.value,
|
|
387
|
-
BackendOperations.DELETE.value,
|
|
388
|
-
entity_type=entity_type,
|
|
389
|
-
entity_name=entity_name,
|
|
390
|
-
)
|
|
391
|
-
return client.delete_object(api, **kwargs)
|
|
392
|
-
|
|
393
|
-
def delete_project_entity(
|
|
394
|
-
self,
|
|
395
|
-
entity_type: str,
|
|
396
|
-
entity_name: str,
|
|
397
|
-
**kwargs,
|
|
398
|
-
) -> dict:
|
|
399
|
-
"""
|
|
400
|
-
Delete object method.
|
|
401
|
-
|
|
402
|
-
Parameters
|
|
403
|
-
----------
|
|
404
|
-
entity_type : str
|
|
405
|
-
Entity type.
|
|
406
|
-
entity_name : str
|
|
407
|
-
Entity name.
|
|
408
|
-
**kwargs : dict
|
|
409
|
-
Parameters to pass to entity builder.
|
|
410
|
-
|
|
411
|
-
Returns
|
|
412
|
-
-------
|
|
413
|
-
dict
|
|
414
|
-
Response from backend.
|
|
415
|
-
"""
|
|
416
|
-
kwargs = self._set_params(**kwargs)
|
|
417
|
-
if cascade := kwargs.pop("cascade", None) is not None:
|
|
418
|
-
kwargs["params"]["cascade"] = str(cascade).lower()
|
|
419
|
-
if kwargs.pop("clean_context", True):
|
|
420
|
-
delete_context(entity_name)
|
|
421
|
-
|
|
422
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
423
|
-
return self._delete_base_entity(
|
|
424
|
-
client,
|
|
425
|
-
entity_type,
|
|
426
|
-
entity_name,
|
|
427
|
-
**kwargs,
|
|
428
|
-
)
|
|
429
|
-
|
|
430
|
-
##############################
|
|
431
|
-
# Base entity operations
|
|
432
|
-
##############################
|
|
433
|
-
|
|
434
|
-
def _build_base_entity_key(
|
|
435
|
-
self,
|
|
436
|
-
client: Client,
|
|
437
|
-
entity_id: str,
|
|
438
|
-
) -> str:
|
|
439
|
-
"""
|
|
440
|
-
Build object key.
|
|
441
|
-
|
|
442
|
-
Parameters
|
|
443
|
-
----------
|
|
444
|
-
client : Client
|
|
445
|
-
Client instance.
|
|
446
|
-
entity_id : str
|
|
447
|
-
Entity ID.
|
|
448
|
-
|
|
449
|
-
Returns
|
|
450
|
-
-------
|
|
451
|
-
str
|
|
452
|
-
Object key.
|
|
453
|
-
"""
|
|
454
|
-
return client.build_key(ApiCategories.BASE.value, entity_id)
|
|
455
|
-
|
|
456
|
-
def build_project_key(
|
|
457
|
-
self,
|
|
458
|
-
entity_id: str,
|
|
459
|
-
**kwargs,
|
|
460
|
-
) -> str:
|
|
461
|
-
"""
|
|
462
|
-
Build object key.
|
|
463
|
-
|
|
464
|
-
Parameters
|
|
465
|
-
----------
|
|
466
|
-
entity_id : str
|
|
467
|
-
Entity ID.
|
|
468
|
-
**kwargs : dict
|
|
469
|
-
Parameters to pass to entity builder.
|
|
470
|
-
|
|
471
|
-
Returns
|
|
472
|
-
-------
|
|
473
|
-
str
|
|
474
|
-
Object key.
|
|
475
|
-
"""
|
|
476
|
-
client = get_client(kwargs.pop("local", False))
|
|
477
|
-
return self._build_base_entity_key(client, entity_id)
|
|
478
|
-
|
|
479
|
-
def share_project_entity(
|
|
480
|
-
self,
|
|
481
|
-
entity_type: str,
|
|
482
|
-
entity_name: str,
|
|
483
|
-
**kwargs,
|
|
484
|
-
) -> None:
|
|
485
|
-
"""
|
|
486
|
-
Share object method.
|
|
487
|
-
|
|
488
|
-
Parameters
|
|
489
|
-
----------
|
|
490
|
-
entity_type : str
|
|
491
|
-
Entity type.
|
|
492
|
-
entity_name : str
|
|
493
|
-
Entity name.
|
|
494
|
-
**kwargs : dict
|
|
495
|
-
Parameters to pass to entity builder.
|
|
496
|
-
|
|
497
|
-
Returns
|
|
498
|
-
-------
|
|
499
|
-
None
|
|
500
|
-
"""
|
|
501
|
-
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
502
|
-
api = client.build_api(
|
|
503
|
-
ApiCategories.BASE.value,
|
|
504
|
-
BackendOperations.SHARE.value,
|
|
505
|
-
entity_type=entity_type,
|
|
506
|
-
entity_name=entity_name,
|
|
507
|
-
)
|
|
508
|
-
user = kwargs.pop("user")
|
|
509
|
-
unshare = kwargs.pop("unshare", False)
|
|
510
|
-
kwargs = self._set_params(**kwargs)
|
|
511
|
-
|
|
512
|
-
# Unshare
|
|
513
|
-
if unshare:
|
|
514
|
-
users = client.read_object(api, **kwargs)
|
|
515
|
-
for u in users:
|
|
516
|
-
if u["user"] == user:
|
|
517
|
-
kwargs["params"]["id"] = u["id"]
|
|
518
|
-
client.delete_object(api, **kwargs)
|
|
519
|
-
break
|
|
520
|
-
return
|
|
521
|
-
|
|
522
|
-
# Share
|
|
523
|
-
kwargs["params"]["user"] = user
|
|
524
|
-
client.create_object(api, obj={}, **kwargs)
|
|
525
|
-
|
|
526
34
|
##############################
|
|
527
35
|
# CRUD context entity
|
|
528
36
|
##############################
|
|
@@ -564,7 +72,7 @@ class OperationsProcessor:
|
|
|
564
72
|
self,
|
|
565
73
|
_entity: ContextEntity | None = None,
|
|
566
74
|
**kwargs,
|
|
567
|
-
) ->
|
|
75
|
+
) -> ContextEntity:
|
|
568
76
|
"""
|
|
569
77
|
Create object in backend.
|
|
570
78
|
|
|
@@ -575,14 +83,14 @@ class OperationsProcessor:
|
|
|
575
83
|
|
|
576
84
|
Returns
|
|
577
85
|
-------
|
|
578
|
-
|
|
86
|
+
ContextEntity
|
|
579
87
|
Object instance.
|
|
580
88
|
"""
|
|
581
89
|
if _entity is not None:
|
|
582
90
|
context = _entity._context()
|
|
583
91
|
obj = _entity
|
|
584
92
|
else:
|
|
585
|
-
context =
|
|
93
|
+
context = get_context_from_project(kwargs["project"])
|
|
586
94
|
obj: ContextEntity = build_entity_from_params(**kwargs)
|
|
587
95
|
new_obj = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
588
96
|
return build_entity_from_dict(new_obj)
|
|
@@ -604,8 +112,8 @@ class OperationsProcessor:
|
|
|
604
112
|
MaterialEntity
|
|
605
113
|
Object instance.
|
|
606
114
|
"""
|
|
607
|
-
source = kwargs.pop("source")
|
|
608
|
-
context =
|
|
115
|
+
source: SourcesOrListOfSources = kwargs.pop("source")
|
|
116
|
+
context = get_context_from_project(kwargs["project"])
|
|
609
117
|
obj = build_entity_from_params(**kwargs)
|
|
610
118
|
if context.is_running:
|
|
611
119
|
obj.add_relationship(Relationship.PRODUCEDBY.value, context.get_run_ctx())
|
|
@@ -647,82 +155,47 @@ class OperationsProcessor:
|
|
|
647
155
|
dict
|
|
648
156
|
Object instance.
|
|
649
157
|
"""
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
158
|
+
project, entity_type, _, entity_name, entity_id = parse_identifier(
|
|
159
|
+
identifier,
|
|
160
|
+
project=project,
|
|
161
|
+
entity_type=entity_type,
|
|
162
|
+
entity_id=entity_id,
|
|
163
|
+
)
|
|
656
164
|
|
|
657
|
-
|
|
165
|
+
if entity_id is None:
|
|
166
|
+
kwargs["entity_name"] = entity_name
|
|
167
|
+
kwargs = context.client.build_parameters(
|
|
168
|
+
ApiCategories.CONTEXT.value,
|
|
169
|
+
BackendOperations.READ.value,
|
|
170
|
+
**kwargs,
|
|
171
|
+
)
|
|
658
172
|
|
|
659
173
|
if entity_id is None:
|
|
660
|
-
kwargs["params"]["name"] = entity_name
|
|
661
174
|
api = context.client.build_api(
|
|
662
175
|
ApiCategories.CONTEXT.value,
|
|
663
176
|
BackendOperations.LIST.value,
|
|
664
177
|
project=context.name,
|
|
665
178
|
entity_type=entity_type,
|
|
666
179
|
)
|
|
667
|
-
return context.client.list_first_object(api, **kwargs)
|
|
668
|
-
|
|
669
|
-
api = context.client.build_api(
|
|
670
|
-
ApiCategories.CONTEXT.value,
|
|
671
|
-
BackendOperations.READ.value,
|
|
672
|
-
project=context.name,
|
|
673
|
-
entity_type=entity_type,
|
|
674
|
-
entity_id=entity_id,
|
|
675
|
-
)
|
|
676
|
-
return context.client.read_object(api, **kwargs)
|
|
677
|
-
|
|
678
|
-
def read_context_entity(
|
|
679
|
-
self,
|
|
680
|
-
identifier: str,
|
|
681
|
-
entity_type: str | None = None,
|
|
682
|
-
project: str | None = None,
|
|
683
|
-
entity_id: str | None = None,
|
|
684
|
-
**kwargs,
|
|
685
|
-
) -> ContextEntity:
|
|
686
|
-
"""
|
|
687
|
-
Read object from backend.
|
|
688
|
-
|
|
689
|
-
Parameters
|
|
690
|
-
----------
|
|
691
|
-
identifier : str
|
|
692
|
-
Entity key (store://...) or entity name.
|
|
693
|
-
entity_type : str
|
|
694
|
-
Entity type.
|
|
695
|
-
project : str
|
|
696
|
-
Project name.
|
|
697
|
-
entity_id : str
|
|
698
|
-
Entity ID.
|
|
699
|
-
**kwargs : dict
|
|
700
|
-
Parameters to pass to the API call.
|
|
701
|
-
|
|
702
|
-
Returns
|
|
703
|
-
-------
|
|
704
|
-
VersionedEntity
|
|
705
|
-
Object instance.
|
|
706
|
-
"""
|
|
707
|
-
context = self._get_context_from_identifier(identifier, project)
|
|
708
|
-
obj = self._read_context_entity(
|
|
709
|
-
context,
|
|
710
|
-
identifier,
|
|
180
|
+
return context.client.list_first_object(api, **kwargs)
|
|
181
|
+
|
|
182
|
+
api = context.client.build_api(
|
|
183
|
+
ApiCategories.CONTEXT.value,
|
|
184
|
+
BackendOperations.READ.value,
|
|
185
|
+
project=context.name,
|
|
711
186
|
entity_type=entity_type,
|
|
712
|
-
project=project,
|
|
713
187
|
entity_id=entity_id,
|
|
714
|
-
**kwargs,
|
|
715
188
|
)
|
|
716
|
-
return
|
|
189
|
+
return context.client.read_object(api, **kwargs)
|
|
717
190
|
|
|
718
|
-
def
|
|
191
|
+
def read_context_entity(
|
|
719
192
|
self,
|
|
720
193
|
identifier: str,
|
|
721
194
|
entity_type: str | None = None,
|
|
722
195
|
project: str | None = None,
|
|
723
196
|
entity_id: str | None = None,
|
|
724
197
|
**kwargs,
|
|
725
|
-
) ->
|
|
198
|
+
) -> ContextEntity:
|
|
726
199
|
"""
|
|
727
200
|
Read object from backend.
|
|
728
201
|
|
|
@@ -741,18 +214,20 @@ class OperationsProcessor:
|
|
|
741
214
|
|
|
742
215
|
Returns
|
|
743
216
|
-------
|
|
744
|
-
|
|
217
|
+
VersionedEntity
|
|
745
218
|
Object instance.
|
|
746
219
|
"""
|
|
747
|
-
|
|
220
|
+
context = get_context_from_identifier(identifier, project)
|
|
221
|
+
obj = self._read_context_entity(
|
|
222
|
+
context,
|
|
748
223
|
identifier,
|
|
749
224
|
entity_type=entity_type,
|
|
750
225
|
project=project,
|
|
751
226
|
entity_id=entity_id,
|
|
752
227
|
**kwargs,
|
|
753
228
|
)
|
|
754
|
-
obj
|
|
755
|
-
return
|
|
229
|
+
entity = build_entity_from_dict(obj)
|
|
230
|
+
return self._post_process_get(entity)
|
|
756
231
|
|
|
757
232
|
def read_unversioned_entity(
|
|
758
233
|
self,
|
|
@@ -816,7 +291,7 @@ class OperationsProcessor:
|
|
|
816
291
|
"""
|
|
817
292
|
dict_obj: dict = read_yaml(file)
|
|
818
293
|
dict_obj["status"] = {}
|
|
819
|
-
context =
|
|
294
|
+
context = get_context_from_project(dict_obj["project"])
|
|
820
295
|
obj = build_entity_from_dict(dict_obj)
|
|
821
296
|
try:
|
|
822
297
|
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
@@ -853,7 +328,7 @@ class OperationsProcessor:
|
|
|
853
328
|
exec_dict = dict_obj
|
|
854
329
|
tsk_dicts = []
|
|
855
330
|
|
|
856
|
-
context =
|
|
331
|
+
context = get_context_from_project(exec_dict["project"])
|
|
857
332
|
obj: ExecutableEntity = build_entity_from_dict(exec_dict)
|
|
858
333
|
try:
|
|
859
334
|
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
@@ -882,7 +357,7 @@ class OperationsProcessor:
|
|
|
882
357
|
Object instance.
|
|
883
358
|
"""
|
|
884
359
|
dict_obj: dict = read_yaml(file)
|
|
885
|
-
context =
|
|
360
|
+
context = get_context_from_project(dict_obj["project"])
|
|
886
361
|
obj: ContextEntity = build_entity_from_dict(dict_obj)
|
|
887
362
|
try:
|
|
888
363
|
self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
|
|
@@ -915,7 +390,7 @@ class OperationsProcessor:
|
|
|
915
390
|
exec_dict = dict_obj
|
|
916
391
|
tsk_dicts = []
|
|
917
392
|
|
|
918
|
-
context =
|
|
393
|
+
context = get_context_from_project(exec_dict["project"])
|
|
919
394
|
obj: ExecutableEntity = build_entity_from_dict(exec_dict)
|
|
920
395
|
|
|
921
396
|
try:
|
|
@@ -954,16 +429,18 @@ class OperationsProcessor:
|
|
|
954
429
|
list[dict]
|
|
955
430
|
Object instances.
|
|
956
431
|
"""
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
project, entity_type, _, entity_name, _ = parse_entity_key(identifier)
|
|
432
|
+
project, entity_type, _, entity_name, _ = parse_identifier(
|
|
433
|
+
identifier,
|
|
434
|
+
project=project,
|
|
435
|
+
entity_type=entity_type,
|
|
436
|
+
)
|
|
963
437
|
|
|
964
|
-
kwargs =
|
|
965
|
-
|
|
966
|
-
|
|
438
|
+
kwargs = context.client.build_parameters(
|
|
439
|
+
ApiCategories.CONTEXT.value,
|
|
440
|
+
BackendOperations.READ_ALL_VERSIONS.value,
|
|
441
|
+
entity_name=entity_name,
|
|
442
|
+
**kwargs,
|
|
443
|
+
)
|
|
967
444
|
|
|
968
445
|
api = context.client.build_api(
|
|
969
446
|
ApiCategories.CONTEXT.value,
|
|
@@ -999,43 +476,7 @@ class OperationsProcessor:
|
|
|
999
476
|
list[ContextEntity]
|
|
1000
477
|
List of object instances.
|
|
1001
478
|
"""
|
|
1002
|
-
context =
|
|
1003
|
-
obj = self._read_context_entity_versions(
|
|
1004
|
-
context,
|
|
1005
|
-
identifier,
|
|
1006
|
-
entity_type=entity_type,
|
|
1007
|
-
project=project,
|
|
1008
|
-
**kwargs,
|
|
1009
|
-
)
|
|
1010
|
-
return [build_entity_from_dict(o) for o in obj]
|
|
1011
|
-
|
|
1012
|
-
def read_material_entity_versions(
|
|
1013
|
-
self,
|
|
1014
|
-
identifier: str,
|
|
1015
|
-
entity_type: str | None = None,
|
|
1016
|
-
project: str | None = None,
|
|
1017
|
-
**kwargs,
|
|
1018
|
-
) -> list[MaterialEntity]:
|
|
1019
|
-
"""
|
|
1020
|
-
Read object versions from backend.
|
|
1021
|
-
|
|
1022
|
-
Parameters
|
|
1023
|
-
----------
|
|
1024
|
-
identifier : str
|
|
1025
|
-
Entity key (store://...) or entity name.
|
|
1026
|
-
entity_type : str
|
|
1027
|
-
Entity type.
|
|
1028
|
-
project : str
|
|
1029
|
-
Project name.
|
|
1030
|
-
**kwargs : dict
|
|
1031
|
-
Parameters to pass to the API call.
|
|
1032
|
-
|
|
1033
|
-
Returns
|
|
1034
|
-
-------
|
|
1035
|
-
list[MaterialEntity]
|
|
1036
|
-
List of object instances.
|
|
1037
|
-
"""
|
|
1038
|
-
context = self._get_context_from_identifier(identifier, project)
|
|
479
|
+
context = get_context_from_identifier(identifier, project)
|
|
1039
480
|
objs = self._read_context_entity_versions(
|
|
1040
481
|
context,
|
|
1041
482
|
identifier,
|
|
@@ -1045,8 +486,8 @@ class OperationsProcessor:
|
|
|
1045
486
|
)
|
|
1046
487
|
objects = []
|
|
1047
488
|
for o in objs:
|
|
1048
|
-
entity:
|
|
1049
|
-
entity.
|
|
489
|
+
entity: ContextEntity = build_entity_from_dict(o)
|
|
490
|
+
entity = self._post_process_get(entity)
|
|
1050
491
|
objects.append(entity)
|
|
1051
492
|
return objects
|
|
1052
493
|
|
|
@@ -1104,39 +545,12 @@ class OperationsProcessor:
|
|
|
1104
545
|
list[ContextEntity]
|
|
1105
546
|
List of object instances.
|
|
1106
547
|
"""
|
|
1107
|
-
context =
|
|
1108
|
-
objs = self._list_context_entities(context, entity_type, **kwargs)
|
|
1109
|
-
return [build_entity_from_dict(obj) for obj in objs]
|
|
1110
|
-
|
|
1111
|
-
def list_material_entities(
|
|
1112
|
-
self,
|
|
1113
|
-
project: str,
|
|
1114
|
-
entity_type: str,
|
|
1115
|
-
**kwargs,
|
|
1116
|
-
) -> list[MaterialEntity]:
|
|
1117
|
-
"""
|
|
1118
|
-
List all latest version objects from backend.
|
|
1119
|
-
|
|
1120
|
-
Parameters
|
|
1121
|
-
----------
|
|
1122
|
-
project : str
|
|
1123
|
-
Project name.
|
|
1124
|
-
entity_type : str
|
|
1125
|
-
Entity type.
|
|
1126
|
-
**kwargs : dict
|
|
1127
|
-
Parameters to pass to the API call.
|
|
1128
|
-
|
|
1129
|
-
Returns
|
|
1130
|
-
-------
|
|
1131
|
-
list[MaterialEntity]
|
|
1132
|
-
List of object instances.
|
|
1133
|
-
"""
|
|
1134
|
-
context = self._get_context(project)
|
|
548
|
+
context = get_context_from_project(project)
|
|
1135
549
|
objs = self._list_context_entities(context, entity_type, **kwargs)
|
|
1136
550
|
objects = []
|
|
1137
551
|
for o in objs:
|
|
1138
|
-
entity:
|
|
1139
|
-
entity.
|
|
552
|
+
entity: ContextEntity = build_entity_from_dict(o)
|
|
553
|
+
entity = self._post_process_get(entity)
|
|
1140
554
|
objects.append(entity)
|
|
1141
555
|
return objects
|
|
1142
556
|
|
|
@@ -1207,7 +621,7 @@ class OperationsProcessor:
|
|
|
1207
621
|
ContextEntity
|
|
1208
622
|
Object instance.
|
|
1209
623
|
"""
|
|
1210
|
-
context =
|
|
624
|
+
context = get_context_from_project(project)
|
|
1211
625
|
obj = self._update_context_entity(
|
|
1212
626
|
context,
|
|
1213
627
|
entity_type,
|
|
@@ -1249,23 +663,23 @@ class OperationsProcessor:
|
|
|
1249
663
|
dict
|
|
1250
664
|
Response from backend.
|
|
1251
665
|
"""
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
kwargs = self._set_params(**kwargs)
|
|
1260
|
-
if cascade := kwargs.pop("cascade", None) is not None:
|
|
1261
|
-
kwargs["params"]["cascade"] = str(cascade).lower()
|
|
666
|
+
project, entity_type, _, entity_name, entity_id = parse_identifier(
|
|
667
|
+
identifier,
|
|
668
|
+
project=project,
|
|
669
|
+
entity_type=entity_type,
|
|
670
|
+
entity_id=entity_id,
|
|
671
|
+
)
|
|
1262
672
|
|
|
1263
673
|
delete_all_versions: bool = kwargs.pop("delete_all_versions", False)
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
674
|
+
kwargs = context.client.build_parameters(
|
|
675
|
+
ApiCategories.CONTEXT.value,
|
|
676
|
+
BackendOperations.DELETE.value,
|
|
677
|
+
entity_id=entity_id,
|
|
678
|
+
entity_name=entity_name,
|
|
679
|
+
cascade=kwargs.pop("cascade", None),
|
|
680
|
+
delete_all_versions=delete_all_versions,
|
|
681
|
+
**kwargs,
|
|
682
|
+
)
|
|
1269
683
|
|
|
1270
684
|
if delete_all_versions:
|
|
1271
685
|
api = context.client.build_api(
|
|
@@ -1274,7 +688,6 @@ class OperationsProcessor:
|
|
|
1274
688
|
project=context.name,
|
|
1275
689
|
entity_type=entity_type,
|
|
1276
690
|
)
|
|
1277
|
-
kwargs["params"]["name"] = entity_name
|
|
1278
691
|
else:
|
|
1279
692
|
api = context.client.build_api(
|
|
1280
693
|
ApiCategories.CONTEXT.value,
|
|
@@ -1314,7 +727,7 @@ class OperationsProcessor:
|
|
|
1314
727
|
dict
|
|
1315
728
|
Response from backend.
|
|
1316
729
|
"""
|
|
1317
|
-
context =
|
|
730
|
+
context = get_context_from_identifier(identifier, project)
|
|
1318
731
|
return self._delete_context_entity(
|
|
1319
732
|
context,
|
|
1320
733
|
identifier,
|
|
@@ -1324,6 +737,26 @@ class OperationsProcessor:
|
|
|
1324
737
|
**kwargs,
|
|
1325
738
|
)
|
|
1326
739
|
|
|
740
|
+
def _post_process_get(self, entity: ContextEntity) -> ContextEntity:
|
|
741
|
+
"""
|
|
742
|
+
Post process get (files, metrics).
|
|
743
|
+
|
|
744
|
+
Parameters
|
|
745
|
+
----------
|
|
746
|
+
entity : ContextEntity
|
|
747
|
+
Entity to post process.
|
|
748
|
+
|
|
749
|
+
Returns
|
|
750
|
+
-------
|
|
751
|
+
ContextEntity
|
|
752
|
+
Post processed entity.
|
|
753
|
+
"""
|
|
754
|
+
if hasattr(entity.status, "metrics"):
|
|
755
|
+
entity._get_metrics()
|
|
756
|
+
if hasattr(entity.status, "files"):
|
|
757
|
+
entity._get_files_info()
|
|
758
|
+
return entity
|
|
759
|
+
|
|
1327
760
|
##############################
|
|
1328
761
|
# Context entity operations
|
|
1329
762
|
##############################
|
|
@@ -1395,7 +828,7 @@ class OperationsProcessor:
|
|
|
1395
828
|
str
|
|
1396
829
|
Object key.
|
|
1397
830
|
"""
|
|
1398
|
-
context =
|
|
831
|
+
context = get_context_from_project(project)
|
|
1399
832
|
return self._build_context_entity_key(context, entity_type, entity_kind, entity_name, entity_id)
|
|
1400
833
|
|
|
1401
834
|
def read_secret_data(
|
|
@@ -1421,7 +854,7 @@ class OperationsProcessor:
|
|
|
1421
854
|
dict
|
|
1422
855
|
Response from backend.
|
|
1423
856
|
"""
|
|
1424
|
-
context =
|
|
857
|
+
context = get_context_from_project(project)
|
|
1425
858
|
api = context.client.build_api(
|
|
1426
859
|
ApiCategories.CONTEXT.value,
|
|
1427
860
|
BackendOperations.DATA.value,
|
|
@@ -1455,14 +888,14 @@ class OperationsProcessor:
|
|
|
1455
888
|
-------
|
|
1456
889
|
None
|
|
1457
890
|
"""
|
|
1458
|
-
context =
|
|
891
|
+
context = get_context_from_project(project)
|
|
1459
892
|
api = context.client.build_api(
|
|
1460
893
|
ApiCategories.CONTEXT.value,
|
|
1461
894
|
BackendOperations.DATA.value,
|
|
1462
895
|
project=context.name,
|
|
1463
896
|
entity_type=entity_type,
|
|
1464
897
|
)
|
|
1465
|
-
|
|
898
|
+
context.client.update_object(api, data, **kwargs)
|
|
1466
899
|
|
|
1467
900
|
def read_run_logs(
|
|
1468
901
|
self,
|
|
@@ -1490,7 +923,7 @@ class OperationsProcessor:
|
|
|
1490
923
|
dict
|
|
1491
924
|
Response from backend.
|
|
1492
925
|
"""
|
|
1493
|
-
context =
|
|
926
|
+
context = get_context_from_project(project)
|
|
1494
927
|
api = context.client.build_api(
|
|
1495
928
|
ApiCategories.CONTEXT.value,
|
|
1496
929
|
BackendOperations.LOGS.value,
|
|
@@ -1525,7 +958,7 @@ class OperationsProcessor:
|
|
|
1525
958
|
-------
|
|
1526
959
|
None
|
|
1527
960
|
"""
|
|
1528
|
-
context =
|
|
961
|
+
context = get_context_from_project(project)
|
|
1529
962
|
api = context.client.build_api(
|
|
1530
963
|
ApiCategories.CONTEXT.value,
|
|
1531
964
|
BackendOperations.STOP.value,
|
|
@@ -1533,7 +966,7 @@ class OperationsProcessor:
|
|
|
1533
966
|
entity_type=entity_type,
|
|
1534
967
|
entity_id=entity_id,
|
|
1535
968
|
)
|
|
1536
|
-
|
|
969
|
+
context.client.create_object(api, **kwargs)
|
|
1537
970
|
|
|
1538
971
|
def resume_run(
|
|
1539
972
|
self,
|
|
@@ -1560,7 +993,7 @@ class OperationsProcessor:
|
|
|
1560
993
|
-------
|
|
1561
994
|
None
|
|
1562
995
|
"""
|
|
1563
|
-
context =
|
|
996
|
+
context = get_context_from_project(project)
|
|
1564
997
|
api = context.client.build_api(
|
|
1565
998
|
ApiCategories.CONTEXT.value,
|
|
1566
999
|
BackendOperations.RESUME.value,
|
|
@@ -1568,7 +1001,7 @@ class OperationsProcessor:
|
|
|
1568
1001
|
entity_type=entity_type,
|
|
1569
1002
|
entity_id=entity_id,
|
|
1570
1003
|
)
|
|
1571
|
-
|
|
1004
|
+
context.client.create_object(api, **kwargs)
|
|
1572
1005
|
|
|
1573
1006
|
def read_files_info(
|
|
1574
1007
|
self,
|
|
@@ -1596,7 +1029,7 @@ class OperationsProcessor:
|
|
|
1596
1029
|
list[dict]
|
|
1597
1030
|
Response from backend.
|
|
1598
1031
|
"""
|
|
1599
|
-
context =
|
|
1032
|
+
context = get_context_from_project(project)
|
|
1600
1033
|
api = context.client.build_api(
|
|
1601
1034
|
ApiCategories.CONTEXT.value,
|
|
1602
1035
|
BackendOperations.FILES.value,
|
|
@@ -1634,7 +1067,7 @@ class OperationsProcessor:
|
|
|
1634
1067
|
-------
|
|
1635
1068
|
None
|
|
1636
1069
|
"""
|
|
1637
|
-
context =
|
|
1070
|
+
context = get_context_from_project(project)
|
|
1638
1071
|
api = context.client.build_api(
|
|
1639
1072
|
ApiCategories.CONTEXT.value,
|
|
1640
1073
|
BackendOperations.FILES.value,
|
|
@@ -1644,18 +1077,25 @@ class OperationsProcessor:
|
|
|
1644
1077
|
)
|
|
1645
1078
|
return context.client.update_object(api, entity_list, **kwargs)
|
|
1646
1079
|
|
|
1647
|
-
def
|
|
1080
|
+
def read_metrics(
|
|
1648
1081
|
self,
|
|
1649
1082
|
project: str,
|
|
1083
|
+
entity_type: str,
|
|
1084
|
+
entity_id: str,
|
|
1085
|
+
metric_name: str | None = None,
|
|
1650
1086
|
**kwargs,
|
|
1651
1087
|
) -> dict:
|
|
1652
1088
|
"""
|
|
1653
|
-
|
|
1089
|
+
Get metrics from backend.
|
|
1654
1090
|
|
|
1655
1091
|
Parameters
|
|
1656
1092
|
----------
|
|
1657
1093
|
project : str
|
|
1658
1094
|
Project name.
|
|
1095
|
+
entity_type : str
|
|
1096
|
+
Entity type.
|
|
1097
|
+
entity_id : str
|
|
1098
|
+
Entity ID.
|
|
1659
1099
|
**kwargs : dict
|
|
1660
1100
|
Parameters to pass to the API call.
|
|
1661
1101
|
|
|
@@ -1664,14 +1104,88 @@ class OperationsProcessor:
|
|
|
1664
1104
|
dict
|
|
1665
1105
|
Response from backend.
|
|
1666
1106
|
"""
|
|
1667
|
-
context =
|
|
1107
|
+
context = get_context_from_project(project)
|
|
1668
1108
|
api = context.client.build_api(
|
|
1669
1109
|
ApiCategories.CONTEXT.value,
|
|
1670
|
-
BackendOperations.
|
|
1110
|
+
BackendOperations.METRICS.value,
|
|
1671
1111
|
project=context.name,
|
|
1112
|
+
entity_type=entity_type,
|
|
1113
|
+
entity_id=entity_id,
|
|
1114
|
+
metric_name=metric_name,
|
|
1672
1115
|
)
|
|
1673
1116
|
return context.client.read_object(api, **kwargs)
|
|
1674
1117
|
|
|
1118
|
+
def update_metric(
|
|
1119
|
+
self,
|
|
1120
|
+
project: str,
|
|
1121
|
+
entity_type: str,
|
|
1122
|
+
entity_id: str,
|
|
1123
|
+
metric_name: str,
|
|
1124
|
+
metric_value: Any,
|
|
1125
|
+
**kwargs,
|
|
1126
|
+
) -> None:
|
|
1127
|
+
"""
|
|
1128
|
+
Get single metric from backend.
|
|
1129
|
+
|
|
1130
|
+
Parameters
|
|
1131
|
+
----------
|
|
1132
|
+
project : str
|
|
1133
|
+
Project name.
|
|
1134
|
+
entity_type : str
|
|
1135
|
+
Entity type.
|
|
1136
|
+
entity_id : str
|
|
1137
|
+
Entity ID.
|
|
1138
|
+
**kwargs : dict
|
|
1139
|
+
Parameters to pass to the API call.
|
|
1140
|
+
|
|
1141
|
+
Returns
|
|
1142
|
+
-------
|
|
1143
|
+
None
|
|
1144
|
+
"""
|
|
1145
|
+
context = get_context_from_project(project)
|
|
1146
|
+
api = context.client.build_api(
|
|
1147
|
+
ApiCategories.CONTEXT.value,
|
|
1148
|
+
BackendOperations.METRICS.value,
|
|
1149
|
+
project=context.name,
|
|
1150
|
+
entity_type=entity_type,
|
|
1151
|
+
entity_id=entity_id,
|
|
1152
|
+
metric_name=metric_name,
|
|
1153
|
+
)
|
|
1154
|
+
context.client.update_object(api, metric_value, **kwargs)
|
|
1155
|
+
|
|
1156
|
+
def _search(
|
|
1157
|
+
self,
|
|
1158
|
+
context: Context,
|
|
1159
|
+
**kwargs,
|
|
1160
|
+
) -> dict:
|
|
1161
|
+
"""
|
|
1162
|
+
Search in backend.
|
|
1163
|
+
|
|
1164
|
+
Parameters
|
|
1165
|
+
----------
|
|
1166
|
+
context : Context
|
|
1167
|
+
Context instance.
|
|
1168
|
+
**kwargs : dict
|
|
1169
|
+
Parameters to pass to the API call.
|
|
1170
|
+
|
|
1171
|
+
Returns
|
|
1172
|
+
-------
|
|
1173
|
+
dict
|
|
1174
|
+
Response from backend.
|
|
1175
|
+
"""
|
|
1176
|
+
kwargs = context.client.build_parameters(
|
|
1177
|
+
ApiCategories.CONTEXT.value,
|
|
1178
|
+
BackendOperations.SEARCH.value,
|
|
1179
|
+
**kwargs,
|
|
1180
|
+
)
|
|
1181
|
+
api = context.client.build_api(
|
|
1182
|
+
ApiCategories.CONTEXT.value,
|
|
1183
|
+
BackendOperations.SEARCH.value,
|
|
1184
|
+
project=context.name,
|
|
1185
|
+
)
|
|
1186
|
+
entities_dict = context.client.read_object(api, **kwargs)
|
|
1187
|
+
return [self.read_context_entity(entity["key"]) for entity in entities_dict["content"]]
|
|
1188
|
+
|
|
1675
1189
|
def search_entity(
|
|
1676
1190
|
self,
|
|
1677
1191
|
project: str,
|
|
@@ -1716,158 +1230,19 @@ class OperationsProcessor:
|
|
|
1716
1230
|
list[ContextEntity]
|
|
1717
1231
|
List of object instances.
|
|
1718
1232
|
"""
|
|
1719
|
-
context =
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
if len(entity_types) == 1:
|
|
1733
|
-
entity_types = entity_types[0]
|
|
1734
|
-
else:
|
|
1735
|
-
entity_types = " OR ".join(entity_types)
|
|
1736
|
-
fq.append(f"type:({entity_types})")
|
|
1737
|
-
|
|
1738
|
-
# Name
|
|
1739
|
-
if name is not None:
|
|
1740
|
-
fq.append(f'metadata.name:"{name}"')
|
|
1741
|
-
|
|
1742
|
-
# Kind
|
|
1743
|
-
if kind is not None:
|
|
1744
|
-
fq.append(f'kind:"{kind}"')
|
|
1745
|
-
|
|
1746
|
-
# Time
|
|
1747
|
-
created = created if created is not None else "*"
|
|
1748
|
-
updated = updated if updated is not None else "*"
|
|
1749
|
-
fq.append(f"metadata.updated:[{created} TO {updated}]")
|
|
1750
|
-
|
|
1751
|
-
# Description
|
|
1752
|
-
if description is not None:
|
|
1753
|
-
fq.append(f'metadata.description:"{description}"')
|
|
1754
|
-
|
|
1755
|
-
# Labels
|
|
1756
|
-
if labels is not None:
|
|
1757
|
-
if len(labels) == 1:
|
|
1758
|
-
labels = labels[0]
|
|
1759
|
-
else:
|
|
1760
|
-
labels = " AND ".join(labels)
|
|
1761
|
-
fq.append(f"metadata.labels:({labels})")
|
|
1762
|
-
|
|
1763
|
-
# Add filters
|
|
1764
|
-
kwargs["params"]["fq"] = fq
|
|
1765
|
-
|
|
1766
|
-
objs = self._search(context, **kwargs)
|
|
1767
|
-
return objs
|
|
1768
|
-
return [build_entity_from_dict(obj) for obj in objs]
|
|
1769
|
-
|
|
1770
|
-
##############################
|
|
1771
|
-
# Helpers
|
|
1772
|
-
##############################
|
|
1773
|
-
|
|
1774
|
-
@staticmethod
|
|
1775
|
-
def _set_params(**kwargs) -> dict:
|
|
1776
|
-
"""
|
|
1777
|
-
Format params parameter.
|
|
1778
|
-
|
|
1779
|
-
Parameters
|
|
1780
|
-
----------
|
|
1781
|
-
**kwargs : dict
|
|
1782
|
-
Keyword arguments.
|
|
1783
|
-
|
|
1784
|
-
Returns
|
|
1785
|
-
-------
|
|
1786
|
-
dict
|
|
1787
|
-
Parameters with initialized params.
|
|
1788
|
-
"""
|
|
1789
|
-
if not kwargs:
|
|
1790
|
-
kwargs = {}
|
|
1791
|
-
if "params" not in kwargs:
|
|
1792
|
-
kwargs["params"] = {}
|
|
1793
|
-
return kwargs
|
|
1794
|
-
|
|
1795
|
-
def _get_context_from_identifier(
|
|
1796
|
-
self,
|
|
1797
|
-
identifier: str,
|
|
1798
|
-
project: str | None = None,
|
|
1799
|
-
) -> Context:
|
|
1800
|
-
"""
|
|
1801
|
-
Get context from project.
|
|
1802
|
-
|
|
1803
|
-
Parameters
|
|
1804
|
-
----------
|
|
1805
|
-
identifier : str
|
|
1806
|
-
Entity key (store://...) or entity name.
|
|
1807
|
-
project : str
|
|
1808
|
-
Project name.
|
|
1809
|
-
|
|
1810
|
-
Returns
|
|
1811
|
-
-------
|
|
1812
|
-
Context
|
|
1813
|
-
Context.
|
|
1814
|
-
"""
|
|
1815
|
-
if not identifier.startswith("store://"):
|
|
1816
|
-
if project is None:
|
|
1817
|
-
raise EntityError("Specify project if you do not specify entity key.")
|
|
1818
|
-
else:
|
|
1819
|
-
project = get_project_from_key(identifier)
|
|
1820
|
-
|
|
1821
|
-
return self._get_context(project)
|
|
1822
|
-
|
|
1823
|
-
def _get_context(
|
|
1824
|
-
self,
|
|
1825
|
-
project: str,
|
|
1826
|
-
) -> Context:
|
|
1827
|
-
"""
|
|
1828
|
-
Check if the given project is in the context.
|
|
1829
|
-
Otherwise try to get the project from remote.
|
|
1830
|
-
Finally return the client.
|
|
1831
|
-
|
|
1832
|
-
Parameters
|
|
1833
|
-
----------
|
|
1834
|
-
project : str
|
|
1835
|
-
Project name.
|
|
1836
|
-
|
|
1837
|
-
Returns
|
|
1838
|
-
-------
|
|
1839
|
-
Context
|
|
1840
|
-
Context.
|
|
1841
|
-
"""
|
|
1842
|
-
try:
|
|
1843
|
-
return get_context(project)
|
|
1844
|
-
except ContextError:
|
|
1845
|
-
return self._get_context_from_remote(project)
|
|
1846
|
-
|
|
1847
|
-
def _get_context_from_remote(
|
|
1848
|
-
self,
|
|
1849
|
-
project: str,
|
|
1850
|
-
) -> Client:
|
|
1851
|
-
"""
|
|
1852
|
-
Get context from remote.
|
|
1853
|
-
|
|
1854
|
-
Parameters
|
|
1855
|
-
----------
|
|
1856
|
-
project : str
|
|
1857
|
-
Project name.
|
|
1858
|
-
|
|
1859
|
-
Returns
|
|
1860
|
-
-------
|
|
1861
|
-
Client
|
|
1862
|
-
Client.
|
|
1863
|
-
"""
|
|
1864
|
-
try:
|
|
1865
|
-
client = get_client()
|
|
1866
|
-
obj = self._read_base_entity(client, EntityTypes.PROJECT.value, project)
|
|
1867
|
-
build_entity_from_dict(obj)
|
|
1868
|
-
return get_context(project)
|
|
1869
|
-
except EntityNotExistsError:
|
|
1870
|
-
raise ContextError(f"Project '{project}' not found.")
|
|
1233
|
+
context = get_context_from_project(project)
|
|
1234
|
+
return self._search(
|
|
1235
|
+
context,
|
|
1236
|
+
query=query,
|
|
1237
|
+
entity_types=entity_types,
|
|
1238
|
+
name=name,
|
|
1239
|
+
kind=kind,
|
|
1240
|
+
created=created,
|
|
1241
|
+
updated=updated,
|
|
1242
|
+
description=description,
|
|
1243
|
+
labels=labels,
|
|
1244
|
+
**kwargs,
|
|
1245
|
+
)
|
|
1871
1246
|
|
|
1872
1247
|
|
|
1873
|
-
|
|
1248
|
+
context_processor = ContextEntityOperationsProcessor()
|