digitalhub 0.8.0b15__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/__init__.py +19 -2
- digitalhub/client/_base/api_builder.py +16 -0
- digitalhub/client/_base/client.py +67 -0
- digitalhub/client/_base/key_builder.py +52 -0
- digitalhub/client/api.py +2 -38
- digitalhub/client/dhcore/api_builder.py +100 -0
- digitalhub/client/dhcore/client.py +100 -48
- digitalhub/client/dhcore/enums.py +27 -0
- digitalhub/client/dhcore/env.py +4 -2
- digitalhub/client/dhcore/key_builder.py +58 -0
- digitalhub/client/dhcore/utils.py +17 -17
- digitalhub/client/local/api_builder.py +100 -0
- digitalhub/client/local/client.py +22 -0
- digitalhub/client/local/key_builder.py +58 -0
- digitalhub/context/api.py +3 -38
- digitalhub/context/builder.py +10 -23
- digitalhub/context/context.py +20 -92
- digitalhub/entities/_base/context/entity.py +30 -22
- digitalhub/entities/_base/entity/_constructors/metadata.py +12 -1
- digitalhub/entities/_base/entity/_constructors/name.py +1 -1
- digitalhub/entities/_base/entity/_constructors/spec.py +1 -1
- digitalhub/entities/_base/entity/_constructors/status.py +3 -2
- digitalhub/entities/_base/entity/_constructors/uuid.py +1 -1
- digitalhub/entities/_base/entity/builder.py +6 -1
- digitalhub/entities/_base/entity/entity.py +32 -10
- digitalhub/entities/_base/entity/metadata.py +22 -0
- digitalhub/entities/_base/entity/spec.py +7 -2
- digitalhub/entities/_base/executable/entity.py +8 -8
- digitalhub/entities/_base/material/entity.py +49 -17
- digitalhub/entities/_base/material/status.py +0 -31
- digitalhub/entities/_base/material/utils.py +106 -0
- digitalhub/entities/_base/project/entity.py +341 -0
- digitalhub/entities/_base/unversioned/entity.py +3 -24
- digitalhub/entities/_base/versioned/entity.py +2 -26
- digitalhub/entities/_commons/enums.py +103 -0
- digitalhub/entities/_commons/utils.py +83 -0
- digitalhub/entities/_operations/processor.py +1873 -0
- digitalhub/entities/artifact/_base/builder.py +1 -1
- digitalhub/entities/artifact/_base/entity.py +1 -1
- digitalhub/entities/artifact/artifact/builder.py +2 -1
- digitalhub/entities/artifact/crud.py +46 -29
- digitalhub/entities/artifact/utils.py +62 -0
- digitalhub/entities/dataitem/_base/builder.py +1 -1
- digitalhub/entities/dataitem/_base/entity.py +6 -6
- digitalhub/entities/dataitem/crud.py +50 -66
- digitalhub/entities/dataitem/dataitem/builder.py +2 -1
- digitalhub/entities/dataitem/iceberg/builder.py +2 -1
- digitalhub/entities/dataitem/table/builder.py +2 -1
- digitalhub/entities/dataitem/table/entity.py +5 -10
- digitalhub/entities/dataitem/table/models.py +4 -5
- digitalhub/entities/dataitem/utils.py +137 -0
- digitalhub/entities/function/_base/builder.py +1 -1
- digitalhub/entities/function/_base/entity.py +6 -2
- digitalhub/entities/function/crud.py +36 -17
- digitalhub/entities/model/_base/builder.py +1 -1
- digitalhub/entities/model/_base/entity.py +1 -1
- digitalhub/entities/model/crud.py +46 -29
- digitalhub/entities/model/huggingface/builder.py +2 -1
- digitalhub/entities/model/huggingface/spec.py +4 -2
- digitalhub/entities/model/mlflow/builder.py +2 -1
- digitalhub/entities/model/mlflow/models.py +17 -9
- digitalhub/entities/model/mlflow/spec.py +6 -1
- digitalhub/entities/model/mlflow/utils.py +4 -2
- digitalhub/entities/model/model/builder.py +2 -1
- digitalhub/entities/model/sklearn/builder.py +2 -1
- digitalhub/entities/model/utils.py +62 -0
- digitalhub/entities/project/_base/builder.py +2 -2
- digitalhub/entities/project/_base/entity.py +82 -272
- digitalhub/entities/project/crud.py +110 -89
- digitalhub/entities/project/utils.py +35 -0
- digitalhub/entities/run/_base/builder.py +3 -1
- digitalhub/entities/run/_base/entity.py +52 -54
- digitalhub/entities/run/_base/spec.py +15 -7
- digitalhub/entities/run/crud.py +35 -17
- digitalhub/entities/secret/_base/builder.py +2 -2
- digitalhub/entities/secret/_base/entity.py +4 -10
- digitalhub/entities/secret/crud.py +36 -21
- digitalhub/entities/task/_base/builder.py +14 -14
- digitalhub/entities/task/_base/entity.py +21 -14
- digitalhub/entities/task/_base/models.py +35 -6
- digitalhub/entities/task/_base/spec.py +50 -13
- digitalhub/entities/task/_base/utils.py +18 -0
- digitalhub/entities/task/crud.py +35 -15
- digitalhub/entities/workflow/_base/builder.py +1 -1
- digitalhub/entities/workflow/_base/entity.py +22 -6
- digitalhub/entities/workflow/crud.py +36 -17
- digitalhub/factory/utils.py +1 -1
- digitalhub/readers/_base/reader.py +2 -2
- digitalhub/readers/_commons/enums.py +13 -0
- digitalhub/readers/api.py +3 -2
- digitalhub/readers/factory.py +12 -6
- digitalhub/readers/pandas/reader.py +20 -8
- digitalhub/runtimes/_base.py +0 -7
- digitalhub/runtimes/enums.py +12 -0
- digitalhub/stores/_base/store.py +59 -11
- digitalhub/stores/builder.py +5 -5
- digitalhub/stores/local/store.py +43 -4
- digitalhub/stores/remote/store.py +31 -5
- digitalhub/stores/s3/store.py +136 -57
- digitalhub/stores/sql/store.py +122 -47
- digitalhub/utils/exceptions.py +6 -0
- digitalhub/utils/file_utils.py +60 -2
- digitalhub/utils/generic_utils.py +45 -4
- digitalhub/utils/io_utils.py +18 -0
- digitalhub/utils/s3_utils.py +17 -0
- digitalhub/utils/uri_utils.py +153 -15
- {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/LICENSE.txt +1 -1
- {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/METADATA +11 -11
- {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/RECORD +117 -115
- {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/WHEEL +1 -1
- test/local/instances/test_validate.py +55 -0
- test/testkfp.py +4 -1
- digitalhub/datastores/_base/datastore.py +0 -85
- digitalhub/datastores/api.py +0 -37
- digitalhub/datastores/builder.py +0 -110
- digitalhub/datastores/local/datastore.py +0 -50
- digitalhub/datastores/remote/__init__.py +0 -0
- digitalhub/datastores/remote/datastore.py +0 -31
- digitalhub/datastores/s3/__init__.py +0 -0
- digitalhub/datastores/s3/datastore.py +0 -46
- digitalhub/datastores/sql/__init__.py +0 -0
- digitalhub/datastores/sql/datastore.py +0 -68
- digitalhub/entities/_base/api_utils.py +0 -620
- digitalhub/entities/_base/crud.py +0 -468
- digitalhub/entities/function/_base/models.py +0 -118
- digitalhub/entities/utils/__init__.py +0 -0
- digitalhub/entities/utils/api.py +0 -346
- digitalhub/entities/utils/entity_types.py +0 -19
- digitalhub/entities/utils/state.py +0 -31
- digitalhub/entities/utils/utils.py +0 -202
- /digitalhub/{context → entities/_base/project}/__init__.py +0 -0
- /digitalhub/{datastores → entities/_commons}/__init__.py +0 -0
- /digitalhub/{datastores/_base → entities/_operations}/__init__.py +0 -0
- /digitalhub/{datastores/local → readers/_commons}/__init__.py +0 -0
- {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,1873 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typing
|
|
4
|
+
|
|
5
|
+
from digitalhub.client.api import get_client
|
|
6
|
+
from digitalhub.context.api import delete_context, get_context
|
|
7
|
+
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, EntityTypes, Relationship
|
|
8
|
+
from digitalhub.entities._commons.utils import get_project_from_key, parse_entity_key
|
|
9
|
+
from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
|
|
10
|
+
from digitalhub.utils.exceptions import ContextError, EntityAlreadyExistsError, EntityError, EntityNotExistsError
|
|
11
|
+
from digitalhub.utils.io_utils import read_yaml
|
|
12
|
+
|
|
13
|
+
if typing.TYPE_CHECKING:
|
|
14
|
+
from digitalhub.client._base.client import Client
|
|
15
|
+
from digitalhub.context.context import Context
|
|
16
|
+
from digitalhub.entities._base.context.entity import ContextEntity
|
|
17
|
+
from digitalhub.entities._base.executable.entity import ExecutableEntity
|
|
18
|
+
from digitalhub.entities._base.material.entity import MaterialEntity
|
|
19
|
+
from digitalhub.entities._base.project.entity import ProjectEntity
|
|
20
|
+
from digitalhub.entities._base.unversioned.entity import UnversionedEntity
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class OperationsProcessor:
|
|
24
|
+
"""
|
|
25
|
+
Processor for Entity operations.
|
|
26
|
+
|
|
27
|
+
This object interacts with the context, check the category of the object,
|
|
28
|
+
and then calls the appropriate method to perform the requested operation.
|
|
29
|
+
Operations can be CRUD, search, list, etc.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
##############################
|
|
33
|
+
# CRUD base entity
|
|
34
|
+
##############################
|
|
35
|
+
|
|
36
|
+
def _create_base_entity(
|
|
37
|
+
self,
|
|
38
|
+
client: Client,
|
|
39
|
+
entity_type: str,
|
|
40
|
+
entity_dict: dict,
|
|
41
|
+
**kwargs,
|
|
42
|
+
) -> dict:
|
|
43
|
+
"""
|
|
44
|
+
Create object in backend.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
client : Client
|
|
49
|
+
Client instance.
|
|
50
|
+
entity_type : str
|
|
51
|
+
Entity type.
|
|
52
|
+
entity_dict : dict
|
|
53
|
+
Object instance.
|
|
54
|
+
**kwargs : dict
|
|
55
|
+
Parameters to pass to the API call.
|
|
56
|
+
|
|
57
|
+
Returns
|
|
58
|
+
-------
|
|
59
|
+
dict
|
|
60
|
+
Object instance.
|
|
61
|
+
"""
|
|
62
|
+
api = client.build_api(
|
|
63
|
+
ApiCategories.BASE.value,
|
|
64
|
+
BackendOperations.CREATE.value,
|
|
65
|
+
entity_type=entity_type,
|
|
66
|
+
)
|
|
67
|
+
return client.create_object(api, entity_dict, **kwargs)
|
|
68
|
+
|
|
69
|
+
def create_project_entity(
|
|
70
|
+
self,
|
|
71
|
+
_entity: ProjectEntity | None = None,
|
|
72
|
+
**kwargs,
|
|
73
|
+
) -> ProjectEntity:
|
|
74
|
+
"""
|
|
75
|
+
Create object in backend.
|
|
76
|
+
|
|
77
|
+
Parameters
|
|
78
|
+
----------
|
|
79
|
+
_entity : ProjectEntity
|
|
80
|
+
Object instance.
|
|
81
|
+
**kwargs : dict
|
|
82
|
+
Parameters to pass to entity builder.
|
|
83
|
+
|
|
84
|
+
Returns
|
|
85
|
+
-------
|
|
86
|
+
Project
|
|
87
|
+
Object instance.
|
|
88
|
+
"""
|
|
89
|
+
if _entity is not None:
|
|
90
|
+
client = _entity._client
|
|
91
|
+
obj = _entity
|
|
92
|
+
else:
|
|
93
|
+
client = get_client(kwargs.get("local"), kwargs.pop("config", None))
|
|
94
|
+
obj = build_entity_from_params(**kwargs)
|
|
95
|
+
ent = self._create_base_entity(client, obj.ENTITY_TYPE, obj.to_dict())
|
|
96
|
+
ent["local"] = client.is_local()
|
|
97
|
+
return build_entity_from_dict(ent)
|
|
98
|
+
|
|
99
|
+
def _read_base_entity(
|
|
100
|
+
self,
|
|
101
|
+
client: Client,
|
|
102
|
+
entity_type: str,
|
|
103
|
+
entity_name: str,
|
|
104
|
+
**kwargs,
|
|
105
|
+
) -> dict:
|
|
106
|
+
"""
|
|
107
|
+
Read object from backend.
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
client : Client
|
|
112
|
+
Client instance.
|
|
113
|
+
entity_type : str
|
|
114
|
+
Entity type.
|
|
115
|
+
entity_name : str
|
|
116
|
+
Entity name.
|
|
117
|
+
**kwargs : dict
|
|
118
|
+
Parameters to pass to the API call.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
dict
|
|
123
|
+
Object instance.
|
|
124
|
+
"""
|
|
125
|
+
api = client.build_api(
|
|
126
|
+
ApiCategories.BASE.value,
|
|
127
|
+
BackendOperations.READ.value,
|
|
128
|
+
entity_type=entity_type,
|
|
129
|
+
entity_name=entity_name,
|
|
130
|
+
)
|
|
131
|
+
return client.read_object(api, **kwargs)
|
|
132
|
+
|
|
133
|
+
def read_project_entity(
|
|
134
|
+
self,
|
|
135
|
+
entity_type: str,
|
|
136
|
+
entity_name: str,
|
|
137
|
+
**kwargs,
|
|
138
|
+
) -> ProjectEntity:
|
|
139
|
+
"""
|
|
140
|
+
Read object from backend.
|
|
141
|
+
|
|
142
|
+
Parameters
|
|
143
|
+
----------
|
|
144
|
+
entity_type : str
|
|
145
|
+
Entity type.
|
|
146
|
+
entity_name : str
|
|
147
|
+
Entity name.
|
|
148
|
+
**kwargs : dict
|
|
149
|
+
Parameters to pass to entity builder.
|
|
150
|
+
|
|
151
|
+
Returns
|
|
152
|
+
-------
|
|
153
|
+
ProjectEntity
|
|
154
|
+
Object instance.
|
|
155
|
+
"""
|
|
156
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
157
|
+
obj = self._read_base_entity(client, entity_type, entity_name, **kwargs)
|
|
158
|
+
obj["local"] = client.is_local()
|
|
159
|
+
return build_entity_from_dict(obj)
|
|
160
|
+
|
|
161
|
+
def import_project_entity(
|
|
162
|
+
self,
|
|
163
|
+
file: str,
|
|
164
|
+
**kwargs,
|
|
165
|
+
) -> ProjectEntity:
|
|
166
|
+
"""
|
|
167
|
+
Import object from a YAML file and create a new object into the backend.
|
|
168
|
+
|
|
169
|
+
Parameters
|
|
170
|
+
----------
|
|
171
|
+
file : str
|
|
172
|
+
Path to YAML file.
|
|
173
|
+
**kwargs : dict
|
|
174
|
+
Additional keyword arguments.
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
ProjectEntity
|
|
179
|
+
Object instance.
|
|
180
|
+
"""
|
|
181
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
182
|
+
obj: dict = read_yaml(file)
|
|
183
|
+
obj["status"] = {}
|
|
184
|
+
obj["local"] = client.is_local()
|
|
185
|
+
ent: ProjectEntity = build_entity_from_dict(obj)
|
|
186
|
+
|
|
187
|
+
try:
|
|
188
|
+
self._create_base_entity(ent._client, ent.ENTITY_TYPE, ent.to_dict())
|
|
189
|
+
except EntityAlreadyExistsError:
|
|
190
|
+
raise EntityError(f"Entity {ent.name} already exists. If you want to update it, use load instead.")
|
|
191
|
+
|
|
192
|
+
# Import related entities
|
|
193
|
+
ent._import_entities(obj)
|
|
194
|
+
ent.refresh()
|
|
195
|
+
return ent
|
|
196
|
+
|
|
197
|
+
def load_project_entity(
|
|
198
|
+
self,
|
|
199
|
+
file: str,
|
|
200
|
+
**kwargs,
|
|
201
|
+
) -> ProjectEntity:
|
|
202
|
+
"""
|
|
203
|
+
Load object from a YAML file and update an existing object into the backend.
|
|
204
|
+
|
|
205
|
+
Parameters
|
|
206
|
+
----------
|
|
207
|
+
file : str
|
|
208
|
+
Path to YAML file.
|
|
209
|
+
**kwargs : dict
|
|
210
|
+
Additional keyword arguments.
|
|
211
|
+
|
|
212
|
+
Returns
|
|
213
|
+
-------
|
|
214
|
+
ProjectEntity
|
|
215
|
+
Object instance.
|
|
216
|
+
"""
|
|
217
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
218
|
+
obj: dict = read_yaml(file)
|
|
219
|
+
obj["local"] = client.is_local()
|
|
220
|
+
ent: ProjectEntity = build_entity_from_dict(obj)
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
self._update_base_entity(ent._client, ent.ENTITY_TYPE, ent.name, ent.to_dict())
|
|
224
|
+
except EntityNotExistsError:
|
|
225
|
+
self._create_base_entity(ent._client, ent.ENTITY_TYPE, ent.to_dict())
|
|
226
|
+
|
|
227
|
+
# Load related entities
|
|
228
|
+
ent._load_entities(obj)
|
|
229
|
+
ent.refresh()
|
|
230
|
+
return ent
|
|
231
|
+
|
|
232
|
+
def _list_base_entities(
|
|
233
|
+
self,
|
|
234
|
+
client: Client,
|
|
235
|
+
entity_type: str,
|
|
236
|
+
**kwargs,
|
|
237
|
+
) -> list[dict]:
|
|
238
|
+
"""
|
|
239
|
+
List objects from backend.
|
|
240
|
+
|
|
241
|
+
Parameters
|
|
242
|
+
----------
|
|
243
|
+
client : Client
|
|
244
|
+
Client instance.
|
|
245
|
+
entity_type : str
|
|
246
|
+
Entity type.
|
|
247
|
+
**kwargs : dict
|
|
248
|
+
Parameters to pass to the API call.
|
|
249
|
+
|
|
250
|
+
Returns
|
|
251
|
+
-------
|
|
252
|
+
list[dict]
|
|
253
|
+
List of objects.
|
|
254
|
+
"""
|
|
255
|
+
api = client.build_api(
|
|
256
|
+
ApiCategories.BASE.value,
|
|
257
|
+
BackendOperations.LIST.value,
|
|
258
|
+
entity_type=entity_type,
|
|
259
|
+
)
|
|
260
|
+
return client.list_objects(api, **kwargs)
|
|
261
|
+
|
|
262
|
+
def list_project_entities(
|
|
263
|
+
self,
|
|
264
|
+
entity_type: str,
|
|
265
|
+
**kwargs,
|
|
266
|
+
) -> list[ProjectEntity]:
|
|
267
|
+
"""
|
|
268
|
+
List objects from backend.
|
|
269
|
+
|
|
270
|
+
Parameters
|
|
271
|
+
----------
|
|
272
|
+
entity_type : str
|
|
273
|
+
Entity type.
|
|
274
|
+
**kwargs : dict
|
|
275
|
+
Parameters to pass to API call.
|
|
276
|
+
|
|
277
|
+
Returns
|
|
278
|
+
-------
|
|
279
|
+
list[ProjectEntity]
|
|
280
|
+
List of objects.
|
|
281
|
+
"""
|
|
282
|
+
client = get_client(kwargs.pop("local", False))
|
|
283
|
+
objs = self._list_base_entities(client, entity_type, **kwargs)
|
|
284
|
+
entities = []
|
|
285
|
+
for obj in objs:
|
|
286
|
+
obj["local"] = client.is_local()
|
|
287
|
+
ent = build_entity_from_dict(obj)
|
|
288
|
+
entities.append(ent)
|
|
289
|
+
return entities
|
|
290
|
+
|
|
291
|
+
def _update_base_entity(
|
|
292
|
+
self,
|
|
293
|
+
client: Client,
|
|
294
|
+
entity_type: str,
|
|
295
|
+
entity_name: str,
|
|
296
|
+
entity_dict: dict,
|
|
297
|
+
**kwargs,
|
|
298
|
+
) -> dict:
|
|
299
|
+
"""
|
|
300
|
+
Update object method.
|
|
301
|
+
|
|
302
|
+
Parameters
|
|
303
|
+
----------
|
|
304
|
+
client : Client
|
|
305
|
+
Client instance.
|
|
306
|
+
entity_type : str
|
|
307
|
+
Entity type.
|
|
308
|
+
entity_name : str
|
|
309
|
+
Entity name.
|
|
310
|
+
entity_dict : dict
|
|
311
|
+
Object instance.
|
|
312
|
+
**kwargs : dict
|
|
313
|
+
Parameters to pass to the API call.
|
|
314
|
+
|
|
315
|
+
Returns
|
|
316
|
+
-------
|
|
317
|
+
dict
|
|
318
|
+
Object instance.
|
|
319
|
+
"""
|
|
320
|
+
api = client.build_api(
|
|
321
|
+
ApiCategories.BASE.value,
|
|
322
|
+
BackendOperations.UPDATE.value,
|
|
323
|
+
entity_type=entity_type,
|
|
324
|
+
entity_name=entity_name,
|
|
325
|
+
)
|
|
326
|
+
return client.update_object(api, entity_dict, **kwargs)
|
|
327
|
+
|
|
328
|
+
def update_project_entity(
|
|
329
|
+
self,
|
|
330
|
+
entity_type: str,
|
|
331
|
+
entity_name: str,
|
|
332
|
+
entity_dict: dict,
|
|
333
|
+
**kwargs,
|
|
334
|
+
) -> ProjectEntity:
|
|
335
|
+
"""
|
|
336
|
+
Update object method.
|
|
337
|
+
|
|
338
|
+
Parameters
|
|
339
|
+
----------
|
|
340
|
+
entity_type : str
|
|
341
|
+
Entity type.
|
|
342
|
+
entity_name : str
|
|
343
|
+
Entity name.
|
|
344
|
+
entity_dict : dict
|
|
345
|
+
Object instance.
|
|
346
|
+
**kwargs : dict
|
|
347
|
+
Parameters to pass to entity builder.
|
|
348
|
+
|
|
349
|
+
Returns
|
|
350
|
+
-------
|
|
351
|
+
ProjectEntity
|
|
352
|
+
Object instance.
|
|
353
|
+
"""
|
|
354
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
355
|
+
obj = self._update_base_entity(client, entity_type, entity_name, entity_dict, **kwargs)
|
|
356
|
+
obj["local"] = client.is_local()
|
|
357
|
+
return build_entity_from_dict(obj)
|
|
358
|
+
|
|
359
|
+
def _delete_base_entity(
|
|
360
|
+
self,
|
|
361
|
+
client: Client,
|
|
362
|
+
entity_type: str,
|
|
363
|
+
entity_name: str,
|
|
364
|
+
**kwargs,
|
|
365
|
+
) -> dict:
|
|
366
|
+
"""
|
|
367
|
+
Delete object method.
|
|
368
|
+
|
|
369
|
+
Parameters
|
|
370
|
+
----------
|
|
371
|
+
client : Client
|
|
372
|
+
Client instance.
|
|
373
|
+
entity_type : str
|
|
374
|
+
Entity type.
|
|
375
|
+
entity_name : str
|
|
376
|
+
Entity name.
|
|
377
|
+
**kwargs : dict
|
|
378
|
+
Parameters to pass to the API call.
|
|
379
|
+
|
|
380
|
+
Returns
|
|
381
|
+
-------
|
|
382
|
+
dict
|
|
383
|
+
Response from backend.
|
|
384
|
+
"""
|
|
385
|
+
api = client.build_api(
|
|
386
|
+
ApiCategories.BASE.value,
|
|
387
|
+
BackendOperations.DELETE.value,
|
|
388
|
+
entity_type=entity_type,
|
|
389
|
+
entity_name=entity_name,
|
|
390
|
+
)
|
|
391
|
+
return client.delete_object(api, **kwargs)
|
|
392
|
+
|
|
393
|
+
def delete_project_entity(
|
|
394
|
+
self,
|
|
395
|
+
entity_type: str,
|
|
396
|
+
entity_name: str,
|
|
397
|
+
**kwargs,
|
|
398
|
+
) -> dict:
|
|
399
|
+
"""
|
|
400
|
+
Delete object method.
|
|
401
|
+
|
|
402
|
+
Parameters
|
|
403
|
+
----------
|
|
404
|
+
entity_type : str
|
|
405
|
+
Entity type.
|
|
406
|
+
entity_name : str
|
|
407
|
+
Entity name.
|
|
408
|
+
**kwargs : dict
|
|
409
|
+
Parameters to pass to entity builder.
|
|
410
|
+
|
|
411
|
+
Returns
|
|
412
|
+
-------
|
|
413
|
+
dict
|
|
414
|
+
Response from backend.
|
|
415
|
+
"""
|
|
416
|
+
kwargs = self._set_params(**kwargs)
|
|
417
|
+
if cascade := kwargs.pop("cascade", None) is not None:
|
|
418
|
+
kwargs["params"]["cascade"] = str(cascade).lower()
|
|
419
|
+
if kwargs.pop("clean_context", True):
|
|
420
|
+
delete_context(entity_name)
|
|
421
|
+
|
|
422
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
423
|
+
return self._delete_base_entity(
|
|
424
|
+
client,
|
|
425
|
+
entity_type,
|
|
426
|
+
entity_name,
|
|
427
|
+
**kwargs,
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
##############################
|
|
431
|
+
# Base entity operations
|
|
432
|
+
##############################
|
|
433
|
+
|
|
434
|
+
def _build_base_entity_key(
|
|
435
|
+
self,
|
|
436
|
+
client: Client,
|
|
437
|
+
entity_id: str,
|
|
438
|
+
) -> str:
|
|
439
|
+
"""
|
|
440
|
+
Build object key.
|
|
441
|
+
|
|
442
|
+
Parameters
|
|
443
|
+
----------
|
|
444
|
+
client : Client
|
|
445
|
+
Client instance.
|
|
446
|
+
entity_id : str
|
|
447
|
+
Entity ID.
|
|
448
|
+
|
|
449
|
+
Returns
|
|
450
|
+
-------
|
|
451
|
+
str
|
|
452
|
+
Object key.
|
|
453
|
+
"""
|
|
454
|
+
return client.build_key(ApiCategories.BASE.value, entity_id)
|
|
455
|
+
|
|
456
|
+
def build_project_key(
|
|
457
|
+
self,
|
|
458
|
+
entity_id: str,
|
|
459
|
+
**kwargs,
|
|
460
|
+
) -> str:
|
|
461
|
+
"""
|
|
462
|
+
Build object key.
|
|
463
|
+
|
|
464
|
+
Parameters
|
|
465
|
+
----------
|
|
466
|
+
entity_id : str
|
|
467
|
+
Entity ID.
|
|
468
|
+
**kwargs : dict
|
|
469
|
+
Parameters to pass to entity builder.
|
|
470
|
+
|
|
471
|
+
Returns
|
|
472
|
+
-------
|
|
473
|
+
str
|
|
474
|
+
Object key.
|
|
475
|
+
"""
|
|
476
|
+
client = get_client(kwargs.pop("local", False))
|
|
477
|
+
return self._build_base_entity_key(client, entity_id)
|
|
478
|
+
|
|
479
|
+
def share_project_entity(
|
|
480
|
+
self,
|
|
481
|
+
entity_type: str,
|
|
482
|
+
entity_name: str,
|
|
483
|
+
**kwargs,
|
|
484
|
+
) -> None:
|
|
485
|
+
"""
|
|
486
|
+
Share object method.
|
|
487
|
+
|
|
488
|
+
Parameters
|
|
489
|
+
----------
|
|
490
|
+
entity_type : str
|
|
491
|
+
Entity type.
|
|
492
|
+
entity_name : str
|
|
493
|
+
Entity name.
|
|
494
|
+
**kwargs : dict
|
|
495
|
+
Parameters to pass to entity builder.
|
|
496
|
+
|
|
497
|
+
Returns
|
|
498
|
+
-------
|
|
499
|
+
None
|
|
500
|
+
"""
|
|
501
|
+
client = get_client(kwargs.pop("local", False), kwargs.pop("config", None))
|
|
502
|
+
api = client.build_api(
|
|
503
|
+
ApiCategories.BASE.value,
|
|
504
|
+
BackendOperations.SHARE.value,
|
|
505
|
+
entity_type=entity_type,
|
|
506
|
+
entity_name=entity_name,
|
|
507
|
+
)
|
|
508
|
+
user = kwargs.pop("user")
|
|
509
|
+
unshare = kwargs.pop("unshare", False)
|
|
510
|
+
kwargs = self._set_params(**kwargs)
|
|
511
|
+
|
|
512
|
+
# Unshare
|
|
513
|
+
if unshare:
|
|
514
|
+
users = client.read_object(api, **kwargs)
|
|
515
|
+
for u in users:
|
|
516
|
+
if u["user"] == user:
|
|
517
|
+
kwargs["params"]["id"] = u["id"]
|
|
518
|
+
client.delete_object(api, **kwargs)
|
|
519
|
+
break
|
|
520
|
+
return
|
|
521
|
+
|
|
522
|
+
# Share
|
|
523
|
+
kwargs["params"]["user"] = user
|
|
524
|
+
client.create_object(api, obj={}, **kwargs)
|
|
525
|
+
|
|
526
|
+
##############################
|
|
527
|
+
# CRUD context entity
|
|
528
|
+
##############################
|
|
529
|
+
|
|
530
|
+
def _create_context_entity(
|
|
531
|
+
self,
|
|
532
|
+
context: Context,
|
|
533
|
+
entity_type: str,
|
|
534
|
+
entity_dict: dict,
|
|
535
|
+
) -> dict:
|
|
536
|
+
"""
|
|
537
|
+
Create object in backend.
|
|
538
|
+
|
|
539
|
+
Parameters
|
|
540
|
+
----------
|
|
541
|
+
context : Context
|
|
542
|
+
Context instance.
|
|
543
|
+
project : str
|
|
544
|
+
Project name.
|
|
545
|
+
entity_type : str
|
|
546
|
+
Entity type.
|
|
547
|
+
entity_dict : dict
|
|
548
|
+
Object instance.
|
|
549
|
+
|
|
550
|
+
Returns
|
|
551
|
+
-------
|
|
552
|
+
dict
|
|
553
|
+
Object instance.
|
|
554
|
+
"""
|
|
555
|
+
api = context.client.build_api(
|
|
556
|
+
ApiCategories.CONTEXT.value,
|
|
557
|
+
BackendOperations.CREATE.value,
|
|
558
|
+
project=context.name,
|
|
559
|
+
entity_type=entity_type,
|
|
560
|
+
)
|
|
561
|
+
return context.client.create_object(api, entity_dict)
|
|
562
|
+
|
|
563
|
+
def create_context_entity(
|
|
564
|
+
self,
|
|
565
|
+
_entity: ContextEntity | None = None,
|
|
566
|
+
**kwargs,
|
|
567
|
+
) -> dict:
|
|
568
|
+
"""
|
|
569
|
+
Create object in backend.
|
|
570
|
+
|
|
571
|
+
Parameters
|
|
572
|
+
----------
|
|
573
|
+
**kwargs : dict
|
|
574
|
+
Parameters to pass to entity builder.
|
|
575
|
+
|
|
576
|
+
Returns
|
|
577
|
+
-------
|
|
578
|
+
dict
|
|
579
|
+
Object instance.
|
|
580
|
+
"""
|
|
581
|
+
if _entity is not None:
|
|
582
|
+
context = _entity._context()
|
|
583
|
+
obj = _entity
|
|
584
|
+
else:
|
|
585
|
+
context = self._get_context(kwargs["project"])
|
|
586
|
+
obj: ContextEntity = build_entity_from_params(**kwargs)
|
|
587
|
+
new_obj = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
588
|
+
return build_entity_from_dict(new_obj)
|
|
589
|
+
|
|
590
|
+
def log_material_entity(
|
|
591
|
+
self,
|
|
592
|
+
**kwargs,
|
|
593
|
+
) -> MaterialEntity:
|
|
594
|
+
"""
|
|
595
|
+
Create object in backend and upload file.
|
|
596
|
+
|
|
597
|
+
Parameters
|
|
598
|
+
----------
|
|
599
|
+
**kwargs : dict
|
|
600
|
+
Parameters to pass to entity builder.
|
|
601
|
+
|
|
602
|
+
Returns
|
|
603
|
+
-------
|
|
604
|
+
MaterialEntity
|
|
605
|
+
Object instance.
|
|
606
|
+
"""
|
|
607
|
+
source = kwargs.pop("source")
|
|
608
|
+
context = self._get_context(kwargs["project"])
|
|
609
|
+
obj = build_entity_from_params(**kwargs)
|
|
610
|
+
if context.is_running:
|
|
611
|
+
obj.add_relationship(Relationship.PRODUCEDBY.value, context.get_run_ctx())
|
|
612
|
+
|
|
613
|
+
new_obj: MaterialEntity = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
614
|
+
new_obj = build_entity_from_dict(new_obj)
|
|
615
|
+
new_obj.upload(source)
|
|
616
|
+
return new_obj
|
|
617
|
+
|
|
618
|
+
def _read_context_entity(
|
|
619
|
+
self,
|
|
620
|
+
context: Context,
|
|
621
|
+
identifier: str,
|
|
622
|
+
entity_type: str | None = None,
|
|
623
|
+
project: str | None = None,
|
|
624
|
+
entity_id: str | None = None,
|
|
625
|
+
**kwargs,
|
|
626
|
+
) -> dict:
|
|
627
|
+
"""
|
|
628
|
+
Read object from backend.
|
|
629
|
+
|
|
630
|
+
Parameters
|
|
631
|
+
----------
|
|
632
|
+
context : Context
|
|
633
|
+
Context instance.
|
|
634
|
+
identifier : str
|
|
635
|
+
Entity key (store://...) or entity name.
|
|
636
|
+
entity_type : str
|
|
637
|
+
Entity type.
|
|
638
|
+
project : str
|
|
639
|
+
Project name.
|
|
640
|
+
entity_id : str
|
|
641
|
+
Entity ID.
|
|
642
|
+
**kwargs : dict
|
|
643
|
+
Parameters to pass to the API call.
|
|
644
|
+
|
|
645
|
+
Returns
|
|
646
|
+
-------
|
|
647
|
+
dict
|
|
648
|
+
Object instance.
|
|
649
|
+
"""
|
|
650
|
+
if not identifier.startswith("store://"):
|
|
651
|
+
if project is None or entity_type is None:
|
|
652
|
+
raise ValueError("Project and entity type must be specified.")
|
|
653
|
+
entity_name = identifier
|
|
654
|
+
else:
|
|
655
|
+
project, entity_type, _, entity_name, entity_id = parse_entity_key(identifier)
|
|
656
|
+
|
|
657
|
+
kwargs = self._set_params(**kwargs)
|
|
658
|
+
|
|
659
|
+
if entity_id is None:
|
|
660
|
+
kwargs["params"]["name"] = entity_name
|
|
661
|
+
api = context.client.build_api(
|
|
662
|
+
ApiCategories.CONTEXT.value,
|
|
663
|
+
BackendOperations.LIST.value,
|
|
664
|
+
project=context.name,
|
|
665
|
+
entity_type=entity_type,
|
|
666
|
+
)
|
|
667
|
+
return context.client.list_first_object(api, **kwargs)
|
|
668
|
+
|
|
669
|
+
api = context.client.build_api(
|
|
670
|
+
ApiCategories.CONTEXT.value,
|
|
671
|
+
BackendOperations.READ.value,
|
|
672
|
+
project=context.name,
|
|
673
|
+
entity_type=entity_type,
|
|
674
|
+
entity_id=entity_id,
|
|
675
|
+
)
|
|
676
|
+
return context.client.read_object(api, **kwargs)
|
|
677
|
+
|
|
678
|
+
def read_context_entity(
|
|
679
|
+
self,
|
|
680
|
+
identifier: str,
|
|
681
|
+
entity_type: str | None = None,
|
|
682
|
+
project: str | None = None,
|
|
683
|
+
entity_id: str | None = None,
|
|
684
|
+
**kwargs,
|
|
685
|
+
) -> ContextEntity:
|
|
686
|
+
"""
|
|
687
|
+
Read object from backend.
|
|
688
|
+
|
|
689
|
+
Parameters
|
|
690
|
+
----------
|
|
691
|
+
identifier : str
|
|
692
|
+
Entity key (store://...) or entity name.
|
|
693
|
+
entity_type : str
|
|
694
|
+
Entity type.
|
|
695
|
+
project : str
|
|
696
|
+
Project name.
|
|
697
|
+
entity_id : str
|
|
698
|
+
Entity ID.
|
|
699
|
+
**kwargs : dict
|
|
700
|
+
Parameters to pass to the API call.
|
|
701
|
+
|
|
702
|
+
Returns
|
|
703
|
+
-------
|
|
704
|
+
VersionedEntity
|
|
705
|
+
Object instance.
|
|
706
|
+
"""
|
|
707
|
+
context = self._get_context_from_identifier(identifier, project)
|
|
708
|
+
obj = self._read_context_entity(
|
|
709
|
+
context,
|
|
710
|
+
identifier,
|
|
711
|
+
entity_type=entity_type,
|
|
712
|
+
project=project,
|
|
713
|
+
entity_id=entity_id,
|
|
714
|
+
**kwargs,
|
|
715
|
+
)
|
|
716
|
+
return build_entity_from_dict(obj)
|
|
717
|
+
|
|
718
|
+
def read_material_entity(
|
|
719
|
+
self,
|
|
720
|
+
identifier: str,
|
|
721
|
+
entity_type: str | None = None,
|
|
722
|
+
project: str | None = None,
|
|
723
|
+
entity_id: str | None = None,
|
|
724
|
+
**kwargs,
|
|
725
|
+
) -> MaterialEntity:
|
|
726
|
+
"""
|
|
727
|
+
Read object from backend.
|
|
728
|
+
|
|
729
|
+
Parameters
|
|
730
|
+
----------
|
|
731
|
+
identifier : str
|
|
732
|
+
Entity key (store://...) or entity name.
|
|
733
|
+
entity_type : str
|
|
734
|
+
Entity type.
|
|
735
|
+
project : str
|
|
736
|
+
Project name.
|
|
737
|
+
entity_id : str
|
|
738
|
+
Entity ID.
|
|
739
|
+
**kwargs : dict
|
|
740
|
+
Parameters to pass to the API call.
|
|
741
|
+
|
|
742
|
+
Returns
|
|
743
|
+
-------
|
|
744
|
+
MaterialEntity
|
|
745
|
+
Object instance.
|
|
746
|
+
"""
|
|
747
|
+
obj: MaterialEntity = self.read_context_entity(
|
|
748
|
+
identifier,
|
|
749
|
+
entity_type=entity_type,
|
|
750
|
+
project=project,
|
|
751
|
+
entity_id=entity_id,
|
|
752
|
+
**kwargs,
|
|
753
|
+
)
|
|
754
|
+
obj._get_files_info()
|
|
755
|
+
return obj
|
|
756
|
+
|
|
757
|
+
def read_unversioned_entity(
|
|
758
|
+
self,
|
|
759
|
+
identifier: str,
|
|
760
|
+
entity_type: str | None = None,
|
|
761
|
+
project: str | None = None,
|
|
762
|
+
entity_id: str | None = None,
|
|
763
|
+
**kwargs,
|
|
764
|
+
) -> UnversionedEntity:
|
|
765
|
+
"""
|
|
766
|
+
Read object from backend.
|
|
767
|
+
|
|
768
|
+
Parameters
|
|
769
|
+
----------
|
|
770
|
+
identifier : str
|
|
771
|
+
Entity key (store://...) or entity name.
|
|
772
|
+
entity_type : str
|
|
773
|
+
Entity type.
|
|
774
|
+
project : str
|
|
775
|
+
Project name.
|
|
776
|
+
entity_id : str
|
|
777
|
+
Entity ID.
|
|
778
|
+
**kwargs : dict
|
|
779
|
+
Parameters to pass to the API call.
|
|
780
|
+
|
|
781
|
+
Returns
|
|
782
|
+
-------
|
|
783
|
+
UnversionedEntity
|
|
784
|
+
Object instance.
|
|
785
|
+
"""
|
|
786
|
+
if not identifier.startswith("store://"):
|
|
787
|
+
entity_id = identifier
|
|
788
|
+
else:
|
|
789
|
+
splt = identifier.split(":")
|
|
790
|
+
if len(splt) == 3:
|
|
791
|
+
identifier = f"{splt[0]}:{splt[1]}"
|
|
792
|
+
return self.read_context_entity(
|
|
793
|
+
identifier,
|
|
794
|
+
entity_type=entity_type,
|
|
795
|
+
project=project,
|
|
796
|
+
entity_id=entity_id,
|
|
797
|
+
**kwargs,
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
def import_context_entity(
|
|
801
|
+
self,
|
|
802
|
+
file: str,
|
|
803
|
+
) -> ContextEntity:
|
|
804
|
+
"""
|
|
805
|
+
Import object from a YAML file and create a new object into the backend.
|
|
806
|
+
|
|
807
|
+
Parameters
|
|
808
|
+
----------
|
|
809
|
+
file : str
|
|
810
|
+
Path to YAML file.
|
|
811
|
+
|
|
812
|
+
Returns
|
|
813
|
+
-------
|
|
814
|
+
ContextEntity
|
|
815
|
+
Object instance.
|
|
816
|
+
"""
|
|
817
|
+
dict_obj: dict = read_yaml(file)
|
|
818
|
+
dict_obj["status"] = {}
|
|
819
|
+
context = self._get_context(dict_obj["project"])
|
|
820
|
+
obj = build_entity_from_dict(dict_obj)
|
|
821
|
+
try:
|
|
822
|
+
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
823
|
+
except EntityAlreadyExistsError:
|
|
824
|
+
raise EntityError(f"Entity {obj.name} already exists. If you want to update it, use load instead.")
|
|
825
|
+
return obj
|
|
826
|
+
|
|
827
|
+
def import_executable_entity(
|
|
828
|
+
self,
|
|
829
|
+
file: str,
|
|
830
|
+
) -> ExecutableEntity:
|
|
831
|
+
"""
|
|
832
|
+
Import object from a YAML file and create a new object into the backend.
|
|
833
|
+
|
|
834
|
+
Parameters
|
|
835
|
+
----------
|
|
836
|
+
file : str
|
|
837
|
+
Path to YAML file.
|
|
838
|
+
|
|
839
|
+
Returns
|
|
840
|
+
-------
|
|
841
|
+
ExecutableEntity
|
|
842
|
+
Object instance.
|
|
843
|
+
"""
|
|
844
|
+
dict_obj: dict | list[dict] = read_yaml(file)
|
|
845
|
+
if isinstance(dict_obj, list):
|
|
846
|
+
exec_dict = dict_obj[0]
|
|
847
|
+
exec_dict["status"] = {}
|
|
848
|
+
tsk_dicts = []
|
|
849
|
+
for i in dict_obj[1:]:
|
|
850
|
+
i["status"] = {}
|
|
851
|
+
tsk_dicts.append(i)
|
|
852
|
+
else:
|
|
853
|
+
exec_dict = dict_obj
|
|
854
|
+
tsk_dicts = []
|
|
855
|
+
|
|
856
|
+
context = self._get_context(exec_dict["project"])
|
|
857
|
+
obj: ExecutableEntity = build_entity_from_dict(exec_dict)
|
|
858
|
+
try:
|
|
859
|
+
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
860
|
+
except EntityAlreadyExistsError:
|
|
861
|
+
raise EntityError(f"Entity {obj.name} already exists. If you want to update it, use load instead.")
|
|
862
|
+
|
|
863
|
+
obj.import_tasks(tsk_dicts)
|
|
864
|
+
|
|
865
|
+
return obj
|
|
866
|
+
|
|
867
|
+
def load_context_entity(
|
|
868
|
+
self,
|
|
869
|
+
file: str,
|
|
870
|
+
) -> ContextEntity:
|
|
871
|
+
"""
|
|
872
|
+
Load object from a YAML file and update an existing object into the backend.
|
|
873
|
+
|
|
874
|
+
Parameters
|
|
875
|
+
----------
|
|
876
|
+
file : str
|
|
877
|
+
Path to YAML file.
|
|
878
|
+
|
|
879
|
+
Returns
|
|
880
|
+
-------
|
|
881
|
+
ContextEntity
|
|
882
|
+
Object instance.
|
|
883
|
+
"""
|
|
884
|
+
dict_obj: dict = read_yaml(file)
|
|
885
|
+
context = self._get_context(dict_obj["project"])
|
|
886
|
+
obj: ContextEntity = build_entity_from_dict(dict_obj)
|
|
887
|
+
try:
|
|
888
|
+
self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
|
|
889
|
+
except EntityNotExistsError:
|
|
890
|
+
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
891
|
+
return obj
|
|
892
|
+
|
|
893
|
+
def load_executable_entity(
|
|
894
|
+
self,
|
|
895
|
+
file: str,
|
|
896
|
+
) -> ExecutableEntity:
|
|
897
|
+
"""
|
|
898
|
+
Load object from a YAML file and update an existing object into the backend.
|
|
899
|
+
|
|
900
|
+
Parameters
|
|
901
|
+
----------
|
|
902
|
+
file : str
|
|
903
|
+
Path to YAML file.
|
|
904
|
+
|
|
905
|
+
Returns
|
|
906
|
+
-------
|
|
907
|
+
ExecutableEntity
|
|
908
|
+
Object instance.
|
|
909
|
+
"""
|
|
910
|
+
dict_obj: dict | list[dict] = read_yaml(file)
|
|
911
|
+
if isinstance(dict_obj, list):
|
|
912
|
+
exec_dict = dict_obj[0]
|
|
913
|
+
tsk_dicts = dict_obj[1:]
|
|
914
|
+
else:
|
|
915
|
+
exec_dict = dict_obj
|
|
916
|
+
tsk_dicts = []
|
|
917
|
+
|
|
918
|
+
context = self._get_context(exec_dict["project"])
|
|
919
|
+
obj: ExecutableEntity = build_entity_from_dict(exec_dict)
|
|
920
|
+
|
|
921
|
+
try:
|
|
922
|
+
self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
|
|
923
|
+
except EntityNotExistsError:
|
|
924
|
+
self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
|
|
925
|
+
obj.import_tasks(tsk_dicts)
|
|
926
|
+
return obj
|
|
927
|
+
|
|
928
|
+
def _read_context_entity_versions(
|
|
929
|
+
self,
|
|
930
|
+
context: Context,
|
|
931
|
+
identifier: str,
|
|
932
|
+
entity_type: str | None = None,
|
|
933
|
+
project: str | None = None,
|
|
934
|
+
**kwargs,
|
|
935
|
+
) -> list[dict]:
|
|
936
|
+
"""
|
|
937
|
+
Get all versions object from backend.
|
|
938
|
+
|
|
939
|
+
Parameters
|
|
940
|
+
----------
|
|
941
|
+
context : Context
|
|
942
|
+
Context instance.
|
|
943
|
+
identifier : str
|
|
944
|
+
Entity key (store://...) or entity name.
|
|
945
|
+
entity_type : str
|
|
946
|
+
Entity type.
|
|
947
|
+
project : str
|
|
948
|
+
Project name.
|
|
949
|
+
**kwargs : dict
|
|
950
|
+
Parameters to pass to the API call.
|
|
951
|
+
|
|
952
|
+
Returns
|
|
953
|
+
-------
|
|
954
|
+
list[dict]
|
|
955
|
+
Object instances.
|
|
956
|
+
"""
|
|
957
|
+
if not identifier.startswith("store://"):
|
|
958
|
+
if project is None or entity_type is None:
|
|
959
|
+
raise ValueError("Project and entity type must be specified.")
|
|
960
|
+
entity_name = identifier
|
|
961
|
+
else:
|
|
962
|
+
project, entity_type, _, entity_name, _ = parse_entity_key(identifier)
|
|
963
|
+
|
|
964
|
+
kwargs = self._set_params(**kwargs)
|
|
965
|
+
kwargs["params"]["name"] = entity_name
|
|
966
|
+
kwargs["params"]["versions"] = "all"
|
|
967
|
+
|
|
968
|
+
api = context.client.build_api(
|
|
969
|
+
ApiCategories.CONTEXT.value,
|
|
970
|
+
BackendOperations.LIST.value,
|
|
971
|
+
project=context.name,
|
|
972
|
+
entity_type=entity_type,
|
|
973
|
+
)
|
|
974
|
+
return context.client.list_objects(api, **kwargs)
|
|
975
|
+
|
|
976
|
+
def read_context_entity_versions(
|
|
977
|
+
self,
|
|
978
|
+
identifier: str,
|
|
979
|
+
entity_type: str | None = None,
|
|
980
|
+
project: str | None = None,
|
|
981
|
+
**kwargs,
|
|
982
|
+
) -> list[ContextEntity]:
|
|
983
|
+
"""
|
|
984
|
+
Read object versions from backend.
|
|
985
|
+
|
|
986
|
+
Parameters
|
|
987
|
+
----------
|
|
988
|
+
identifier : str
|
|
989
|
+
Entity key (store://...) or entity name.
|
|
990
|
+
entity_type : str
|
|
991
|
+
Entity type.
|
|
992
|
+
project : str
|
|
993
|
+
Project name.
|
|
994
|
+
**kwargs : dict
|
|
995
|
+
Parameters to pass to the API call.
|
|
996
|
+
|
|
997
|
+
Returns
|
|
998
|
+
-------
|
|
999
|
+
list[ContextEntity]
|
|
1000
|
+
List of object instances.
|
|
1001
|
+
"""
|
|
1002
|
+
context = self._get_context_from_identifier(identifier, project)
|
|
1003
|
+
obj = self._read_context_entity_versions(
|
|
1004
|
+
context,
|
|
1005
|
+
identifier,
|
|
1006
|
+
entity_type=entity_type,
|
|
1007
|
+
project=project,
|
|
1008
|
+
**kwargs,
|
|
1009
|
+
)
|
|
1010
|
+
return [build_entity_from_dict(o) for o in obj]
|
|
1011
|
+
|
|
1012
|
+
def read_material_entity_versions(
|
|
1013
|
+
self,
|
|
1014
|
+
identifier: str,
|
|
1015
|
+
entity_type: str | None = None,
|
|
1016
|
+
project: str | None = None,
|
|
1017
|
+
**kwargs,
|
|
1018
|
+
) -> list[MaterialEntity]:
|
|
1019
|
+
"""
|
|
1020
|
+
Read object versions from backend.
|
|
1021
|
+
|
|
1022
|
+
Parameters
|
|
1023
|
+
----------
|
|
1024
|
+
identifier : str
|
|
1025
|
+
Entity key (store://...) or entity name.
|
|
1026
|
+
entity_type : str
|
|
1027
|
+
Entity type.
|
|
1028
|
+
project : str
|
|
1029
|
+
Project name.
|
|
1030
|
+
**kwargs : dict
|
|
1031
|
+
Parameters to pass to the API call.
|
|
1032
|
+
|
|
1033
|
+
Returns
|
|
1034
|
+
-------
|
|
1035
|
+
list[MaterialEntity]
|
|
1036
|
+
List of object instances.
|
|
1037
|
+
"""
|
|
1038
|
+
context = self._get_context_from_identifier(identifier, project)
|
|
1039
|
+
objs = self._read_context_entity_versions(
|
|
1040
|
+
context,
|
|
1041
|
+
identifier,
|
|
1042
|
+
entity_type=entity_type,
|
|
1043
|
+
project=project,
|
|
1044
|
+
**kwargs,
|
|
1045
|
+
)
|
|
1046
|
+
objects = []
|
|
1047
|
+
for o in objs:
|
|
1048
|
+
entity: MaterialEntity = build_entity_from_dict(o)
|
|
1049
|
+
entity._get_files_info()
|
|
1050
|
+
objects.append(entity)
|
|
1051
|
+
return objects
|
|
1052
|
+
|
|
1053
|
+
def _list_context_entities(
|
|
1054
|
+
self,
|
|
1055
|
+
context: Context,
|
|
1056
|
+
entity_type: str,
|
|
1057
|
+
**kwargs,
|
|
1058
|
+
) -> list[dict]:
|
|
1059
|
+
"""
|
|
1060
|
+
List objects from backend.
|
|
1061
|
+
|
|
1062
|
+
Parameters
|
|
1063
|
+
----------
|
|
1064
|
+
context : Context
|
|
1065
|
+
Context instance.
|
|
1066
|
+
entity_type : str
|
|
1067
|
+
Entity type.
|
|
1068
|
+
**kwargs : dict
|
|
1069
|
+
Parameters to pass to the API call.
|
|
1070
|
+
|
|
1071
|
+
Returns
|
|
1072
|
+
-------
|
|
1073
|
+
list[dict]
|
|
1074
|
+
List of objects.
|
|
1075
|
+
"""
|
|
1076
|
+
api = context.client.build_api(
|
|
1077
|
+
ApiCategories.CONTEXT.value,
|
|
1078
|
+
BackendOperations.LIST.value,
|
|
1079
|
+
project=context.name,
|
|
1080
|
+
entity_type=entity_type,
|
|
1081
|
+
)
|
|
1082
|
+
return context.client.list_objects(api, **kwargs)
|
|
1083
|
+
|
|
1084
|
+
def list_context_entities(
|
|
1085
|
+
self,
|
|
1086
|
+
project: str,
|
|
1087
|
+
entity_type: str,
|
|
1088
|
+
**kwargs,
|
|
1089
|
+
) -> list[ContextEntity]:
|
|
1090
|
+
"""
|
|
1091
|
+
List all latest version objects from backend.
|
|
1092
|
+
|
|
1093
|
+
Parameters
|
|
1094
|
+
----------
|
|
1095
|
+
project : str
|
|
1096
|
+
Project name.
|
|
1097
|
+
entity_type : str
|
|
1098
|
+
Entity type.
|
|
1099
|
+
**kwargs : dict
|
|
1100
|
+
Parameters to pass to the API call.
|
|
1101
|
+
|
|
1102
|
+
Returns
|
|
1103
|
+
-------
|
|
1104
|
+
list[ContextEntity]
|
|
1105
|
+
List of object instances.
|
|
1106
|
+
"""
|
|
1107
|
+
context = self._get_context(project)
|
|
1108
|
+
objs = self._list_context_entities(context, entity_type, **kwargs)
|
|
1109
|
+
return [build_entity_from_dict(obj) for obj in objs]
|
|
1110
|
+
|
|
1111
|
+
def list_material_entities(
|
|
1112
|
+
self,
|
|
1113
|
+
project: str,
|
|
1114
|
+
entity_type: str,
|
|
1115
|
+
**kwargs,
|
|
1116
|
+
) -> list[MaterialEntity]:
|
|
1117
|
+
"""
|
|
1118
|
+
List all latest version objects from backend.
|
|
1119
|
+
|
|
1120
|
+
Parameters
|
|
1121
|
+
----------
|
|
1122
|
+
project : str
|
|
1123
|
+
Project name.
|
|
1124
|
+
entity_type : str
|
|
1125
|
+
Entity type.
|
|
1126
|
+
**kwargs : dict
|
|
1127
|
+
Parameters to pass to the API call.
|
|
1128
|
+
|
|
1129
|
+
Returns
|
|
1130
|
+
-------
|
|
1131
|
+
list[MaterialEntity]
|
|
1132
|
+
List of object instances.
|
|
1133
|
+
"""
|
|
1134
|
+
context = self._get_context(project)
|
|
1135
|
+
objs = self._list_context_entities(context, entity_type, **kwargs)
|
|
1136
|
+
objects = []
|
|
1137
|
+
for o in objs:
|
|
1138
|
+
entity: MaterialEntity = build_entity_from_dict(o)
|
|
1139
|
+
entity._get_files_info()
|
|
1140
|
+
objects.append(entity)
|
|
1141
|
+
return objects
|
|
1142
|
+
|
|
1143
|
+
def _update_context_entity(
|
|
1144
|
+
self,
|
|
1145
|
+
context: Context,
|
|
1146
|
+
entity_type: str,
|
|
1147
|
+
entity_id: str,
|
|
1148
|
+
entity_dict: dict,
|
|
1149
|
+
**kwargs,
|
|
1150
|
+
) -> dict:
|
|
1151
|
+
"""
|
|
1152
|
+
Update object. Note that object spec are immutable.
|
|
1153
|
+
|
|
1154
|
+
Parameters
|
|
1155
|
+
----------
|
|
1156
|
+
context : Context
|
|
1157
|
+
Context instance.
|
|
1158
|
+
entity_type : str
|
|
1159
|
+
Entity type.
|
|
1160
|
+
entity_id : str
|
|
1161
|
+
Entity ID.
|
|
1162
|
+
entity_dict : dict
|
|
1163
|
+
Entity dictionary.
|
|
1164
|
+
**kwargs : dict
|
|
1165
|
+
Parameters to pass to the API call.
|
|
1166
|
+
|
|
1167
|
+
Returns
|
|
1168
|
+
-------
|
|
1169
|
+
dict
|
|
1170
|
+
Response from backend.
|
|
1171
|
+
"""
|
|
1172
|
+
api = context.client.build_api(
|
|
1173
|
+
ApiCategories.CONTEXT.value,
|
|
1174
|
+
BackendOperations.UPDATE.value,
|
|
1175
|
+
project=context.name,
|
|
1176
|
+
entity_type=entity_type,
|
|
1177
|
+
entity_id=entity_id,
|
|
1178
|
+
)
|
|
1179
|
+
return context.client.update_object(api, entity_dict, **kwargs)
|
|
1180
|
+
|
|
1181
|
+
def update_context_entity(
|
|
1182
|
+
self,
|
|
1183
|
+
project: str,
|
|
1184
|
+
entity_type: str,
|
|
1185
|
+
entity_id: str,
|
|
1186
|
+
entity_dict: dict,
|
|
1187
|
+
**kwargs,
|
|
1188
|
+
) -> ContextEntity:
|
|
1189
|
+
"""
|
|
1190
|
+
Update object. Note that object spec are immutable.
|
|
1191
|
+
|
|
1192
|
+
Parameters
|
|
1193
|
+
----------
|
|
1194
|
+
project : str
|
|
1195
|
+
Project name.
|
|
1196
|
+
entity_type : str
|
|
1197
|
+
Entity type.
|
|
1198
|
+
entity_id : str
|
|
1199
|
+
Entity ID.
|
|
1200
|
+
entity_dict : dict
|
|
1201
|
+
Entity dictionary.
|
|
1202
|
+
**kwargs : dict
|
|
1203
|
+
Parameters to pass to the API call.
|
|
1204
|
+
|
|
1205
|
+
Returns
|
|
1206
|
+
-------
|
|
1207
|
+
ContextEntity
|
|
1208
|
+
Object instance.
|
|
1209
|
+
"""
|
|
1210
|
+
context = self._get_context(project)
|
|
1211
|
+
obj = self._update_context_entity(
|
|
1212
|
+
context,
|
|
1213
|
+
entity_type,
|
|
1214
|
+
entity_id,
|
|
1215
|
+
entity_dict,
|
|
1216
|
+
**kwargs,
|
|
1217
|
+
)
|
|
1218
|
+
return build_entity_from_dict(obj)
|
|
1219
|
+
|
|
1220
|
+
def _delete_context_entity(
|
|
1221
|
+
self,
|
|
1222
|
+
context: Context,
|
|
1223
|
+
identifier: str,
|
|
1224
|
+
entity_type: str | None = None,
|
|
1225
|
+
project: str | None = None,
|
|
1226
|
+
entity_id: str | None = None,
|
|
1227
|
+
**kwargs,
|
|
1228
|
+
) -> dict:
|
|
1229
|
+
"""
|
|
1230
|
+
Delete object from backend.
|
|
1231
|
+
|
|
1232
|
+
Parameters
|
|
1233
|
+
----------
|
|
1234
|
+
context : Context
|
|
1235
|
+
Context instance.
|
|
1236
|
+
identifier : str
|
|
1237
|
+
Entity key (store://...) or entity name.
|
|
1238
|
+
entity_type : str
|
|
1239
|
+
Entity type.
|
|
1240
|
+
project : str
|
|
1241
|
+
Project name.
|
|
1242
|
+
entity_id : str
|
|
1243
|
+
Entity ID.
|
|
1244
|
+
**kwargs : dict
|
|
1245
|
+
Parameters to pass to the API call.
|
|
1246
|
+
|
|
1247
|
+
Returns
|
|
1248
|
+
-------
|
|
1249
|
+
dict
|
|
1250
|
+
Response from backend.
|
|
1251
|
+
"""
|
|
1252
|
+
if not identifier.startswith("store://"):
|
|
1253
|
+
if project is None or entity_type is None:
|
|
1254
|
+
raise ValueError("Project must be provided.")
|
|
1255
|
+
entity_name = identifier
|
|
1256
|
+
else:
|
|
1257
|
+
project, _, _, entity_name, entity_id = parse_entity_key(identifier)
|
|
1258
|
+
|
|
1259
|
+
kwargs = self._set_params(**kwargs)
|
|
1260
|
+
if cascade := kwargs.pop("cascade", None) is not None:
|
|
1261
|
+
kwargs["params"]["cascade"] = str(cascade).lower()
|
|
1262
|
+
|
|
1263
|
+
delete_all_versions: bool = kwargs.pop("delete_all_versions", False)
|
|
1264
|
+
|
|
1265
|
+
if not delete_all_versions and entity_id is None:
|
|
1266
|
+
raise ValueError(
|
|
1267
|
+
"If `delete_all_versions` is False, `entity_id` must be provided, either as an argument or in key `identifier`."
|
|
1268
|
+
)
|
|
1269
|
+
|
|
1270
|
+
if delete_all_versions:
|
|
1271
|
+
api = context.client.build_api(
|
|
1272
|
+
ApiCategories.CONTEXT.value,
|
|
1273
|
+
BackendOperations.LIST.value,
|
|
1274
|
+
project=context.name,
|
|
1275
|
+
entity_type=entity_type,
|
|
1276
|
+
)
|
|
1277
|
+
kwargs["params"]["name"] = entity_name
|
|
1278
|
+
else:
|
|
1279
|
+
api = context.client.build_api(
|
|
1280
|
+
ApiCategories.CONTEXT.value,
|
|
1281
|
+
BackendOperations.DELETE.value,
|
|
1282
|
+
project=context.name,
|
|
1283
|
+
entity_type=entity_type,
|
|
1284
|
+
entity_id=entity_id,
|
|
1285
|
+
)
|
|
1286
|
+
return context.client.delete_object(api, **kwargs)
|
|
1287
|
+
|
|
1288
|
+
def delete_context_entity(
|
|
1289
|
+
self,
|
|
1290
|
+
identifier: str,
|
|
1291
|
+
project: str | None = None,
|
|
1292
|
+
entity_type: str | None = None,
|
|
1293
|
+
entity_id: str | None = None,
|
|
1294
|
+
**kwargs,
|
|
1295
|
+
) -> dict:
|
|
1296
|
+
"""
|
|
1297
|
+
Delete object from backend.
|
|
1298
|
+
|
|
1299
|
+
Parameters
|
|
1300
|
+
----------
|
|
1301
|
+
identifier : str
|
|
1302
|
+
Entity key (store://...) or entity name.
|
|
1303
|
+
project : str
|
|
1304
|
+
Project name.
|
|
1305
|
+
entity_type : str
|
|
1306
|
+
Entity type.
|
|
1307
|
+
entity_id : str
|
|
1308
|
+
Entity ID.
|
|
1309
|
+
**kwargs : dict
|
|
1310
|
+
Parameters to pass to the API call.
|
|
1311
|
+
|
|
1312
|
+
Returns
|
|
1313
|
+
-------
|
|
1314
|
+
dict
|
|
1315
|
+
Response from backend.
|
|
1316
|
+
"""
|
|
1317
|
+
context = self._get_context_from_identifier(identifier, project)
|
|
1318
|
+
return self._delete_context_entity(
|
|
1319
|
+
context,
|
|
1320
|
+
identifier,
|
|
1321
|
+
entity_type,
|
|
1322
|
+
context.name,
|
|
1323
|
+
entity_id,
|
|
1324
|
+
**kwargs,
|
|
1325
|
+
)
|
|
1326
|
+
|
|
1327
|
+
##############################
|
|
1328
|
+
# Context entity operations
|
|
1329
|
+
##############################
|
|
1330
|
+
|
|
1331
|
+
def _build_context_entity_key(
|
|
1332
|
+
self,
|
|
1333
|
+
context: Context,
|
|
1334
|
+
entity_type: str,
|
|
1335
|
+
entity_kind: str,
|
|
1336
|
+
entity_name: str,
|
|
1337
|
+
entity_id: str | None = None,
|
|
1338
|
+
) -> str:
|
|
1339
|
+
"""
|
|
1340
|
+
Build object key.
|
|
1341
|
+
|
|
1342
|
+
Parameters
|
|
1343
|
+
----------
|
|
1344
|
+
context : Context
|
|
1345
|
+
Context instance.
|
|
1346
|
+
entity_type : str
|
|
1347
|
+
Entity type.
|
|
1348
|
+
entity_kind : str
|
|
1349
|
+
Entity kind.
|
|
1350
|
+
entity_name : str
|
|
1351
|
+
Entity name.
|
|
1352
|
+
entity_id : str
|
|
1353
|
+
Entity ID.
|
|
1354
|
+
|
|
1355
|
+
Returns
|
|
1356
|
+
-------
|
|
1357
|
+
str
|
|
1358
|
+
Object key.
|
|
1359
|
+
"""
|
|
1360
|
+
return context.client.build_key(
|
|
1361
|
+
ApiCategories.CONTEXT.value,
|
|
1362
|
+
project=context.name,
|
|
1363
|
+
entity_type=entity_type,
|
|
1364
|
+
entity_kind=entity_kind,
|
|
1365
|
+
entity_name=entity_name,
|
|
1366
|
+
entity_id=entity_id,
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
def build_context_entity_key(
|
|
1370
|
+
self,
|
|
1371
|
+
project: str,
|
|
1372
|
+
entity_type: str,
|
|
1373
|
+
entity_kind: str,
|
|
1374
|
+
entity_name: str,
|
|
1375
|
+
entity_id: str | None = None,
|
|
1376
|
+
) -> str:
|
|
1377
|
+
"""
|
|
1378
|
+
Build object key.
|
|
1379
|
+
|
|
1380
|
+
Parameters
|
|
1381
|
+
----------
|
|
1382
|
+
project : str
|
|
1383
|
+
Project name.
|
|
1384
|
+
entity_type : str
|
|
1385
|
+
Entity type.
|
|
1386
|
+
entity_kind : str
|
|
1387
|
+
Entity kind.
|
|
1388
|
+
entity_name : str
|
|
1389
|
+
Entity name.
|
|
1390
|
+
entity_id : str
|
|
1391
|
+
Entity ID.
|
|
1392
|
+
|
|
1393
|
+
Returns
|
|
1394
|
+
-------
|
|
1395
|
+
str
|
|
1396
|
+
Object key.
|
|
1397
|
+
"""
|
|
1398
|
+
context = self._get_context(project)
|
|
1399
|
+
return self._build_context_entity_key(context, entity_type, entity_kind, entity_name, entity_id)
|
|
1400
|
+
|
|
1401
|
+
def read_secret_data(
|
|
1402
|
+
self,
|
|
1403
|
+
project: str,
|
|
1404
|
+
entity_type: str,
|
|
1405
|
+
**kwargs,
|
|
1406
|
+
) -> dict:
|
|
1407
|
+
"""
|
|
1408
|
+
Get data from backend.
|
|
1409
|
+
|
|
1410
|
+
Parameters
|
|
1411
|
+
----------
|
|
1412
|
+
project : str
|
|
1413
|
+
Project name.
|
|
1414
|
+
entity_type : str
|
|
1415
|
+
Entity type.
|
|
1416
|
+
**kwargs : dict
|
|
1417
|
+
Parameters to pass to the API call.
|
|
1418
|
+
|
|
1419
|
+
Returns
|
|
1420
|
+
-------
|
|
1421
|
+
dict
|
|
1422
|
+
Response from backend.
|
|
1423
|
+
"""
|
|
1424
|
+
context = self._get_context(project)
|
|
1425
|
+
api = context.client.build_api(
|
|
1426
|
+
ApiCategories.CONTEXT.value,
|
|
1427
|
+
BackendOperations.DATA.value,
|
|
1428
|
+
project=context.name,
|
|
1429
|
+
entity_type=entity_type,
|
|
1430
|
+
)
|
|
1431
|
+
return context.client.read_object(api, **kwargs)
|
|
1432
|
+
|
|
1433
|
+
def update_secret_data(
|
|
1434
|
+
self,
|
|
1435
|
+
project: str,
|
|
1436
|
+
entity_type: str,
|
|
1437
|
+
data: dict,
|
|
1438
|
+
**kwargs,
|
|
1439
|
+
) -> None:
|
|
1440
|
+
"""
|
|
1441
|
+
Set data in backend.
|
|
1442
|
+
|
|
1443
|
+
Parameters
|
|
1444
|
+
----------
|
|
1445
|
+
project : str
|
|
1446
|
+
Project name.
|
|
1447
|
+
entity_type : str
|
|
1448
|
+
Entity type.
|
|
1449
|
+
data : dict
|
|
1450
|
+
Data dictionary.
|
|
1451
|
+
**kwargs : dict
|
|
1452
|
+
Parameters to pass to the API call.
|
|
1453
|
+
|
|
1454
|
+
Returns
|
|
1455
|
+
-------
|
|
1456
|
+
None
|
|
1457
|
+
"""
|
|
1458
|
+
context = self._get_context(project)
|
|
1459
|
+
api = context.client.build_api(
|
|
1460
|
+
ApiCategories.CONTEXT.value,
|
|
1461
|
+
BackendOperations.DATA.value,
|
|
1462
|
+
project=context.name,
|
|
1463
|
+
entity_type=entity_type,
|
|
1464
|
+
)
|
|
1465
|
+
return context.client.update_object(api, data, **kwargs)
|
|
1466
|
+
|
|
1467
|
+
def read_run_logs(
|
|
1468
|
+
self,
|
|
1469
|
+
project: str,
|
|
1470
|
+
entity_type: str,
|
|
1471
|
+
entity_id: str,
|
|
1472
|
+
**kwargs,
|
|
1473
|
+
) -> dict:
|
|
1474
|
+
"""
|
|
1475
|
+
Get logs from backend.
|
|
1476
|
+
|
|
1477
|
+
Parameters
|
|
1478
|
+
----------
|
|
1479
|
+
project : str
|
|
1480
|
+
Project name.
|
|
1481
|
+
entity_type : str
|
|
1482
|
+
Entity type.
|
|
1483
|
+
entity_id : str
|
|
1484
|
+
Entity ID.
|
|
1485
|
+
**kwargs : dict
|
|
1486
|
+
Parameters to pass to the API call.
|
|
1487
|
+
|
|
1488
|
+
Returns
|
|
1489
|
+
-------
|
|
1490
|
+
dict
|
|
1491
|
+
Response from backend.
|
|
1492
|
+
"""
|
|
1493
|
+
context = self._get_context(project)
|
|
1494
|
+
api = context.client.build_api(
|
|
1495
|
+
ApiCategories.CONTEXT.value,
|
|
1496
|
+
BackendOperations.LOGS.value,
|
|
1497
|
+
project=context.name,
|
|
1498
|
+
entity_type=entity_type,
|
|
1499
|
+
entity_id=entity_id,
|
|
1500
|
+
)
|
|
1501
|
+
return context.client.read_object(api, **kwargs)
|
|
1502
|
+
|
|
1503
|
+
def stop_run(
|
|
1504
|
+
self,
|
|
1505
|
+
project: str,
|
|
1506
|
+
entity_type: str,
|
|
1507
|
+
entity_id: str,
|
|
1508
|
+
**kwargs,
|
|
1509
|
+
) -> None:
|
|
1510
|
+
"""
|
|
1511
|
+
Stop object in backend.
|
|
1512
|
+
|
|
1513
|
+
Parameters
|
|
1514
|
+
----------
|
|
1515
|
+
project : str
|
|
1516
|
+
Project name.
|
|
1517
|
+
entity_type : str
|
|
1518
|
+
Entity type.
|
|
1519
|
+
entity_id : str
|
|
1520
|
+
Entity ID.
|
|
1521
|
+
**kwargs : dict
|
|
1522
|
+
Parameters to pass to the API call.
|
|
1523
|
+
|
|
1524
|
+
Returns
|
|
1525
|
+
-------
|
|
1526
|
+
None
|
|
1527
|
+
"""
|
|
1528
|
+
context = self._get_context(project)
|
|
1529
|
+
api = context.client.build_api(
|
|
1530
|
+
ApiCategories.CONTEXT.value,
|
|
1531
|
+
BackendOperations.STOP.value,
|
|
1532
|
+
project=context.name,
|
|
1533
|
+
entity_type=entity_type,
|
|
1534
|
+
entity_id=entity_id,
|
|
1535
|
+
)
|
|
1536
|
+
return context.client.create_object(api, **kwargs)
|
|
1537
|
+
|
|
1538
|
+
def resume_run(
|
|
1539
|
+
self,
|
|
1540
|
+
project: str,
|
|
1541
|
+
entity_type: str,
|
|
1542
|
+
entity_id: str,
|
|
1543
|
+
**kwargs,
|
|
1544
|
+
) -> None:
|
|
1545
|
+
"""
|
|
1546
|
+
Resume object in backend.
|
|
1547
|
+
|
|
1548
|
+
Parameters
|
|
1549
|
+
----------
|
|
1550
|
+
project : str
|
|
1551
|
+
Project name.
|
|
1552
|
+
entity_type : str
|
|
1553
|
+
Entity type.
|
|
1554
|
+
entity_id : str
|
|
1555
|
+
Entity ID.
|
|
1556
|
+
**kwargs : dict
|
|
1557
|
+
Parameters to pass to the API call.
|
|
1558
|
+
|
|
1559
|
+
Returns
|
|
1560
|
+
-------
|
|
1561
|
+
None
|
|
1562
|
+
"""
|
|
1563
|
+
context = self._get_context(project)
|
|
1564
|
+
api = context.client.build_api(
|
|
1565
|
+
ApiCategories.CONTEXT.value,
|
|
1566
|
+
BackendOperations.RESUME.value,
|
|
1567
|
+
project=context.name,
|
|
1568
|
+
entity_type=entity_type,
|
|
1569
|
+
entity_id=entity_id,
|
|
1570
|
+
)
|
|
1571
|
+
return context.client.create_object(api, **kwargs)
|
|
1572
|
+
|
|
1573
|
+
def read_files_info(
|
|
1574
|
+
self,
|
|
1575
|
+
project: str,
|
|
1576
|
+
entity_type: str,
|
|
1577
|
+
entity_id: str,
|
|
1578
|
+
**kwargs,
|
|
1579
|
+
) -> list[dict]:
|
|
1580
|
+
"""
|
|
1581
|
+
Get files info from backend.
|
|
1582
|
+
|
|
1583
|
+
Parameters
|
|
1584
|
+
----------
|
|
1585
|
+
project : str
|
|
1586
|
+
Project name.
|
|
1587
|
+
entity_type : str
|
|
1588
|
+
Entity type.
|
|
1589
|
+
entity_id : str
|
|
1590
|
+
Entity ID.
|
|
1591
|
+
**kwargs : dict
|
|
1592
|
+
Parameters to pass to the API call.
|
|
1593
|
+
|
|
1594
|
+
Returns
|
|
1595
|
+
-------
|
|
1596
|
+
list[dict]
|
|
1597
|
+
Response from backend.
|
|
1598
|
+
"""
|
|
1599
|
+
context = self._get_context(project)
|
|
1600
|
+
api = context.client.build_api(
|
|
1601
|
+
ApiCategories.CONTEXT.value,
|
|
1602
|
+
BackendOperations.FILES.value,
|
|
1603
|
+
project=context.name,
|
|
1604
|
+
entity_type=entity_type,
|
|
1605
|
+
entity_id=entity_id,
|
|
1606
|
+
)
|
|
1607
|
+
return context.client.read_object(api, **kwargs)
|
|
1608
|
+
|
|
1609
|
+
def update_files_info(
|
|
1610
|
+
self,
|
|
1611
|
+
project: str,
|
|
1612
|
+
entity_type: str,
|
|
1613
|
+
entity_id: str,
|
|
1614
|
+
entity_list: list[dict],
|
|
1615
|
+
**kwargs,
|
|
1616
|
+
) -> None:
|
|
1617
|
+
"""
|
|
1618
|
+
Get files info from backend.
|
|
1619
|
+
|
|
1620
|
+
Parameters
|
|
1621
|
+
----------
|
|
1622
|
+
project : str
|
|
1623
|
+
Project name.
|
|
1624
|
+
entity_type : str
|
|
1625
|
+
Entity type.
|
|
1626
|
+
entity_id : str
|
|
1627
|
+
Entity ID.
|
|
1628
|
+
entity_list : list[dict]
|
|
1629
|
+
Entity list.
|
|
1630
|
+
**kwargs : dict
|
|
1631
|
+
Parameters to pass to the API call.
|
|
1632
|
+
|
|
1633
|
+
Returns
|
|
1634
|
+
-------
|
|
1635
|
+
None
|
|
1636
|
+
"""
|
|
1637
|
+
context = self._get_context(project)
|
|
1638
|
+
api = context.client.build_api(
|
|
1639
|
+
ApiCategories.CONTEXT.value,
|
|
1640
|
+
BackendOperations.FILES.value,
|
|
1641
|
+
project=context.name,
|
|
1642
|
+
entity_type=entity_type,
|
|
1643
|
+
entity_id=entity_id,
|
|
1644
|
+
)
|
|
1645
|
+
return context.client.update_object(api, entity_list, **kwargs)
|
|
1646
|
+
|
|
1647
|
+
def _search(
|
|
1648
|
+
self,
|
|
1649
|
+
project: str,
|
|
1650
|
+
**kwargs,
|
|
1651
|
+
) -> dict:
|
|
1652
|
+
"""
|
|
1653
|
+
Search in backend.
|
|
1654
|
+
|
|
1655
|
+
Parameters
|
|
1656
|
+
----------
|
|
1657
|
+
project : str
|
|
1658
|
+
Project name.
|
|
1659
|
+
**kwargs : dict
|
|
1660
|
+
Parameters to pass to the API call.
|
|
1661
|
+
|
|
1662
|
+
Returns
|
|
1663
|
+
-------
|
|
1664
|
+
dict
|
|
1665
|
+
Response from backend.
|
|
1666
|
+
"""
|
|
1667
|
+
context = self._get_context(project)
|
|
1668
|
+
api = context.client.build_api(
|
|
1669
|
+
ApiCategories.CONTEXT.value,
|
|
1670
|
+
BackendOperations.SEARCH.value,
|
|
1671
|
+
project=context.name,
|
|
1672
|
+
)
|
|
1673
|
+
return context.client.read_object(api, **kwargs)
|
|
1674
|
+
|
|
1675
|
+
def search_entity(
|
|
1676
|
+
self,
|
|
1677
|
+
project: str,
|
|
1678
|
+
query: str | None = None,
|
|
1679
|
+
entity_types: list[str] | None = None,
|
|
1680
|
+
name: str | None = None,
|
|
1681
|
+
kind: str | None = None,
|
|
1682
|
+
created: str | None = None,
|
|
1683
|
+
updated: str | None = None,
|
|
1684
|
+
description: str | None = None,
|
|
1685
|
+
labels: list[str] | None = None,
|
|
1686
|
+
**kwargs,
|
|
1687
|
+
) -> list[ContextEntity]:
|
|
1688
|
+
"""
|
|
1689
|
+
Search objects from backend.
|
|
1690
|
+
|
|
1691
|
+
Parameters
|
|
1692
|
+
----------
|
|
1693
|
+
project : str
|
|
1694
|
+
Project name.
|
|
1695
|
+
query : str
|
|
1696
|
+
Search query.
|
|
1697
|
+
entity_types : list[str]
|
|
1698
|
+
Entity types.
|
|
1699
|
+
name : str
|
|
1700
|
+
Entity name.
|
|
1701
|
+
kind : str
|
|
1702
|
+
Entity kind.
|
|
1703
|
+
created : str
|
|
1704
|
+
Entity creation date.
|
|
1705
|
+
updated : str
|
|
1706
|
+
Entity update date.
|
|
1707
|
+
description : str
|
|
1708
|
+
Entity description.
|
|
1709
|
+
labels : list[str]
|
|
1710
|
+
Entity labels.
|
|
1711
|
+
**kwargs : dict
|
|
1712
|
+
Parameters to pass to the API call.
|
|
1713
|
+
|
|
1714
|
+
Returns
|
|
1715
|
+
-------
|
|
1716
|
+
list[ContextEntity]
|
|
1717
|
+
List of object instances.
|
|
1718
|
+
"""
|
|
1719
|
+
context = self._get_context(project)
|
|
1720
|
+
|
|
1721
|
+
kwargs = self._set_params(**kwargs)
|
|
1722
|
+
|
|
1723
|
+
# Add search query
|
|
1724
|
+
if query is not None:
|
|
1725
|
+
kwargs["params"]["q"] = query
|
|
1726
|
+
|
|
1727
|
+
# Add search filters
|
|
1728
|
+
fq = []
|
|
1729
|
+
|
|
1730
|
+
# Entity types
|
|
1731
|
+
if entity_types is not None:
|
|
1732
|
+
if len(entity_types) == 1:
|
|
1733
|
+
entity_types = entity_types[0]
|
|
1734
|
+
else:
|
|
1735
|
+
entity_types = " OR ".join(entity_types)
|
|
1736
|
+
fq.append(f"type:({entity_types})")
|
|
1737
|
+
|
|
1738
|
+
# Name
|
|
1739
|
+
if name is not None:
|
|
1740
|
+
fq.append(f'metadata.name:"{name}"')
|
|
1741
|
+
|
|
1742
|
+
# Kind
|
|
1743
|
+
if kind is not None:
|
|
1744
|
+
fq.append(f'kind:"{kind}"')
|
|
1745
|
+
|
|
1746
|
+
# Time
|
|
1747
|
+
created = created if created is not None else "*"
|
|
1748
|
+
updated = updated if updated is not None else "*"
|
|
1749
|
+
fq.append(f"metadata.updated:[{created} TO {updated}]")
|
|
1750
|
+
|
|
1751
|
+
# Description
|
|
1752
|
+
if description is not None:
|
|
1753
|
+
fq.append(f'metadata.description:"{description}"')
|
|
1754
|
+
|
|
1755
|
+
# Labels
|
|
1756
|
+
if labels is not None:
|
|
1757
|
+
if len(labels) == 1:
|
|
1758
|
+
labels = labels[0]
|
|
1759
|
+
else:
|
|
1760
|
+
labels = " AND ".join(labels)
|
|
1761
|
+
fq.append(f"metadata.labels:({labels})")
|
|
1762
|
+
|
|
1763
|
+
# Add filters
|
|
1764
|
+
kwargs["params"]["fq"] = fq
|
|
1765
|
+
|
|
1766
|
+
objs = self._search(context, **kwargs)
|
|
1767
|
+
return objs
|
|
1768
|
+
return [build_entity_from_dict(obj) for obj in objs]
|
|
1769
|
+
|
|
1770
|
+
##############################
|
|
1771
|
+
# Helpers
|
|
1772
|
+
##############################
|
|
1773
|
+
|
|
1774
|
+
@staticmethod
|
|
1775
|
+
def _set_params(**kwargs) -> dict:
|
|
1776
|
+
"""
|
|
1777
|
+
Format params parameter.
|
|
1778
|
+
|
|
1779
|
+
Parameters
|
|
1780
|
+
----------
|
|
1781
|
+
**kwargs : dict
|
|
1782
|
+
Keyword arguments.
|
|
1783
|
+
|
|
1784
|
+
Returns
|
|
1785
|
+
-------
|
|
1786
|
+
dict
|
|
1787
|
+
Parameters with initialized params.
|
|
1788
|
+
"""
|
|
1789
|
+
if not kwargs:
|
|
1790
|
+
kwargs = {}
|
|
1791
|
+
if "params" not in kwargs:
|
|
1792
|
+
kwargs["params"] = {}
|
|
1793
|
+
return kwargs
|
|
1794
|
+
|
|
1795
|
+
def _get_context_from_identifier(
|
|
1796
|
+
self,
|
|
1797
|
+
identifier: str,
|
|
1798
|
+
project: str | None = None,
|
|
1799
|
+
) -> Context:
|
|
1800
|
+
"""
|
|
1801
|
+
Get context from project.
|
|
1802
|
+
|
|
1803
|
+
Parameters
|
|
1804
|
+
----------
|
|
1805
|
+
identifier : str
|
|
1806
|
+
Entity key (store://...) or entity name.
|
|
1807
|
+
project : str
|
|
1808
|
+
Project name.
|
|
1809
|
+
|
|
1810
|
+
Returns
|
|
1811
|
+
-------
|
|
1812
|
+
Context
|
|
1813
|
+
Context.
|
|
1814
|
+
"""
|
|
1815
|
+
if not identifier.startswith("store://"):
|
|
1816
|
+
if project is None:
|
|
1817
|
+
raise EntityError("Specify project if you do not specify entity key.")
|
|
1818
|
+
else:
|
|
1819
|
+
project = get_project_from_key(identifier)
|
|
1820
|
+
|
|
1821
|
+
return self._get_context(project)
|
|
1822
|
+
|
|
1823
|
+
def _get_context(
|
|
1824
|
+
self,
|
|
1825
|
+
project: str,
|
|
1826
|
+
) -> Context:
|
|
1827
|
+
"""
|
|
1828
|
+
Check if the given project is in the context.
|
|
1829
|
+
Otherwise try to get the project from remote.
|
|
1830
|
+
Finally return the client.
|
|
1831
|
+
|
|
1832
|
+
Parameters
|
|
1833
|
+
----------
|
|
1834
|
+
project : str
|
|
1835
|
+
Project name.
|
|
1836
|
+
|
|
1837
|
+
Returns
|
|
1838
|
+
-------
|
|
1839
|
+
Context
|
|
1840
|
+
Context.
|
|
1841
|
+
"""
|
|
1842
|
+
try:
|
|
1843
|
+
return get_context(project)
|
|
1844
|
+
except ContextError:
|
|
1845
|
+
return self._get_context_from_remote(project)
|
|
1846
|
+
|
|
1847
|
+
def _get_context_from_remote(
|
|
1848
|
+
self,
|
|
1849
|
+
project: str,
|
|
1850
|
+
) -> Client:
|
|
1851
|
+
"""
|
|
1852
|
+
Get context from remote.
|
|
1853
|
+
|
|
1854
|
+
Parameters
|
|
1855
|
+
----------
|
|
1856
|
+
project : str
|
|
1857
|
+
Project name.
|
|
1858
|
+
|
|
1859
|
+
Returns
|
|
1860
|
+
-------
|
|
1861
|
+
Client
|
|
1862
|
+
Client.
|
|
1863
|
+
"""
|
|
1864
|
+
try:
|
|
1865
|
+
client = get_client()
|
|
1866
|
+
obj = self._read_base_entity(client, EntityTypes.PROJECT.value, project)
|
|
1867
|
+
build_entity_from_dict(obj)
|
|
1868
|
+
return get_context(project)
|
|
1869
|
+
except EntityNotExistsError:
|
|
1870
|
+
raise ContextError(f"Project '{project}' not found.")
|
|
1871
|
+
|
|
1872
|
+
|
|
1873
|
+
processor = OperationsProcessor()
|