digitalhub 0.8.1__py3-none-any.whl → 0.9.0b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/__init__.py +19 -2
- digitalhub/client/_base/api_builder.py +16 -0
- digitalhub/client/_base/client.py +31 -0
- digitalhub/client/api.py +2 -38
- digitalhub/client/dhcore/api_builder.py +100 -0
- digitalhub/client/dhcore/client.py +77 -24
- digitalhub/client/dhcore/enums.py +26 -0
- digitalhub/client/dhcore/env.py +2 -2
- digitalhub/client/dhcore/utils.py +17 -17
- digitalhub/client/local/api_builder.py +100 -0
- digitalhub/client/local/client.py +20 -0
- digitalhub/context/api.py +3 -38
- digitalhub/context/builder.py +10 -23
- digitalhub/context/context.py +20 -92
- digitalhub/entities/_base/context/entity.py +30 -22
- digitalhub/entities/_base/entity/_constructors/metadata.py +12 -1
- digitalhub/entities/_base/entity/_constructors/name.py +1 -1
- digitalhub/entities/_base/entity/_constructors/spec.py +1 -1
- digitalhub/entities/_base/entity/_constructors/status.py +3 -2
- digitalhub/entities/_base/entity/builder.py +6 -1
- digitalhub/entities/_base/entity/entity.py +30 -10
- digitalhub/entities/_base/entity/metadata.py +22 -0
- digitalhub/entities/_base/entity/spec.py +7 -2
- digitalhub/entities/_base/executable/entity.py +8 -8
- digitalhub/entities/_base/material/entity.py +48 -16
- digitalhub/entities/_base/material/status.py +0 -31
- digitalhub/entities/_base/material/utils.py +106 -0
- digitalhub/entities/_base/project/entity.py +341 -0
- digitalhub/entities/_base/unversioned/entity.py +1 -23
- digitalhub/entities/_base/versioned/entity.py +0 -25
- digitalhub/entities/_commons/enums.py +103 -0
- digitalhub/entities/_commons/utils.py +83 -0
- digitalhub/entities/_operations/processor.py +1747 -0
- digitalhub/entities/artifact/_base/builder.py +1 -1
- digitalhub/entities/artifact/_base/entity.py +1 -1
- digitalhub/entities/artifact/artifact/builder.py +2 -1
- digitalhub/entities/artifact/crud.py +46 -29
- digitalhub/entities/artifact/utils.py +62 -0
- digitalhub/entities/dataitem/_base/builder.py +1 -1
- digitalhub/entities/dataitem/_base/entity.py +6 -6
- digitalhub/entities/dataitem/crud.py +50 -66
- digitalhub/entities/dataitem/dataitem/builder.py +2 -1
- digitalhub/entities/dataitem/iceberg/builder.py +2 -1
- digitalhub/entities/dataitem/table/builder.py +2 -1
- digitalhub/entities/dataitem/table/entity.py +5 -10
- digitalhub/entities/dataitem/table/models.py +4 -5
- digitalhub/entities/dataitem/utils.py +137 -0
- digitalhub/entities/function/_base/builder.py +1 -1
- digitalhub/entities/function/_base/entity.py +5 -1
- digitalhub/entities/function/crud.py +36 -17
- digitalhub/entities/model/_base/builder.py +1 -1
- digitalhub/entities/model/_base/entity.py +1 -1
- digitalhub/entities/model/crud.py +46 -29
- digitalhub/entities/model/huggingface/builder.py +2 -1
- digitalhub/entities/model/huggingface/spec.py +4 -2
- digitalhub/entities/model/mlflow/builder.py +2 -1
- digitalhub/entities/model/mlflow/models.py +17 -9
- digitalhub/entities/model/mlflow/spec.py +6 -1
- digitalhub/entities/model/mlflow/utils.py +4 -2
- digitalhub/entities/model/model/builder.py +2 -1
- digitalhub/entities/model/sklearn/builder.py +2 -1
- digitalhub/entities/model/utils.py +62 -0
- digitalhub/entities/project/_base/builder.py +2 -2
- digitalhub/entities/project/_base/entity.py +82 -272
- digitalhub/entities/project/crud.py +110 -91
- digitalhub/entities/project/utils.py +35 -0
- digitalhub/entities/run/_base/builder.py +3 -1
- digitalhub/entities/run/_base/entity.py +52 -54
- digitalhub/entities/run/_base/spec.py +11 -7
- digitalhub/entities/run/crud.py +35 -17
- digitalhub/entities/secret/_base/builder.py +2 -2
- digitalhub/entities/secret/_base/entity.py +4 -10
- digitalhub/entities/secret/crud.py +36 -21
- digitalhub/entities/task/_base/builder.py +14 -14
- digitalhub/entities/task/_base/entity.py +6 -6
- digitalhub/entities/task/_base/models.py +29 -6
- digitalhub/entities/task/_base/spec.py +44 -13
- digitalhub/entities/task/_base/utils.py +18 -0
- digitalhub/entities/task/crud.py +35 -15
- digitalhub/entities/workflow/_base/builder.py +1 -1
- digitalhub/entities/workflow/_base/entity.py +14 -6
- digitalhub/entities/workflow/crud.py +36 -17
- digitalhub/factory/utils.py +1 -1
- digitalhub/readers/_base/reader.py +2 -2
- digitalhub/readers/_commons/enums.py +13 -0
- digitalhub/readers/api.py +3 -2
- digitalhub/readers/factory.py +12 -6
- digitalhub/readers/pandas/reader.py +20 -8
- digitalhub/runtimes/_base.py +0 -7
- digitalhub/stores/_base/store.py +53 -9
- digitalhub/stores/builder.py +5 -5
- digitalhub/stores/local/store.py +37 -2
- digitalhub/stores/remote/store.py +25 -3
- digitalhub/stores/s3/store.py +34 -7
- digitalhub/stores/sql/store.py +112 -45
- digitalhub/utils/exceptions.py +6 -0
- digitalhub/utils/file_utils.py +60 -2
- digitalhub/utils/generic_utils.py +45 -4
- digitalhub/utils/io_utils.py +18 -0
- digitalhub/utils/uri_utils.py +153 -15
- {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/METADATA +2 -2
- {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/RECORD +110 -113
- test/testkfp.py +4 -1
- digitalhub/datastores/_base/datastore.py +0 -85
- digitalhub/datastores/api.py +0 -37
- digitalhub/datastores/builder.py +0 -110
- digitalhub/datastores/local/datastore.py +0 -50
- digitalhub/datastores/remote/__init__.py +0 -0
- digitalhub/datastores/remote/datastore.py +0 -31
- digitalhub/datastores/s3/__init__.py +0 -0
- digitalhub/datastores/s3/datastore.py +0 -46
- digitalhub/datastores/sql/__init__.py +0 -0
- digitalhub/datastores/sql/datastore.py +0 -68
- digitalhub/entities/_base/api_utils.py +0 -620
- digitalhub/entities/_base/crud.py +0 -468
- digitalhub/entities/function/_base/models.py +0 -118
- digitalhub/entities/utils/__init__.py +0 -0
- digitalhub/entities/utils/api.py +0 -346
- digitalhub/entities/utils/entity_types.py +0 -19
- digitalhub/entities/utils/state.py +0 -31
- digitalhub/entities/utils/utils.py +0 -202
- /digitalhub/{context → entities/_base/project}/__init__.py +0 -0
- /digitalhub/{datastores → entities/_commons}/__init__.py +0 -0
- /digitalhub/{datastores/_base → entities/_operations}/__init__.py +0 -0
- /digitalhub/{datastores/local → readers/_commons}/__init__.py +0 -0
- {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/LICENSE.txt +0 -0
- {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/WHEEL +0 -0
- {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/top_level.txt +0 -0
digitalhub/entities/utils/api.py
DELETED
|
@@ -1,346 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
API_BASE = "/api/v1"
|
|
4
|
-
API_CONTEXT = f"{API_BASE}/-"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
##############################
|
|
8
|
-
# Context APIs
|
|
9
|
-
##############################
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def api_ctx_create(
|
|
13
|
-
project: str,
|
|
14
|
-
entity_type: str,
|
|
15
|
-
) -> str:
|
|
16
|
-
"""
|
|
17
|
-
Create context API.
|
|
18
|
-
|
|
19
|
-
Parameters
|
|
20
|
-
----------
|
|
21
|
-
project : str
|
|
22
|
-
Project name.
|
|
23
|
-
entity_type : str
|
|
24
|
-
The name of the entity_type.
|
|
25
|
-
|
|
26
|
-
Returns
|
|
27
|
-
-------
|
|
28
|
-
str
|
|
29
|
-
The API string formatted.
|
|
30
|
-
"""
|
|
31
|
-
if not entity_type.endswith("s"):
|
|
32
|
-
entity_type += "s"
|
|
33
|
-
return f"{API_CONTEXT}/{project}/{entity_type}"
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def api_ctx_read(project: str, entity_type: str, entity_id: str) -> str:
|
|
37
|
-
"""
|
|
38
|
-
Read context API.
|
|
39
|
-
|
|
40
|
-
Parameters
|
|
41
|
-
----------
|
|
42
|
-
project : str
|
|
43
|
-
Project name.
|
|
44
|
-
entity_type : str
|
|
45
|
-
Entity type.
|
|
46
|
-
entity_id : str
|
|
47
|
-
Entity ID.
|
|
48
|
-
|
|
49
|
-
Returns
|
|
50
|
-
-------
|
|
51
|
-
str
|
|
52
|
-
The API string formatted.
|
|
53
|
-
"""
|
|
54
|
-
if not entity_type.endswith("s"):
|
|
55
|
-
entity_type += "s"
|
|
56
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def api_ctx_update(
|
|
60
|
-
project: str,
|
|
61
|
-
entity_type: str,
|
|
62
|
-
entity_id: str,
|
|
63
|
-
) -> str:
|
|
64
|
-
"""
|
|
65
|
-
Update context API.
|
|
66
|
-
|
|
67
|
-
Parameters
|
|
68
|
-
----------
|
|
69
|
-
project : str
|
|
70
|
-
Project name.
|
|
71
|
-
entity_type : str
|
|
72
|
-
Entity type.
|
|
73
|
-
entity_id : str
|
|
74
|
-
Entity ID.
|
|
75
|
-
|
|
76
|
-
Returns
|
|
77
|
-
-------
|
|
78
|
-
str
|
|
79
|
-
The API string formatted.
|
|
80
|
-
"""
|
|
81
|
-
if not entity_type.endswith("s"):
|
|
82
|
-
entity_type += "s"
|
|
83
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def api_ctx_delete(project: str, entity_type: str, entity_id: str) -> str:
|
|
87
|
-
"""
|
|
88
|
-
Delete context API.
|
|
89
|
-
|
|
90
|
-
Parameters
|
|
91
|
-
----------
|
|
92
|
-
project : str
|
|
93
|
-
Project name.
|
|
94
|
-
entity_type : str
|
|
95
|
-
Entity type.
|
|
96
|
-
entity_id : str
|
|
97
|
-
Entity ID.
|
|
98
|
-
|
|
99
|
-
Returns
|
|
100
|
-
-------
|
|
101
|
-
str
|
|
102
|
-
The API string formatted.
|
|
103
|
-
"""
|
|
104
|
-
if not entity_type.endswith("s"):
|
|
105
|
-
entity_type += "s"
|
|
106
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def api_ctx_list(project: str, entity_type: str) -> str:
|
|
110
|
-
"""
|
|
111
|
-
List context API.
|
|
112
|
-
|
|
113
|
-
Parameters
|
|
114
|
-
----------
|
|
115
|
-
project : str
|
|
116
|
-
Project name.
|
|
117
|
-
entity_type : str
|
|
118
|
-
Entity type.
|
|
119
|
-
|
|
120
|
-
Returns
|
|
121
|
-
-------
|
|
122
|
-
str
|
|
123
|
-
The API string formatted.
|
|
124
|
-
"""
|
|
125
|
-
if not entity_type.endswith("s"):
|
|
126
|
-
entity_type += "s"
|
|
127
|
-
return f"{API_CONTEXT}/{project}/{entity_type}"
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def api_ctx_data(project: str, entity_type: str) -> str:
|
|
131
|
-
"""
|
|
132
|
-
Data context API.
|
|
133
|
-
|
|
134
|
-
Parameters
|
|
135
|
-
----------
|
|
136
|
-
project : str
|
|
137
|
-
Project name.
|
|
138
|
-
entity_type : str
|
|
139
|
-
Entity type.
|
|
140
|
-
|
|
141
|
-
Returns
|
|
142
|
-
-------
|
|
143
|
-
str
|
|
144
|
-
The API string formatted.
|
|
145
|
-
"""
|
|
146
|
-
if not entity_type.endswith("s"):
|
|
147
|
-
entity_type += "s"
|
|
148
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/data"
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def api_ctx_logs(project: str, entity_type: str, entity_id: str) -> str:
|
|
152
|
-
"""
|
|
153
|
-
Logs context API.
|
|
154
|
-
|
|
155
|
-
Parameters
|
|
156
|
-
----------
|
|
157
|
-
project : str
|
|
158
|
-
Project name.
|
|
159
|
-
entity_type : str
|
|
160
|
-
Entity type.
|
|
161
|
-
entity_id : str
|
|
162
|
-
Entity ID.
|
|
163
|
-
|
|
164
|
-
Returns
|
|
165
|
-
-------
|
|
166
|
-
str
|
|
167
|
-
The API string formatted.
|
|
168
|
-
"""
|
|
169
|
-
if not entity_type.endswith("s"):
|
|
170
|
-
entity_type += "s"
|
|
171
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/logs"
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
def api_ctx_stop(project: str, entity_type: str, entity_id: str) -> str:
|
|
175
|
-
"""
|
|
176
|
-
Stop context API.
|
|
177
|
-
|
|
178
|
-
Parameters
|
|
179
|
-
----------
|
|
180
|
-
project : str
|
|
181
|
-
Project name.
|
|
182
|
-
entity_type : str
|
|
183
|
-
Entity type.
|
|
184
|
-
entity_id : str
|
|
185
|
-
Entity ID.
|
|
186
|
-
|
|
187
|
-
Returns
|
|
188
|
-
-------
|
|
189
|
-
str
|
|
190
|
-
The API string formatted.
|
|
191
|
-
"""
|
|
192
|
-
if not entity_type.endswith("s"):
|
|
193
|
-
entity_type += "s"
|
|
194
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/stop"
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
def api_ctx_resume(project: str, entity_type: str, entity_id: str) -> str:
|
|
198
|
-
"""
|
|
199
|
-
Resume context API.
|
|
200
|
-
|
|
201
|
-
Parameters
|
|
202
|
-
----------
|
|
203
|
-
project : str
|
|
204
|
-
Project name.
|
|
205
|
-
entity_type : str
|
|
206
|
-
Entity type.
|
|
207
|
-
entity_id : str
|
|
208
|
-
Entity ID.
|
|
209
|
-
|
|
210
|
-
Returns
|
|
211
|
-
-------
|
|
212
|
-
str
|
|
213
|
-
The API string formatted.
|
|
214
|
-
"""
|
|
215
|
-
if not entity_type.endswith("s"):
|
|
216
|
-
entity_type += "s"
|
|
217
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/resume"
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
def api_ctx_files(project: str, entity_type: str, entity_id: str) -> str:
|
|
221
|
-
"""
|
|
222
|
-
Files context API.
|
|
223
|
-
|
|
224
|
-
Parameters
|
|
225
|
-
----------
|
|
226
|
-
project : str
|
|
227
|
-
Project name.
|
|
228
|
-
entity_type : str
|
|
229
|
-
Entity type.
|
|
230
|
-
entity_id : str
|
|
231
|
-
Entity ID.
|
|
232
|
-
|
|
233
|
-
Returns
|
|
234
|
-
-------
|
|
235
|
-
str
|
|
236
|
-
The API string formatted.
|
|
237
|
-
"""
|
|
238
|
-
if not entity_type.endswith("s"):
|
|
239
|
-
entity_type += "s"
|
|
240
|
-
return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/files/info"
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
##############################
|
|
244
|
-
# Base APIs
|
|
245
|
-
##############################
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
def api_base_list(entity_type: str) -> str:
|
|
249
|
-
"""
|
|
250
|
-
List base API.
|
|
251
|
-
|
|
252
|
-
Parameters
|
|
253
|
-
----------
|
|
254
|
-
entity_type : str
|
|
255
|
-
Entity type.
|
|
256
|
-
|
|
257
|
-
Returns
|
|
258
|
-
-------
|
|
259
|
-
str
|
|
260
|
-
The API string formatted.
|
|
261
|
-
"""
|
|
262
|
-
if not entity_type.endswith("s"):
|
|
263
|
-
entity_type += "s"
|
|
264
|
-
return f"{API_BASE}/{entity_type}"
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def api_base_create(entity_type: str) -> str:
|
|
268
|
-
"""
|
|
269
|
-
Create base API.
|
|
270
|
-
|
|
271
|
-
Parameters
|
|
272
|
-
----------
|
|
273
|
-
entity_type : str
|
|
274
|
-
Entity type.
|
|
275
|
-
|
|
276
|
-
Returns
|
|
277
|
-
-------
|
|
278
|
-
str
|
|
279
|
-
The API string formatted.
|
|
280
|
-
"""
|
|
281
|
-
if not entity_type.endswith("s"):
|
|
282
|
-
entity_type += "s"
|
|
283
|
-
return f"{API_BASE}/{entity_type}"
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
def api_base_read(entity_type: str, entity_id: str) -> str:
|
|
287
|
-
"""
|
|
288
|
-
Read base API.
|
|
289
|
-
|
|
290
|
-
Parameters
|
|
291
|
-
----------
|
|
292
|
-
entity_type : str
|
|
293
|
-
Entity type.
|
|
294
|
-
entity_id : str
|
|
295
|
-
Entity ID.
|
|
296
|
-
|
|
297
|
-
Returns
|
|
298
|
-
-------
|
|
299
|
-
str
|
|
300
|
-
The API string formatted.
|
|
301
|
-
"""
|
|
302
|
-
if not entity_type.endswith("s"):
|
|
303
|
-
entity_type += "s"
|
|
304
|
-
return f"{API_BASE}/{entity_type}/{entity_id}"
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
def api_base_update(entity_type: str, entity_id: str) -> str:
|
|
308
|
-
"""
|
|
309
|
-
Update base API.
|
|
310
|
-
|
|
311
|
-
Parameters
|
|
312
|
-
----------
|
|
313
|
-
entity_type : str
|
|
314
|
-
Entity type.
|
|
315
|
-
entity_id : str
|
|
316
|
-
Entity ID.
|
|
317
|
-
|
|
318
|
-
Returns
|
|
319
|
-
-------
|
|
320
|
-
str
|
|
321
|
-
The API string formatted.
|
|
322
|
-
"""
|
|
323
|
-
if not entity_type.endswith("s"):
|
|
324
|
-
entity_type += "s"
|
|
325
|
-
return f"{API_BASE}/{entity_type}/{entity_id}"
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
def api_base_delete(entity_type: str, entity_id: str) -> str:
|
|
329
|
-
"""
|
|
330
|
-
Delete base API.
|
|
331
|
-
|
|
332
|
-
Parameters
|
|
333
|
-
----------
|
|
334
|
-
entity_type : str
|
|
335
|
-
Entity type.
|
|
336
|
-
entity_id : str
|
|
337
|
-
Entity ID.
|
|
338
|
-
|
|
339
|
-
Returns
|
|
340
|
-
-------
|
|
341
|
-
str
|
|
342
|
-
The API string formatted.
|
|
343
|
-
"""
|
|
344
|
-
if not entity_type.endswith("s"):
|
|
345
|
-
entity_type += "s"
|
|
346
|
-
return f"{API_BASE}/{entity_type}/{entity_id}"
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from enum import Enum
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class EntityTypes(Enum):
|
|
7
|
-
"""
|
|
8
|
-
Entity types.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
PROJECT = "project"
|
|
12
|
-
ARTIFACT = "artifact"
|
|
13
|
-
DATAITEM = "dataitem"
|
|
14
|
-
MODEL = "model"
|
|
15
|
-
SECRET = "secret"
|
|
16
|
-
FUNCTION = "function"
|
|
17
|
-
WORKFLOW = "workflow"
|
|
18
|
-
TASK = "task"
|
|
19
|
-
RUN = "run"
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from enum import Enum
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class State(Enum):
|
|
7
|
-
"""
|
|
8
|
-
State enumeration.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
BUILT = "BUILT"
|
|
12
|
-
CANCELLED = "CANCELLED"
|
|
13
|
-
COMPLETED = "COMPLETED"
|
|
14
|
-
CREATED = "CREATED"
|
|
15
|
-
CREATING = "CREATING"
|
|
16
|
-
DELETED = "DELETED"
|
|
17
|
-
DELETING = "DELETING"
|
|
18
|
-
ERROR = "ERROR"
|
|
19
|
-
FSM_ERROR = "FSM_ERROR"
|
|
20
|
-
IDLE = "IDLE"
|
|
21
|
-
NONE = "NONE"
|
|
22
|
-
ONLINE = "ONLINE"
|
|
23
|
-
PENDING = "PENDING"
|
|
24
|
-
READY = "READY"
|
|
25
|
-
RESUME = "RESUME"
|
|
26
|
-
RUN_ERROR = "RUN_ERROR"
|
|
27
|
-
RUNNING = "RUNNING"
|
|
28
|
-
STOP = "STOP"
|
|
29
|
-
STOPPED = "STOPPED"
|
|
30
|
-
SUCCESS = "SUCCESS"
|
|
31
|
-
UNKNOWN = "UNKNOWN"
|
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
from digitalhub.entities.utils.entity_types import EntityTypes
|
|
6
|
-
from digitalhub.utils.file_utils import get_file_mime_type
|
|
7
|
-
from digitalhub.utils.s3_utils import get_s3_bucket
|
|
8
|
-
from digitalhub.utils.uri_utils import check_local_path
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def parse_entity_key(key: str) -> tuple[str]:
|
|
12
|
-
"""
|
|
13
|
-
Parse the entity key. Returns project, entity type, kind, name and uuid.
|
|
14
|
-
|
|
15
|
-
Parameters
|
|
16
|
-
----------
|
|
17
|
-
key : str
|
|
18
|
-
The entity key.
|
|
19
|
-
|
|
20
|
-
Returns
|
|
21
|
-
-------
|
|
22
|
-
tuple[str]
|
|
23
|
-
Project, entity type, kind, name and uuid.
|
|
24
|
-
"""
|
|
25
|
-
try:
|
|
26
|
-
# Remove "store://" from the key
|
|
27
|
-
key = key.replace("store://", "")
|
|
28
|
-
|
|
29
|
-
# Split the key into parts
|
|
30
|
-
parts = key.split("/")
|
|
31
|
-
|
|
32
|
-
# The project is the first part
|
|
33
|
-
project = parts[0]
|
|
34
|
-
|
|
35
|
-
# The entity type is the second part
|
|
36
|
-
entity_type = parts[1]
|
|
37
|
-
|
|
38
|
-
# The kind is the third part
|
|
39
|
-
kind = parts[2]
|
|
40
|
-
|
|
41
|
-
# Tasks and runs have no name and uuid
|
|
42
|
-
if entity_type in (EntityTypes.TASK.value, EntityTypes.RUN.value):
|
|
43
|
-
name = None
|
|
44
|
-
uuid = parts[3]
|
|
45
|
-
|
|
46
|
-
# The name and uuid are separated by a colon in the last part
|
|
47
|
-
else:
|
|
48
|
-
name, uuid = parts[3].split(":")
|
|
49
|
-
|
|
50
|
-
return project, entity_type, kind, name, uuid
|
|
51
|
-
except Exception as e:
|
|
52
|
-
raise ValueError("Invalid key format.") from e
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def eval_local_source(source: str | list[str]) -> None:
|
|
56
|
-
"""
|
|
57
|
-
Evaluate if source is local.
|
|
58
|
-
|
|
59
|
-
Parameters
|
|
60
|
-
----------
|
|
61
|
-
source : str | list[str]
|
|
62
|
-
Source(s).
|
|
63
|
-
|
|
64
|
-
Returns
|
|
65
|
-
-------
|
|
66
|
-
None
|
|
67
|
-
"""
|
|
68
|
-
if isinstance(source, list):
|
|
69
|
-
if not source:
|
|
70
|
-
raise ValueError("Empty list of sources.")
|
|
71
|
-
source_is_local = all(check_local_path(s) for s in source)
|
|
72
|
-
for s in source:
|
|
73
|
-
if Path(s).is_dir():
|
|
74
|
-
raise ValueError(f"Invalid source path: {s}. List of paths must be list of files, not directories.")
|
|
75
|
-
else:
|
|
76
|
-
source_is_local = check_local_path(source)
|
|
77
|
-
|
|
78
|
-
if not source_is_local:
|
|
79
|
-
raise ValueError("Invalid source path. Source must be a local path.")
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def eval_zip_type(source: str | list[str]) -> bool:
|
|
83
|
-
"""
|
|
84
|
-
Evaluate zip type.
|
|
85
|
-
|
|
86
|
-
Parameters
|
|
87
|
-
----------
|
|
88
|
-
source : str | list[str]
|
|
89
|
-
Source(s).
|
|
90
|
-
|
|
91
|
-
Returns
|
|
92
|
-
-------
|
|
93
|
-
bool
|
|
94
|
-
True if path is zip.
|
|
95
|
-
"""
|
|
96
|
-
if isinstance(source, list):
|
|
97
|
-
if len(source) > 1:
|
|
98
|
-
return False
|
|
99
|
-
else:
|
|
100
|
-
path = source[0]
|
|
101
|
-
else:
|
|
102
|
-
if Path(source).is_dir():
|
|
103
|
-
return False
|
|
104
|
-
path = source
|
|
105
|
-
|
|
106
|
-
extension = path.endswith(".zip")
|
|
107
|
-
mime_zip = get_file_mime_type(path) == "application/zip"
|
|
108
|
-
return extension or mime_zip
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def build_log_path_from_filename(
|
|
112
|
-
project: str,
|
|
113
|
-
entity_type: str,
|
|
114
|
-
name: str,
|
|
115
|
-
uuid: str,
|
|
116
|
-
filename: str,
|
|
117
|
-
) -> str:
|
|
118
|
-
"""
|
|
119
|
-
Build log path.
|
|
120
|
-
|
|
121
|
-
Parameters
|
|
122
|
-
----------
|
|
123
|
-
project : str
|
|
124
|
-
Project name.
|
|
125
|
-
entity_type : str
|
|
126
|
-
Entity type.
|
|
127
|
-
name : str
|
|
128
|
-
Object name.
|
|
129
|
-
uuid : str
|
|
130
|
-
Object UUID.
|
|
131
|
-
filename : str
|
|
132
|
-
Filename.
|
|
133
|
-
|
|
134
|
-
Returns
|
|
135
|
-
-------
|
|
136
|
-
str
|
|
137
|
-
Log path.
|
|
138
|
-
"""
|
|
139
|
-
return f"s3://{get_s3_bucket()}/{project}/{entity_type}/{name}/{uuid}/{filename}"
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def build_log_path_from_source(
|
|
143
|
-
project: str,
|
|
144
|
-
entity_type: str,
|
|
145
|
-
name: str,
|
|
146
|
-
uuid: str,
|
|
147
|
-
source: str | list[str],
|
|
148
|
-
) -> str:
|
|
149
|
-
"""
|
|
150
|
-
Build log path.
|
|
151
|
-
|
|
152
|
-
Parameters
|
|
153
|
-
----------
|
|
154
|
-
project : str
|
|
155
|
-
Project name.
|
|
156
|
-
entity_type : str
|
|
157
|
-
Entity type.
|
|
158
|
-
name : str
|
|
159
|
-
Object name.
|
|
160
|
-
uuid : str
|
|
161
|
-
Object UUID.
|
|
162
|
-
source : str | list[str]
|
|
163
|
-
Source(s).
|
|
164
|
-
|
|
165
|
-
Returns
|
|
166
|
-
-------
|
|
167
|
-
str
|
|
168
|
-
Log path.
|
|
169
|
-
"""
|
|
170
|
-
is_zip = eval_zip_type(source)
|
|
171
|
-
scheme = "zip+s3" if is_zip else "s3"
|
|
172
|
-
path = f"{scheme}://{get_s3_bucket()}/{project}/{entity_type}/{name}/{uuid}"
|
|
173
|
-
|
|
174
|
-
if isinstance(source, list) and len(source) >= 1:
|
|
175
|
-
if len(source) > 1:
|
|
176
|
-
path += "/"
|
|
177
|
-
else:
|
|
178
|
-
path += f"/{Path(source[0]).name}"
|
|
179
|
-
elif Path(source).is_dir():
|
|
180
|
-
path += "/"
|
|
181
|
-
elif Path(source).is_file():
|
|
182
|
-
path += f"/{Path(source).name}"
|
|
183
|
-
|
|
184
|
-
return path
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
def get_entity_type_from_key(key: str) -> str:
|
|
188
|
-
"""
|
|
189
|
-
Get entity type.
|
|
190
|
-
|
|
191
|
-
Parameters
|
|
192
|
-
----------
|
|
193
|
-
key : str
|
|
194
|
-
The key of the entity.
|
|
195
|
-
|
|
196
|
-
Returns
|
|
197
|
-
-------
|
|
198
|
-
str
|
|
199
|
-
The entity type.
|
|
200
|
-
"""
|
|
201
|
-
_, entity_type, _, _, _ = parse_entity_key(key)
|
|
202
|
-
return entity_type
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|