digitalhub 0.8.1__py3-none-any.whl → 0.9.0b0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of digitalhub might be problematic. Click here for more details.

Files changed (128) hide show
  1. digitalhub/__init__.py +19 -2
  2. digitalhub/client/_base/api_builder.py +16 -0
  3. digitalhub/client/_base/client.py +31 -0
  4. digitalhub/client/api.py +2 -38
  5. digitalhub/client/dhcore/api_builder.py +100 -0
  6. digitalhub/client/dhcore/client.py +77 -24
  7. digitalhub/client/dhcore/enums.py +26 -0
  8. digitalhub/client/dhcore/env.py +2 -2
  9. digitalhub/client/dhcore/utils.py +17 -17
  10. digitalhub/client/local/api_builder.py +100 -0
  11. digitalhub/client/local/client.py +20 -0
  12. digitalhub/context/api.py +3 -38
  13. digitalhub/context/builder.py +10 -23
  14. digitalhub/context/context.py +20 -92
  15. digitalhub/entities/_base/context/entity.py +30 -22
  16. digitalhub/entities/_base/entity/_constructors/metadata.py +12 -1
  17. digitalhub/entities/_base/entity/_constructors/name.py +1 -1
  18. digitalhub/entities/_base/entity/_constructors/spec.py +1 -1
  19. digitalhub/entities/_base/entity/_constructors/status.py +3 -2
  20. digitalhub/entities/_base/entity/builder.py +6 -1
  21. digitalhub/entities/_base/entity/entity.py +30 -10
  22. digitalhub/entities/_base/entity/metadata.py +22 -0
  23. digitalhub/entities/_base/entity/spec.py +7 -2
  24. digitalhub/entities/_base/executable/entity.py +8 -8
  25. digitalhub/entities/_base/material/entity.py +48 -16
  26. digitalhub/entities/_base/material/status.py +0 -31
  27. digitalhub/entities/_base/material/utils.py +106 -0
  28. digitalhub/entities/_base/project/entity.py +341 -0
  29. digitalhub/entities/_base/unversioned/entity.py +1 -23
  30. digitalhub/entities/_base/versioned/entity.py +0 -25
  31. digitalhub/entities/_commons/enums.py +103 -0
  32. digitalhub/entities/_commons/utils.py +83 -0
  33. digitalhub/entities/_operations/processor.py +1747 -0
  34. digitalhub/entities/artifact/_base/builder.py +1 -1
  35. digitalhub/entities/artifact/_base/entity.py +1 -1
  36. digitalhub/entities/artifact/artifact/builder.py +2 -1
  37. digitalhub/entities/artifact/crud.py +46 -29
  38. digitalhub/entities/artifact/utils.py +62 -0
  39. digitalhub/entities/dataitem/_base/builder.py +1 -1
  40. digitalhub/entities/dataitem/_base/entity.py +6 -6
  41. digitalhub/entities/dataitem/crud.py +50 -66
  42. digitalhub/entities/dataitem/dataitem/builder.py +2 -1
  43. digitalhub/entities/dataitem/iceberg/builder.py +2 -1
  44. digitalhub/entities/dataitem/table/builder.py +2 -1
  45. digitalhub/entities/dataitem/table/entity.py +5 -10
  46. digitalhub/entities/dataitem/table/models.py +4 -5
  47. digitalhub/entities/dataitem/utils.py +137 -0
  48. digitalhub/entities/function/_base/builder.py +1 -1
  49. digitalhub/entities/function/_base/entity.py +5 -1
  50. digitalhub/entities/function/crud.py +36 -17
  51. digitalhub/entities/model/_base/builder.py +1 -1
  52. digitalhub/entities/model/_base/entity.py +1 -1
  53. digitalhub/entities/model/crud.py +46 -29
  54. digitalhub/entities/model/huggingface/builder.py +2 -1
  55. digitalhub/entities/model/huggingface/spec.py +4 -2
  56. digitalhub/entities/model/mlflow/builder.py +2 -1
  57. digitalhub/entities/model/mlflow/models.py +17 -9
  58. digitalhub/entities/model/mlflow/spec.py +6 -1
  59. digitalhub/entities/model/mlflow/utils.py +4 -2
  60. digitalhub/entities/model/model/builder.py +2 -1
  61. digitalhub/entities/model/sklearn/builder.py +2 -1
  62. digitalhub/entities/model/utils.py +62 -0
  63. digitalhub/entities/project/_base/builder.py +2 -2
  64. digitalhub/entities/project/_base/entity.py +82 -272
  65. digitalhub/entities/project/crud.py +110 -91
  66. digitalhub/entities/project/utils.py +35 -0
  67. digitalhub/entities/run/_base/builder.py +3 -1
  68. digitalhub/entities/run/_base/entity.py +52 -54
  69. digitalhub/entities/run/_base/spec.py +11 -7
  70. digitalhub/entities/run/crud.py +35 -17
  71. digitalhub/entities/secret/_base/builder.py +2 -2
  72. digitalhub/entities/secret/_base/entity.py +4 -10
  73. digitalhub/entities/secret/crud.py +36 -21
  74. digitalhub/entities/task/_base/builder.py +14 -14
  75. digitalhub/entities/task/_base/entity.py +6 -6
  76. digitalhub/entities/task/_base/models.py +29 -6
  77. digitalhub/entities/task/_base/spec.py +44 -13
  78. digitalhub/entities/task/_base/utils.py +18 -0
  79. digitalhub/entities/task/crud.py +35 -15
  80. digitalhub/entities/workflow/_base/builder.py +1 -1
  81. digitalhub/entities/workflow/_base/entity.py +14 -6
  82. digitalhub/entities/workflow/crud.py +36 -17
  83. digitalhub/factory/utils.py +1 -1
  84. digitalhub/readers/_base/reader.py +2 -2
  85. digitalhub/readers/_commons/enums.py +13 -0
  86. digitalhub/readers/api.py +3 -2
  87. digitalhub/readers/factory.py +12 -6
  88. digitalhub/readers/pandas/reader.py +20 -8
  89. digitalhub/runtimes/_base.py +0 -7
  90. digitalhub/stores/_base/store.py +53 -9
  91. digitalhub/stores/builder.py +5 -5
  92. digitalhub/stores/local/store.py +37 -2
  93. digitalhub/stores/remote/store.py +25 -3
  94. digitalhub/stores/s3/store.py +34 -7
  95. digitalhub/stores/sql/store.py +112 -45
  96. digitalhub/utils/exceptions.py +6 -0
  97. digitalhub/utils/file_utils.py +60 -2
  98. digitalhub/utils/generic_utils.py +45 -4
  99. digitalhub/utils/io_utils.py +18 -0
  100. digitalhub/utils/uri_utils.py +153 -15
  101. {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/METADATA +2 -2
  102. {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/RECORD +110 -113
  103. test/testkfp.py +4 -1
  104. digitalhub/datastores/_base/datastore.py +0 -85
  105. digitalhub/datastores/api.py +0 -37
  106. digitalhub/datastores/builder.py +0 -110
  107. digitalhub/datastores/local/datastore.py +0 -50
  108. digitalhub/datastores/remote/__init__.py +0 -0
  109. digitalhub/datastores/remote/datastore.py +0 -31
  110. digitalhub/datastores/s3/__init__.py +0 -0
  111. digitalhub/datastores/s3/datastore.py +0 -46
  112. digitalhub/datastores/sql/__init__.py +0 -0
  113. digitalhub/datastores/sql/datastore.py +0 -68
  114. digitalhub/entities/_base/api_utils.py +0 -620
  115. digitalhub/entities/_base/crud.py +0 -468
  116. digitalhub/entities/function/_base/models.py +0 -118
  117. digitalhub/entities/utils/__init__.py +0 -0
  118. digitalhub/entities/utils/api.py +0 -346
  119. digitalhub/entities/utils/entity_types.py +0 -19
  120. digitalhub/entities/utils/state.py +0 -31
  121. digitalhub/entities/utils/utils.py +0 -202
  122. /digitalhub/{context → entities/_base/project}/__init__.py +0 -0
  123. /digitalhub/{datastores → entities/_commons}/__init__.py +0 -0
  124. /digitalhub/{datastores/_base → entities/_operations}/__init__.py +0 -0
  125. /digitalhub/{datastores/local → readers/_commons}/__init__.py +0 -0
  126. {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/LICENSE.txt +0 -0
  127. {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/WHEEL +0 -0
  128. {digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/top_level.txt +0 -0
@@ -1,346 +0,0 @@
1
- from __future__ import annotations
2
-
3
- API_BASE = "/api/v1"
4
- API_CONTEXT = f"{API_BASE}/-"
5
-
6
-
7
- ##############################
8
- # Context APIs
9
- ##############################
10
-
11
-
12
- def api_ctx_create(
13
- project: str,
14
- entity_type: str,
15
- ) -> str:
16
- """
17
- Create context API.
18
-
19
- Parameters
20
- ----------
21
- project : str
22
- Project name.
23
- entity_type : str
24
- The name of the entity_type.
25
-
26
- Returns
27
- -------
28
- str
29
- The API string formatted.
30
- """
31
- if not entity_type.endswith("s"):
32
- entity_type += "s"
33
- return f"{API_CONTEXT}/{project}/{entity_type}"
34
-
35
-
36
- def api_ctx_read(project: str, entity_type: str, entity_id: str) -> str:
37
- """
38
- Read context API.
39
-
40
- Parameters
41
- ----------
42
- project : str
43
- Project name.
44
- entity_type : str
45
- Entity type.
46
- entity_id : str
47
- Entity ID.
48
-
49
- Returns
50
- -------
51
- str
52
- The API string formatted.
53
- """
54
- if not entity_type.endswith("s"):
55
- entity_type += "s"
56
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
57
-
58
-
59
- def api_ctx_update(
60
- project: str,
61
- entity_type: str,
62
- entity_id: str,
63
- ) -> str:
64
- """
65
- Update context API.
66
-
67
- Parameters
68
- ----------
69
- project : str
70
- Project name.
71
- entity_type : str
72
- Entity type.
73
- entity_id : str
74
- Entity ID.
75
-
76
- Returns
77
- -------
78
- str
79
- The API string formatted.
80
- """
81
- if not entity_type.endswith("s"):
82
- entity_type += "s"
83
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
84
-
85
-
86
- def api_ctx_delete(project: str, entity_type: str, entity_id: str) -> str:
87
- """
88
- Delete context API.
89
-
90
- Parameters
91
- ----------
92
- project : str
93
- Project name.
94
- entity_type : str
95
- Entity type.
96
- entity_id : str
97
- Entity ID.
98
-
99
- Returns
100
- -------
101
- str
102
- The API string formatted.
103
- """
104
- if not entity_type.endswith("s"):
105
- entity_type += "s"
106
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}"
107
-
108
-
109
- def api_ctx_list(project: str, entity_type: str) -> str:
110
- """
111
- List context API.
112
-
113
- Parameters
114
- ----------
115
- project : str
116
- Project name.
117
- entity_type : str
118
- Entity type.
119
-
120
- Returns
121
- -------
122
- str
123
- The API string formatted.
124
- """
125
- if not entity_type.endswith("s"):
126
- entity_type += "s"
127
- return f"{API_CONTEXT}/{project}/{entity_type}"
128
-
129
-
130
- def api_ctx_data(project: str, entity_type: str) -> str:
131
- """
132
- Data context API.
133
-
134
- Parameters
135
- ----------
136
- project : str
137
- Project name.
138
- entity_type : str
139
- Entity type.
140
-
141
- Returns
142
- -------
143
- str
144
- The API string formatted.
145
- """
146
- if not entity_type.endswith("s"):
147
- entity_type += "s"
148
- return f"{API_CONTEXT}/{project}/{entity_type}/data"
149
-
150
-
151
- def api_ctx_logs(project: str, entity_type: str, entity_id: str) -> str:
152
- """
153
- Logs context API.
154
-
155
- Parameters
156
- ----------
157
- project : str
158
- Project name.
159
- entity_type : str
160
- Entity type.
161
- entity_id : str
162
- Entity ID.
163
-
164
- Returns
165
- -------
166
- str
167
- The API string formatted.
168
- """
169
- if not entity_type.endswith("s"):
170
- entity_type += "s"
171
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/logs"
172
-
173
-
174
- def api_ctx_stop(project: str, entity_type: str, entity_id: str) -> str:
175
- """
176
- Stop context API.
177
-
178
- Parameters
179
- ----------
180
- project : str
181
- Project name.
182
- entity_type : str
183
- Entity type.
184
- entity_id : str
185
- Entity ID.
186
-
187
- Returns
188
- -------
189
- str
190
- The API string formatted.
191
- """
192
- if not entity_type.endswith("s"):
193
- entity_type += "s"
194
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/stop"
195
-
196
-
197
- def api_ctx_resume(project: str, entity_type: str, entity_id: str) -> str:
198
- """
199
- Resume context API.
200
-
201
- Parameters
202
- ----------
203
- project : str
204
- Project name.
205
- entity_type : str
206
- Entity type.
207
- entity_id : str
208
- Entity ID.
209
-
210
- Returns
211
- -------
212
- str
213
- The API string formatted.
214
- """
215
- if not entity_type.endswith("s"):
216
- entity_type += "s"
217
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/resume"
218
-
219
-
220
- def api_ctx_files(project: str, entity_type: str, entity_id: str) -> str:
221
- """
222
- Files context API.
223
-
224
- Parameters
225
- ----------
226
- project : str
227
- Project name.
228
- entity_type : str
229
- Entity type.
230
- entity_id : str
231
- Entity ID.
232
-
233
- Returns
234
- -------
235
- str
236
- The API string formatted.
237
- """
238
- if not entity_type.endswith("s"):
239
- entity_type += "s"
240
- return f"{API_CONTEXT}/{project}/{entity_type}/{entity_id}/files/info"
241
-
242
-
243
- ##############################
244
- # Base APIs
245
- ##############################
246
-
247
-
248
- def api_base_list(entity_type: str) -> str:
249
- """
250
- List base API.
251
-
252
- Parameters
253
- ----------
254
- entity_type : str
255
- Entity type.
256
-
257
- Returns
258
- -------
259
- str
260
- The API string formatted.
261
- """
262
- if not entity_type.endswith("s"):
263
- entity_type += "s"
264
- return f"{API_BASE}/{entity_type}"
265
-
266
-
267
- def api_base_create(entity_type: str) -> str:
268
- """
269
- Create base API.
270
-
271
- Parameters
272
- ----------
273
- entity_type : str
274
- Entity type.
275
-
276
- Returns
277
- -------
278
- str
279
- The API string formatted.
280
- """
281
- if not entity_type.endswith("s"):
282
- entity_type += "s"
283
- return f"{API_BASE}/{entity_type}"
284
-
285
-
286
- def api_base_read(entity_type: str, entity_id: str) -> str:
287
- """
288
- Read base API.
289
-
290
- Parameters
291
- ----------
292
- entity_type : str
293
- Entity type.
294
- entity_id : str
295
- Entity ID.
296
-
297
- Returns
298
- -------
299
- str
300
- The API string formatted.
301
- """
302
- if not entity_type.endswith("s"):
303
- entity_type += "s"
304
- return f"{API_BASE}/{entity_type}/{entity_id}"
305
-
306
-
307
- def api_base_update(entity_type: str, entity_id: str) -> str:
308
- """
309
- Update base API.
310
-
311
- Parameters
312
- ----------
313
- entity_type : str
314
- Entity type.
315
- entity_id : str
316
- Entity ID.
317
-
318
- Returns
319
- -------
320
- str
321
- The API string formatted.
322
- """
323
- if not entity_type.endswith("s"):
324
- entity_type += "s"
325
- return f"{API_BASE}/{entity_type}/{entity_id}"
326
-
327
-
328
- def api_base_delete(entity_type: str, entity_id: str) -> str:
329
- """
330
- Delete base API.
331
-
332
- Parameters
333
- ----------
334
- entity_type : str
335
- Entity type.
336
- entity_id : str
337
- Entity ID.
338
-
339
- Returns
340
- -------
341
- str
342
- The API string formatted.
343
- """
344
- if not entity_type.endswith("s"):
345
- entity_type += "s"
346
- return f"{API_BASE}/{entity_type}/{entity_id}"
@@ -1,19 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from enum import Enum
4
-
5
-
6
- class EntityTypes(Enum):
7
- """
8
- Entity types.
9
- """
10
-
11
- PROJECT = "project"
12
- ARTIFACT = "artifact"
13
- DATAITEM = "dataitem"
14
- MODEL = "model"
15
- SECRET = "secret"
16
- FUNCTION = "function"
17
- WORKFLOW = "workflow"
18
- TASK = "task"
19
- RUN = "run"
@@ -1,31 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from enum import Enum
4
-
5
-
6
- class State(Enum):
7
- """
8
- State enumeration.
9
- """
10
-
11
- BUILT = "BUILT"
12
- CANCELLED = "CANCELLED"
13
- COMPLETED = "COMPLETED"
14
- CREATED = "CREATED"
15
- CREATING = "CREATING"
16
- DELETED = "DELETED"
17
- DELETING = "DELETING"
18
- ERROR = "ERROR"
19
- FSM_ERROR = "FSM_ERROR"
20
- IDLE = "IDLE"
21
- NONE = "NONE"
22
- ONLINE = "ONLINE"
23
- PENDING = "PENDING"
24
- READY = "READY"
25
- RESUME = "RESUME"
26
- RUN_ERROR = "RUN_ERROR"
27
- RUNNING = "RUNNING"
28
- STOP = "STOP"
29
- STOPPED = "STOPPED"
30
- SUCCESS = "SUCCESS"
31
- UNKNOWN = "UNKNOWN"
@@ -1,202 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from pathlib import Path
4
-
5
- from digitalhub.entities.utils.entity_types import EntityTypes
6
- from digitalhub.utils.file_utils import get_file_mime_type
7
- from digitalhub.utils.s3_utils import get_s3_bucket
8
- from digitalhub.utils.uri_utils import check_local_path
9
-
10
-
11
- def parse_entity_key(key: str) -> tuple[str]:
12
- """
13
- Parse the entity key. Returns project, entity type, kind, name and uuid.
14
-
15
- Parameters
16
- ----------
17
- key : str
18
- The entity key.
19
-
20
- Returns
21
- -------
22
- tuple[str]
23
- Project, entity type, kind, name and uuid.
24
- """
25
- try:
26
- # Remove "store://" from the key
27
- key = key.replace("store://", "")
28
-
29
- # Split the key into parts
30
- parts = key.split("/")
31
-
32
- # The project is the first part
33
- project = parts[0]
34
-
35
- # The entity type is the second part
36
- entity_type = parts[1]
37
-
38
- # The kind is the third part
39
- kind = parts[2]
40
-
41
- # Tasks and runs have no name and uuid
42
- if entity_type in (EntityTypes.TASK.value, EntityTypes.RUN.value):
43
- name = None
44
- uuid = parts[3]
45
-
46
- # The name and uuid are separated by a colon in the last part
47
- else:
48
- name, uuid = parts[3].split(":")
49
-
50
- return project, entity_type, kind, name, uuid
51
- except Exception as e:
52
- raise ValueError("Invalid key format.") from e
53
-
54
-
55
- def eval_local_source(source: str | list[str]) -> None:
56
- """
57
- Evaluate if source is local.
58
-
59
- Parameters
60
- ----------
61
- source : str | list[str]
62
- Source(s).
63
-
64
- Returns
65
- -------
66
- None
67
- """
68
- if isinstance(source, list):
69
- if not source:
70
- raise ValueError("Empty list of sources.")
71
- source_is_local = all(check_local_path(s) for s in source)
72
- for s in source:
73
- if Path(s).is_dir():
74
- raise ValueError(f"Invalid source path: {s}. List of paths must be list of files, not directories.")
75
- else:
76
- source_is_local = check_local_path(source)
77
-
78
- if not source_is_local:
79
- raise ValueError("Invalid source path. Source must be a local path.")
80
-
81
-
82
- def eval_zip_type(source: str | list[str]) -> bool:
83
- """
84
- Evaluate zip type.
85
-
86
- Parameters
87
- ----------
88
- source : str | list[str]
89
- Source(s).
90
-
91
- Returns
92
- -------
93
- bool
94
- True if path is zip.
95
- """
96
- if isinstance(source, list):
97
- if len(source) > 1:
98
- return False
99
- else:
100
- path = source[0]
101
- else:
102
- if Path(source).is_dir():
103
- return False
104
- path = source
105
-
106
- extension = path.endswith(".zip")
107
- mime_zip = get_file_mime_type(path) == "application/zip"
108
- return extension or mime_zip
109
-
110
-
111
- def build_log_path_from_filename(
112
- project: str,
113
- entity_type: str,
114
- name: str,
115
- uuid: str,
116
- filename: str,
117
- ) -> str:
118
- """
119
- Build log path.
120
-
121
- Parameters
122
- ----------
123
- project : str
124
- Project name.
125
- entity_type : str
126
- Entity type.
127
- name : str
128
- Object name.
129
- uuid : str
130
- Object UUID.
131
- filename : str
132
- Filename.
133
-
134
- Returns
135
- -------
136
- str
137
- Log path.
138
- """
139
- return f"s3://{get_s3_bucket()}/{project}/{entity_type}/{name}/{uuid}/{filename}"
140
-
141
-
142
- def build_log_path_from_source(
143
- project: str,
144
- entity_type: str,
145
- name: str,
146
- uuid: str,
147
- source: str | list[str],
148
- ) -> str:
149
- """
150
- Build log path.
151
-
152
- Parameters
153
- ----------
154
- project : str
155
- Project name.
156
- entity_type : str
157
- Entity type.
158
- name : str
159
- Object name.
160
- uuid : str
161
- Object UUID.
162
- source : str | list[str]
163
- Source(s).
164
-
165
- Returns
166
- -------
167
- str
168
- Log path.
169
- """
170
- is_zip = eval_zip_type(source)
171
- scheme = "zip+s3" if is_zip else "s3"
172
- path = f"{scheme}://{get_s3_bucket()}/{project}/{entity_type}/{name}/{uuid}"
173
-
174
- if isinstance(source, list) and len(source) >= 1:
175
- if len(source) > 1:
176
- path += "/"
177
- else:
178
- path += f"/{Path(source[0]).name}"
179
- elif Path(source).is_dir():
180
- path += "/"
181
- elif Path(source).is_file():
182
- path += f"/{Path(source).name}"
183
-
184
- return path
185
-
186
-
187
- def get_entity_type_from_key(key: str) -> str:
188
- """
189
- Get entity type.
190
-
191
- Parameters
192
- ----------
193
- key : str
194
- The key of the entity.
195
-
196
- Returns
197
- -------
198
- str
199
- The entity type.
200
- """
201
- _, entity_type, _, _, _ = parse_entity_key(key)
202
- return entity_type