digitalhub 0.8.0b15__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of digitalhub might be problematic. Click here for more details.

Files changed (135) hide show
  1. digitalhub/__init__.py +19 -2
  2. digitalhub/client/_base/api_builder.py +16 -0
  3. digitalhub/client/_base/client.py +67 -0
  4. digitalhub/client/_base/key_builder.py +52 -0
  5. digitalhub/client/api.py +2 -38
  6. digitalhub/client/dhcore/api_builder.py +100 -0
  7. digitalhub/client/dhcore/client.py +100 -48
  8. digitalhub/client/dhcore/enums.py +27 -0
  9. digitalhub/client/dhcore/env.py +4 -2
  10. digitalhub/client/dhcore/key_builder.py +58 -0
  11. digitalhub/client/dhcore/utils.py +17 -17
  12. digitalhub/client/local/api_builder.py +100 -0
  13. digitalhub/client/local/client.py +22 -0
  14. digitalhub/client/local/key_builder.py +58 -0
  15. digitalhub/context/api.py +3 -38
  16. digitalhub/context/builder.py +10 -23
  17. digitalhub/context/context.py +20 -92
  18. digitalhub/entities/_base/context/entity.py +30 -22
  19. digitalhub/entities/_base/entity/_constructors/metadata.py +12 -1
  20. digitalhub/entities/_base/entity/_constructors/name.py +1 -1
  21. digitalhub/entities/_base/entity/_constructors/spec.py +1 -1
  22. digitalhub/entities/_base/entity/_constructors/status.py +3 -2
  23. digitalhub/entities/_base/entity/_constructors/uuid.py +1 -1
  24. digitalhub/entities/_base/entity/builder.py +6 -1
  25. digitalhub/entities/_base/entity/entity.py +32 -10
  26. digitalhub/entities/_base/entity/metadata.py +22 -0
  27. digitalhub/entities/_base/entity/spec.py +7 -2
  28. digitalhub/entities/_base/executable/entity.py +8 -8
  29. digitalhub/entities/_base/material/entity.py +49 -17
  30. digitalhub/entities/_base/material/status.py +0 -31
  31. digitalhub/entities/_base/material/utils.py +106 -0
  32. digitalhub/entities/_base/project/entity.py +341 -0
  33. digitalhub/entities/_base/unversioned/entity.py +3 -24
  34. digitalhub/entities/_base/versioned/entity.py +2 -26
  35. digitalhub/entities/_commons/enums.py +103 -0
  36. digitalhub/entities/_commons/utils.py +83 -0
  37. digitalhub/entities/_operations/processor.py +1873 -0
  38. digitalhub/entities/artifact/_base/builder.py +1 -1
  39. digitalhub/entities/artifact/_base/entity.py +1 -1
  40. digitalhub/entities/artifact/artifact/builder.py +2 -1
  41. digitalhub/entities/artifact/crud.py +46 -29
  42. digitalhub/entities/artifact/utils.py +62 -0
  43. digitalhub/entities/dataitem/_base/builder.py +1 -1
  44. digitalhub/entities/dataitem/_base/entity.py +6 -6
  45. digitalhub/entities/dataitem/crud.py +50 -66
  46. digitalhub/entities/dataitem/dataitem/builder.py +2 -1
  47. digitalhub/entities/dataitem/iceberg/builder.py +2 -1
  48. digitalhub/entities/dataitem/table/builder.py +2 -1
  49. digitalhub/entities/dataitem/table/entity.py +5 -10
  50. digitalhub/entities/dataitem/table/models.py +4 -5
  51. digitalhub/entities/dataitem/utils.py +137 -0
  52. digitalhub/entities/function/_base/builder.py +1 -1
  53. digitalhub/entities/function/_base/entity.py +6 -2
  54. digitalhub/entities/function/crud.py +36 -17
  55. digitalhub/entities/model/_base/builder.py +1 -1
  56. digitalhub/entities/model/_base/entity.py +1 -1
  57. digitalhub/entities/model/crud.py +46 -29
  58. digitalhub/entities/model/huggingface/builder.py +2 -1
  59. digitalhub/entities/model/huggingface/spec.py +4 -2
  60. digitalhub/entities/model/mlflow/builder.py +2 -1
  61. digitalhub/entities/model/mlflow/models.py +17 -9
  62. digitalhub/entities/model/mlflow/spec.py +6 -1
  63. digitalhub/entities/model/mlflow/utils.py +4 -2
  64. digitalhub/entities/model/model/builder.py +2 -1
  65. digitalhub/entities/model/sklearn/builder.py +2 -1
  66. digitalhub/entities/model/utils.py +62 -0
  67. digitalhub/entities/project/_base/builder.py +2 -2
  68. digitalhub/entities/project/_base/entity.py +82 -272
  69. digitalhub/entities/project/crud.py +110 -89
  70. digitalhub/entities/project/utils.py +35 -0
  71. digitalhub/entities/run/_base/builder.py +3 -1
  72. digitalhub/entities/run/_base/entity.py +52 -54
  73. digitalhub/entities/run/_base/spec.py +15 -7
  74. digitalhub/entities/run/crud.py +35 -17
  75. digitalhub/entities/secret/_base/builder.py +2 -2
  76. digitalhub/entities/secret/_base/entity.py +4 -10
  77. digitalhub/entities/secret/crud.py +36 -21
  78. digitalhub/entities/task/_base/builder.py +14 -14
  79. digitalhub/entities/task/_base/entity.py +21 -14
  80. digitalhub/entities/task/_base/models.py +35 -6
  81. digitalhub/entities/task/_base/spec.py +50 -13
  82. digitalhub/entities/task/_base/utils.py +18 -0
  83. digitalhub/entities/task/crud.py +35 -15
  84. digitalhub/entities/workflow/_base/builder.py +1 -1
  85. digitalhub/entities/workflow/_base/entity.py +22 -6
  86. digitalhub/entities/workflow/crud.py +36 -17
  87. digitalhub/factory/utils.py +1 -1
  88. digitalhub/readers/_base/reader.py +2 -2
  89. digitalhub/readers/_commons/enums.py +13 -0
  90. digitalhub/readers/api.py +3 -2
  91. digitalhub/readers/factory.py +12 -6
  92. digitalhub/readers/pandas/reader.py +20 -8
  93. digitalhub/runtimes/_base.py +0 -7
  94. digitalhub/runtimes/enums.py +12 -0
  95. digitalhub/stores/_base/store.py +59 -11
  96. digitalhub/stores/builder.py +5 -5
  97. digitalhub/stores/local/store.py +43 -4
  98. digitalhub/stores/remote/store.py +31 -5
  99. digitalhub/stores/s3/store.py +136 -57
  100. digitalhub/stores/sql/store.py +122 -47
  101. digitalhub/utils/exceptions.py +6 -0
  102. digitalhub/utils/file_utils.py +60 -2
  103. digitalhub/utils/generic_utils.py +45 -4
  104. digitalhub/utils/io_utils.py +18 -0
  105. digitalhub/utils/s3_utils.py +17 -0
  106. digitalhub/utils/uri_utils.py +153 -15
  107. {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/LICENSE.txt +1 -1
  108. {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/METADATA +11 -11
  109. {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/RECORD +117 -115
  110. {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/WHEEL +1 -1
  111. test/local/instances/test_validate.py +55 -0
  112. test/testkfp.py +4 -1
  113. digitalhub/datastores/_base/datastore.py +0 -85
  114. digitalhub/datastores/api.py +0 -37
  115. digitalhub/datastores/builder.py +0 -110
  116. digitalhub/datastores/local/datastore.py +0 -50
  117. digitalhub/datastores/remote/__init__.py +0 -0
  118. digitalhub/datastores/remote/datastore.py +0 -31
  119. digitalhub/datastores/s3/__init__.py +0 -0
  120. digitalhub/datastores/s3/datastore.py +0 -46
  121. digitalhub/datastores/sql/__init__.py +0 -0
  122. digitalhub/datastores/sql/datastore.py +0 -68
  123. digitalhub/entities/_base/api_utils.py +0 -620
  124. digitalhub/entities/_base/crud.py +0 -468
  125. digitalhub/entities/function/_base/models.py +0 -118
  126. digitalhub/entities/utils/__init__.py +0 -0
  127. digitalhub/entities/utils/api.py +0 -346
  128. digitalhub/entities/utils/entity_types.py +0 -19
  129. digitalhub/entities/utils/state.py +0 -31
  130. digitalhub/entities/utils/utils.py +0 -202
  131. /digitalhub/{context → entities/_base/project}/__init__.py +0 -0
  132. /digitalhub/{datastores → entities/_commons}/__init__.py +0 -0
  133. /digitalhub/{datastores/_base → entities/_operations}/__init__.py +0 -0
  134. /digitalhub/{datastores/local → readers/_commons}/__init__.py +0 -0
  135. {digitalhub-0.8.0b15.dist-info → digitalhub-0.9.0.dist-info}/top_level.txt +0 -0
@@ -1,10 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from digitalhub.entities._base.entity.builder import EntityBuilder
4
+ from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
4
5
  from digitalhub.entities.project._base.entity import Project
5
6
  from digitalhub.entities.project._base.spec import ProjectSpec, ProjectValidator
6
7
  from digitalhub.entities.project._base.status import ProjectStatus
7
- from digitalhub.entities.utils.entity_types import EntityTypes
8
8
 
9
9
 
10
10
  class ProjectProjectBuilder(EntityBuilder):
@@ -17,7 +17,7 @@ class ProjectProjectBuilder(EntityBuilder):
17
17
  ENTITY_SPEC_CLASS = ProjectSpec
18
18
  ENTITY_SPEC_VALIDATOR = ProjectValidator
19
19
  ENTITY_STATUS_CLASS = ProjectStatus
20
- ENTITY_KIND = "project"
20
+ ENTITY_KIND = EntityKinds.PROJECT_PROJECT.value
21
21
 
22
22
  def build(
23
23
  self,
@@ -1,19 +1,11 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import typing
4
- from pathlib import Path
5
4
  from typing import Any
6
5
 
7
- from digitalhub.client.api import get_client
8
- from digitalhub.context.api import set_context
9
- from digitalhub.entities._base.api_utils import (
10
- create_entity_api_base,
11
- read_entity_api_base,
12
- read_entity_api_ctx,
13
- update_entity_api_base,
14
- )
15
- from digitalhub.entities._base.crud import import_context_entity, import_executable_entity
16
- from digitalhub.entities._base.entity.entity import Entity
6
+ from digitalhub.entities._base.project.entity import ProjectEntity
7
+ from digitalhub.entities._commons.enums import EntityTypes
8
+ from digitalhub.entities._operations.processor import processor
17
9
  from digitalhub.entities.artifact.crud import (
18
10
  delete_artifact,
19
11
  get_artifact,
@@ -63,7 +55,6 @@ from digitalhub.entities.secret.crud import (
63
55
  new_secret,
64
56
  update_secret,
65
57
  )
66
- from digitalhub.entities.utils.entity_types import EntityTypes
67
58
  from digitalhub.entities.workflow.crud import (
68
59
  delete_workflow,
69
60
  get_workflow,
@@ -73,11 +64,7 @@ from digitalhub.entities.workflow.crud import (
73
64
  new_workflow,
74
65
  update_workflow,
75
66
  )
76
- from digitalhub.factory.api import build_entity_from_dict
77
- from digitalhub.utils.exceptions import BackendError, EntityAlreadyExistsError, EntityError
78
- from digitalhub.utils.generic_utils import get_timestamp
79
- from digitalhub.utils.io_utils import write_yaml
80
- from digitalhub.utils.uri_utils import map_uri_scheme
67
+ from digitalhub.utils.exceptions import EntityError
81
68
 
82
69
  if typing.TYPE_CHECKING:
83
70
  from digitalhub.entities._base.entity.metadata import Metadata
@@ -92,7 +79,7 @@ if typing.TYPE_CHECKING:
92
79
  from digitalhub.entities.workflow._base.entity import Workflow
93
80
 
94
81
 
95
- class Project(Entity):
82
+ class Project(ProjectEntity):
96
83
  """
97
84
  A class representing a project.
98
85
  """
@@ -109,257 +96,10 @@ class Project(Entity):
109
96
  user: str | None = None,
110
97
  local: bool = False,
111
98
  ) -> None:
112
- super().__init__(kind, metadata, spec, status, user)
113
- self.id = name
114
- self.name = name
115
- self.key = f"store://{name}"
99
+ super().__init__(name, kind, metadata, spec, status, user, local)
116
100
  self.spec: ProjectSpec
117
101
  self.status: ProjectStatus
118
102
 
119
- # Add attributes to be used in the to_dict method
120
- self._obj_attr.extend(["id", "name"])
121
-
122
- # Set client
123
- self._client = get_client(local)
124
-
125
- # Set context
126
- set_context(self)
127
-
128
- ##############################
129
- # Save / Refresh / Export
130
- ##############################
131
-
132
- def save(self, update: bool = False) -> Project:
133
- """
134
- Save entity into backend.
135
-
136
- Parameters
137
- ----------
138
- update : bool
139
- If True, the object will be updated.
140
-
141
- Returns
142
- -------
143
- Project
144
- Entity saved.
145
- """
146
- obj = self._refresh_to_dict()
147
-
148
- if not update:
149
- new_obj = create_entity_api_base(self._client, self.ENTITY_TYPE, obj)
150
- new_obj["local"] = self._client.is_local()
151
- self._update_attributes(new_obj)
152
- return self
153
-
154
- self.metadata.updated = obj["metadata"]["updated"] = get_timestamp()
155
- new_obj = update_entity_api_base(self._client, self.ENTITY_TYPE, obj)
156
- new_obj["local"] = self._client.is_local()
157
- self._update_attributes(new_obj)
158
- return self
159
-
160
- def refresh(self) -> Project:
161
- """
162
- Refresh object from backend.
163
-
164
- Returns
165
- -------
166
- Project
167
- Project object.
168
- """
169
- new_obj = read_entity_api_base(self._client, self.ENTITY_TYPE, self.name)
170
- new_obj["local"] = self._client.is_local()
171
- self._update_attributes(new_obj)
172
- return self
173
-
174
- def export(self, filename: str | None = None) -> str:
175
- """
176
- Export object as a YAML file. If the objects are not embedded, the objects are
177
- exported as a YAML file.
178
-
179
- Parameters
180
- ----------
181
- filename : str
182
- Name of the export YAML file. If not specified, the default value is used.
183
-
184
- Returns
185
- -------
186
- str
187
- Exported file.
188
- """
189
- obj = self._refresh_to_dict()
190
-
191
- if filename is None:
192
- filename = f"{self.kind}_{self.name}.yml"
193
- pth = Path(self.spec.context) / filename
194
- pth.parent.mkdir(parents=True, exist_ok=True)
195
-
196
- obj = self._export_not_embedded(obj)
197
- write_yaml(pth, obj)
198
- return str(pth)
199
-
200
- def _refresh_to_dict(self) -> dict:
201
- """
202
- Try to refresh object to collect entities related to project.
203
-
204
- Returns
205
- -------
206
- dict
207
- Entity object in dictionary format.
208
- """
209
- try:
210
- return self.refresh().to_dict()
211
- except BackendError:
212
- return self.to_dict()
213
-
214
- def _export_not_embedded(self, obj: dict) -> dict:
215
- """
216
- Export project objects if not embedded.
217
-
218
- Parameters
219
- ----------
220
- obj : dict
221
- Project object in dictionary format.
222
-
223
- Returns
224
- -------
225
- dict
226
- Updatated project object in dictionary format with referenced entities.
227
- """
228
- # Cycle over entity types
229
- for entity_type in self._get_entity_types():
230
- # Entity types are stored as a list of entities
231
- for idx, entity in enumerate(obj.get("spec", {}).get(entity_type, [])):
232
- # Export entity if not embedded is in metadata, else do nothing
233
- if not self._is_embedded(entity):
234
- # Get entity object from backend
235
- obj_dict: dict = read_entity_api_ctx(entity["key"])
236
-
237
- # Create from dict (not need to new method, we do not save to backend)
238
- ent = build_entity_from_dict(obj_dict)
239
-
240
- # Export and stor ref in object metadata inside project
241
- pth = ent.export()
242
- obj["spec"][entity_type][idx]["metadata"]["ref"] = pth
243
-
244
- # Return updated object
245
- return obj
246
-
247
- def _import_entities(self, obj: dict) -> None:
248
- """
249
- Import project entities.
250
-
251
- Parameters
252
- ----------
253
- obj : dict
254
- Project object in dictionary format.
255
-
256
- Returns
257
- -------
258
- None
259
- """
260
- entity_types = self._get_entity_types()
261
-
262
- # Cycle over entity types
263
- for entity_type in entity_types:
264
- # Entity types are stored as a list of entities
265
- for entity in obj.get("spec", {}).get(entity_type, []):
266
- embedded = self._is_embedded(entity)
267
- ref = entity["metadata"].get("ref")
268
-
269
- # Import entity if not embedded and there is a ref
270
- if not embedded and ref is not None:
271
- # Import entity from local ref
272
- if map_uri_scheme(ref) == "local":
273
- try:
274
- # Artifacts, Dataitems and Models
275
- if entity_type in entity_types[:3]:
276
- import_context_entity(ref)
277
-
278
- # Functions and Workflows
279
- elif entity_type in entity_types[3:]:
280
- import_executable_entity(ref)
281
-
282
- except FileNotFoundError:
283
- msg = f"File not found: {ref}."
284
- raise EntityError(msg)
285
-
286
- # If entity is embedded, create it and try to save
287
- elif embedded:
288
- # It's possible that embedded field in metadata is not shown
289
- if entity["metadata"].get("embedded") is None:
290
- entity["metadata"]["embedded"] = True
291
-
292
- try:
293
- build_entity_from_dict(entity).save()
294
- except EntityAlreadyExistsError:
295
- pass
296
-
297
- def _is_embedded(self, entity: dict) -> bool:
298
- """
299
- Check if entity is embedded.
300
-
301
- Parameters
302
- ----------
303
- entity : dict
304
- Entity in dictionary format.
305
-
306
- Returns
307
- -------
308
- bool
309
- True if entity is embedded.
310
- """
311
- metadata_embedded = entity["metadata"].get("embedded", False)
312
- no_status = entity.get("status", None) is None
313
- no_spec = entity.get("spec", None) is None
314
- return metadata_embedded or not (no_status and no_spec)
315
-
316
- def _get_entity_types(self) -> list[str]:
317
- """
318
- Get entity types.
319
-
320
- Returns
321
- -------
322
- list
323
- Entity types.
324
- """
325
- return [
326
- f"{EntityTypes.ARTIFACT.value}s",
327
- f"{EntityTypes.DATAITEM.value}s",
328
- f"{EntityTypes.MODEL.value}s",
329
- f"{EntityTypes.FUNCTION.value}s",
330
- f"{EntityTypes.WORKFLOW.value}s",
331
- ]
332
-
333
- def run(self, workflow: str | None = None, **kwargs) -> Run:
334
- """
335
- Run workflow project.
336
-
337
- Parameters
338
- ----------
339
- workflow : str
340
- Workflow name.
341
- **kwargs : dict
342
- Keyword arguments passed to workflow.run().
343
-
344
- Returns
345
- -------
346
- Run
347
- Run instance.
348
- """
349
- self.refresh()
350
-
351
- workflow = workflow if workflow is not None else "main"
352
-
353
- for i in self.spec.workflows:
354
- if workflow in [i["name"], i["key"]]:
355
- entity = self.get_workflow(i["key"])
356
- break
357
- else:
358
- msg = f"Workflow {workflow} not found."
359
- raise EntityError(msg)
360
-
361
- return entity.run(**kwargs)
362
-
363
103
  ##############################
364
104
  # Artifacts
365
105
  ##############################
@@ -565,7 +305,7 @@ class Project(Entity):
565
305
  **kwargs,
566
306
  ) -> Artifact:
567
307
  """
568
- Import object from a YAML file.
308
+ Import object from a YAML file and create a new object into the backend.
569
309
 
570
310
  Parameters
571
311
  ----------
@@ -868,7 +608,7 @@ class Project(Entity):
868
608
  **kwargs,
869
609
  ) -> Dataitem:
870
610
  """
871
- Import object from a YAML file.
611
+ Import object from a YAML file and create a new object into the backend.
872
612
 
873
613
  Parameters
874
614
  ----------
@@ -1160,7 +900,7 @@ class Project(Entity):
1160
900
  **kwargs,
1161
901
  ) -> Model:
1162
902
  """
1163
- Import object from a YAML file.
903
+ Import object from a YAML file and create a new object into the backend.
1164
904
 
1165
905
  Parameters
1166
906
  ----------
@@ -1403,7 +1143,7 @@ class Project(Entity):
1403
1143
  **kwargs,
1404
1144
  ) -> Function:
1405
1145
  """
1406
- Import object from a YAML file.
1146
+ Import object from a YAML file and create a new object into the backend.
1407
1147
 
1408
1148
  Parameters
1409
1149
  ----------
@@ -1649,7 +1389,7 @@ class Project(Entity):
1649
1389
  **kwargs,
1650
1390
  ) -> Workflow:
1651
1391
  """
1652
- Import object from a YAML file.
1392
+ Import object from a YAML file and create a new object into the backend.
1653
1393
 
1654
1394
  Parameters
1655
1395
  ----------
@@ -1893,7 +1633,7 @@ class Project(Entity):
1893
1633
  **kwargs,
1894
1634
  ) -> Secret:
1895
1635
  """
1896
- Import object from a YAML file.
1636
+ Import object from a YAML file and create a new object into the backend.
1897
1637
 
1898
1638
  Parameters
1899
1639
  ----------
@@ -2076,3 +1816,73 @@ class Project(Entity):
2076
1816
  ##############################
2077
1817
  # Project methods
2078
1818
  ##############################
1819
+
1820
+ def run(self, workflow: str | None = None, **kwargs) -> Run:
1821
+ """
1822
+ Run workflow project.
1823
+
1824
+ Parameters
1825
+ ----------
1826
+ workflow : str
1827
+ Workflow name.
1828
+ **kwargs : dict
1829
+ Keyword arguments passed to workflow.run().
1830
+
1831
+ Returns
1832
+ -------
1833
+ Run
1834
+ Run instance.
1835
+ """
1836
+ self.refresh()
1837
+
1838
+ workflow = workflow if workflow is not None else "main"
1839
+
1840
+ for i in self.spec.workflows:
1841
+ if workflow in [i["name"], i["key"]]:
1842
+ entity = self.get_workflow(i["key"])
1843
+ break
1844
+ else:
1845
+ msg = f"Workflow {workflow} not found."
1846
+ raise EntityError(msg)
1847
+
1848
+ return entity.run(**kwargs)
1849
+
1850
+ def share(self, user: str) -> None:
1851
+ """
1852
+ Share project.
1853
+
1854
+ Parameters
1855
+ ----------
1856
+ user : str
1857
+ User to share project with.
1858
+ Returns
1859
+ -------
1860
+ None
1861
+ """
1862
+ return processor.share_project_entity(
1863
+ entity_type=self.ENTITY_TYPE,
1864
+ entity_name=self.name,
1865
+ user=user,
1866
+ unshare=False,
1867
+ local=self._client.is_local(),
1868
+ )
1869
+
1870
+ def unshare(self, user: str) -> None:
1871
+ """
1872
+ Unshare project.
1873
+
1874
+ Parameters
1875
+ ----------
1876
+ user : str
1877
+ User to unshare project with.
1878
+ Returns
1879
+ -------
1880
+ None
1881
+ """
1882
+ return processor.share_project_entity(
1883
+ entity_type=self.ENTITY_TYPE,
1884
+ entity_name=self.name,
1885
+ user=user,
1886
+ unshare=True,
1887
+ local=self._client.is_local(),
1888
+ )