digitalhub 0.13.0b3__py3-none-any.whl → 0.14.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- digitalhub/__init__.py +3 -8
- digitalhub/context/api.py +43 -6
- digitalhub/context/builder.py +1 -5
- digitalhub/context/context.py +28 -13
- digitalhub/entities/_base/_base/entity.py +0 -15
- digitalhub/entities/_base/context/entity.py +1 -4
- digitalhub/entities/_base/entity/builder.py +5 -5
- digitalhub/entities/_base/entity/entity.py +0 -8
- digitalhub/entities/_base/executable/entity.py +195 -87
- digitalhub/entities/_base/material/entity.py +11 -23
- digitalhub/entities/_base/material/utils.py +28 -4
- digitalhub/entities/_base/runtime_entity/builder.py +53 -18
- digitalhub/entities/_base/unversioned/entity.py +1 -1
- digitalhub/entities/_base/versioned/entity.py +1 -1
- digitalhub/entities/_commons/enums.py +1 -31
- digitalhub/entities/_commons/metrics.py +64 -30
- digitalhub/entities/_commons/utils.py +119 -30
- digitalhub/entities/_constructors/_resources.py +151 -0
- digitalhub/entities/{_base/entity/_constructors → _constructors}/name.py +18 -0
- digitalhub/entities/_processors/base/crud.py +381 -0
- digitalhub/entities/_processors/base/import_export.py +118 -0
- digitalhub/entities/_processors/base/processor.py +299 -0
- digitalhub/entities/_processors/base/special_ops.py +104 -0
- digitalhub/entities/_processors/context/crud.py +652 -0
- digitalhub/entities/_processors/context/import_export.py +242 -0
- digitalhub/entities/_processors/context/material.py +123 -0
- digitalhub/entities/_processors/context/processor.py +400 -0
- digitalhub/entities/_processors/context/special_ops.py +476 -0
- digitalhub/entities/_processors/processors.py +12 -0
- digitalhub/entities/_processors/utils.py +38 -102
- digitalhub/entities/artifact/crud.py +58 -22
- digitalhub/entities/artifact/utils.py +28 -13
- digitalhub/entities/builders.py +2 -0
- digitalhub/entities/dataitem/crud.py +63 -20
- digitalhub/entities/dataitem/table/entity.py +27 -22
- digitalhub/entities/dataitem/utils.py +82 -32
- digitalhub/entities/function/_base/entity.py +3 -6
- digitalhub/entities/function/crud.py +55 -24
- digitalhub/entities/model/_base/entity.py +62 -20
- digitalhub/entities/model/crud.py +59 -23
- digitalhub/entities/model/mlflow/utils.py +29 -20
- digitalhub/entities/model/utils.py +28 -13
- digitalhub/entities/project/_base/builder.py +0 -6
- digitalhub/entities/project/_base/entity.py +337 -164
- digitalhub/entities/project/_base/spec.py +4 -4
- digitalhub/entities/project/crud.py +28 -71
- digitalhub/entities/project/utils.py +7 -3
- digitalhub/entities/run/_base/builder.py +0 -4
- digitalhub/entities/run/_base/entity.py +70 -63
- digitalhub/entities/run/crud.py +79 -26
- digitalhub/entities/secret/_base/entity.py +1 -5
- digitalhub/entities/secret/crud.py +31 -28
- digitalhub/entities/task/_base/builder.py +0 -4
- digitalhub/entities/task/_base/entity.py +5 -5
- digitalhub/entities/task/_base/models.py +13 -16
- digitalhub/entities/task/crud.py +61 -29
- digitalhub/entities/trigger/_base/entity.py +1 -5
- digitalhub/entities/trigger/crud.py +89 -30
- digitalhub/entities/workflow/_base/entity.py +3 -8
- digitalhub/entities/workflow/crud.py +55 -24
- digitalhub/factory/entity.py +283 -0
- digitalhub/factory/enums.py +18 -0
- digitalhub/factory/registry.py +197 -0
- digitalhub/factory/runtime.py +44 -0
- digitalhub/factory/utils.py +3 -54
- digitalhub/runtimes/_base.py +2 -2
- digitalhub/stores/client/{dhcore/api_builder.py → api_builder.py} +3 -3
- digitalhub/stores/client/builder.py +19 -31
- digitalhub/stores/client/client.py +322 -0
- digitalhub/stores/client/configurator.py +408 -0
- digitalhub/stores/client/enums.py +50 -0
- digitalhub/stores/client/{dhcore/error_parser.py → error_parser.py} +0 -4
- digitalhub/stores/client/header_manager.py +61 -0
- digitalhub/stores/client/http_handler.py +152 -0
- digitalhub/stores/client/{_base/key_builder.py → key_builder.py} +14 -14
- digitalhub/stores/client/params_builder.py +330 -0
- digitalhub/stores/client/response_processor.py +102 -0
- digitalhub/stores/client/utils.py +35 -0
- digitalhub/stores/{credentials → configurator}/api.py +5 -9
- digitalhub/stores/configurator/configurator.py +123 -0
- digitalhub/stores/{credentials → configurator}/enums.py +27 -10
- digitalhub/stores/configurator/handler.py +213 -0
- digitalhub/stores/{credentials → configurator}/ini_module.py +31 -22
- digitalhub/stores/data/_base/store.py +0 -20
- digitalhub/stores/data/api.py +5 -7
- digitalhub/stores/data/builder.py +53 -27
- digitalhub/stores/data/local/store.py +0 -103
- digitalhub/stores/data/remote/store.py +0 -4
- digitalhub/stores/data/s3/configurator.py +39 -77
- digitalhub/stores/data/s3/store.py +57 -37
- digitalhub/stores/data/sql/configurator.py +66 -46
- digitalhub/stores/data/sql/store.py +171 -104
- digitalhub/stores/readers/data/factory.py +0 -8
- digitalhub/stores/readers/data/pandas/reader.py +9 -19
- digitalhub/utils/file_utils.py +0 -17
- digitalhub/utils/generic_utils.py +1 -14
- digitalhub/utils/git_utils.py +0 -8
- digitalhub/utils/io_utils.py +0 -12
- digitalhub/utils/store_utils.py +44 -0
- {digitalhub-0.13.0b3.dist-info → digitalhub-0.14.9.dist-info}/METADATA +5 -4
- {digitalhub-0.13.0b3.dist-info → digitalhub-0.14.9.dist-info}/RECORD +112 -113
- {digitalhub-0.13.0b3.dist-info → digitalhub-0.14.9.dist-info}/WHEEL +1 -1
- digitalhub/entities/_commons/types.py +0 -9
- digitalhub/entities/_processors/base.py +0 -531
- digitalhub/entities/_processors/context.py +0 -1299
- digitalhub/entities/task/_base/utils.py +0 -22
- digitalhub/factory/factory.py +0 -381
- digitalhub/stores/client/_base/api_builder.py +0 -34
- digitalhub/stores/client/_base/client.py +0 -243
- digitalhub/stores/client/_base/params_builder.py +0 -34
- digitalhub/stores/client/api.py +0 -36
- digitalhub/stores/client/dhcore/client.py +0 -613
- digitalhub/stores/client/dhcore/configurator.py +0 -675
- digitalhub/stores/client/dhcore/enums.py +0 -34
- digitalhub/stores/client/dhcore/key_builder.py +0 -62
- digitalhub/stores/client/dhcore/models.py +0 -40
- digitalhub/stores/client/dhcore/params_builder.py +0 -278
- digitalhub/stores/client/dhcore/utils.py +0 -94
- digitalhub/stores/client/local/api_builder.py +0 -116
- digitalhub/stores/client/local/client.py +0 -573
- digitalhub/stores/client/local/enums.py +0 -15
- digitalhub/stores/client/local/key_builder.py +0 -62
- digitalhub/stores/client/local/params_builder.py +0 -120
- digitalhub/stores/credentials/__init__.py +0 -3
- digitalhub/stores/credentials/configurator.py +0 -210
- digitalhub/stores/credentials/handler.py +0 -176
- digitalhub/stores/credentials/store.py +0 -81
- digitalhub/stores/data/enums.py +0 -15
- digitalhub/stores/data/s3/utils.py +0 -78
- /digitalhub/entities/{_base/entity/_constructors → _constructors}/__init__.py +0 -0
- /digitalhub/entities/{_base/entity/_constructors → _constructors}/metadata.py +0 -0
- /digitalhub/entities/{_base/entity/_constructors → _constructors}/spec.py +0 -0
- /digitalhub/entities/{_base/entity/_constructors → _constructors}/status.py +0 -0
- /digitalhub/entities/{_base/entity/_constructors → _constructors}/uuid.py +0 -0
- /digitalhub/{stores/client/_base → entities/_processors/base}/__init__.py +0 -0
- /digitalhub/{stores/client/dhcore → entities/_processors/context}/__init__.py +0 -0
- /digitalhub/stores/{client/local → configurator}/__init__.py +0 -0
- {digitalhub-0.13.0b3.dist-info → digitalhub-0.14.9.dist-info}/licenses/AUTHORS +0 -0
- {digitalhub-0.13.0b3.dist-info → digitalhub-0.14.9.dist-info}/licenses/LICENSE +0 -0
digitalhub/__init__.py
CHANGED
|
@@ -95,16 +95,11 @@ from digitalhub.entities.workflow.crud import (
|
|
|
95
95
|
new_workflow,
|
|
96
96
|
update_workflow,
|
|
97
97
|
)
|
|
98
|
+
from digitalhub.stores.client.utils import get_credentials_and_config, refresh_token
|
|
99
|
+
from digitalhub.stores.configurator.api import get_current_profile, set_current_profile
|
|
100
|
+
from digitalhub.utils.store_utils import get_s3_client, get_sql_engine
|
|
98
101
|
|
|
99
102
|
try:
|
|
100
103
|
from digitalhub.entities.model.mlflow.utils import from_mlflow_run, get_mlflow_model_metrics
|
|
101
104
|
except ImportError:
|
|
102
105
|
...
|
|
103
|
-
|
|
104
|
-
# Register entities into registry
|
|
105
|
-
from digitalhub.factory.utils import register_entities, register_runtimes_entities
|
|
106
|
-
from digitalhub.stores.client.dhcore.utils import refresh_token, set_dhcore_env
|
|
107
|
-
from digitalhub.stores.credentials.api import get_current_profile, set_current_profile
|
|
108
|
-
|
|
109
|
-
register_entities()
|
|
110
|
-
register_runtimes_entities()
|
digitalhub/context/api.py
CHANGED
|
@@ -7,6 +7,11 @@ from __future__ import annotations
|
|
|
7
7
|
import typing
|
|
8
8
|
|
|
9
9
|
from digitalhub.context.builder import context_builder
|
|
10
|
+
from digitalhub.entities._commons.enums import EntityTypes
|
|
11
|
+
from digitalhub.factory.entity import entity_factory
|
|
12
|
+
from digitalhub.stores.client.builder import get_client
|
|
13
|
+
from digitalhub.stores.client.enums import ApiCategories, BackendOperations
|
|
14
|
+
from digitalhub.utils.exceptions import ContextError, EntityNotExistsError
|
|
10
15
|
|
|
11
16
|
if typing.TYPE_CHECKING:
|
|
12
17
|
from digitalhub.context.context import Context
|
|
@@ -24,7 +29,7 @@ def build_context(project: Project, overwrite: bool = False) -> Context:
|
|
|
24
29
|
----------
|
|
25
30
|
project : Project
|
|
26
31
|
The project object used to build the context.
|
|
27
|
-
overwrite : bool
|
|
32
|
+
overwrite : bool
|
|
28
33
|
If True, overwrites existing context if it exists. Default is False.
|
|
29
34
|
|
|
30
35
|
Returns
|
|
@@ -49,20 +54,52 @@ def get_context(project: str) -> Context:
|
|
|
49
54
|
Context
|
|
50
55
|
The context for the given project name.
|
|
51
56
|
"""
|
|
52
|
-
|
|
57
|
+
try:
|
|
58
|
+
return context_builder.get(project)
|
|
59
|
+
except ContextError:
|
|
60
|
+
try:
|
|
61
|
+
return get_context_from_remote(project)
|
|
62
|
+
except EntityNotExistsError as e:
|
|
63
|
+
raise ContextError(f"Context '{project}' not found remotely nor locally.") from e
|
|
53
64
|
|
|
54
65
|
|
|
55
|
-
def
|
|
66
|
+
def get_context_from_remote(project: str) -> Context:
|
|
56
67
|
"""
|
|
57
|
-
|
|
68
|
+
Fetch project context from remote backend and create local context.
|
|
69
|
+
|
|
58
70
|
|
|
59
71
|
Parameters
|
|
60
72
|
----------
|
|
61
73
|
project : str
|
|
62
|
-
|
|
74
|
+
The name of the project to fetch from remote.
|
|
63
75
|
|
|
64
76
|
Returns
|
|
65
77
|
-------
|
|
66
|
-
|
|
78
|
+
Context
|
|
79
|
+
The context instance created from the remote project data.
|
|
80
|
+
"""
|
|
81
|
+
try:
|
|
82
|
+
client = get_client()
|
|
83
|
+
api = client.build_api(
|
|
84
|
+
ApiCategories.BASE.value,
|
|
85
|
+
BackendOperations.READ.value,
|
|
86
|
+
entity_type=EntityTypes.PROJECT.value,
|
|
87
|
+
entity_name=project,
|
|
88
|
+
)
|
|
89
|
+
obj = client.read_object(api)
|
|
90
|
+
entity_factory.build_entity_from_dict(obj)
|
|
91
|
+
return context_builder.get(project)
|
|
92
|
+
except EntityNotExistsError:
|
|
93
|
+
raise ContextError(f"Project '{project}' not found.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def delete_context(project: str) -> None:
|
|
97
|
+
"""
|
|
98
|
+
Delete the context for a given project name.
|
|
99
|
+
|
|
100
|
+
Parameters
|
|
101
|
+
----------
|
|
102
|
+
project : str
|
|
103
|
+
Project name.
|
|
67
104
|
"""
|
|
68
105
|
context_builder.remove(project)
|
digitalhub/context/builder.py
CHANGED
|
@@ -38,7 +38,7 @@ class ContextBuilder:
|
|
|
38
38
|
----------
|
|
39
39
|
project : Project
|
|
40
40
|
The project instance to create a context for.
|
|
41
|
-
overwrite : bool
|
|
41
|
+
overwrite : bool
|
|
42
42
|
If True, overwrites existing context if project name already exists. Default is False.
|
|
43
43
|
|
|
44
44
|
Returns
|
|
@@ -82,10 +82,6 @@ class ContextBuilder:
|
|
|
82
82
|
----------
|
|
83
83
|
project : str
|
|
84
84
|
The name of the project whose context should be removed.
|
|
85
|
-
|
|
86
|
-
Returns
|
|
87
|
-
-------
|
|
88
|
-
None
|
|
89
85
|
This method does not return anything.
|
|
90
86
|
|
|
91
87
|
Notes
|
digitalhub/context/context.py
CHANGED
|
@@ -4,12 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import os
|
|
7
8
|
import typing
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
|
|
11
|
+
from digitalhub.entities._commons.enums import EntityTypes
|
|
12
|
+
from digitalhub.runtimes.enums import RuntimeEnvVar
|
|
13
|
+
from digitalhub.stores.client.enums import ApiCategories, BackendOperations
|
|
14
|
+
from digitalhub.utils.exceptions import BackendError
|
|
15
|
+
|
|
10
16
|
if typing.TYPE_CHECKING:
|
|
11
17
|
from digitalhub.entities.project._base.entity import Project
|
|
12
|
-
from digitalhub.stores.client.
|
|
18
|
+
from digitalhub.stores.client.client import Client
|
|
13
19
|
|
|
14
20
|
|
|
15
21
|
class Context:
|
|
@@ -27,8 +33,6 @@ class Context:
|
|
|
27
33
|
The client instance (local or remote) associated with the project.
|
|
28
34
|
config : dict
|
|
29
35
|
Project configuration profile.
|
|
30
|
-
local : bool
|
|
31
|
-
Whether the client is local or remote.
|
|
32
36
|
root : Path
|
|
33
37
|
The local context project path.
|
|
34
38
|
is_running : bool
|
|
@@ -41,12 +45,31 @@ class Context:
|
|
|
41
45
|
self.name: str = project.name
|
|
42
46
|
self.client: Client = project._client
|
|
43
47
|
self.config: dict = project.spec.config
|
|
44
|
-
self.
|
|
45
|
-
self.root: Path = Path(project.spec.context)
|
|
48
|
+
self.root: Path = Path(project.spec.source)
|
|
46
49
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
47
50
|
|
|
48
51
|
self.is_running: bool = False
|
|
49
52
|
self._run_ctx: str | None = None
|
|
53
|
+
self._search_run_ctx()
|
|
54
|
+
|
|
55
|
+
def _search_run_ctx(self) -> None:
|
|
56
|
+
"""
|
|
57
|
+
Search for an existing run id in env.
|
|
58
|
+
"""
|
|
59
|
+
run_id = os.getenv(RuntimeEnvVar.RUN_ID.value)
|
|
60
|
+
if run_id is not None:
|
|
61
|
+
try:
|
|
62
|
+
api = self.client.build_api(
|
|
63
|
+
category=ApiCategories.CONTEXT.value,
|
|
64
|
+
operation=BackendOperations.READ.value,
|
|
65
|
+
project=self.name,
|
|
66
|
+
entity_type=EntityTypes.RUN.value,
|
|
67
|
+
entity_id=run_id,
|
|
68
|
+
)
|
|
69
|
+
run_key = self.client.read_object(api=api)["key"]
|
|
70
|
+
self.set_run(run_key)
|
|
71
|
+
except BackendError:
|
|
72
|
+
pass
|
|
50
73
|
|
|
51
74
|
def set_run(self, run_ctx: str) -> None:
|
|
52
75
|
"""
|
|
@@ -56,10 +79,6 @@ class Context:
|
|
|
56
79
|
----------
|
|
57
80
|
run_ctx : str
|
|
58
81
|
The run key to set.
|
|
59
|
-
|
|
60
|
-
Returns
|
|
61
|
-
-------
|
|
62
|
-
None
|
|
63
82
|
"""
|
|
64
83
|
self.is_running = True
|
|
65
84
|
self._run_ctx = run_ctx
|
|
@@ -67,10 +86,6 @@ class Context:
|
|
|
67
86
|
def unset_run(self) -> None:
|
|
68
87
|
"""
|
|
69
88
|
Clear the current run key and reset running state.
|
|
70
|
-
|
|
71
|
-
Returns
|
|
72
|
-
-------
|
|
73
|
-
None
|
|
74
89
|
"""
|
|
75
90
|
self.is_running = False
|
|
76
91
|
self._run_ctx = None
|
|
@@ -54,26 +54,11 @@ class Base:
|
|
|
54
54
|
----------
|
|
55
55
|
**kwargs : dict
|
|
56
56
|
Keyword arguments to be set as attributes.
|
|
57
|
-
|
|
58
|
-
Returns
|
|
59
|
-
-------
|
|
60
|
-
None
|
|
61
57
|
"""
|
|
62
58
|
for k, v in kwargs.items():
|
|
63
59
|
if k not in self.__dict__:
|
|
64
60
|
setattr(self, k, v)
|
|
65
61
|
|
|
66
|
-
def _get_private_attrs(self) -> dict:
|
|
67
|
-
"""
|
|
68
|
-
Return all private attributes of the object.
|
|
69
|
-
|
|
70
|
-
Returns
|
|
71
|
-
-------
|
|
72
|
-
dict
|
|
73
|
-
A dictionary containing the private attributes of the entity instance.
|
|
74
|
-
"""
|
|
75
|
-
return {k: v for k, v in self.__dict__.items() if k.startswith("_")}
|
|
76
|
-
|
|
77
62
|
def __repr__(self) -> str:
|
|
78
63
|
"""
|
|
79
64
|
Return string representation of the entity object.
|
|
@@ -8,8 +8,7 @@ import typing
|
|
|
8
8
|
|
|
9
9
|
from digitalhub.context.api import get_context
|
|
10
10
|
from digitalhub.entities._base.entity.entity import Entity
|
|
11
|
-
from digitalhub.entities._processors.
|
|
12
|
-
from digitalhub.utils.generic_utils import get_timestamp
|
|
11
|
+
from digitalhub.entities._processors.processors import context_processor
|
|
13
12
|
from digitalhub.utils.io_utils import write_yaml
|
|
14
13
|
|
|
15
14
|
if typing.TYPE_CHECKING:
|
|
@@ -81,8 +80,6 @@ class ContextEntity(Entity):
|
|
|
81
80
|
ContextEntity
|
|
82
81
|
Entity updated.
|
|
83
82
|
"""
|
|
84
|
-
if self._context().local:
|
|
85
|
-
self.metadata.updated = self.metadata.updated = get_timestamp()
|
|
86
83
|
new_obj = context_processor.update_context_entity(self.project, self.ENTITY_TYPE, self.id, self.to_dict())
|
|
87
84
|
self._update_attributes(new_obj)
|
|
88
85
|
return self
|
|
@@ -7,11 +7,11 @@ from __future__ import annotations
|
|
|
7
7
|
import typing
|
|
8
8
|
from abc import abstractmethod
|
|
9
9
|
|
|
10
|
-
from digitalhub.entities.
|
|
11
|
-
from digitalhub.entities.
|
|
12
|
-
from digitalhub.entities.
|
|
13
|
-
from digitalhub.entities.
|
|
14
|
-
from digitalhub.entities.
|
|
10
|
+
from digitalhub.entities._constructors.metadata import build_metadata
|
|
11
|
+
from digitalhub.entities._constructors.name import build_name
|
|
12
|
+
from digitalhub.entities._constructors.spec import build_spec
|
|
13
|
+
from digitalhub.entities._constructors.status import build_status
|
|
14
|
+
from digitalhub.entities._constructors.uuid import build_uuid
|
|
15
15
|
from digitalhub.utils.exceptions import BuilderError
|
|
16
16
|
|
|
17
17
|
if typing.TYPE_CHECKING:
|
|
@@ -67,10 +67,6 @@ class Entity(Base):
|
|
|
67
67
|
----------
|
|
68
68
|
obj : dict
|
|
69
69
|
Mapping representation of object.
|
|
70
|
-
|
|
71
|
-
Returns
|
|
72
|
-
-------
|
|
73
|
-
None
|
|
74
70
|
"""
|
|
75
71
|
self.metadata = obj.metadata
|
|
76
72
|
self.spec = obj.spec
|
|
@@ -95,10 +91,6 @@ class Entity(Base):
|
|
|
95
91
|
The target entity.
|
|
96
92
|
source : str
|
|
97
93
|
The source entity.
|
|
98
|
-
|
|
99
|
-
Returns
|
|
100
|
-
-------
|
|
101
|
-
None
|
|
102
94
|
"""
|
|
103
95
|
if self.metadata.relationships is None:
|
|
104
96
|
self.metadata.relationships = []
|