digitalhub 0.14.0b5__py3-none-any.whl → 0.14.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- digitalhub/__init__.py +2 -2
- digitalhub/context/api.py +43 -6
- digitalhub/context/builder.py +1 -1
- digitalhub/context/context.py +3 -6
- digitalhub/entities/_base/context/entity.py +0 -3
- digitalhub/entities/_base/executable/entity.py +29 -11
- digitalhub/entities/_base/material/entity.py +2 -2
- digitalhub/entities/_base/material/utils.py +0 -4
- digitalhub/entities/_commons/enums.py +1 -0
- digitalhub/entities/_commons/utils.py +19 -0
- digitalhub/entities/_processors/base/crud.py +15 -24
- digitalhub/entities/_processors/base/import_export.py +3 -7
- digitalhub/entities/_processors/base/processor.py +4 -7
- digitalhub/entities/_processors/base/special_ops.py +4 -8
- digitalhub/entities/_processors/context/crud.py +27 -29
- digitalhub/entities/_processors/context/import_export.py +7 -7
- digitalhub/entities/_processors/context/material.py +2 -2
- digitalhub/entities/_processors/context/special_ops.py +25 -25
- digitalhub/entities/_processors/utils.py +7 -116
- digitalhub/entities/artifact/crud.py +3 -3
- digitalhub/entities/artifact/utils.py +2 -2
- digitalhub/entities/builders.py +2 -0
- digitalhub/entities/dataitem/crud.py +3 -3
- digitalhub/entities/dataitem/utils.py +10 -14
- digitalhub/entities/function/_base/entity.py +0 -3
- digitalhub/entities/function/crud.py +3 -3
- digitalhub/entities/model/crud.py +3 -3
- digitalhub/entities/model/mlflow/utils.py +29 -20
- digitalhub/entities/model/utils.py +2 -2
- digitalhub/entities/project/_base/builder.py +0 -6
- digitalhub/entities/project/_base/entity.py +264 -114
- digitalhub/entities/project/_base/spec.py +4 -4
- digitalhub/entities/project/crud.py +16 -51
- digitalhub/entities/project/utils.py +7 -3
- digitalhub/entities/secret/crud.py +2 -2
- digitalhub/entities/task/_base/models.py +13 -16
- digitalhub/entities/trigger/crud.py +28 -9
- digitalhub/entities/workflow/_base/entity.py +0 -5
- digitalhub/entities/workflow/crud.py +3 -6
- digitalhub/stores/client/{dhcore/api_builder.py → api_builder.py} +2 -3
- digitalhub/stores/client/builder.py +20 -32
- digitalhub/stores/client/client.py +322 -0
- digitalhub/stores/client/{dhcore/configurator.py → configurator.py} +148 -195
- digitalhub/stores/client/{_base/enums.py → enums.py} +11 -0
- digitalhub/stores/client/header_manager.py +61 -0
- digitalhub/stores/client/http_handler.py +152 -0
- digitalhub/stores/client/{_base/key_builder.py → key_builder.py} +14 -14
- digitalhub/stores/client/{dhcore/params_builder.py → params_builder.py} +51 -12
- digitalhub/stores/client/response_processor.py +102 -0
- digitalhub/stores/client/utils.py +35 -0
- digitalhub/stores/{credentials → configurator}/api.py +5 -9
- digitalhub/stores/configurator/configurator.py +123 -0
- digitalhub/stores/{credentials → configurator}/enums.py +26 -10
- digitalhub/stores/configurator/handler.py +213 -0
- digitalhub/stores/{credentials → configurator}/ini_module.py +31 -6
- digitalhub/stores/data/_base/store.py +0 -4
- digitalhub/stores/data/api.py +4 -6
- digitalhub/stores/data/builder.py +6 -38
- digitalhub/stores/data/s3/configurator.py +30 -114
- digitalhub/stores/data/s3/store.py +9 -22
- digitalhub/stores/data/sql/configurator.py +49 -71
- digitalhub/stores/data/sql/store.py +26 -61
- digitalhub/utils/generic_utils.py +0 -12
- digitalhub/utils/git_utils.py +0 -8
- digitalhub/utils/io_utils.py +0 -8
- digitalhub/utils/store_utils.py +1 -1
- {digitalhub-0.14.0b5.dist-info → digitalhub-0.14.9.dist-info}/METADATA +3 -3
- {digitalhub-0.14.0b5.dist-info → digitalhub-0.14.9.dist-info}/RECORD +73 -86
- {digitalhub-0.14.0b5.dist-info → digitalhub-0.14.9.dist-info}/WHEEL +1 -1
- digitalhub/stores/client/_base/api_builder.py +0 -34
- digitalhub/stores/client/_base/client.py +0 -243
- digitalhub/stores/client/_base/params_builder.py +0 -82
- digitalhub/stores/client/api.py +0 -32
- digitalhub/stores/client/dhcore/__init__.py +0 -3
- digitalhub/stores/client/dhcore/client.py +0 -553
- digitalhub/stores/client/dhcore/enums.py +0 -18
- digitalhub/stores/client/dhcore/key_builder.py +0 -62
- digitalhub/stores/client/dhcore/utils.py +0 -86
- digitalhub/stores/client/local/__init__.py +0 -3
- digitalhub/stores/client/local/api_builder.py +0 -116
- digitalhub/stores/client/local/client.py +0 -605
- digitalhub/stores/client/local/enums.py +0 -15
- digitalhub/stores/client/local/key_builder.py +0 -62
- digitalhub/stores/client/local/params_builder.py +0 -97
- digitalhub/stores/credentials/__init__.py +0 -3
- digitalhub/stores/credentials/configurator.py +0 -185
- digitalhub/stores/credentials/handler.py +0 -164
- digitalhub/stores/credentials/store.py +0 -77
- digitalhub/stores/data/enums.py +0 -15
- /digitalhub/stores/client/{dhcore/error_parser.py → error_parser.py} +0 -0
- /digitalhub/stores/{client/_base → configurator}/__init__.py +0 -0
- {digitalhub-0.14.0b5.dist-info → digitalhub-0.14.9.dist-info}/licenses/AUTHORS +0 -0
- {digitalhub-0.14.0b5.dist-info → digitalhub-0.14.9.dist-info}/licenses/LICENSE +0 -0
|
@@ -41,17 +41,17 @@ def eval_source(
|
|
|
41
41
|
|
|
42
42
|
Parameters
|
|
43
43
|
----------
|
|
44
|
-
source : SourcesOrListOfSources
|
|
44
|
+
source : SourcesOrListOfSources
|
|
45
45
|
The source specification(s) for the dataitem. Can be file paths,
|
|
46
46
|
URLs, or other source identifiers.
|
|
47
|
-
data : Any
|
|
47
|
+
data : Any
|
|
48
48
|
The data object to process (e.g., DataFrame). Alternative to source.
|
|
49
49
|
Exactly one of source or data must be provided.
|
|
50
|
-
kind : str
|
|
50
|
+
kind : str
|
|
51
51
|
The kind of dataitem being created (e.g., 'table').
|
|
52
|
-
name : str
|
|
52
|
+
name : str
|
|
53
53
|
The name of the dataitem, used for generating file paths.
|
|
54
|
-
project : str
|
|
54
|
+
project : str
|
|
55
55
|
The project name, used for context and path generation.
|
|
56
56
|
|
|
57
57
|
Returns
|
|
@@ -105,13 +105,13 @@ def eval_data(
|
|
|
105
105
|
how data should be processed.
|
|
106
106
|
source : SourcesOrListOfSources
|
|
107
107
|
The source specification(s) to load data from.
|
|
108
|
-
data : Any
|
|
108
|
+
data : Any
|
|
109
109
|
Pre-loaded data object. If provided, may be returned directly
|
|
110
110
|
depending on the dataitem kind.
|
|
111
|
-
file_format : str
|
|
111
|
+
file_format : str
|
|
112
112
|
The file format specification for reading the source
|
|
113
113
|
(e.g., 'parquet', 'csv').
|
|
114
|
-
engine : str
|
|
114
|
+
engine : str
|
|
115
115
|
The engine to use for reading the data (e.g., 'pandas', 'polars').
|
|
116
116
|
|
|
117
117
|
Returns
|
|
@@ -159,10 +159,10 @@ def process_kwargs(
|
|
|
159
159
|
The kind of dataitem being created (e.g., 'table').
|
|
160
160
|
source : SourcesOrListOfSources
|
|
161
161
|
The source specification(s) for the dataitem content.
|
|
162
|
-
data : Any
|
|
162
|
+
data : Any
|
|
163
163
|
The data object for schema extraction and processing.
|
|
164
164
|
Used as an alternative to source for table dataitems.
|
|
165
|
-
path : str
|
|
165
|
+
path : str
|
|
166
166
|
The destination path for the dataitem entity.
|
|
167
167
|
If None, a path will be automatically generated.
|
|
168
168
|
**kwargs : dict
|
|
@@ -201,10 +201,6 @@ def clean_tmp_path(pth: SourcesOrListOfSources) -> None:
|
|
|
201
201
|
pth : SourcesOrListOfSources
|
|
202
202
|
The path or list of paths to clean up. Can be file paths
|
|
203
203
|
or directory paths that need to be removed.
|
|
204
|
-
|
|
205
|
-
Returns
|
|
206
|
-
-------
|
|
207
|
-
None
|
|
208
204
|
"""
|
|
209
205
|
if isinstance(pth, list):
|
|
210
206
|
for p in pth:
|
|
@@ -10,7 +10,6 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
10
10
|
from digitalhub.entities._base.executable.entity import ExecutableEntity
|
|
11
11
|
from digitalhub.entities._commons.enums import EntityTypes, Relationship
|
|
12
12
|
from digitalhub.factory.entity import entity_factory
|
|
13
|
-
from digitalhub.utils.exceptions import BackendError
|
|
14
13
|
|
|
15
14
|
if typing.TYPE_CHECKING:
|
|
16
15
|
from digitalhub.entities._base.entity.metadata import Metadata
|
|
@@ -91,8 +90,6 @@ class Function(ExecutableEntity):
|
|
|
91
90
|
|
|
92
91
|
# If execution is done by DHCore backend, return the object
|
|
93
92
|
if not local_execution:
|
|
94
|
-
if self._context().local:
|
|
95
|
-
raise BackendError("Cannot run remote function with local backend.")
|
|
96
93
|
if wait:
|
|
97
94
|
return run.wait(log_info=log_info)
|
|
98
95
|
return run
|
|
@@ -156,7 +156,7 @@ def list_functions(
|
|
|
156
156
|
state: str | None = None,
|
|
157
157
|
created: str | None = None,
|
|
158
158
|
updated: str | None = None,
|
|
159
|
-
|
|
159
|
+
versions: str | None = None,
|
|
160
160
|
) -> list[Function]:
|
|
161
161
|
"""
|
|
162
162
|
List all latest version objects from backend.
|
|
@@ -179,7 +179,7 @@ def list_functions(
|
|
|
179
179
|
Creation date filter.
|
|
180
180
|
updated : str
|
|
181
181
|
Update date filter.
|
|
182
|
-
|
|
182
|
+
versions : str
|
|
183
183
|
Object version, default is latest.
|
|
184
184
|
|
|
185
185
|
Returns
|
|
@@ -201,7 +201,7 @@ def list_functions(
|
|
|
201
201
|
state=state,
|
|
202
202
|
created=created,
|
|
203
203
|
updated=updated,
|
|
204
|
-
|
|
204
|
+
versions=versions,
|
|
205
205
|
)
|
|
206
206
|
|
|
207
207
|
|
|
@@ -211,7 +211,7 @@ def list_models(
|
|
|
211
211
|
state: str | None = None,
|
|
212
212
|
created: str | None = None,
|
|
213
213
|
updated: str | None = None,
|
|
214
|
-
|
|
214
|
+
versions: str | None = None,
|
|
215
215
|
) -> list[Model]:
|
|
216
216
|
"""
|
|
217
217
|
List all latest version objects from backend.
|
|
@@ -234,7 +234,7 @@ def list_models(
|
|
|
234
234
|
Creation date filter.
|
|
235
235
|
updated : str
|
|
236
236
|
Update date filter.
|
|
237
|
-
|
|
237
|
+
versions : str
|
|
238
238
|
Object version, default is latest.
|
|
239
239
|
|
|
240
240
|
Returns
|
|
@@ -256,7 +256,7 @@ def list_models(
|
|
|
256
256
|
state=state,
|
|
257
257
|
created=created,
|
|
258
258
|
updated=updated,
|
|
259
|
-
|
|
259
|
+
versions=versions,
|
|
260
260
|
)
|
|
261
261
|
|
|
262
262
|
|
|
@@ -53,10 +53,13 @@ def from_mlflow_run(run_id: str) -> dict:
|
|
|
53
53
|
# Extract spec
|
|
54
54
|
data = run.data
|
|
55
55
|
parameters = data.params
|
|
56
|
-
source_path = urlparse(run.info.artifact_uri).path
|
|
56
|
+
source_path = urlparse(run.info.artifact_uri).path + "/model"
|
|
57
57
|
model_uri = f"runs:/{run_id}/model"
|
|
58
58
|
model = mlflow.pyfunc.load_model(model_uri=model_uri)
|
|
59
|
-
|
|
59
|
+
try:
|
|
60
|
+
model_config = model.model_config
|
|
61
|
+
except Exception:
|
|
62
|
+
model_config = {}
|
|
60
63
|
flavor = None
|
|
61
64
|
for f in model.metadata.flavors:
|
|
62
65
|
if f != "python_function":
|
|
@@ -64,27 +67,33 @@ def from_mlflow_run(run_id: str) -> dict:
|
|
|
64
67
|
break
|
|
65
68
|
|
|
66
69
|
# Extract signature
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
try:
|
|
71
|
+
mlflow_signature = model.metadata.signature
|
|
72
|
+
signature = Signature(
|
|
73
|
+
inputs=mlflow_signature.inputs.to_json() if mlflow_signature.inputs else None,
|
|
74
|
+
outputs=mlflow_signature.outputs.to_json() if mlflow_signature.outputs else None,
|
|
75
|
+
params=mlflow_signature.params.to_json() if mlflow_signature.params else None,
|
|
76
|
+
).to_dict()
|
|
77
|
+
except Exception:
|
|
78
|
+
signature = None
|
|
73
79
|
|
|
74
80
|
# Extract datasets
|
|
75
81
|
datasets = []
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
82
|
+
try:
|
|
83
|
+
if run.inputs and run.inputs.dataset_inputs:
|
|
84
|
+
datasets = [
|
|
85
|
+
Dataset(
|
|
86
|
+
name=d.dataset.name,
|
|
87
|
+
digest=d.dataset.digest,
|
|
88
|
+
profile=d.dataset.profile,
|
|
89
|
+
dataset_schema=d.dataset.schema,
|
|
90
|
+
source=d.dataset.source,
|
|
91
|
+
source_type=d.dataset.source_type,
|
|
92
|
+
).to_dict()
|
|
93
|
+
for d in run.inputs.dataset_inputs
|
|
94
|
+
]
|
|
95
|
+
except Exception:
|
|
96
|
+
datasets = []
|
|
88
97
|
|
|
89
98
|
# Create model params
|
|
90
99
|
model_params = {}
|
|
@@ -23,7 +23,7 @@ def eval_source(
|
|
|
23
23
|
|
|
24
24
|
Parameters
|
|
25
25
|
----------
|
|
26
|
-
source : str, list[str], or None
|
|
26
|
+
source : str, list[str], or None
|
|
27
27
|
The source specification(s) to evaluate. Can be a single
|
|
28
28
|
source string, a list of source strings, or None.
|
|
29
29
|
|
|
@@ -59,7 +59,7 @@ def process_kwargs(
|
|
|
59
59
|
source : str or list[str]
|
|
60
60
|
The source specification(s) for the model content.
|
|
61
61
|
Can be a single source or multiple sources.
|
|
62
|
-
path : str
|
|
62
|
+
path : str
|
|
63
63
|
The destination path for the model entity.
|
|
64
64
|
If None, a path will be automatically generated.
|
|
65
65
|
**kwargs : dict
|
|
@@ -29,7 +29,6 @@ class ProjectProjectBuilder(EntityBuilder):
|
|
|
29
29
|
kind: str,
|
|
30
30
|
description: str | None = None,
|
|
31
31
|
labels: list[str] | None = None,
|
|
32
|
-
local: bool = False,
|
|
33
32
|
config: dict | None = None,
|
|
34
33
|
context: str | None = None,
|
|
35
34
|
**kwargs,
|
|
@@ -47,8 +46,6 @@ class ProjectProjectBuilder(EntityBuilder):
|
|
|
47
46
|
Description of the object (human readable).
|
|
48
47
|
labels : list[str]
|
|
49
48
|
List of labels.
|
|
50
|
-
local : bool
|
|
51
|
-
If True, use local backend, if False use DHCore backend. Default to False.
|
|
52
49
|
config : dict
|
|
53
50
|
DHCore environment configuration.
|
|
54
51
|
context : str
|
|
@@ -80,7 +77,6 @@ class ProjectProjectBuilder(EntityBuilder):
|
|
|
80
77
|
metadata=metadata,
|
|
81
78
|
spec=spec,
|
|
82
79
|
status=status,
|
|
83
|
-
local=local,
|
|
84
80
|
)
|
|
85
81
|
|
|
86
82
|
def from_dict(self, obj: dict, validate: bool = True) -> Project:
|
|
@@ -120,7 +116,6 @@ class ProjectProjectBuilder(EntityBuilder):
|
|
|
120
116
|
"""
|
|
121
117
|
name = self.build_name(obj.get("name"))
|
|
122
118
|
kind = obj.get("kind")
|
|
123
|
-
local = obj.get("local", False)
|
|
124
119
|
metadata = self.build_metadata(**obj.get("metadata", {}))
|
|
125
120
|
spec = self.build_spec(validate=validate, **obj.get("spec", {}))
|
|
126
121
|
status = self.build_status(**obj.get("status", {}))
|
|
@@ -132,5 +127,4 @@ class ProjectProjectBuilder(EntityBuilder):
|
|
|
132
127
|
"spec": spec,
|
|
133
128
|
"status": status,
|
|
134
129
|
"user": user,
|
|
135
|
-
"local": local,
|
|
136
130
|
}
|