zenml-nightly 0.54.1.dev20240118__py3-none-any.whl → 0.54.1.dev20240120__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/__init__.py +10 -4
- zenml/artifacts/artifact_config.py +12 -12
- zenml/artifacts/external_artifact.py +3 -3
- zenml/artifacts/external_artifact_config.py +8 -8
- zenml/artifacts/utils.py +4 -4
- zenml/cli/__init__.py +4 -4
- zenml/cli/artifact.py +38 -18
- zenml/cli/base.py +3 -3
- zenml/cli/model.py +24 -16
- zenml/cli/server.py +9 -0
- zenml/cli/utils.py +3 -3
- zenml/client.py +13 -2
- zenml/config/compiler.py +1 -1
- zenml/config/pipeline_configurations.py +2 -2
- zenml/config/pipeline_run_configuration.py +2 -2
- zenml/config/step_configurations.py +2 -2
- zenml/integrations/__init__.py +2 -4
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +11 -9
- zenml/metadata/lazy_load.py +5 -5
- zenml/model/lazy_load.py +2 -2
- zenml/model/{model_version.py → model.py} +47 -38
- zenml/model/utils.py +33 -33
- zenml/model_registries/base_model_registry.py +10 -8
- zenml/models/__init__.py +2 -0
- zenml/models/v2/base/filter.py +3 -0
- zenml/models/v2/base/scoped.py +59 -0
- zenml/models/v2/core/artifact.py +2 -2
- zenml/models/v2/core/artifact_version.py +6 -6
- zenml/models/v2/core/model.py +6 -6
- zenml/models/v2/core/model_version.py +9 -9
- zenml/models/v2/core/run_metadata.py +2 -2
- zenml/new/pipelines/model_utils.py +20 -20
- zenml/new/pipelines/pipeline.py +47 -54
- zenml/new/pipelines/pipeline_context.py +1 -1
- zenml/new/pipelines/pipeline_decorator.py +4 -4
- zenml/new/steps/step_context.py +15 -15
- zenml/new/steps/step_decorator.py +5 -5
- zenml/orchestrators/input_utils.py +5 -7
- zenml/orchestrators/step_launcher.py +12 -19
- zenml/orchestrators/step_runner.py +8 -10
- zenml/pipelines/base_pipeline.py +1 -1
- zenml/pipelines/pipeline_decorator.py +6 -6
- zenml/steps/base_step.py +15 -15
- zenml/steps/step_decorator.py +6 -6
- zenml/steps/utils.py +68 -0
- zenml/zen_server/deploy/helm/templates/server-db-job.yaml +1 -1
- zenml/zen_server/deploy/helm/templates/server-secret.yaml +1 -1
- zenml/zen_server/deploy/helm/templates/serviceaccount.yaml +1 -1
- zenml/zen_server/utils.py +19 -1
- zenml/zen_stores/migrations/versions/4d688d8f7aff_rename_model_version_to_model.py +94 -0
- zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +16 -4
- zenml/zen_stores/rest_zen_store.py +2 -2
- zenml/zen_stores/sql_zen_store.py +4 -1
- {zenml_nightly-0.54.1.dev20240118.dist-info → zenml_nightly-0.54.1.dev20240120.dist-info}/METADATA +1 -1
- {zenml_nightly-0.54.1.dev20240118.dist-info → zenml_nightly-0.54.1.dev20240120.dist-info}/RECORD +58 -57
- {zenml_nightly-0.54.1.dev20240118.dist-info → zenml_nightly-0.54.1.dev20240120.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.54.1.dev20240118.dist-info → zenml_nightly-0.54.1.dev20240120.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.54.1.dev20240118.dist-info → zenml_nightly-0.54.1.dev20240120.dist-info}/entry_points.txt +0 -0
zenml/new/steps/step_context.py
CHANGED
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
|
|
32
32
|
from zenml.config.step_run_info import StepRunInfo
|
33
33
|
from zenml.materializers.base_materializer import BaseMaterializer
|
34
34
|
from zenml.metadata.metadata_types import MetadataType
|
35
|
-
from zenml.model.
|
35
|
+
from zenml.model.model import Model
|
36
36
|
from zenml.models import (
|
37
37
|
ArtifactVersionResponse,
|
38
38
|
PipelineResponse,
|
@@ -163,33 +163,33 @@ class StepContext(metaclass=SingletonMetaClass):
|
|
163
163
|
)
|
164
164
|
|
165
165
|
@property
|
166
|
-
def
|
167
|
-
"""Returns configured
|
166
|
+
def model(self) -> "Model":
|
167
|
+
"""Returns configured Model.
|
168
168
|
|
169
|
-
Order of resolution to search for
|
170
|
-
1.
|
171
|
-
2.
|
169
|
+
Order of resolution to search for Model is:
|
170
|
+
1. Model from @step
|
171
|
+
2. Model from @pipeline
|
172
172
|
|
173
173
|
Returns:
|
174
|
-
The `
|
174
|
+
The `Model` object associated with the current step.
|
175
175
|
|
176
176
|
Raises:
|
177
|
-
StepContextError: If the `
|
177
|
+
StepContextError: If the `Model` object is not set in `@step` or `@pipeline`.
|
178
178
|
"""
|
179
|
-
if self.step_run.config.
|
180
|
-
|
181
|
-
elif self.pipeline_run.config.
|
182
|
-
|
179
|
+
if self.step_run.config.model is not None:
|
180
|
+
model = self.step_run.config.model
|
181
|
+
elif self.pipeline_run.config.model is not None:
|
182
|
+
model = self.pipeline_run.config.model
|
183
183
|
else:
|
184
184
|
raise StepContextError(
|
185
|
-
f"Unable to get
|
185
|
+
f"Unable to get Model in step '{self.step_name}' of pipeline "
|
186
186
|
f"run '{self.pipeline_run.id}': it was not set in `@step` or `@pipeline`."
|
187
187
|
)
|
188
188
|
|
189
189
|
# warm-up the model version
|
190
|
-
|
190
|
+
model._get_or_create_model_version()
|
191
191
|
|
192
|
-
return
|
192
|
+
return model
|
193
193
|
|
194
194
|
@property
|
195
195
|
def inputs(self) -> Dict[str, "ArtifactVersionResponse"]:
|
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
|
|
33
33
|
from zenml.config.base_settings import SettingsOrDict
|
34
34
|
from zenml.config.source import Source
|
35
35
|
from zenml.materializers.base_materializer import BaseMaterializer
|
36
|
-
from zenml.model.
|
36
|
+
from zenml.model.model import Model
|
37
37
|
from zenml.steps import BaseStep
|
38
38
|
|
39
39
|
MaterializerClassOrSource = Union[str, Source, Type[BaseMaterializer]]
|
@@ -67,7 +67,7 @@ def step(
|
|
67
67
|
extra: Optional[Dict[str, Any]] = None,
|
68
68
|
on_failure: Optional["HookSpecification"] = None,
|
69
69
|
on_success: Optional["HookSpecification"] = None,
|
70
|
-
|
70
|
+
model: Optional["Model"] = None,
|
71
71
|
) -> Callable[["F"], "BaseStep"]:
|
72
72
|
...
|
73
73
|
|
@@ -87,7 +87,7 @@ def step(
|
|
87
87
|
extra: Optional[Dict[str, Any]] = None,
|
88
88
|
on_failure: Optional["HookSpecification"] = None,
|
89
89
|
on_success: Optional["HookSpecification"] = None,
|
90
|
-
|
90
|
+
model: Optional["Model"] = None,
|
91
91
|
) -> Union["BaseStep", Callable[["F"], "BaseStep"]]:
|
92
92
|
"""Decorator to create a ZenML step.
|
93
93
|
|
@@ -117,7 +117,7 @@ def step(
|
|
117
117
|
on_success: Callback function in event of success of the step. Can be a
|
118
118
|
function with no arguments, or a source path to such a function
|
119
119
|
(e.g. `module.my_function`).
|
120
|
-
|
120
|
+
model: configuration of the model in the Model Control Plane.
|
121
121
|
|
122
122
|
Returns:
|
123
123
|
The step instance.
|
@@ -149,7 +149,7 @@ def step(
|
|
149
149
|
extra=extra,
|
150
150
|
on_failure=on_failure,
|
151
151
|
on_success=on_success,
|
152
|
-
|
152
|
+
model=model,
|
153
153
|
)
|
154
154
|
|
155
155
|
return step_instance
|
@@ -87,7 +87,7 @@ def resolve_step_inputs(
|
|
87
87
|
issue_found = False
|
88
88
|
try:
|
89
89
|
if config_.metadata_name is None and config_.artifact_name:
|
90
|
-
if artifact_ := config_.
|
90
|
+
if artifact_ := config_.model.get_artifact(
|
91
91
|
config_.artifact_name, config_.artifact_version
|
92
92
|
):
|
93
93
|
input_artifacts[name] = artifact_
|
@@ -95,14 +95,12 @@ def resolve_step_inputs(
|
|
95
95
|
issue_found = True
|
96
96
|
elif config_.artifact_name is None and config_.metadata_name:
|
97
97
|
# metadata values should go directly in parameters, as primitive types
|
98
|
-
step.config.parameters[
|
99
|
-
name
|
100
|
-
] = config_.model_version.run_metadata[
|
98
|
+
step.config.parameters[name] = config_.model.run_metadata[
|
101
99
|
config_.metadata_name
|
102
100
|
].value
|
103
101
|
elif config_.metadata_name and config_.artifact_name:
|
104
102
|
# metadata values should go directly in parameters, as primitive types
|
105
|
-
if artifact_ := config_.
|
103
|
+
if artifact_ := config_.model.get_artifact(
|
106
104
|
config_.artifact_name, config_.artifact_version
|
107
105
|
):
|
108
106
|
step.config.parameters[name] = artifact_.run_metadata[
|
@@ -118,8 +116,8 @@ def resolve_step_inputs(
|
|
118
116
|
if issue_found:
|
119
117
|
raise ValueError(
|
120
118
|
"Cannot fetch requested information from model "
|
121
|
-
f"`{config_.
|
122
|
-
f"`{config_.
|
119
|
+
f"`{config_.model.name}` version "
|
120
|
+
f"`{config_.model.version}` given artifact "
|
123
121
|
f"`{config_.artifact_name}`, artifact version "
|
124
122
|
f"`{config_.artifact_version}`, and metadata "
|
125
123
|
f"key `{config_.metadata_name}` passed into "
|
@@ -31,7 +31,7 @@ from zenml.enums import ExecutionStatus
|
|
31
31
|
from zenml.environment import get_run_environment_dict
|
32
32
|
from zenml.logger import get_logger
|
33
33
|
from zenml.logging import step_logging
|
34
|
-
from zenml.model.utils import
|
34
|
+
from zenml.model.utils import link_artifact_config_to_model
|
35
35
|
from zenml.models import (
|
36
36
|
ArtifactVersionResponse,
|
37
37
|
LogsRequest,
|
@@ -53,7 +53,7 @@ from zenml.stack import Stack
|
|
53
53
|
from zenml.utils import string_utils
|
54
54
|
|
55
55
|
if TYPE_CHECKING:
|
56
|
-
from zenml.model.
|
56
|
+
from zenml.model.model import Model
|
57
57
|
from zenml.step_operators import BaseStepOperator
|
58
58
|
|
59
59
|
logger = get_logger(__name__)
|
@@ -310,11 +310,9 @@ class StepLauncher:
|
|
310
310
|
Tuple that specifies whether the step needs to be executed as
|
311
311
|
well as the response model of the registered step run.
|
312
312
|
"""
|
313
|
-
|
314
|
-
self._deployment.step_configurations[
|
315
|
-
|
316
|
-
].config.model_version
|
317
|
-
or self._deployment.pipeline_configuration.model_version
|
313
|
+
model = (
|
314
|
+
self._deployment.step_configurations[step_run.name].config.model
|
315
|
+
or self._deployment.pipeline_configuration.model
|
318
316
|
)
|
319
317
|
input_artifacts, parent_step_ids = input_utils.resolve_step_inputs(
|
320
318
|
step=self._step,
|
@@ -362,8 +360,8 @@ class StepLauncher:
|
|
362
360
|
output_name: artifact.id
|
363
361
|
for output_name, artifact in cached_outputs.items()
|
364
362
|
}
|
365
|
-
self.
|
366
|
-
|
363
|
+
self._link_cached_artifacts_to_model(
|
364
|
+
model_from_context=model,
|
367
365
|
step_run=step_run,
|
368
366
|
)
|
369
367
|
step_run.status = ExecutionStatus.CACHED
|
@@ -371,15 +369,15 @@ class StepLauncher:
|
|
371
369
|
|
372
370
|
return execution_needed, step_run
|
373
371
|
|
374
|
-
def
|
372
|
+
def _link_cached_artifacts_to_model(
|
375
373
|
self,
|
376
|
-
|
374
|
+
model_from_context: Optional["Model"],
|
377
375
|
step_run: StepRunRequest,
|
378
376
|
) -> None:
|
379
377
|
"""Links the output artifacts of the cached step to the model version in Control Plane.
|
380
378
|
|
381
379
|
Args:
|
382
|
-
|
380
|
+
model_from_context: The model version of the current step.
|
383
381
|
step_run: The step to run.
|
384
382
|
"""
|
385
383
|
from zenml.artifacts.artifact_config import ArtifactConfig
|
@@ -397,15 +395,10 @@ class StepLauncher:
|
|
397
395
|
artifact_config_ = annotation.artifact_config.copy()
|
398
396
|
else:
|
399
397
|
artifact_config_ = ArtifactConfig(name=output_name_)
|
400
|
-
logger.info(
|
401
|
-
f"Linking artifact `{artifact_config_.name}` to "
|
402
|
-
f"model `{artifact_config_.model_name}` version "
|
403
|
-
f"`{artifact_config_.model_version}` implicitly."
|
404
|
-
)
|
405
398
|
|
406
|
-
|
399
|
+
link_artifact_config_to_model(
|
407
400
|
artifact_config=artifact_config_,
|
408
|
-
|
401
|
+
model=model_from_context,
|
409
402
|
artifact_version_id=output_id,
|
410
403
|
)
|
411
404
|
|
@@ -634,8 +634,8 @@ class StepRunner:
|
|
634
634
|
|
635
635
|
def _prepare_model_context_for_step(self) -> None:
|
636
636
|
try:
|
637
|
-
|
638
|
-
|
637
|
+
model = get_step_context().model
|
638
|
+
model._get_or_create_model_version()
|
639
639
|
except StepContextError:
|
640
640
|
return
|
641
641
|
|
@@ -657,11 +657,9 @@ class StepRunner:
|
|
657
657
|
get_step_context()._get_output(artifact_name).artifact_config
|
658
658
|
)
|
659
659
|
if artifact_config is not None:
|
660
|
-
if (
|
661
|
-
model_version := artifact_config._model_version
|
662
|
-
) is not None:
|
660
|
+
if (model := artifact_config._model) is not None:
|
663
661
|
model_version_response = (
|
664
|
-
|
662
|
+
model._get_or_create_model_version()
|
665
663
|
)
|
666
664
|
models.add(
|
667
665
|
(
|
@@ -680,7 +678,7 @@ class StepRunner:
|
|
680
678
|
Set of tuples of (model_id, model_version_id).
|
681
679
|
"""
|
682
680
|
try:
|
683
|
-
mc = get_step_context().
|
681
|
+
mc = get_step_context().model
|
684
682
|
model_version = mc._get_or_create_model_version()
|
685
683
|
return {(model_version.model.id, model_version.id)}
|
686
684
|
except StepContextError:
|
@@ -731,11 +729,11 @@ class StepRunner:
|
|
731
729
|
|
732
730
|
# Add models from external artifacts
|
733
731
|
for external_artifact in external_artifacts:
|
734
|
-
if external_artifact.
|
732
|
+
if external_artifact.model:
|
735
733
|
models.add(
|
736
734
|
(
|
737
|
-
external_artifact.
|
738
|
-
external_artifact.
|
735
|
+
external_artifact.model.model_id,
|
736
|
+
external_artifact.model.id,
|
739
737
|
)
|
740
738
|
)
|
741
739
|
|
zenml/pipelines/base_pipeline.py
CHANGED
@@ -26,7 +26,7 @@ from typing import (
|
|
26
26
|
)
|
27
27
|
|
28
28
|
from zenml.logger import get_logger
|
29
|
-
from zenml.model.
|
29
|
+
from zenml.model.model import Model
|
30
30
|
from zenml.pipelines.base_pipeline import (
|
31
31
|
CLASS_CONFIGURATION,
|
32
32
|
PARAM_ENABLE_ARTIFACT_METADATA,
|
@@ -34,7 +34,7 @@ from zenml.pipelines.base_pipeline import (
|
|
34
34
|
PARAM_ENABLE_CACHE,
|
35
35
|
PARAM_ENABLE_STEP_LOGS,
|
36
36
|
PARAM_EXTRA_OPTIONS,
|
37
|
-
|
37
|
+
PARAM_MODEL,
|
38
38
|
PARAM_ON_FAILURE,
|
39
39
|
PARAM_ON_SUCCESS,
|
40
40
|
PARAM_PIPELINE_NAME,
|
@@ -68,7 +68,7 @@ def pipeline(
|
|
68
68
|
enable_step_logs: Optional[bool] = None,
|
69
69
|
settings: Optional[Dict[str, "SettingsOrDict"]] = None,
|
70
70
|
extra: Optional[Dict[str, Any]] = None,
|
71
|
-
|
71
|
+
model: Optional["Model"] = None,
|
72
72
|
) -> Callable[[F], Type[BasePipeline]]:
|
73
73
|
...
|
74
74
|
|
@@ -85,7 +85,7 @@ def pipeline(
|
|
85
85
|
extra: Optional[Dict[str, Any]] = None,
|
86
86
|
on_failure: Optional["HookSpecification"] = None,
|
87
87
|
on_success: Optional["HookSpecification"] = None,
|
88
|
-
|
88
|
+
model: Optional["Model"] = None,
|
89
89
|
) -> Union[Type[BasePipeline], Callable[[F], Type[BasePipeline]]]:
|
90
90
|
"""Outer decorator function for the creation of a ZenML pipeline.
|
91
91
|
|
@@ -105,7 +105,7 @@ def pipeline(
|
|
105
105
|
on_success: Callback function in event of success of the step. Can be a
|
106
106
|
function with no arguments, or a source path to such a function
|
107
107
|
(e.g. `module.my_function`).
|
108
|
-
|
108
|
+
model: configuration of the model in the Model Control Plane.
|
109
109
|
|
110
110
|
Returns:
|
111
111
|
the inner decorator which creates the pipeline class based on the
|
@@ -137,7 +137,7 @@ def pipeline(
|
|
137
137
|
PARAM_EXTRA_OPTIONS: extra,
|
138
138
|
PARAM_ON_FAILURE: on_failure,
|
139
139
|
PARAM_ON_SUCCESS: on_success,
|
140
|
-
|
140
|
+
PARAM_MODEL: model,
|
141
141
|
},
|
142
142
|
"__module__": func.__module__,
|
143
143
|
"__doc__": func.__doc__,
|
zenml/steps/base_step.py
CHANGED
@@ -71,7 +71,7 @@ if TYPE_CHECKING:
|
|
71
71
|
StepConfigurationUpdate,
|
72
72
|
)
|
73
73
|
from zenml.model.lazy_load import ModelVersionDataLazyLoader
|
74
|
-
from zenml.model.
|
74
|
+
from zenml.model.model import Model
|
75
75
|
|
76
76
|
ParametersOrDict = Union["BaseParameters", Dict[str, Any]]
|
77
77
|
MaterializerClassOrSource = Union[str, Source, Type["BaseMaterializer"]]
|
@@ -137,7 +137,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
137
137
|
extra: Optional[Dict[str, Any]] = None,
|
138
138
|
on_failure: Optional["HookSpecification"] = None,
|
139
139
|
on_success: Optional["HookSpecification"] = None,
|
140
|
-
|
140
|
+
model: Optional["Model"] = None,
|
141
141
|
**kwargs: Any,
|
142
142
|
) -> None:
|
143
143
|
"""Initializes a step.
|
@@ -166,7 +166,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
166
166
|
on_success: Callback function in event of success of the step. Can
|
167
167
|
be a function with no arguments, or a source path to such a
|
168
168
|
function (e.g. `module.my_function`).
|
169
|
-
|
169
|
+
model: configuration of the model version in the Model Control Plane.
|
170
170
|
**kwargs: Keyword arguments passed to the step.
|
171
171
|
"""
|
172
172
|
from zenml.config.step_configurations import PartialStepConfiguration
|
@@ -213,13 +213,13 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
213
213
|
name,
|
214
214
|
"enabled" if enable_step_logs is not False else "disabled",
|
215
215
|
)
|
216
|
-
if
|
216
|
+
if model is not None:
|
217
217
|
logger.debug(
|
218
218
|
"Step '%s': Is in Model context %s.",
|
219
219
|
name,
|
220
220
|
{
|
221
|
-
"model":
|
222
|
-
"version":
|
221
|
+
"model": model.name,
|
222
|
+
"version": model.version,
|
223
223
|
},
|
224
224
|
)
|
225
225
|
|
@@ -239,7 +239,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
239
239
|
extra=extra,
|
240
240
|
on_failure=on_failure,
|
241
241
|
on_success=on_success,
|
242
|
-
|
242
|
+
model=model,
|
243
243
|
)
|
244
244
|
self._verify_and_apply_init_params(*args, **kwargs)
|
245
245
|
|
@@ -505,14 +505,14 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
505
505
|
)
|
506
506
|
elif isinstance(value, LazyArtifactVersionResponse):
|
507
507
|
model_artifacts_or_metadata[key] = ModelVersionDataLazyLoader(
|
508
|
-
|
508
|
+
model=value._lazy_load_model,
|
509
509
|
artifact_name=value._lazy_load_name,
|
510
510
|
artifact_version=value._lazy_load_version,
|
511
511
|
metadata_name=None,
|
512
512
|
)
|
513
513
|
elif isinstance(value, LazyRunMetadataResponse):
|
514
514
|
model_artifacts_or_metadata[key] = ModelVersionDataLazyLoader(
|
515
|
-
|
515
|
+
model=value._lazy_load_model,
|
516
516
|
artifact_name=value._lazy_load_artifact_name,
|
517
517
|
artifact_version=value._lazy_load_artifact_version,
|
518
518
|
metadata_name=value._lazy_load_metadata_name,
|
@@ -684,7 +684,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
684
684
|
extra: Optional[Dict[str, Any]] = None,
|
685
685
|
on_failure: Optional["HookSpecification"] = None,
|
686
686
|
on_success: Optional["HookSpecification"] = None,
|
687
|
-
|
687
|
+
model: Optional["Model"] = None,
|
688
688
|
merge: bool = True,
|
689
689
|
) -> T:
|
690
690
|
"""Configures the step.
|
@@ -722,7 +722,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
722
722
|
on_success: Callback function in event of success of the step. Can
|
723
723
|
be a function with no arguments, or a source path to such a
|
724
724
|
function (e.g. `module.my_function`).
|
725
|
-
|
725
|
+
model: configuration of the model version in the Model Control Plane.
|
726
726
|
merge: If `True`, will merge the given dictionary configurations
|
727
727
|
like `parameters` and `settings` with existing
|
728
728
|
configurations. If `False` the given configurations will
|
@@ -796,7 +796,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
796
796
|
"extra": extra,
|
797
797
|
"failure_hook_source": failure_hook_source,
|
798
798
|
"success_hook_source": success_hook_source,
|
799
|
-
"
|
799
|
+
"model": model,
|
800
800
|
}
|
801
801
|
)
|
802
802
|
config = StepConfigurationUpdate(**values)
|
@@ -819,7 +819,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
819
819
|
extra: Optional[Dict[str, Any]] = None,
|
820
820
|
on_failure: Optional["HookSpecification"] = None,
|
821
821
|
on_success: Optional["HookSpecification"] = None,
|
822
|
-
|
822
|
+
model: Optional["Model"] = None,
|
823
823
|
merge: bool = True,
|
824
824
|
) -> "BaseStep":
|
825
825
|
"""Copies the step and applies the given configurations.
|
@@ -846,7 +846,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
846
846
|
on_success: Callback function in event of success of the step. Can
|
847
847
|
be a function with no arguments, or a source path to such a
|
848
848
|
function (e.g. `module.my_function`).
|
849
|
-
|
849
|
+
model: configuration of the model version in the Model Control Plane.
|
850
850
|
merge: If `True`, will merge the given dictionary configurations
|
851
851
|
like `parameters` and `settings` with existing
|
852
852
|
configurations. If `False` the given configurations will
|
@@ -870,7 +870,7 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
870
870
|
extra=extra,
|
871
871
|
on_failure=on_failure,
|
872
872
|
on_success=on_success,
|
873
|
-
|
873
|
+
model=model,
|
874
874
|
merge=merge,
|
875
875
|
)
|
876
876
|
return step_copy
|
zenml/steps/step_decorator.py
CHANGED
@@ -36,7 +36,7 @@ if TYPE_CHECKING:
|
|
36
36
|
from zenml.config.base_settings import SettingsOrDict
|
37
37
|
from zenml.config.source import Source
|
38
38
|
from zenml.materializers.base_materializer import BaseMaterializer
|
39
|
-
from zenml.model.
|
39
|
+
from zenml.model.model import Model
|
40
40
|
|
41
41
|
MaterializerClassOrSource = Union[str, "Source", Type["BaseMaterializer"]]
|
42
42
|
HookSpecification = Union[str, "Source", FunctionType]
|
@@ -63,7 +63,7 @@ PARAM_SETTINGS = "settings"
|
|
63
63
|
PARAM_EXTRA_OPTIONS = "extra"
|
64
64
|
PARAM_ON_FAILURE = "on_failure"
|
65
65
|
PARAM_ON_SUCCESS = "on_success"
|
66
|
-
|
66
|
+
PARAM_MODEL = "model"
|
67
67
|
|
68
68
|
logger = get_logger(__name__)
|
69
69
|
|
@@ -108,7 +108,7 @@ def step(
|
|
108
108
|
extra: Optional[Dict[str, Any]] = None,
|
109
109
|
on_failure: Optional["HookSpecification"] = None,
|
110
110
|
on_success: Optional["HookSpecification"] = None,
|
111
|
-
|
111
|
+
model: Optional["Model"] = None,
|
112
112
|
) -> Callable[[F], Type[BaseStep]]:
|
113
113
|
...
|
114
114
|
|
@@ -128,7 +128,7 @@ def step(
|
|
128
128
|
extra: Optional[Dict[str, Any]] = None,
|
129
129
|
on_failure: Optional["HookSpecification"] = None,
|
130
130
|
on_success: Optional["HookSpecification"] = None,
|
131
|
-
|
131
|
+
model: Optional["Model"] = None,
|
132
132
|
) -> Union[Type[BaseStep], Callable[[F], Type[BaseStep]]]:
|
133
133
|
"""Outer decorator function for the creation of a ZenML step.
|
134
134
|
|
@@ -161,7 +161,7 @@ def step(
|
|
161
161
|
on_success: Callback function in event of success of the step. Can be a
|
162
162
|
function with no arguments, or a source path to such a function
|
163
163
|
(e.g. `module.my_function`).
|
164
|
-
|
164
|
+
model: configuration of the model version in the Model Control Plane.
|
165
165
|
|
166
166
|
Returns:
|
167
167
|
The inner decorator which creates the step class based on the
|
@@ -204,7 +204,7 @@ def step(
|
|
204
204
|
PARAM_EXTRA_OPTIONS: extra,
|
205
205
|
PARAM_ON_FAILURE: on_failure,
|
206
206
|
PARAM_ON_SUCCESS: on_success,
|
207
|
-
|
207
|
+
PARAM_MODEL: model,
|
208
208
|
},
|
209
209
|
"__module__": func.__module__,
|
210
210
|
"__doc__": func.__doc__,
|
zenml/steps/utils.py
CHANGED
@@ -14,17 +14,24 @@
|
|
14
14
|
|
15
15
|
"""Utility functions and classes to run ZenML steps."""
|
16
16
|
|
17
|
+
|
17
18
|
import ast
|
19
|
+
import contextlib
|
18
20
|
import inspect
|
19
21
|
import textwrap
|
20
22
|
from typing import Any, Callable, Dict, Optional, Tuple, Union
|
23
|
+
from uuid import UUID
|
21
24
|
|
22
25
|
import pydantic.typing as pydantic_typing
|
23
26
|
from pydantic import BaseModel
|
24
27
|
from typing_extensions import Annotated
|
25
28
|
|
26
29
|
from zenml.artifacts.artifact_config import ArtifactConfig
|
30
|
+
from zenml.client import Client
|
31
|
+
from zenml.enums import MetadataResourceTypes
|
27
32
|
from zenml.logger import get_logger
|
33
|
+
from zenml.metadata.metadata_types import MetadataType
|
34
|
+
from zenml.new.steps.step_context import get_step_context
|
28
35
|
from zenml.steps.step_output import Output
|
29
36
|
from zenml.utils import source_code_utils
|
30
37
|
|
@@ -403,3 +410,64 @@ def has_only_none_returns(func: Callable[..., Any]) -> bool:
|
|
403
410
|
visitor.visit(tree)
|
404
411
|
|
405
412
|
return visitor.has_only_none_returns
|
413
|
+
|
414
|
+
|
415
|
+
def log_step_metadata(
|
416
|
+
metadata: Dict[str, "MetadataType"],
|
417
|
+
step_name: Optional[str] = None,
|
418
|
+
pipeline_name_id_or_prefix: Optional[Union[str, UUID]] = None,
|
419
|
+
pipeline_version: Optional[str] = None,
|
420
|
+
run_id: Optional[str] = None,
|
421
|
+
) -> None:
|
422
|
+
"""Logs step metadata.
|
423
|
+
|
424
|
+
Args:
|
425
|
+
metadata: The metadata to log.
|
426
|
+
step_name: The name of the step to log metadata for. Can be omitted
|
427
|
+
when being called inside a step.
|
428
|
+
pipeline_name_id_or_prefix: The name of the pipeline to log metadata
|
429
|
+
for. Can be omitted when being called inside a step.
|
430
|
+
pipeline_version: The version of the pipeline to log metadata for.
|
431
|
+
Can be omitted when being called inside a step.
|
432
|
+
run_id: The ID of the run to log metadata for. Can be omitted when
|
433
|
+
being called inside a step.
|
434
|
+
|
435
|
+
Raises:
|
436
|
+
ValueError: If no step name is provided and the function is not called
|
437
|
+
from within a step or if no pipeline name or ID is provided and
|
438
|
+
the function is not called from within a step.
|
439
|
+
"""
|
440
|
+
step_context = None
|
441
|
+
if not step_name:
|
442
|
+
with contextlib.suppress(RuntimeError):
|
443
|
+
step_context = get_step_context()
|
444
|
+
step_name = step_context.step_name
|
445
|
+
# not running within a step and no user-provided step name
|
446
|
+
if not step_name:
|
447
|
+
raise ValueError(
|
448
|
+
"No step name provided and you are not running "
|
449
|
+
"within a step. Please provide a step name."
|
450
|
+
)
|
451
|
+
|
452
|
+
client = Client()
|
453
|
+
if step_context:
|
454
|
+
step_run_id = step_context.step_run.id
|
455
|
+
elif run_id:
|
456
|
+
step_run_id = UUID(int=int(run_id))
|
457
|
+
else:
|
458
|
+
if not pipeline_name_id_or_prefix:
|
459
|
+
raise ValueError(
|
460
|
+
"No pipeline name or ID provided and you are not running "
|
461
|
+
"within a step. Please provide a pipeline name or ID, or "
|
462
|
+
"provide a run ID."
|
463
|
+
)
|
464
|
+
pipeline_run = client.get_pipeline(
|
465
|
+
name_id_or_prefix=pipeline_name_id_or_prefix,
|
466
|
+
version=pipeline_version,
|
467
|
+
).last_run
|
468
|
+
step_run_id = pipeline_run.steps[step_name].id
|
469
|
+
client.create_run_metadata(
|
470
|
+
metadata=metadata,
|
471
|
+
resource_id=step_run_id,
|
472
|
+
resource_type=MetadataResourceTypes.STEP_RUN,
|
473
|
+
)
|
@@ -8,7 +8,7 @@ metadata:
|
|
8
8
|
annotations:
|
9
9
|
"helm.sh/hook": pre-install,pre-upgrade
|
10
10
|
"helm.sh/hook-weight": "-1"
|
11
|
-
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
11
|
+
"helm.sh/hook-delete-policy": before-hook-creation{{ if not .Values.zenml.debug }},hook-succeeded{{ end }}
|
12
12
|
spec:
|
13
13
|
backoffLimit: 2
|
14
14
|
template:
|
@@ -55,5 +55,5 @@ metadata:
|
|
55
55
|
annotations:
|
56
56
|
"helm.sh/hook": "pre-install,pre-upgrade"
|
57
57
|
"helm.sh/hook-weight": "-3"
|
58
|
-
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
58
|
+
"helm.sh/hook-delete-policy": before-hook-creation{{ if not .Values.zenml.debug }},hook-succeeded{{ end }}
|
59
59
|
{{ include "secret.content" . }}
|
@@ -20,7 +20,7 @@ metadata:
|
|
20
20
|
annotations:
|
21
21
|
"helm.sh/hook": "pre-install,pre-upgrade"
|
22
22
|
"helm.sh/hook-weight": "-2" # Set your desired hook weight here
|
23
|
-
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
23
|
+
"helm.sh/hook-delete-policy": before-hook-creation{{ if not .Values.zenml.debug }},hook-succeeded{{ end }}
|
24
24
|
{{- with .Values.serviceAccount.annotations }}
|
25
25
|
{{- toYaml . | nindent 4 }}
|
26
26
|
{{- end }}
|
zenml/zen_server/utils.py
CHANGED
@@ -272,12 +272,17 @@ def make_dependable(cls: Type[BaseModel]) -> Callable[..., Any]:
|
|
272
272
|
def f(model: Model = Depends(make_dependable(Model))):
|
273
273
|
...
|
274
274
|
|
275
|
+
UPDATE: Function from above mentioned Github issue was extended to support
|
276
|
+
multi-input parameters, e.g. tags: List[str]. It needs a default set to Query(<default>),
|
277
|
+
rather just plain <default>.
|
278
|
+
|
275
279
|
Args:
|
276
280
|
cls: The model class.
|
277
281
|
|
278
282
|
Returns:
|
279
283
|
Function to use in FastAPI `Depends`.
|
280
284
|
"""
|
285
|
+
from fastapi import Query
|
281
286
|
|
282
287
|
def init_cls_and_handle_errors(*args: Any, **kwargs: Any) -> BaseModel:
|
283
288
|
from fastapi import HTTPException
|
@@ -290,7 +295,20 @@ def make_dependable(cls: Type[BaseModel]) -> Callable[..., Any]:
|
|
290
295
|
error["loc"] = tuple(["query"] + list(error["loc"]))
|
291
296
|
raise HTTPException(422, detail=e.errors())
|
292
297
|
|
293
|
-
|
298
|
+
params = {v.name: v for v in inspect.signature(cls).parameters.values()}
|
299
|
+
query_params = getattr(cls, "API_MULTI_INPUT_PARAMS", [])
|
300
|
+
for qp in query_params:
|
301
|
+
if qp in params:
|
302
|
+
params[qp] = inspect.Parameter(
|
303
|
+
name=params[qp].name,
|
304
|
+
default=Query(params[qp].default),
|
305
|
+
kind=params[qp].kind,
|
306
|
+
annotation=params[qp].annotation,
|
307
|
+
)
|
308
|
+
|
309
|
+
init_cls_and_handle_errors.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
310
|
+
parameters=[v for v in params.values()]
|
311
|
+
)
|
294
312
|
|
295
313
|
return init_cls_and_handle_errors
|
296
314
|
|