zenml-nightly 0.70.0.dev20241129__py3-none-any.whl → 0.70.0.dev20241130__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/artifacts/artifact_config.py +0 -14
- zenml/artifacts/utils.py +50 -29
- zenml/cli/__init__.py +15 -0
- zenml/cli/base.py +4 -4
- zenml/cli/server.py +1 -1
- zenml/cli/stack.py +0 -3
- zenml/cli/stack_components.py +0 -1
- zenml/cli/utils.py +0 -5
- zenml/client.py +8 -18
- zenml/model/model.py +8 -4
- zenml/model/utils.py +18 -16
- zenml/models/__init__.py +6 -0
- zenml/models/v2/core/artifact_version.py +6 -3
- zenml/models/v2/core/component.py +0 -22
- zenml/models/v2/core/model_version.py +6 -3
- zenml/models/v2/core/pipeline_run.py +16 -4
- zenml/models/v2/core/run_metadata.py +30 -9
- zenml/models/v2/core/step_run.py +5 -3
- zenml/models/v2/misc/run_metadata.py +38 -0
- zenml/orchestrators/input_utils.py +19 -6
- zenml/orchestrators/publish_utils.py +11 -4
- zenml/orchestrators/step_launcher.py +2 -2
- zenml/orchestrators/step_run_utils.py +3 -0
- zenml/orchestrators/step_runner.py +7 -8
- zenml/orchestrators/utils.py +0 -24
- zenml/pipelines/run_utils.py +3 -4
- zenml/steps/utils.py +6 -2
- zenml/utils/metadata_utils.py +186 -153
- zenml/utils/string_utils.py +15 -8
- zenml/zen_server/routers/workspaces_endpoints.py +19 -19
- zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
- zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
- zenml/zen_stores/schemas/__init__.py +5 -1
- zenml/zen_stores/schemas/artifact_schemas.py +10 -10
- zenml/zen_stores/schemas/component_schemas.py +0 -3
- zenml/zen_stores/schemas/model_schemas.py +13 -11
- zenml/zen_stores/schemas/pipeline_run_schemas.py +32 -15
- zenml/zen_stores/schemas/run_metadata_schemas.py +66 -31
- zenml/zen_stores/schemas/step_run_schemas.py +13 -12
- zenml/zen_stores/schemas/utils.py +47 -3
- zenml/zen_stores/sql_zen_store.py +102 -23
- {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/METADATA +1 -1
- {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/RECORD +47 -44
- {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/entry_points.txt +0 -0
@@ -13,16 +13,16 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Models representing run metadata."""
|
15
15
|
|
16
|
-
from typing import Dict, Optional
|
16
|
+
from typing import Dict, List, Optional
|
17
17
|
from uuid import UUID
|
18
18
|
|
19
|
-
from pydantic import Field
|
19
|
+
from pydantic import Field, model_validator
|
20
20
|
|
21
|
-
from zenml.enums import MetadataResourceTypes
|
22
21
|
from zenml.metadata.metadata_types import MetadataType, MetadataTypeEnum
|
23
22
|
from zenml.models.v2.base.scoped import (
|
24
23
|
WorkspaceScopedRequest,
|
25
24
|
)
|
25
|
+
from zenml.models.v2.misc.run_metadata import RunMetadataResource
|
26
26
|
|
27
27
|
# ------------------ Request Model ------------------
|
28
28
|
|
@@ -30,14 +30,12 @@ from zenml.models.v2.base.scoped import (
|
|
30
30
|
class RunMetadataRequest(WorkspaceScopedRequest):
|
31
31
|
"""Request model for run metadata."""
|
32
32
|
|
33
|
-
|
34
|
-
title="The
|
35
|
-
)
|
36
|
-
resource_type: MetadataResourceTypes = Field(
|
37
|
-
title="The type of the resource that this metadata belongs to.",
|
33
|
+
resources: List[RunMetadataResource] = Field(
|
34
|
+
title="The list of resources that this metadata belongs to."
|
38
35
|
)
|
39
36
|
stack_component_id: Optional[UUID] = Field(
|
40
|
-
title="The ID of the stack component that this metadata belongs to."
|
37
|
+
title="The ID of the stack component that this metadata belongs to.",
|
38
|
+
default=None,
|
41
39
|
)
|
42
40
|
values: Dict[str, "MetadataType"] = Field(
|
43
41
|
title="The metadata to be created.",
|
@@ -45,3 +43,26 @@ class RunMetadataRequest(WorkspaceScopedRequest):
|
|
45
43
|
types: Dict[str, "MetadataTypeEnum"] = Field(
|
46
44
|
title="The types of the metadata to be created.",
|
47
45
|
)
|
46
|
+
publisher_step_id: Optional[UUID] = Field(
|
47
|
+
title="The ID of the step execution that published this metadata.",
|
48
|
+
default=None,
|
49
|
+
)
|
50
|
+
|
51
|
+
@model_validator(mode="after")
|
52
|
+
def validate_values_keys(self) -> "RunMetadataRequest":
|
53
|
+
"""Validates if the keys in the metadata are properly defined.
|
54
|
+
|
55
|
+
Returns:
|
56
|
+
self
|
57
|
+
|
58
|
+
Raises:
|
59
|
+
ValueError: if one of the key in the metadata contains `:`
|
60
|
+
"""
|
61
|
+
invalid_keys = [key for key in self.values.keys() if ":" in key]
|
62
|
+
if invalid_keys:
|
63
|
+
raise ValueError(
|
64
|
+
"You can not use colons (`:`) in the key names when you "
|
65
|
+
"are creating metadata for your ZenML objects. Please change "
|
66
|
+
f"the following keys: {invalid_keys}"
|
67
|
+
)
|
68
|
+
return self
|
zenml/models/v2/core/step_run.py
CHANGED
@@ -594,6 +594,7 @@ class StepRunFilter(WorkspaceScopedFilter):
|
|
594
594
|
from zenml.zen_stores.schemas import (
|
595
595
|
ModelSchema,
|
596
596
|
ModelVersionSchema,
|
597
|
+
RunMetadataResourceSchema,
|
597
598
|
RunMetadataSchema,
|
598
599
|
StepRunSchema,
|
599
600
|
)
|
@@ -612,10 +613,11 @@ class StepRunFilter(WorkspaceScopedFilter):
|
|
612
613
|
|
613
614
|
for key, value in self.run_metadata.items():
|
614
615
|
additional_filter = and_(
|
615
|
-
|
616
|
-
|
616
|
+
RunMetadataResourceSchema.resource_id == StepRunSchema.id,
|
617
|
+
RunMetadataResourceSchema.resource_type
|
617
618
|
== MetadataResourceTypes.STEP_RUN,
|
618
|
-
|
619
|
+
RunMetadataResourceSchema.run_metadata_id
|
620
|
+
== RunMetadataSchema.id,
|
619
621
|
self.generate_custom_query_conditions_for_column(
|
620
622
|
value=value,
|
621
623
|
table=RunMetadataSchema,
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Utility classes for modeling run metadata."""
|
15
|
+
|
16
|
+
from datetime import datetime
|
17
|
+
from uuid import UUID
|
18
|
+
|
19
|
+
from pydantic import BaseModel, Field
|
20
|
+
|
21
|
+
from zenml.enums import MetadataResourceTypes
|
22
|
+
from zenml.metadata.metadata_types import MetadataType
|
23
|
+
|
24
|
+
|
25
|
+
class RunMetadataResource(BaseModel):
|
26
|
+
"""Utility class to help identify resources to tag metadata to."""
|
27
|
+
|
28
|
+
id: UUID = Field(title="The ID of the resource.")
|
29
|
+
type: MetadataResourceTypes = Field(title="The type of the resource.")
|
30
|
+
|
31
|
+
|
32
|
+
class RunMetadataEntry(BaseModel):
|
33
|
+
"""Utility class to sort/list run metadata entries."""
|
34
|
+
|
35
|
+
value: MetadataType = Field(title="The value for the run metadata entry")
|
36
|
+
created: datetime = Field(
|
37
|
+
title="The timestamp when this resource was created."
|
38
|
+
)
|
@@ -20,7 +20,7 @@ from zenml.client import Client
|
|
20
20
|
from zenml.config.step_configurations import Step
|
21
21
|
from zenml.enums import ArtifactSaveType, StepRunInputArtifactType
|
22
22
|
from zenml.exceptions import InputResolutionError
|
23
|
-
from zenml.utils import pagination_utils
|
23
|
+
from zenml.utils import pagination_utils, string_utils
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
26
|
from zenml.models import PipelineRunResponse
|
@@ -53,7 +53,8 @@ def resolve_step_inputs(
|
|
53
53
|
current_run_steps = {
|
54
54
|
run_step.name: run_step
|
55
55
|
for run_step in pagination_utils.depaginate(
|
56
|
-
Client().list_run_steps,
|
56
|
+
Client().list_run_steps,
|
57
|
+
pipeline_run_id=pipeline_run.id,
|
57
58
|
)
|
58
59
|
}
|
59
60
|
|
@@ -66,11 +67,23 @@ def resolve_step_inputs(
|
|
66
67
|
f"No step `{input_.step_name}` found in current run."
|
67
68
|
)
|
68
69
|
|
70
|
+
# Try to get the substitutions from the pipeline run first, as we
|
71
|
+
# already have a hydrated version of that. In the unlikely case
|
72
|
+
# that the pipeline run is outdated, we fetch it from the step
|
73
|
+
# run instead which will costs us one hydration call.
|
74
|
+
substitutions = (
|
75
|
+
pipeline_run.step_substitutions.get(step_run.name)
|
76
|
+
or step_run.config.substitutions
|
77
|
+
)
|
78
|
+
output_name = string_utils.format_name_template(
|
79
|
+
input_.output_name, substitutions=substitutions
|
80
|
+
)
|
81
|
+
|
69
82
|
try:
|
70
|
-
outputs = step_run.outputs[
|
83
|
+
outputs = step_run.outputs[output_name]
|
71
84
|
except KeyError:
|
72
85
|
raise InputResolutionError(
|
73
|
-
f"No step output `{
|
86
|
+
f"No step output `{output_name}` found for step "
|
74
87
|
f"`{input_.step_name}`."
|
75
88
|
)
|
76
89
|
|
@@ -83,12 +96,12 @@ def resolve_step_inputs(
|
|
83
96
|
# This should never happen, there can only be a single regular step
|
84
97
|
# output for a name
|
85
98
|
raise InputResolutionError(
|
86
|
-
f"Too many step outputs for output `{
|
99
|
+
f"Too many step outputs for output `{output_name}` of "
|
87
100
|
f"step `{input_.step_name}`."
|
88
101
|
)
|
89
102
|
elif len(step_outputs) == 0:
|
90
103
|
raise InputResolutionError(
|
91
|
-
f"No step output `{
|
104
|
+
f"No step output `{output_name}` found for step "
|
92
105
|
f"`{input_.step_name}`."
|
93
106
|
)
|
94
107
|
|
@@ -21,6 +21,7 @@ from zenml.enums import ExecutionStatus, MetadataResourceTypes
|
|
21
21
|
from zenml.models import (
|
22
22
|
PipelineRunResponse,
|
23
23
|
PipelineRunUpdate,
|
24
|
+
RunMetadataResource,
|
24
25
|
StepRunResponse,
|
25
26
|
StepRunUpdate,
|
26
27
|
)
|
@@ -129,8 +130,11 @@ def publish_pipeline_run_metadata(
|
|
129
130
|
for stack_component_id, metadata in pipeline_run_metadata.items():
|
130
131
|
client.create_run_metadata(
|
131
132
|
metadata=metadata,
|
132
|
-
|
133
|
-
|
133
|
+
resources=[
|
134
|
+
RunMetadataResource(
|
135
|
+
id=pipeline_run_id, type=MetadataResourceTypes.PIPELINE_RUN
|
136
|
+
)
|
137
|
+
],
|
134
138
|
stack_component_id=stack_component_id,
|
135
139
|
)
|
136
140
|
|
@@ -150,7 +154,10 @@ def publish_step_run_metadata(
|
|
150
154
|
for stack_component_id, metadata in step_run_metadata.items():
|
151
155
|
client.create_run_metadata(
|
152
156
|
metadata=metadata,
|
153
|
-
|
154
|
-
|
157
|
+
resources=[
|
158
|
+
RunMetadataResource(
|
159
|
+
id=step_run_id, type=MetadataResourceTypes.STEP_RUN
|
160
|
+
)
|
161
|
+
],
|
155
162
|
stack_component_id=stack_component_id,
|
156
163
|
)
|
@@ -310,8 +310,8 @@ class StepLauncher:
|
|
310
310
|
and a boolean indicating whether the run was created or reused.
|
311
311
|
"""
|
312
312
|
start_time = datetime.utcnow()
|
313
|
-
run_name =
|
314
|
-
|
313
|
+
run_name = string_utils.format_name_template(
|
314
|
+
name_template=self._deployment.run_name_template,
|
315
315
|
substitutions=self._deployment.pipeline_configuration._get_full_substitutions(
|
316
316
|
start_time
|
317
317
|
),
|
@@ -309,6 +309,9 @@ def create_cached_step_runs(
|
|
309
309
|
for invocation_id in cache_candidates:
|
310
310
|
visited_invocations.add(invocation_id)
|
311
311
|
|
312
|
+
# Make sure the request factory has the most up to date pipeline
|
313
|
+
# run to avoid hydration calls
|
314
|
+
request_factory.pipeline_run = pipeline_run
|
312
315
|
try:
|
313
316
|
step_run_request = request_factory.create_request(
|
314
317
|
invocation_id
|
@@ -56,7 +56,7 @@ from zenml.steps.utils import (
|
|
56
56
|
parse_return_type_annotations,
|
57
57
|
resolve_type_annotation,
|
58
58
|
)
|
59
|
-
from zenml.utils import materializer_utils, source_utils
|
59
|
+
from zenml.utils import materializer_utils, source_utils, string_utils
|
60
60
|
from zenml.utils.typing_utils import get_origin, is_union
|
61
61
|
|
62
62
|
if TYPE_CHECKING:
|
@@ -292,16 +292,15 @@ class StepRunner:
|
|
292
292
|
"""
|
293
293
|
collections.append(output_annotations)
|
294
294
|
for k, v in list(output_annotations.items()):
|
295
|
-
|
296
|
-
if v.artifact_config:
|
297
|
-
|
298
|
-
|
295
|
+
name = k
|
296
|
+
if v.artifact_config and v.artifact_config.name:
|
297
|
+
name = string_utils.format_name_template(
|
298
|
+
v.artifact_config.name,
|
299
|
+
substitutions=step_run.config.substitutions,
|
299
300
|
)
|
300
|
-
if _evaluated_name is None:
|
301
|
-
_evaluated_name = k
|
302
301
|
|
303
302
|
for d in collections:
|
304
|
-
d[
|
303
|
+
d[name] = d.pop(k)
|
305
304
|
|
306
305
|
def _load_step(self) -> "BaseStep":
|
307
306
|
"""Load the step instance.
|
zenml/orchestrators/utils.py
CHANGED
@@ -32,7 +32,6 @@ from zenml.constants import (
|
|
32
32
|
from zenml.enums import AuthScheme, StackComponentType, StoreType
|
33
33
|
from zenml.logger import get_logger
|
34
34
|
from zenml.stack import StackComponent
|
35
|
-
from zenml.utils.string_utils import format_name_template
|
36
35
|
|
37
36
|
logger = get_logger(__name__)
|
38
37
|
|
@@ -196,29 +195,6 @@ def get_config_environment_vars(
|
|
196
195
|
return environment_vars
|
197
196
|
|
198
197
|
|
199
|
-
def get_run_name(run_name_template: str, substitutions: Dict[str, str]) -> str:
|
200
|
-
"""Fill out the run name template to get a complete run name.
|
201
|
-
|
202
|
-
Args:
|
203
|
-
run_name_template: The run name template to fill out.
|
204
|
-
substitutions: The substitutions to use in the template.
|
205
|
-
|
206
|
-
Raises:
|
207
|
-
ValueError: If the run name is empty.
|
208
|
-
|
209
|
-
Returns:
|
210
|
-
The run name derived from the template.
|
211
|
-
"""
|
212
|
-
run_name = format_name_template(
|
213
|
-
run_name_template, substitutions=substitutions
|
214
|
-
)
|
215
|
-
|
216
|
-
if run_name == "":
|
217
|
-
raise ValueError("Empty run names are not allowed.")
|
218
|
-
|
219
|
-
return run_name
|
220
|
-
|
221
|
-
|
222
198
|
class register_artifact_store_filesystem:
|
223
199
|
"""Context manager for the artifact_store/filesystem_registry dependency.
|
224
200
|
|
zenml/pipelines/run_utils.py
CHANGED
@@ -23,9 +23,8 @@ from zenml.models import (
|
|
23
23
|
StackResponse,
|
24
24
|
)
|
25
25
|
from zenml.orchestrators.publish_utils import publish_failed_pipeline_run
|
26
|
-
from zenml.orchestrators.utils import get_run_name
|
27
26
|
from zenml.stack import Flavor, Stack
|
28
|
-
from zenml.utils import code_utils, notebook_utils, source_utils
|
27
|
+
from zenml.utils import code_utils, notebook_utils, source_utils, string_utils
|
29
28
|
from zenml.zen_stores.base_zen_store import BaseZenStore
|
30
29
|
|
31
30
|
if TYPE_CHECKING:
|
@@ -68,8 +67,8 @@ def create_placeholder_run(
|
|
68
67
|
return None
|
69
68
|
start_time = datetime.utcnow()
|
70
69
|
run_request = PipelineRunRequest(
|
71
|
-
name=
|
72
|
-
|
70
|
+
name=string_utils.format_name_template(
|
71
|
+
name_template=deployment.run_name_template,
|
73
72
|
substitutions=deployment.pipeline_configuration._get_full_substitutions(
|
74
73
|
start_time
|
75
74
|
),
|
zenml/steps/utils.py
CHANGED
@@ -42,6 +42,7 @@ from zenml.enums import (
|
|
42
42
|
from zenml.exceptions import StepInterfaceError
|
43
43
|
from zenml.logger import get_logger
|
44
44
|
from zenml.metadata.metadata_types import MetadataType
|
45
|
+
from zenml.models import RunMetadataResource
|
45
46
|
from zenml.steps.step_context import get_step_context
|
46
47
|
from zenml.utils import settings_utils, source_code_utils, typing_utils
|
47
48
|
|
@@ -489,8 +490,11 @@ def log_step_metadata(
|
|
489
490
|
step_run_id = pipeline_run.steps[step_name].id
|
490
491
|
client.create_run_metadata(
|
491
492
|
metadata=metadata,
|
492
|
-
|
493
|
-
|
493
|
+
resources=[
|
494
|
+
RunMetadataResource(
|
495
|
+
id=step_run_id, type=MetadataResourceTypes.STEP_RUN
|
496
|
+
)
|
497
|
+
],
|
494
498
|
)
|
495
499
|
|
496
500
|
|