zenml-nightly 0.70.0.dev20241129__py3-none-any.whl → 0.70.0.dev20241130__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. zenml/VERSION +1 -1
  2. zenml/artifacts/artifact_config.py +0 -14
  3. zenml/artifacts/utils.py +50 -29
  4. zenml/cli/__init__.py +15 -0
  5. zenml/cli/base.py +4 -4
  6. zenml/cli/server.py +1 -1
  7. zenml/cli/stack.py +0 -3
  8. zenml/cli/stack_components.py +0 -1
  9. zenml/cli/utils.py +0 -5
  10. zenml/client.py +8 -18
  11. zenml/model/model.py +8 -4
  12. zenml/model/utils.py +18 -16
  13. zenml/models/__init__.py +6 -0
  14. zenml/models/v2/core/artifact_version.py +6 -3
  15. zenml/models/v2/core/component.py +0 -22
  16. zenml/models/v2/core/model_version.py +6 -3
  17. zenml/models/v2/core/pipeline_run.py +16 -4
  18. zenml/models/v2/core/run_metadata.py +30 -9
  19. zenml/models/v2/core/step_run.py +5 -3
  20. zenml/models/v2/misc/run_metadata.py +38 -0
  21. zenml/orchestrators/input_utils.py +19 -6
  22. zenml/orchestrators/publish_utils.py +11 -4
  23. zenml/orchestrators/step_launcher.py +2 -2
  24. zenml/orchestrators/step_run_utils.py +3 -0
  25. zenml/orchestrators/step_runner.py +7 -8
  26. zenml/orchestrators/utils.py +0 -24
  27. zenml/pipelines/run_utils.py +3 -4
  28. zenml/steps/utils.py +6 -2
  29. zenml/utils/metadata_utils.py +186 -153
  30. zenml/utils/string_utils.py +15 -8
  31. zenml/zen_server/routers/workspaces_endpoints.py +19 -19
  32. zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
  33. zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
  34. zenml/zen_stores/schemas/__init__.py +5 -1
  35. zenml/zen_stores/schemas/artifact_schemas.py +10 -10
  36. zenml/zen_stores/schemas/component_schemas.py +0 -3
  37. zenml/zen_stores/schemas/model_schemas.py +13 -11
  38. zenml/zen_stores/schemas/pipeline_run_schemas.py +32 -15
  39. zenml/zen_stores/schemas/run_metadata_schemas.py +66 -31
  40. zenml/zen_stores/schemas/step_run_schemas.py +13 -12
  41. zenml/zen_stores/schemas/utils.py +47 -3
  42. zenml/zen_stores/sql_zen_store.py +102 -23
  43. {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/METADATA +1 -1
  44. {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/RECORD +47 -44
  45. {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/LICENSE +0 -0
  46. {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/WHEEL +0 -0
  47. {zenml_nightly-0.70.0.dev20241129.dist-info → zenml_nightly-0.70.0.dev20241130.dist-info}/entry_points.txt +0 -0
@@ -13,16 +13,16 @@
13
13
  # permissions and limitations under the License.
14
14
  """Models representing run metadata."""
15
15
 
16
- from typing import Dict, Optional
16
+ from typing import Dict, List, Optional
17
17
  from uuid import UUID
18
18
 
19
- from pydantic import Field
19
+ from pydantic import Field, model_validator
20
20
 
21
- from zenml.enums import MetadataResourceTypes
22
21
  from zenml.metadata.metadata_types import MetadataType, MetadataTypeEnum
23
22
  from zenml.models.v2.base.scoped import (
24
23
  WorkspaceScopedRequest,
25
24
  )
25
+ from zenml.models.v2.misc.run_metadata import RunMetadataResource
26
26
 
27
27
  # ------------------ Request Model ------------------
28
28
 
@@ -30,14 +30,12 @@ from zenml.models.v2.base.scoped import (
30
30
  class RunMetadataRequest(WorkspaceScopedRequest):
31
31
  """Request model for run metadata."""
32
32
 
33
- resource_id: UUID = Field(
34
- title="The ID of the resource that this metadata belongs to.",
35
- )
36
- resource_type: MetadataResourceTypes = Field(
37
- title="The type of the resource that this metadata belongs to.",
33
+ resources: List[RunMetadataResource] = Field(
34
+ title="The list of resources that this metadata belongs to."
38
35
  )
39
36
  stack_component_id: Optional[UUID] = Field(
40
- title="The ID of the stack component that this metadata belongs to."
37
+ title="The ID of the stack component that this metadata belongs to.",
38
+ default=None,
41
39
  )
42
40
  values: Dict[str, "MetadataType"] = Field(
43
41
  title="The metadata to be created.",
@@ -45,3 +43,26 @@ class RunMetadataRequest(WorkspaceScopedRequest):
45
43
  types: Dict[str, "MetadataTypeEnum"] = Field(
46
44
  title="The types of the metadata to be created.",
47
45
  )
46
+ publisher_step_id: Optional[UUID] = Field(
47
+ title="The ID of the step execution that published this metadata.",
48
+ default=None,
49
+ )
50
+
51
+ @model_validator(mode="after")
52
+ def validate_values_keys(self) -> "RunMetadataRequest":
53
+ """Validates if the keys in the metadata are properly defined.
54
+
55
+ Returns:
56
+ self
57
+
58
+ Raises:
59
+ ValueError: if one of the key in the metadata contains `:`
60
+ """
61
+ invalid_keys = [key for key in self.values.keys() if ":" in key]
62
+ if invalid_keys:
63
+ raise ValueError(
64
+ "You can not use colons (`:`) in the key names when you "
65
+ "are creating metadata for your ZenML objects. Please change "
66
+ f"the following keys: {invalid_keys}"
67
+ )
68
+ return self
@@ -594,6 +594,7 @@ class StepRunFilter(WorkspaceScopedFilter):
594
594
  from zenml.zen_stores.schemas import (
595
595
  ModelSchema,
596
596
  ModelVersionSchema,
597
+ RunMetadataResourceSchema,
597
598
  RunMetadataSchema,
598
599
  StepRunSchema,
599
600
  )
@@ -612,10 +613,11 @@ class StepRunFilter(WorkspaceScopedFilter):
612
613
 
613
614
  for key, value in self.run_metadata.items():
614
615
  additional_filter = and_(
615
- RunMetadataSchema.resource_id == StepRunSchema.id,
616
- RunMetadataSchema.resource_type
616
+ RunMetadataResourceSchema.resource_id == StepRunSchema.id,
617
+ RunMetadataResourceSchema.resource_type
617
618
  == MetadataResourceTypes.STEP_RUN,
618
- RunMetadataSchema.key == key,
619
+ RunMetadataResourceSchema.run_metadata_id
620
+ == RunMetadataSchema.id,
619
621
  self.generate_custom_query_conditions_for_column(
620
622
  value=value,
621
623
  table=RunMetadataSchema,
@@ -0,0 +1,38 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Utility classes for modeling run metadata."""
15
+
16
+ from datetime import datetime
17
+ from uuid import UUID
18
+
19
+ from pydantic import BaseModel, Field
20
+
21
+ from zenml.enums import MetadataResourceTypes
22
+ from zenml.metadata.metadata_types import MetadataType
23
+
24
+
25
+ class RunMetadataResource(BaseModel):
26
+ """Utility class to help identify resources to tag metadata to."""
27
+
28
+ id: UUID = Field(title="The ID of the resource.")
29
+ type: MetadataResourceTypes = Field(title="The type of the resource.")
30
+
31
+
32
+ class RunMetadataEntry(BaseModel):
33
+ """Utility class to sort/list run metadata entries."""
34
+
35
+ value: MetadataType = Field(title="The value for the run metadata entry")
36
+ created: datetime = Field(
37
+ title="The timestamp when this resource was created."
38
+ )
@@ -20,7 +20,7 @@ from zenml.client import Client
20
20
  from zenml.config.step_configurations import Step
21
21
  from zenml.enums import ArtifactSaveType, StepRunInputArtifactType
22
22
  from zenml.exceptions import InputResolutionError
23
- from zenml.utils import pagination_utils
23
+ from zenml.utils import pagination_utils, string_utils
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  from zenml.models import PipelineRunResponse
@@ -53,7 +53,8 @@ def resolve_step_inputs(
53
53
  current_run_steps = {
54
54
  run_step.name: run_step
55
55
  for run_step in pagination_utils.depaginate(
56
- Client().list_run_steps, pipeline_run_id=pipeline_run.id
56
+ Client().list_run_steps,
57
+ pipeline_run_id=pipeline_run.id,
57
58
  )
58
59
  }
59
60
 
@@ -66,11 +67,23 @@ def resolve_step_inputs(
66
67
  f"No step `{input_.step_name}` found in current run."
67
68
  )
68
69
 
70
+ # Try to get the substitutions from the pipeline run first, as we
71
+ # already have a hydrated version of that. In the unlikely case
72
+ # that the pipeline run is outdated, we fetch it from the step
73
+ # run instead which will costs us one hydration call.
74
+ substitutions = (
75
+ pipeline_run.step_substitutions.get(step_run.name)
76
+ or step_run.config.substitutions
77
+ )
78
+ output_name = string_utils.format_name_template(
79
+ input_.output_name, substitutions=substitutions
80
+ )
81
+
69
82
  try:
70
- outputs = step_run.outputs[input_.output_name]
83
+ outputs = step_run.outputs[output_name]
71
84
  except KeyError:
72
85
  raise InputResolutionError(
73
- f"No step output `{input_.output_name}` found for step "
86
+ f"No step output `{output_name}` found for step "
74
87
  f"`{input_.step_name}`."
75
88
  )
76
89
 
@@ -83,12 +96,12 @@ def resolve_step_inputs(
83
96
  # This should never happen, there can only be a single regular step
84
97
  # output for a name
85
98
  raise InputResolutionError(
86
- f"Too many step outputs for output `{input_.output_name}` of "
99
+ f"Too many step outputs for output `{output_name}` of "
87
100
  f"step `{input_.step_name}`."
88
101
  )
89
102
  elif len(step_outputs) == 0:
90
103
  raise InputResolutionError(
91
- f"No step output `{input_.output_name}` found for step "
104
+ f"No step output `{output_name}` found for step "
92
105
  f"`{input_.step_name}`."
93
106
  )
94
107
 
@@ -21,6 +21,7 @@ from zenml.enums import ExecutionStatus, MetadataResourceTypes
21
21
  from zenml.models import (
22
22
  PipelineRunResponse,
23
23
  PipelineRunUpdate,
24
+ RunMetadataResource,
24
25
  StepRunResponse,
25
26
  StepRunUpdate,
26
27
  )
@@ -129,8 +130,11 @@ def publish_pipeline_run_metadata(
129
130
  for stack_component_id, metadata in pipeline_run_metadata.items():
130
131
  client.create_run_metadata(
131
132
  metadata=metadata,
132
- resource_id=pipeline_run_id,
133
- resource_type=MetadataResourceTypes.PIPELINE_RUN,
133
+ resources=[
134
+ RunMetadataResource(
135
+ id=pipeline_run_id, type=MetadataResourceTypes.PIPELINE_RUN
136
+ )
137
+ ],
134
138
  stack_component_id=stack_component_id,
135
139
  )
136
140
 
@@ -150,7 +154,10 @@ def publish_step_run_metadata(
150
154
  for stack_component_id, metadata in step_run_metadata.items():
151
155
  client.create_run_metadata(
152
156
  metadata=metadata,
153
- resource_id=step_run_id,
154
- resource_type=MetadataResourceTypes.STEP_RUN,
157
+ resources=[
158
+ RunMetadataResource(
159
+ id=step_run_id, type=MetadataResourceTypes.STEP_RUN
160
+ )
161
+ ],
155
162
  stack_component_id=stack_component_id,
156
163
  )
@@ -310,8 +310,8 @@ class StepLauncher:
310
310
  and a boolean indicating whether the run was created or reused.
311
311
  """
312
312
  start_time = datetime.utcnow()
313
- run_name = orchestrator_utils.get_run_name(
314
- run_name_template=self._deployment.run_name_template,
313
+ run_name = string_utils.format_name_template(
314
+ name_template=self._deployment.run_name_template,
315
315
  substitutions=self._deployment.pipeline_configuration._get_full_substitutions(
316
316
  start_time
317
317
  ),
@@ -309,6 +309,9 @@ def create_cached_step_runs(
309
309
  for invocation_id in cache_candidates:
310
310
  visited_invocations.add(invocation_id)
311
311
 
312
+ # Make sure the request factory has the most up to date pipeline
313
+ # run to avoid hydration calls
314
+ request_factory.pipeline_run = pipeline_run
312
315
  try:
313
316
  step_run_request = request_factory.create_request(
314
317
  invocation_id
@@ -56,7 +56,7 @@ from zenml.steps.utils import (
56
56
  parse_return_type_annotations,
57
57
  resolve_type_annotation,
58
58
  )
59
- from zenml.utils import materializer_utils, source_utils
59
+ from zenml.utils import materializer_utils, source_utils, string_utils
60
60
  from zenml.utils.typing_utils import get_origin, is_union
61
61
 
62
62
  if TYPE_CHECKING:
@@ -292,16 +292,15 @@ class StepRunner:
292
292
  """
293
293
  collections.append(output_annotations)
294
294
  for k, v in list(output_annotations.items()):
295
- _evaluated_name = None
296
- if v.artifact_config:
297
- _evaluated_name = v.artifact_config._evaluated_name(
298
- step_run.config.substitutions
295
+ name = k
296
+ if v.artifact_config and v.artifact_config.name:
297
+ name = string_utils.format_name_template(
298
+ v.artifact_config.name,
299
+ substitutions=step_run.config.substitutions,
299
300
  )
300
- if _evaluated_name is None:
301
- _evaluated_name = k
302
301
 
303
302
  for d in collections:
304
- d[_evaluated_name] = d.pop(k)
303
+ d[name] = d.pop(k)
305
304
 
306
305
  def _load_step(self) -> "BaseStep":
307
306
  """Load the step instance.
@@ -32,7 +32,6 @@ from zenml.constants import (
32
32
  from zenml.enums import AuthScheme, StackComponentType, StoreType
33
33
  from zenml.logger import get_logger
34
34
  from zenml.stack import StackComponent
35
- from zenml.utils.string_utils import format_name_template
36
35
 
37
36
  logger = get_logger(__name__)
38
37
 
@@ -196,29 +195,6 @@ def get_config_environment_vars(
196
195
  return environment_vars
197
196
 
198
197
 
199
- def get_run_name(run_name_template: str, substitutions: Dict[str, str]) -> str:
200
- """Fill out the run name template to get a complete run name.
201
-
202
- Args:
203
- run_name_template: The run name template to fill out.
204
- substitutions: The substitutions to use in the template.
205
-
206
- Raises:
207
- ValueError: If the run name is empty.
208
-
209
- Returns:
210
- The run name derived from the template.
211
- """
212
- run_name = format_name_template(
213
- run_name_template, substitutions=substitutions
214
- )
215
-
216
- if run_name == "":
217
- raise ValueError("Empty run names are not allowed.")
218
-
219
- return run_name
220
-
221
-
222
198
  class register_artifact_store_filesystem:
223
199
  """Context manager for the artifact_store/filesystem_registry dependency.
224
200
 
@@ -23,9 +23,8 @@ from zenml.models import (
23
23
  StackResponse,
24
24
  )
25
25
  from zenml.orchestrators.publish_utils import publish_failed_pipeline_run
26
- from zenml.orchestrators.utils import get_run_name
27
26
  from zenml.stack import Flavor, Stack
28
- from zenml.utils import code_utils, notebook_utils, source_utils
27
+ from zenml.utils import code_utils, notebook_utils, source_utils, string_utils
29
28
  from zenml.zen_stores.base_zen_store import BaseZenStore
30
29
 
31
30
  if TYPE_CHECKING:
@@ -68,8 +67,8 @@ def create_placeholder_run(
68
67
  return None
69
68
  start_time = datetime.utcnow()
70
69
  run_request = PipelineRunRequest(
71
- name=get_run_name(
72
- run_name_template=deployment.run_name_template,
70
+ name=string_utils.format_name_template(
71
+ name_template=deployment.run_name_template,
73
72
  substitutions=deployment.pipeline_configuration._get_full_substitutions(
74
73
  start_time
75
74
  ),
zenml/steps/utils.py CHANGED
@@ -42,6 +42,7 @@ from zenml.enums import (
42
42
  from zenml.exceptions import StepInterfaceError
43
43
  from zenml.logger import get_logger
44
44
  from zenml.metadata.metadata_types import MetadataType
45
+ from zenml.models import RunMetadataResource
45
46
  from zenml.steps.step_context import get_step_context
46
47
  from zenml.utils import settings_utils, source_code_utils, typing_utils
47
48
 
@@ -489,8 +490,11 @@ def log_step_metadata(
489
490
  step_run_id = pipeline_run.steps[step_name].id
490
491
  client.create_run_metadata(
491
492
  metadata=metadata,
492
- resource_id=step_run_id,
493
- resource_type=MetadataResourceTypes.STEP_RUN,
493
+ resources=[
494
+ RunMetadataResource(
495
+ id=step_run_id, type=MetadataResourceTypes.STEP_RUN
496
+ )
497
+ ],
494
498
  )
495
499
 
496
500