zenml-nightly 0.70.0.dev20241129__py3-none-any.whl → 0.70.0.dev20241130__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.70.0.dev20241129
+0.70.0.dev20241130

zenml/artifacts/artifact_config.py

@@ -21,7 +21,6 @@ from zenml.enums import ArtifactType
 from zenml.logger import get_logger
 from zenml.metadata.metadata_types import MetadataType
 from zenml.utils.pydantic_utils import before_validator_handler
-from zenml.utils.string_utils import format_name_template
 
 logger = get_logger(__name__)
 
@@ -118,16 +117,3 @@ class ArtifactConfig(BaseModel):
             data.setdefault("artifact_type", ArtifactType.SERVICE)
 
         return data
-
-    def _evaluated_name(self, substitutions: Dict[str, str]) -> Optional[str]:
-        """Evaluated name of the artifact.
-
-        Args:
-            substitutions: Extra placeholders to use in the name template.
-
-        Returns:
-            The evaluated name of the artifact.
-        """
-        if self.name:
-            return format_name_template(self.name, substitutions=substitutions)
-        return self.name

zenml/artifacts/utils.py

@@ -14,6 +14,7 @@
 """Utility functions for handling artifacts."""
 
 import base64
+import contextlib
 import os
 import tempfile
 import zipfile
@@ -41,7 +42,6 @@ from zenml.enums import (
     ArtifactSaveType,
     ArtifactType,
     ExecutionStatus,
-    MetadataResourceTypes,
     StackComponentType,
     VisualizationType,
 )
@@ -404,50 +404,71 @@ def log_artifact_metadata(
         artifact_version: The version of the artifact to log metadata for. If
             not provided, when being called inside a step that produces an
             artifact named `artifact_name`, the metadata will be associated to
-            the corresponding newly created artifact. Or, if not provided when
-            being called outside a step, or in a step that does not produce
-            any artifact named `artifact_name`, the metadata will be associated
-            to the latest version of that artifact.
+            the corresponding newly created artifact.
 
     Raises:
         ValueError: If no artifact name is provided and the function is not
             called inside a step with a single output, or, if neither an
             artifact nor an output with the given name exists.
+
     """
     logger.warning(
         "The `log_artifact_metadata` function is deprecated and will soon be "
         "removed. Please use `log_metadata` instead."
     )
-    try:
+
+    from zenml import log_metadata
+
+    if artifact_name and artifact_version:
+        assert artifact_name is not None
+
+        log_metadata(
+            metadata=metadata,
+            artifact_name=artifact_name,
+            artifact_version=artifact_version,
+        )
+
+    step_context = None
+    with contextlib.suppress(RuntimeError):
         step_context = get_step_context()
-        in_step_outputs = (artifact_name in step_context._outputs) or (
-            not artifact_name and len(step_context._outputs) == 1
+
+    if step_context and artifact_name in step_context._outputs.keys():
+        log_metadata(
+            metadata=metadata,
+            artifact_name=artifact_name,
+            infer_artifact=True,
         )
-    except RuntimeError:
-        step_context = None
-        in_step_outputs = False
-
-    if not step_context or not in_step_outputs or artifact_version:
-        if not artifact_name:
-            raise ValueError(
-                "Artifact name must be provided unless the function is called "
-                "inside a step with a single output."
-            )
+    elif step_context and len(step_context._outputs) == 1:
+        single_output_name = list(step_context._outputs.keys())[0]
+
+        log_metadata(
+            metadata=metadata,
+            artifact_name=single_output_name,
+            infer_artifact=True,
+        )
+    elif artifact_name:
         client = Client()
-        response = client.get_artifact_version(artifact_name, artifact_version)
-        client.create_run_metadata(
+        logger.warning(
+            "Deprecation warning! Currently, you are calling "
+            "`log_artifact_metadata` from a context, where we use the "
+            "`artifact_name` to fetch it and link the metadata to its "
+            "latest version. This behavior is deprecated and will be "
+            "removed in the future. To circumvent this, please check"
+            "the `log_metadata` function."
+        )
+        artifact_version_model = client.get_artifact_version(
+            name_id_or_prefix=artifact_name
+        )
+        log_metadata(
             metadata=metadata,
-            resource_id=response.id,
-            resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
+            artifact_version_id=artifact_version_model.id,
         )
-
     else:
-        try:
-            step_context.add_output_metadata(
-                metadata=metadata, output_name=artifact_name
-            )
-        except StepContextError as e:
-            raise ValueError(e)
+        raise ValueError(
+            "You need to call `log_artifact_metadata` either within a step "
+            "(potentially with an artifact name) or outside of a step with an "
+            "artifact name (and/or version)."
+        )
 
 
 # -----------------

zenml/cli/__init__.py

@@ -2410,6 +2410,21 @@ stack, use the `--stack` option.
 zenml pipeline run <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
 ```
 
+If you want to create a run template based on your pipeline that can later be used to trigger a run either from the dashboard or through an HTTP request:
+
+```bash
+zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
+   --name=<TEMPLATE_NAME>
+
+To specify a config file, use the `--config/-c` option. If you would like to use a different stack than the active one, use the `--stack` option.
+
+```bash
+zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
+   --name=<TEMPLATE_NAME> \
+   --config=<PATH_TO_CONFIG_YAML> \
+   --stack=<STACK_ID_OR_NAME>
+```
+
 Tagging your resources with ZenML
 ---------------------------------
 

zenml/cli/base.py

@@ -79,19 +79,19 @@ class ZenMLProjectTemplateLocation(BaseModel):
 ZENML_PROJECT_TEMPLATES = dict(
     e2e_batch=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-e2e-batch",
-        github_tag="2024.11.20",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     starter=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-starter",
-        github_tag="2024.10.30",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",
-        github_tag="2024.10.30",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",
-        github_tag="2024.11.08",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
 )
 

zenml/cli/server.py

@@ -469,7 +469,7 @@ def logs(
     if server is None:
         cli_utils.error(
             "The local ZenML dashboard is not running. Please call `zenml "
-            "up` first to start the ZenML dashboard locally."
+            "login --local` first to start the ZenML dashboard locally."
         )
 
     from zenml.zen_server.deploy.deployer import LocalServerDeployer

zenml/cli/stack.py

@@ -1129,14 +1129,12 @@ def export_stack(
 def _import_stack_component(
     component_type: StackComponentType,
     component_dict: Dict[str, Any],
-    component_spec_path: Optional[str] = None,
 ) -> UUID:
     """Import a single stack component with given type/config.
 
     Args:
         component_type: The type of component to import.
         component_dict: Dict representation of the component to import.
-        component_spec_path: Path to the component spec file.
 
     Returns:
         The ID of the imported component.
@@ -1172,7 +1170,6 @@ def _import_stack_component(
         component_type=component_type,
         flavor=flavor,
         configuration=config,
-        component_spec_path=component_spec_path,
     )
     return component.id
 

zenml/cli/stack_components.py

@@ -573,7 +573,6 @@ def generate_stack_component_copy_command(
                 component_type=component_to_copy.type,
                 configuration=component_to_copy.configuration,
                 labels=component_to_copy.labels,
-                component_spec_path=component_to_copy.component_spec_path,
             )
             print_model_url(get_component_url(copied_component))
 

zenml/cli/utils.py

@@ -711,11 +711,6 @@ def print_stack_component_configuration(
 
         console.print(rich_table)
 
-    if component.component_spec_path:
-        declare(
-            f"Component spec path for `mlstacks`: {component.component_spec_path}"
-        )
-
 
 def expand_argument_value_from_file(name: str, value: str) -> str:
     """Expands the value of an argument pointing to a file into the contents of that file.

zenml/client.py

@@ -60,7 +60,6 @@ from zenml.constants import (
 from zenml.enums import (
     ArtifactType,
     LogicalOperators,
-    MetadataResourceTypes,
     ModelStages,
     OAuthDeviceStatus,
     PluginSubType,
@@ -137,6 +136,7 @@ from zenml.models import (
     PipelineRunFilter,
     PipelineRunResponse,
     RunMetadataRequest,
+    RunMetadataResource,
     RunTemplateFilter,
     RunTemplateRequest,
     RunTemplateResponse,
@@ -1979,7 +1979,6 @@ class Client(metaclass=ClientMetaClass):
         flavor: str,
         component_type: StackComponentType,
         configuration: Dict[str, str],
-        component_spec_path: Optional[str] = None,
         labels: Optional[Dict[str, Any]] = None,
     ) -> "ComponentResponse":
         """Registers a stack component.
@@ -1987,7 +1986,6 @@ class Client(metaclass=ClientMetaClass):
         Args:
             name: The name of the stack component.
             flavor: The flavor of the stack component.
-            component_spec_path: The path to the stack spec file.
             component_type: The type of the stack component.
             configuration: The configuration of the stack component.
             labels: The labels of the stack component.
@@ -2016,7 +2014,6 @@ class Client(metaclass=ClientMetaClass):
             name=name,
             type=component_type,
             flavor=flavor,
-            component_spec_path=component_spec_path,
             configuration=configuration,
             user=self.active_user.id,
             workspace=self.active_workspace.id,
@@ -2033,7 +2030,6 @@ class Client(metaclass=ClientMetaClass):
         name_id_or_prefix: Optional[Union[UUID, str]],
         component_type: StackComponentType,
         name: Optional[str] = None,
-        component_spec_path: Optional[str] = None,
         configuration: Optional[Dict[str, Any]] = None,
         labels: Optional[Dict[str, Any]] = None,
         disconnect: Optional[bool] = None,
@@ -2047,7 +2043,6 @@ class Client(metaclass=ClientMetaClass):
                 update.
             component_type: The type of the stack component to update.
             name: The new name of the stack component.
-            component_spec_path: The new path to the stack spec file.
             configuration: The new configuration of the stack component.
             labels: The new labels of the stack component.
             disconnect: Whether to disconnect the stack component from its
@@ -2072,7 +2067,6 @@ class Client(metaclass=ClientMetaClass):
         update_model = ComponentUpdate(
             workspace=self.active_workspace.id,
             user=self.active_user.id,
-            component_spec_path=component_spec_path,
         )
 
         if name is not None:
@@ -4438,23 +4432,20 @@ class Client(metaclass=ClientMetaClass):
     def create_run_metadata(
         self,
         metadata: Dict[str, "MetadataType"],
-        resource_id: UUID,
-        resource_type: MetadataResourceTypes,
+        resources: List[RunMetadataResource],
         stack_component_id: Optional[UUID] = None,
+        publisher_step_id: Optional[UUID] = None,
     ) -> None:
         """Create run metadata.
 
         Args:
             metadata: The metadata to create as a dictionary of key-value pairs.
-            resource_id: The ID of the resource for which the
-                metadata was produced.
-            resource_type: The type of the resource for which the
+            resources: The list of IDs and types of the resources for that the
                 metadata was produced.
             stack_component_id: The ID of the stack component that produced
                 the metadata.
-
-        Returns:
-            None
+            publisher_step_id: The ID of the step execution that publishes
+                this metadata automatically.
         """
         from zenml.metadata.metadata_types import get_metadata_type
 
@@ -4483,14 +4474,13 @@ class Client(metaclass=ClientMetaClass):
         run_metadata = RunMetadataRequest(
             workspace=self.active_workspace.id,
             user=self.active_user.id,
-            resource_id=resource_id,
-            resource_type=resource_type,
+            resources=resources,
             stack_component_id=stack_component_id,
+            publisher_step_id=publisher_step_id,
             values=values,
             types=types,
         )
         self.zen_store.create_run_metadata(run_metadata)
-        return None
 
     # -------------------------------- Secrets ---------------------------------
 

zenml/model/model.py

@@ -336,12 +336,16 @@ class Model(BaseModel):
             metadata: The metadata to log.
         """
         from zenml.client import Client
+        from zenml.models import RunMetadataResource
 
         response = self._get_or_create_model_version()
         Client().create_run_metadata(
             metadata=metadata,
-            resource_id=response.id,
-            resource_type=MetadataResourceTypes.MODEL_VERSION,
+            resources=[
+                RunMetadataResource(
+                    id=response.id, type=MetadataResourceTypes.MODEL_VERSION
+                )
+            ],
         )
 
     @property
@@ -537,7 +541,7 @@ class Model(BaseModel):
 
         zenml_client = Client()
         # backup logic, if the Model class is used directly from the code
-        self.name = format_name_template(self.name, substitutions={})
+        self.name = format_name_template(self.name)
         if self.model_version_id:
             mv = zenml_client.get_model_version(
                 model_version_name_or_number_or_id=self.model_version_id,
@@ -667,7 +671,7 @@ class Model(BaseModel):
 
         # backup logic, if the Model class is used directly from the code
         if isinstance(self.version, str):
-            self.version = format_name_template(self.version, substitutions={})
+            self.version = format_name_template(self.version)
 
         try:
             if self.version or self.model_version_id:

zenml/model/utils.py

@@ -52,29 +52,31 @@ def log_model_metadata(
             `model` in decorator.
 
     Raises:
-        ValueError: If no model name/version is provided and the function is not
-            called inside a step with configured `model` in decorator.
+        ValueError: If the function is not called with proper input.
     """
     logger.warning(
         "The `log_model_metadata` function is deprecated and will soon be "
         "removed. Please use `log_metadata` instead."
     )
 
-    if model_name and model_version:
-        from zenml import Model
+    from zenml import log_metadata
 
-        mv = Model(name=model_name, version=model_version)
+    if model_name and model_version:
+        log_metadata(
+            metadata=metadata,
+            model_version=model_version,
+            model_name=model_name,
+        )
+    elif model_name is None and model_version is None:
+        log_metadata(
+            metadata=metadata,
+            infer_model=True,
+        )
     else:
-        try:
-            step_context = get_step_context()
-        except RuntimeError:
-            raise ValueError(
-                "Model name and version must be provided unless the function is "
-                "called inside a step with configured `model` in decorator."
-            )
-        mv = step_context.model
-
-    mv.log_metadata(metadata)
+        raise ValueError(
+            "You can call `log_model_metadata` by either providing both "
+            "`model_name` and `model_version` or keeping both of them None."
+        )
 
 
 def link_artifact_version_to_model_version(
@@ -107,7 +109,7 @@ def link_artifact_to_model(
         model: The model to link to.
 
     Raises:
-        RuntimeError: If called outside of a step.
+        RuntimeError: If called outside a step.
     """
     if not model:
         is_issue = False

zenml/models/__init__.py

@@ -372,6 +372,10 @@ from zenml.models.v2.misc.auth_models import (
     OAuthRedirectResponse,
     OAuthTokenResponse,
 )
+from zenml.models.v2.misc.run_metadata import (
+    RunMetadataEntry,
+    RunMetadataResource,
+)
 from zenml.models.v2.misc.server_models import (
     ServerModel,
     ServerDatabaseType,
@@ -752,4 +756,6 @@ __all__ = [
     "ServiceConnectorInfo",
     "ServiceConnectorResourcesInfo",
     "ResourcesInfo",
+    "RunMetadataEntry",
+    "RunMetadataResource",
 ]

zenml/models/v2/core/artifact_version.py

@@ -576,6 +576,7 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
             ModelVersionArtifactSchema,
             ModelVersionSchema,
             PipelineRunSchema,
+            RunMetadataResourceSchema,
             RunMetadataSchema,
             StepRunInputArtifactSchema,
             StepRunOutputArtifactSchema,
@@ -679,10 +680,12 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
 
             for key, value in self.run_metadata.items():
                 additional_filter = and_(
-                    RunMetadataSchema.resource_id == ArtifactVersionSchema.id,
-                    RunMetadataSchema.resource_type
+                    RunMetadataResourceSchema.resource_id
+                    == ArtifactVersionSchema.id,
+                    RunMetadataResourceSchema.resource_type
                     == MetadataResourceTypes.ARTIFACT_VERSION,
-                    RunMetadataSchema.key == key,
+                    RunMetadataResourceSchema.run_metadata_id
+                    == RunMetadataSchema.id,
                     self.generate_custom_query_conditions_for_column(
                         value=value,
                         table=RunMetadataSchema,

zenml/models/v2/core/component.py

@@ -81,11 +81,6 @@ class ComponentBase(BaseModel):
         title="The stack component labels.",
     )
 
-    component_spec_path: Optional[str] = Field(
-        default=None,
-        title="The path to the component spec used for mlstacks deployments.",
-    )
-
 
 # ------------------ Request Model ------------------
 
@@ -155,10 +150,6 @@ class ComponentUpdate(BaseUpdate):
         title="The stack component labels.",
         default=None,
     )
-    component_spec_path: Optional[str] = Field(
-        title="The path to the component spec used for mlstacks deployments.",
-        default=None,
-    )
     connector: Optional[UUID] = Field(
         title="The service connector linked to this stack component.",
         default=None,
@@ -201,10 +192,6 @@ class ComponentResponseMetadata(WorkspaceScopedResponseMetadata):
         default=None,
         title="The stack component labels.",
     )
-    component_spec_path: Optional[str] = Field(
-        default=None,
-        title="The path to the component spec used for mlstacks deployments.",
-    )
     connector_resource_id: Optional[str] = Field(
         default=None,
         description="The ID of a specific resource instance to "
@@ -325,15 +312,6 @@ class ComponentResponse(
         """
         return self.get_metadata().labels
 
-    @property
-    def component_spec_path(self) -> Optional[str]:
-        """The `component_spec_path` property.
-
-        Returns:
-            the value of the property.
-        """
-        return self.get_metadata().component_spec_path
-
     @property
     def connector_resource_id(self) -> Optional[str]:
         """The `connector_resource_id` property.

zenml/models/v2/core/model_version.py

@@ -652,6 +652,7 @@ class ModelVersionFilter(WorkspaceScopedTaggableFilter):
 
         from zenml.zen_stores.schemas import (
             ModelVersionSchema,
+            RunMetadataResourceSchema,
             RunMetadataSchema,
             UserSchema,
         )
@@ -672,10 +673,12 @@ class ModelVersionFilter(WorkspaceScopedTaggableFilter):
 
             for key, value in self.run_metadata.items():
                 additional_filter = and_(
-                    RunMetadataSchema.resource_id == ModelVersionSchema.id,
-                    RunMetadataSchema.resource_type
+                    RunMetadataResourceSchema.resource_id
+                    == ModelVersionSchema.id,
+                    RunMetadataResourceSchema.resource_type
                     == MetadataResourceTypes.MODEL_VERSION,
-                    RunMetadataSchema.key == key,
+                    RunMetadataResourceSchema.run_metadata_id
+                    == RunMetadataSchema.id,
                     self.generate_custom_query_conditions_for_column(
                         value=value,
                         table=RunMetadataSchema,

zenml/models/v2/core/pipeline_run.py

@@ -237,7 +237,7 @@ class PipelineRunResponseMetadata(WorkspaceScopedResponseMetadata):
         default=False,
         description="Whether a template can be created from this run.",
     )
-    steps_substitutions: Dict[str, Dict[str, str]] = Field(
+    step_substitutions: Dict[str, Dict[str, str]] = Field(
         title="Substitutions used in the step runs of this pipeline run.",
         default_factory=dict,
     )
@@ -550,6 +550,15 @@ class PipelineRunResponse(
         """
         return self.get_metadata().is_templatable
 
+    @property
+    def step_substitutions(self) -> Dict[str, Dict[str, str]]:
+        """The `step_substitutions` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().step_substitutions
+
     @property
     def model_version(self) -> Optional[ModelVersionResponse]:
         """The `model_version` property.
@@ -726,6 +735,7 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
             PipelineDeploymentSchema,
             PipelineRunSchema,
             PipelineSchema,
+            RunMetadataResourceSchema,
             RunMetadataSchema,
             ScheduleSchema,
             StackComponentSchema,
@@ -901,10 +911,12 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
 
             for key, value in self.run_metadata.items():
                 additional_filter = and_(
-                    RunMetadataSchema.resource_id == PipelineRunSchema.id,
-                    RunMetadataSchema.resource_type
+                    RunMetadataResourceSchema.resource_id
+                    == PipelineRunSchema.id,
+                    RunMetadataResourceSchema.resource_type
                     == MetadataResourceTypes.PIPELINE_RUN,
-                    RunMetadataSchema.key == key,
+                    RunMetadataResourceSchema.run_metadata_id
+                    == RunMetadataSchema.id,
                     self.generate_custom_query_conditions_for_column(
                         value=value,
                         table=RunMetadataSchema,