zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl

zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py}

@@ -12,14 +12,12 @@
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
 """Definition of a ZenML pipeline."""
+
 import copy
 import hashlib
 import inspect
-from collections import defaultdict
 from contextlib import contextmanager
-from datetime import datetime
 from pathlib import Path
-from types import FunctionType
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -38,7 +36,8 @@ from typing import (
 from uuid import UUID
 
 import yaml
-from pydantic import ValidationError
+from pydantic import ConfigDict, ValidationError
+from typing_extensions import Self
 
 from zenml import constants
 from zenml.analytics.enums import AnalyticsEvent
@@ -53,7 +52,8 @@ from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.config.pipeline_spec import PipelineSpec
 from zenml.config.schedule import Schedule
 from zenml.config.step_configurations import StepConfigurationUpdate
-from zenml.enums import ExecutionStatus, StackComponentType
+from zenml.enums import StackComponentType
+from zenml.exceptions import EntityExistsError
 from zenml.hooks.hook_validators import resolve_and_validate_hook
 from zenml.logger import get_logger
 from zenml.models import (
@@ -65,13 +65,16 @@ from zenml.models import (
     PipelineDeploymentResponse,
     PipelineRequest,
     PipelineResponse,
-    PipelineRunRequest,
     PipelineRunResponse,
+    RunTemplateResponse,
     ScheduleRequest,
 )
-from zenml.new.pipelines import build_utils
-from zenml.new.pipelines.model_utils import NewModelRequest
-from zenml.orchestrators.utils import get_run_name
+from zenml.pipelines import build_utils
+from zenml.pipelines.run_utils import (
+    create_placeholder_run,
+    deploy_pipeline,
+    upload_notebook_cell_code_if_necessary,
+)
 from zenml.stack import Stack
 from zenml.steps import BaseStep
 from zenml.steps.entrypoint_function_utils import (
@@ -80,29 +83,33 @@ from zenml.steps.entrypoint_function_utils import (
 from zenml.steps.step_invocation import StepInvocation
 from zenml.utils import (
     code_repository_utils,
+    code_utils,
     dashboard_utils,
     dict_utils,
+    env_utils,
     pydantic_utils,
     settings_utils,
     source_utils,
     yaml_utils,
 )
+from zenml.utils.string_utils import format_name_template
 
 if TYPE_CHECKING:
     from zenml.artifacts.external_artifact import ExternalArtifact
+    from zenml.client_lazy_loader import ClientLazyLoader
     from zenml.config.base_settings import SettingsOrDict
     from zenml.config.source import Source
     from zenml.model.lazy_load import ModelVersionDataLazyLoader
     from zenml.model.model import Model
+    from zenml.models import ArtifactVersionResponse
+    from zenml.types import HookSpecification
 
     StepConfigurationUpdateOrDict = Union[
         Dict[str, Any], StepConfigurationUpdate
     ]
-    HookSpecification = Union[str, "Source", FunctionType]
 
 logger = get_logger(__name__)
 
-T = TypeVar("T", bound="Pipeline")
 F = TypeVar("F", bound=Callable[..., None])
 
 
@@ -123,10 +130,12 @@ class Pipeline:
         enable_artifact_visualization: Optional[bool] = None,
         enable_step_logs: Optional[bool] = None,
         settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
+        tags: Optional[List[str]] = None,
         extra: Optional[Dict[str, Any]] = None,
         on_failure: Optional["HookSpecification"] = None,
         on_success: Optional["HookSpecification"] = None,
         model: Optional["Model"] = None,
+        substitutions: Optional[Dict[str, str]] = None,
     ) -> None:
         """Initializes a pipeline.
 
@@ -139,7 +148,8 @@ class Pipeline:
             enable_artifact_visualization: If artifact visualization should be
                 enabled for this pipeline.
             enable_step_logs: If step logs should be enabled for this pipeline.
-            settings: settings for this pipeline.
+            settings: Settings for this pipeline.
+            tags: Tags to apply to runs of this pipeline.
             extra: Extra configurations for this pipeline.
             on_failure: Callback function in event of failure of the step. Can
                 be a function with a single argument of type `BaseException`, or
@@ -148,6 +158,7 @@ class Pipeline:
                 be a function with no arguments, or a source path to such a
                 function (e.g. `module.my_function`).
             model: configuration of the model in the Model Control Plane.
+            substitutions: Extra placeholders to use in the name templates.
         """
         self._invocations: Dict[str, StepInvocation] = {}
         self._run_args: Dict[str, Any] = {}
@@ -163,16 +174,17 @@ class Pipeline:
                 enable_artifact_visualization=enable_artifact_visualization,
                 enable_step_logs=enable_step_logs,
                 settings=settings,
+                tags=tags,
                 extra=extra,
                 on_failure=on_failure,
                 on_success=on_success,
                 model=model,
+                substitutions=substitutions,
             )
         self.entrypoint = entrypoint
         self._parameters: Dict[str, Any] = {}
 
         self.__suppress_warnings_flag__ = False
-        self.__new_unnamed_model_versions_in_current_run__: Dict[str, int] = {}
 
     @property
     def name(self) -> str:
@@ -239,20 +251,6 @@ class Pipeline:
         """
         return inspect.getsource(self.source_object)
 
-    @classmethod
-    def from_model(cls, model: "PipelineResponse") -> "Pipeline":
-        """Creates a pipeline instance from a model.
-
-        Args:
-            model: The model to load the pipeline instance from.
-
-        Returns:
-            The pipeline instance.
-        """
-        from zenml.new.pipelines.deserialization_utils import load_pipeline
-
-        return load_pipeline(model=model)
-
     @property
     def model(self) -> "PipelineResponse":
         """Gets the registered pipeline model for this instance.
@@ -265,21 +263,14 @@ class Pipeline:
         """
         self._prepare_if_possible()
 
-        pipeline_spec = Compiler().compile_spec(self)
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
-        pipelines = Client().list_pipelines(
-            name=self.name, version_hash=version_hash
-        )
+        pipelines = Client().list_pipelines(name=self.name)
         if len(pipelines) == 1:
             return pipelines.items[0]
 
         raise RuntimeError(
             f"Cannot get the model of pipeline '{self.name}' because it has "
             f"not been registered yet. Please ensure that the pipeline has "
-            f"been run and that at least one step has been executed."
+            f"been run or built and try again."
         )
 
     @contextmanager
@@ -296,19 +287,21 @@ class Pipeline:
         self.__suppress_warnings_flag__ = False
 
     def configure(
-        self: T,
+        self,
         enable_cache: Optional[bool] = None,
         enable_artifact_metadata: Optional[bool] = None,
         enable_artifact_visualization: Optional[bool] = None,
         enable_step_logs: Optional[bool] = None,
         settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
+        tags: Optional[List[str]] = None,
         extra: Optional[Dict[str, Any]] = None,
         on_failure: Optional["HookSpecification"] = None,
         on_success: Optional["HookSpecification"] = None,
         model: Optional["Model"] = None,
         parameters: Optional[Dict[str, Any]] = None,
         merge: bool = True,
-    ) -> T:
+        substitutions: Optional[Dict[str, str]] = None,
+    ) -> Self:
         """Configures the pipeline.
 
         Configuration merging example:
@@ -329,6 +322,7 @@ class Pipeline:
                 enabled for this pipeline.
             enable_step_logs: If step logs should be enabled for this pipeline.
             settings: settings for this pipeline.
+            tags: Tags to apply to runs of this pipeline.
             extra: Extra configurations for this pipeline.
             on_failure: Callback function in event of failure of the step. Can
                 be a function with a single argument of type `BaseException`, or
@@ -343,6 +337,7 @@ class Pipeline:
                 method for an example.
             model: configuration of the model version in the Model Control Plane.
             parameters: input parameters for the pipeline.
+            substitutions: Extra placeholders to use in the name templates.
 
         Returns:
             The pipeline instance that this method was called on.
@@ -357,6 +352,11 @@ class Pipeline:
             # string of on_success hook function to be used for this pipeline
             success_hook_source = resolve_and_validate_hook(on_success)
 
+        if merge and tags and self._configuration.tags:
+            # Merge tags explicitly here as the recursive update later only
+            # merges dicts
+            tags = self._configuration.tags + tags
+
         values = dict_utils.remove_none_values(
             {
                 "enable_cache": enable_cache,
@@ -364,18 +364,20 @@ class Pipeline:
                 "enable_artifact_visualization": enable_artifact_visualization,
                 "enable_step_logs": enable_step_logs,
                 "settings": settings,
+                "tags": tags,
                 "extra": extra,
                 "failure_hook_source": failure_hook_source,
                 "success_hook_source": success_hook_source,
                 "model": model,
                 "parameters": parameters,
+                "substitutions": substitutions,
             }
         )
         if not self.__suppress_warnings_flag__:
             to_be_reapplied = []
             for param_, value_ in values.items():
                 if (
-                    param_ in PipelineRunConfiguration.__fields__
+                    param_ in PipelineRunConfiguration.model_fields
                     and param_ in self._from_config_file
                     and value_ != self._from_config_file[param_]
                 ):
@@ -386,16 +388,17 @@ class Pipeline:
                 msg = ""
                 reapply_during_run_warning = (
                     "The value of parameter '{name}' has changed from "
-                    "'{file_value}' to '{new_value}' set in your configuration file.\n"
+                    "'{file_value}' to '{new_value}' set in your configuration "
+                    "file.\n"
                 )
                 for name, file_value, new_value in to_be_reapplied:
                     msg += reapply_during_run_warning.format(
                         name=name, file_value=file_value, new_value=new_value
                     )
                 msg += (
-                    "Configuration file value will be used during pipeline run, "
-                    "so you change will not be efficient. Consider updating your "
-                    "configuration file instead."
+                    "Configuration file value will be used during pipeline "
+                    "run, so you change will not be efficient. Consider "
+                    "updating your configuration file instead."
                 )
                 logger.warning(msg)
 
@@ -498,7 +501,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
         self._prepare_if_possible()
         integration_registry.activate_integrations()
 
-        if self.configuration.dict(exclude_defaults=True, exclude={"name"}):
+        if self.configuration.model_dump(
+            exclude_defaults=True, exclude={"name"}
+        ):
             logger.warning(
                 f"The pipeline `{self.name}` that you're registering has "
                 "custom configurations applied to it. These will not be "
@@ -508,8 +513,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 "pipeline build/run` commands."
             )
 
-        pipeline_spec = Compiler().compile_spec(self)
-        return self._register(pipeline_spec=pipeline_spec)
+        return self._register()
 
     def build(
         self,
@@ -536,12 +540,19 @@ To avoid this consider setting pipeline parameters only in one place (config or
         """
         with track_handler(event=AnalyticsEvent.BUILD_PIPELINE):
             self._prepare_if_possible()
-            deployment, pipeline_spec, _, _ = self._compile(
-                config_path=config_path,
-                steps=step_configurations,
-                settings=settings,
-            )
-            pipeline_id = self._register(pipeline_spec=pipeline_spec).id
+
+            compile_args = self._run_args.copy()
+            compile_args.pop("unlisted", None)
+            compile_args.pop("prevent_build_reuse", None)
+            if config_path:
+                compile_args["config_path"] = config_path
+            if step_configurations:
+                compile_args["step_configurations"] = step_configurations
+            if settings:
+                compile_args["settings"] = settings
+
+            deployment, _, _ = self._compile(**compile_args)
+            pipeline_id = self._register().id
 
             local_repo = code_repository_utils.find_active_code_repository()
             code_repository = build_utils.verify_local_repository_context(
@@ -554,7 +565,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 code_repository=code_repository,
             )
 
-    def _run(
+    def _create_deployment(
         self,
         *,
         run_name: Optional[str] = None,
@@ -572,8 +583,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
         config_path: Optional[str] = None,
         unlisted: bool = False,
         prevent_build_reuse: bool = False,
-    ) -> Optional[PipelineRunResponse]:
-        """Runs the pipeline on the active stack.
+        skip_schedule_registration: bool = False,
+    ) -> PipelineDeploymentResponse:
+        """Create a pipeline deployment.
 
         Args:
             run_name: Name of the pipeline run.
@@ -596,14 +608,163 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 method.
             unlisted: Whether the pipeline run should be unlisted (not assigned
                 to any pipeline).
-            prevent_build_reuse: Whether to prevent the reuse of a build.
+            prevent_build_reuse: DEPRECATED: Use
+                `DockerSettings.prevent_build_reuse` instead.
+            skip_schedule_registration: Whether to skip schedule registration.
+
+        Returns:
+            The pipeline deployment.
 
         Raises:
-            Exception: bypass any exception from pipeline up.
+            ValueError: If the orchestrator doesn't support scheduling, but a
+                schedule was given
+        """
+        deployment, schedule, build = self._compile(
+            config_path=config_path,
+            run_name=run_name,
+            enable_cache=enable_cache,
+            enable_artifact_metadata=enable_artifact_metadata,
+            enable_artifact_visualization=enable_artifact_visualization,
+            enable_step_logs=enable_step_logs,
+            steps=step_configurations,
+            settings=settings,
+            schedule=schedule,
+            build=build,
+            extra=extra,
+        )
+
+        skip_pipeline_registration = constants.handle_bool_env_var(
+            constants.ENV_ZENML_SKIP_PIPELINE_REGISTRATION,
+            default=False,
+        )
+
+        register_pipeline = not (skip_pipeline_registration or unlisted)
+
+        pipeline_id = None
+        if register_pipeline:
+            pipeline_id = self._register().id
+
+        else:
+            logger.debug(f"Pipeline {self.name} is unlisted.")
+
+        stack = Client().active_stack
+        stack.validate()
+
+        schedule_id = None
+        if schedule and not skip_schedule_registration:
+            if not stack.orchestrator.config.is_schedulable:
+                raise ValueError(
+                    f"Stack {stack.name} does not support scheduling. "
+                    "Not all orchestrator types support scheduling, "
+                    "kindly consult with "
+                    "https://docs.zenml.io/how-to/build-pipelines/schedule-a-pipeline "
+                    "for details."
+                )
+            if schedule.name:
+                schedule_name = schedule.name
+            else:
+                schedule_name = format_name_template(
+                    deployment.run_name_template,
+                    substitutions=deployment.pipeline_configuration.substitutions,
+                )
+            components = Client().active_stack_model.components
+            orchestrator = components[StackComponentType.ORCHESTRATOR][0]
+            schedule_model = ScheduleRequest(
+                workspace=Client().active_workspace.id,
+                user=Client().active_user.id,
+                pipeline_id=pipeline_id,
+                orchestrator_id=orchestrator.id,
+                name=schedule_name,
+                active=True,
+                cron_expression=schedule.cron_expression,
+                start_time=schedule.start_time,
+                end_time=schedule.end_time,
+                interval_second=schedule.interval_second,
+                catchup=schedule.catchup,
+                run_once_start_time=schedule.run_once_start_time,
+            )
+            schedule_id = Client().zen_store.create_schedule(schedule_model).id
+            logger.info(
+                f"Created schedule `{schedule_name}` for pipeline "
+                f"`{deployment.pipeline_configuration.name}`."
+            )
+
+        stack = Client().active_stack
+        stack.validate()
+        upload_notebook_cell_code_if_necessary(
+            deployment=deployment, stack=stack
+        )
+
+        local_repo_context = (
+            code_repository_utils.find_active_code_repository()
+        )
+        code_repository = build_utils.verify_local_repository_context(
+            deployment=deployment, local_repo_context=local_repo_context
+        )
+
+        if prevent_build_reuse:
+            logger.warning(
+                "Passing `prevent_build_reuse=True` to "
+                "`pipeline.with_opitions(...)` is deprecated. Use "
+                "`DockerSettings.prevent_build_reuse` instead."
+            )
+
+        build_model = build_utils.reuse_or_create_pipeline_build(
+            deployment=deployment,
+            pipeline_id=pipeline_id,
+            allow_build_reuse=not prevent_build_reuse,
+            build=build,
+            code_repository=code_repository,
+        )
+        build_id = build_model.id if build_model else None
+
+        code_reference = None
+        if local_repo_context and not local_repo_context.is_dirty:
+            source_root = source_utils.get_source_root()
+            subdirectory = (
+                Path(source_root)
+                .resolve()
+                .relative_to(local_repo_context.root)
+            )
+
+            code_reference = CodeReferenceRequest(
+                commit=local_repo_context.current_commit,
+                subdirectory=subdirectory.as_posix(),
+                code_repository=local_repo_context.code_repository_id,
+            )
+
+        code_path = None
+        if build_utils.should_upload_code(
+            deployment=deployment,
+            build=build_model,
+            code_reference=code_reference,
+        ):
+            code_archive = code_utils.CodeArchive(
+                root=source_utils.get_source_root()
+            )
+            logger.info("Archiving pipeline code...")
+            code_path = code_utils.upload_code_if_necessary(code_archive)
+
+        request = PipelineDeploymentRequest(
+            user=Client().active_user.id,
+            workspace=Client().active_workspace.id,
+            stack=stack.id,
+            pipeline=pipeline_id,
+            build=build_id,
+            schedule=schedule_id,
+            code_reference=code_reference,
+            code_path=code_path,
+            **deployment.model_dump(),
+        )
+        return Client().zen_store.create_deployment(deployment=request)
+
+    def _run(
+        self,
+    ) -> Optional[PipelineRunResponse]:
+        """Runs the pipeline on the active stack.
 
         Returns:
-            Model of the pipeline run if running without a schedule, `None` if
-            running with a schedule.
+            The pipeline run or `None` if running with a schedule.
         """
         if constants.SHOULD_PREVENT_PIPELINE_EXECUTION:
             # An environment variable was set to stop the execution of
@@ -621,182 +782,35 @@ To avoid this consider setting pipeline parameters only in one place (config or
         logger.info(f"Initiating a new run for the pipeline: `{self.name}`.")
 
         with track_handler(AnalyticsEvent.RUN_PIPELINE) as analytics_handler:
-            deployment, pipeline_spec, schedule, build = self._compile(
-                config_path=config_path,
-                run_name=run_name,
-                enable_cache=enable_cache,
-                enable_artifact_metadata=enable_artifact_metadata,
-                enable_artifact_visualization=enable_artifact_visualization,
-                enable_step_logs=enable_step_logs,
-                steps=step_configurations,
-                settings=settings,
-                schedule=schedule,
-                build=build,
-                extra=extra,
-            )
-
-            skip_pipeline_registration = constants.handle_bool_env_var(
-                constants.ENV_ZENML_SKIP_PIPELINE_REGISTRATION,
-                default=False,
-            )
-
-            register_pipeline = not (skip_pipeline_registration or unlisted)
-
-            pipeline_id = None
-            if register_pipeline:
-                pipeline_id = self._register(pipeline_spec=pipeline_spec).id
-
-            else:
-                logger.debug(f"Pipeline {self.name} is unlisted.")
-
-            # TODO: check whether orchestrator even support scheduling before
-            #   registering the schedule
-            schedule_id = None
-            if schedule:
-                if schedule.name:
-                    schedule_name = schedule.name
-                else:
-                    date = datetime.utcnow().strftime("%Y_%m_%d")
-                    time = datetime.utcnow().strftime("%H_%M_%S_%f")
-                    schedule_name = deployment.run_name_template.format(
-                        date=date, time=time
-                    )
-                components = Client().active_stack_model.components
-                orchestrator = components[StackComponentType.ORCHESTRATOR][0]
-                schedule_model = ScheduleRequest(
-                    workspace=Client().active_workspace.id,
-                    user=Client().active_user.id,
-                    pipeline_id=pipeline_id,
-                    orchestrator_id=orchestrator.id,
-                    name=schedule_name,
-                    active=True,
-                    cron_expression=schedule.cron_expression,
-                    start_time=schedule.start_time,
-                    end_time=schedule.end_time,
-                    interval_second=schedule.interval_second,
-                    catchup=schedule.catchup,
-                )
-                schedule_id = (
-                    Client().zen_store.create_schedule(schedule_model).id
-                )
-                logger.info(
-                    f"Created schedule `{schedule_name}` for pipeline "
-                    f"`{deployment.pipeline_configuration.name}`."
-                )
-
             stack = Client().active_stack
-            stack.validate()
+            deployment = self._create_deployment(**self._run_args)
 
-            self.prepare_model_versions(deployment)
-
-            local_repo_context = (
-                code_repository_utils.find_active_code_repository()
-            )
-            code_repository = build_utils.verify_local_repository_context(
-                deployment=deployment, local_repo_context=local_repo_context
-            )
-
-            build_model = build_utils.reuse_or_create_pipeline_build(
-                deployment=deployment,
-                pipeline_id=pipeline_id,
-                allow_build_reuse=not prevent_build_reuse,
-                build=build,
-                code_repository=code_repository,
-            )
-            build_id = build_model.id if build_model else None
-
-            code_reference = None
-            if local_repo_context and not local_repo_context.is_dirty:
-                source_root = source_utils.get_source_root()
-                subdirectory = (
-                    Path(source_root)
-                    .resolve()
-                    .relative_to(local_repo_context.root)
-                )
-
-                code_reference = CodeReferenceRequest(
-                    commit=local_repo_context.current_commit,
-                    subdirectory=subdirectory.as_posix(),
-                    code_repository=local_repo_context.code_repository_id,
-                )
-
-            deployment_request = PipelineDeploymentRequest(
-                user=Client().active_user.id,
-                workspace=Client().active_workspace.id,
-                stack=stack.id,
-                pipeline=pipeline_id,
-                build=build_id,
-                schedule=schedule_id,
-                code_reference=code_reference,
-                **deployment.dict(),
-            )
-            deployment_model = Client().zen_store.create_deployment(
-                deployment=deployment_request
-            )
+            self.log_pipeline_deployment_metadata(deployment)
+            run = create_placeholder_run(deployment=deployment)
 
             analytics_handler.metadata = self._get_pipeline_analytics_metadata(
-                deployment=deployment_model, stack=stack
+                deployment=deployment,
+                stack=stack,
+                run_id=run.id if run else None,
             )
-            stack.prepare_pipeline_deployment(deployment=deployment_model)
-
-            self.log_pipeline_deployment_metadata(deployment_model)
-
-            run = None
-            if not schedule:
-                run_request = PipelineRunRequest(
-                    name=get_run_name(
-                        run_name_template=deployment_model.run_name_template
-                    ),
-                    # We set the start time on the placeholder run already to
-                    # make it consistent with the {time} placeholder in the
-                    # run name. This means the placeholder run will usually
-                    # have longer durations than scheduled runs, as for them
-                    # the start_time is only set once the first step starts
-                    # running.
-                    start_time=datetime.utcnow(),
-                    orchestrator_run_id=None,
-                    user=Client().active_user.id,
-                    workspace=deployment_model.workspace.id,
-                    deployment=deployment_model.id,
-                    pipeline=deployment_model.pipeline.id
-                    if deployment_model.pipeline
-                    else None,
-                    status=ExecutionStatus.INITIALIZING,
-                )
-                run = Client().zen_store.create_run(run_request)
-
-            # Prevent execution of nested pipelines which might lead to
-            # unexpected behavior
-            constants.SHOULD_PREVENT_PIPELINE_EXECUTION = True
-            try:
-                stack.deploy_pipeline(deployment=deployment_model)
-            except Exception as e:
-                if (
-                    run
-                    and Client().get_pipeline_run(run.id).status
-                    == ExecutionStatus.INITIALIZING
-                ):
-                    # The run hasn't actually started yet, which means that we
-                    # failed during initialization -> We don't want the
-                    # placeholder run to stay in the database
-                    Client().delete_pipeline_run(run.id)
-
-                raise e
-            finally:
-                constants.SHOULD_PREVENT_PIPELINE_EXECUTION = False
 
             if run:
                 run_url = dashboard_utils.get_run_url(run)
                 if run_url:
-                    logger.info(f"Dashboard URL: {run_url}")
+                    logger.info(f"Dashboard URL for Pipeline Run: {run_url}")
                 else:
                     logger.info(
                         "You can visualize your pipeline runs in the `ZenML "
                         "Dashboard`. In order to try it locally, please run "
-                        "`zenml up`."
+                        "`zenml login --local`."
                     )
 
-            return run
+            deploy_pipeline(
+                deployment=deployment, stack=stack, placeholder_run=run
+            )
+            if run:
+                return Client().get_pipeline_run(run.id)
+            return None
 
     @staticmethod
     def log_pipeline_deployment_metadata(
@@ -808,15 +822,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
             deployment_model: The model for the pipeline deployment
         """
         try:
-            # Log about the schedule/run
-            if deployment_model.schedule:
-                logger.info(
-                    "Scheduling a run with the schedule: "
-                    f"`{deployment_model.schedule.name}`"
-                )
-            else:
-                logger.info("Executing a new run.")
-
             # Log about the caching status
             if deployment_model.pipeline_configuration.enable_cache is False:
                 logger.info(
@@ -869,123 +874,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
         except Exception as e:
             logger.debug(f"Logging pipeline deployment metadata failed: {e}")
 
-    def _update_new_requesters(
-        self,
-        requester_name: str,
-        model: "Model",
-        new_versions_requested: Dict[
-            Tuple[str, Optional[str]], NewModelRequest
-        ],
-        other_models: Set["Model"],
-    ) -> None:
-        key = (
-            model.name,
-            str(model.version) if model.version else None,
-        )
-        if model.version is None:
-            version_existed = False
-        else:
-            try:
-                model._get_model_version()
-                version_existed = key not in new_versions_requested
-            except KeyError:
-                version_existed = False
-        if not version_existed:
-            model.was_created_in_this_run = True
-            new_versions_requested[key].update_request(
-                model,
-                NewModelRequest.Requester(source="step", name=requester_name),
-            )
-        else:
-            other_models.add(model)
-
-    def prepare_model_versions(
-        self, deployment: "PipelineDeploymentBase"
-    ) -> None:
-        """Create model versions which are missing and validate existing ones that are used in the pipeline run.
-
-        Args:
-            deployment: The pipeline deployment configuration.
-        """
-        new_versions_requested: Dict[
-            Tuple[str, Optional[str]], NewModelRequest
-        ] = defaultdict(NewModelRequest)
-        other_models: Set["Model"] = set()
-        all_steps_have_own_config = True
-        for step in deployment.step_configurations.values():
-            step_model = step.config.model
-            all_steps_have_own_config = (
-                all_steps_have_own_config and step.config.model is not None
-            )
-            if step_model:
-                self._update_new_requesters(
-                    model=step_model,
-                    requester_name=step.config.name,
-                    new_versions_requested=new_versions_requested,
-                    other_models=other_models,
-                )
-        if not all_steps_have_own_config:
-            pipeline_model = deployment.pipeline_configuration.model
-            if pipeline_model:
-                self._update_new_requesters(
-                    model=pipeline_model,
-                    requester_name=self.name,
-                    new_versions_requested=new_versions_requested,
-                    other_models=other_models,
-                )
-        elif deployment.pipeline_configuration.model is not None:
-            logger.warning(
-                f"ModelConfig of pipeline `{self.name}` is overridden in all "
-                f"steps. "
-            )
-
-        self._validate_new_version_requests(new_versions_requested)
-
-        for other_model in other_models:
-            other_model._validate_config_in_runtime()
-
-    def _validate_new_version_requests(
-        self,
-        new_versions_requested: Dict[
-            Tuple[str, Optional[str]], NewModelRequest
-        ],
-    ) -> None:
-        """Validate the model version that are used in the pipeline run.
-
-        Args:
-            new_versions_requested: A dict of new model version request objects.
-
-        """
-        for key, data in new_versions_requested.items():
-            model_name, model_version = key
-            if len(data.requesters) > 1:
-                logger.warning(
-                    f"New version of model version `{model_name}::{model_version or 'NEW'}` "
-                    f"requested in multiple decorators:\n{data.requesters}\n We recommend "
-                    "that `Model` requesting new version is configured only in one "
-                    "place of the pipeline."
-                )
-            data.model._validate_config_in_runtime()
-            self.__new_unnamed_model_versions_in_current_run__[
-                data.model.name
-            ] = data.model.number
-
-    def get_runs(self, **kwargs: Any) -> List["PipelineRunResponse"]:
-        """(Deprecated) Get runs of this pipeline.
-
-        Args:
-            **kwargs: Further arguments for filtering or pagination that are
-                passed to `client.list_pipeline_runs()`.
-
-        Returns:
-            List of runs of this pipeline.
-        """
-        logger.warning(
-            "`Pipeline.get_runs()` is deprecated and will be removed in a "
-            "future version. Please use `Pipeline.model.get_runs()` instead."
-        )
-        return self.model.get_runs(**kwargs)
-
     def write_run_configuration_template(
         self, path: str, stack: Optional["Stack"] = None
     ) -> None:
@@ -1020,19 +908,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
         steps = {}
         for step_name, invocation in self.invocations.items():
             step = invocation.step
-            parameters = (
-                pydantic_utils.TemplateGenerator(
-                    step.entrypoint_definition.legacy_params.annotation
-                ).run()
-                if step.entrypoint_definition.legacy_params
-                else {}
-            )
             outputs = {
                 name: PartialArtifactConfiguration()
                 for name in step.entrypoint_definition.outputs
             }
             step_template = StepConfigurationUpdate(
-                parameters=parameters,
+                parameters={},
                 settings=step_settings,
                 outputs=outputs,
             )
@@ -1081,12 +962,14 @@ To avoid this consider setting pipeline parameters only in one place (config or
         self,
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
+        run_id: Optional[UUID] = None,
     ) -> Dict[str, Any]:
         """Returns the pipeline deployment metadata.
 
         Args:
             deployment: The pipeline deployment to track.
             stack: The stack on which the pipeline will be deployed.
+            run_id: The ID of the pipeline run.
 
         Returns:
             the metadata about the pipeline deployment
@@ -1113,13 +996,13 @@ To avoid this consider setting pipeline parameters only in one place (config or
             "schedule": bool(deployment.schedule),
             "custom_materializer": custom_materializer,
             "own_stack": own_stack,
+            "pipeline_run_id": str(run_id) if run_id else None,
         }
 
     def _compile(
         self, config_path: Optional[str] = None, **run_configuration_args: Any
     ) -> Tuple[
         "PipelineDeploymentBase",
-        "PipelineSpec",
         Optional["Schedule"],
         Union["PipelineBuildBase", UUID, None],
     ]:
@@ -1130,7 +1013,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             **run_configuration_args: Configurations for the pipeline run.
 
         Returns:
-            A tuple containing the deployment, spec, schedule and build of
+            A tuple containing the deployment, schedule and build of
             the compiled pipeline.
         """
         # Activating the built-in integrations to load all materializers
@@ -1138,76 +1021,71 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         integration_registry.activate_integrations()
 
-        self._parse_config_file(
+        _from_config_file = self._parse_config_file(
             config_path=config_path,
-            matcher=list(PipelineRunConfiguration.__fields__.keys()),
+            matcher=list(PipelineRunConfiguration.model_fields.keys()),
         )
 
-        run_config = PipelineRunConfiguration(**self._from_config_file)
+        self._reconfigure_from_file_with_overrides(config_path=config_path)
+
+        run_config = PipelineRunConfiguration(**_from_config_file)
 
         new_values = dict_utils.remove_none_values(run_configuration_args)
-        update = PipelineRunConfiguration.parse_obj(new_values)
+        update = PipelineRunConfiguration.model_validate(new_values)
 
         # Update with the values in code so they take precedence
         run_config = pydantic_utils.update_model(run_config, update=update)
+        run_config = env_utils.substitute_env_variable_placeholders(run_config)
 
-        deployment, pipeline_spec = Compiler().compile(
+        deployment = Compiler().compile(
             pipeline=self,
             stack=Client().active_stack,
             run_configuration=run_config,
         )
+        deployment = env_utils.substitute_env_variable_placeholders(deployment)
 
-        return deployment, pipeline_spec, run_config.schedule, run_config.build
+        return deployment, run_config.schedule, run_config.build
 
-    def _register(self, pipeline_spec: "PipelineSpec") -> "PipelineResponse":
+    def _register(self) -> "PipelineResponse":
         """Register the pipeline in the server.
 
-        Args:
-            pipeline_spec: The pipeline spec to register.
-
         Returns:
             The registered pipeline model.
         """
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
         client = Client()
-        matching_pipelines = client.list_pipelines(
-            name=self.name,
-            version_hash=version_hash,
-            size=1,
-            sort_by="desc:created",
-        )
-        if matching_pipelines.total:
-            registered_pipeline = matching_pipelines.items[0]
-            logger.info(
-                "Reusing registered pipeline version: `(version: %s)`.",
-                registered_pipeline.version,
+
+        def _get() -> PipelineResponse:
+            matching_pipelines = client.list_pipelines(
+                name=self.name,
+                size=1,
+                sort_by="desc:created",
             )
-            return registered_pipeline
-
-        latest_version = self._get_latest_version() or 0
-        version = str(latest_version + 1)
-
-        request = PipelineRequest(
-            workspace=client.active_workspace.id,
-            user=client.active_user.id,
-            name=self.name,
-            version=version,
-            version_hash=version_hash,
-            spec=pipeline_spec,
-            docstring=self.__doc__,
-        )
 
-        registered_pipeline = client.zen_store.create_pipeline(
-            pipeline=request
-        )
-        logger.info(
-            "Registered new version: `(version %s)`.",
-            registered_pipeline.version,
-        )
-        return registered_pipeline
+            if matching_pipelines.total:
+                registered_pipeline = matching_pipelines.items[0]
+                return registered_pipeline
+            raise RuntimeError("No matching pipelines found.")
+
+        try:
+            return _get()
+        except RuntimeError:
+            request = PipelineRequest(
+                workspace=client.active_workspace.id,
+                user=client.active_user.id,
+                name=self.name,
+            )
+
+            try:
+                registered_pipeline = client.zen_store.create_pipeline(
+                    pipeline=request
+                )
+                logger.info(
+                    "Registered new pipeline: `%s`.",
+                    registered_pipeline.name,
+                )
+                return registered_pipeline
+            except EntityExistsError:
+                return _get()
 
     def _compute_unique_identifier(self, pipeline_spec: PipelineSpec) -> str:
         """Computes a unique identifier from the pipeline spec and steps.
@@ -1234,28 +1112,15 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         return hash_.hexdigest()
 
-    def _get_latest_version(self) -> Optional[int]:
-        """Gets the latest version of this pipeline.
-
-        Returns:
-            The latest version or `None` if no version exists.
-        """
-        all_pipelines = Client().list_pipelines(
-            name=self.name, sort_by="desc:created", size=1
-        )
-        if not all_pipelines.total:
-            return None
-        pipeline = all_pipelines.items[0]
-        if pipeline.version == "UNVERSIONED":
-            return None
-        return int(all_pipelines.items[0].version)
-
     def add_step_invocation(
         self,
         step: "BaseStep",
         input_artifacts: Dict[str, StepArtifact],
-        external_artifacts: Dict[str, "ExternalArtifact"],
+        external_artifacts: Dict[
+            str, Union["ExternalArtifact", "ArtifactVersionResponse"]
+        ],
         model_artifacts_or_metadata: Dict[str, "ModelVersionDataLazyLoader"],
+        client_lazy_loaders: Dict[str, "ClientLazyLoader"],
         parameters: Dict[str, Any],
         default_parameters: Dict[str, Any],
         upstream_steps: Set[str],
@@ -1270,6 +1135,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             external_artifacts: The external artifacts for the invocation.
             model_artifacts_or_metadata: The model artifacts or metadata for
                 the invocation.
+            client_lazy_loaders: The client lazy loaders for the invocation.
             parameters: The parameters for the invocation.
             default_parameters: The default parameters for the invocation.
             upstream_steps: The upstream steps for the invocation.
@@ -1307,6 +1173,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             input_artifacts=input_artifacts,
             external_artifacts=external_artifacts,
             model_artifacts_or_metadata=model_artifacts_or_metadata,
+            client_lazy_loaders=client_lazy_loaders,
             parameters=parameters,
             default_parameters=default_parameters,
             upstream_steps=upstream_steps,
@@ -1352,7 +1219,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         raise RuntimeError("Unable to find step ID")
 
-    def __enter__(self: T) -> T:
+    def __enter__(self) -> Self:
         """Activate the pipeline context.
 
         Raises:
@@ -1380,17 +1247,21 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
     def _parse_config_file(
         self, config_path: Optional[str], matcher: List[str]
-    ) -> None:
+    ) -> Dict[str, Any]:
         """Parses the given configuration file and sets `self._from_config_file`.
 
         Args:
             config_path: Path to a yaml configuration file.
             matcher: List of keys to match in the configuration file.
+
+        Returns:
+            Parsed config file according to matcher settings.
         """
         _from_config_file: Dict[str, Any] = {}
         if config_path:
             with open(config_path, "r") as f:
                 _from_config_file = yaml.load(f, Loader=yaml.SafeLoader)
+
             _from_config_file = dict_utils.remove_none_values(
                 {k: v for k, v in _from_config_file.items() if k in matcher}
             )
@@ -1403,10 +1274,10 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 else:
                     from zenml.model.model import Model
 
-                    _from_config_file["model"] = Model.parse_obj(
+                    _from_config_file["model"] = Model.model_validate(
                         _from_config_file["model"]
                     )
-        self._from_config_file = _from_config_file
+        return _from_config_file
 
     def with_options(
         self,
@@ -1416,6 +1287,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
         step_configurations: Optional[
             Mapping[str, "StepConfigurationUpdateOrDict"]
         ] = None,
+        steps: Optional[Mapping[str, "StepConfigurationUpdateOrDict"]] = None,
         config_path: Optional[str] = None,
         unlisted: bool = False,
         prevent_build_reuse: bool = False,
@@ -1428,6 +1300,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
             schedule: Optional schedule to use for the run.
             build: Optional build to use for the run.
             step_configurations: Configurations for steps of the pipeline.
+            steps: Configurations for steps of the pipeline. This is equivalent
+                to `step_configurations`, and will be ignored if
+                `step_configurations` is set as well.
             config_path: Path to a yaml configuration file. This file will
                 be parsed as a
                 `zenml.config.pipeline_configurations.PipelineRunConfiguration`
@@ -1436,32 +1311,33 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 method.
             unlisted: Whether the pipeline run should be unlisted (not assigned
                 to any pipeline).
-            prevent_build_reuse: Whether to prevent the reuse of a build.
+            prevent_build_reuse: DEPRECATED: Use
+                `DockerSettings.prevent_build_reuse` instead.
             **kwargs: Pipeline configuration options. These will be passed
                 to the `pipeline.configure(...)` method.
 
         Returns:
             The copied pipeline instance.
         """
+        if steps and step_configurations:
+            logger.warning(
+                "Step configurations were passed using both the "
+                "`step_configurations` and `steps` keywords, ignoring the "
+                "values passed using the `steps` keyword."
+            )
+
         pipeline_copy = self.copy()
 
-        pipeline_copy._parse_config_file(
-            config_path=config_path,
-            matcher=inspect.getfullargspec(self.configure)[0],
-        )
-        pipeline_copy._from_config_file = dict_utils.recursive_update(
-            pipeline_copy._from_config_file, kwargs
+        pipeline_copy._reconfigure_from_file_with_overrides(
+            config_path=config_path, **kwargs
         )
 
-        with pipeline_copy.__suppress_configure_warnings__():
-            pipeline_copy.configure(**pipeline_copy._from_config_file)
-
         run_args = dict_utils.remove_none_values(
             {
                 "run_name": run_name,
                 "schedule": schedule,
                 "build": build,
-                "step_configurations": step_configurations,
+                "step_configurations": step_configurations or steps,
                 "config_path": config_path,
                 "unlisted": unlisted,
                 "prevent_build_reuse": prevent_build_reuse,
@@ -1478,7 +1354,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
         """
         return copy.deepcopy(self)
 
-    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+    def __call__(
+        self, *args: Any, **kwargs: Any
+    ) -> Optional[PipelineRunResponse]:
         """Handle a call of the pipeline.
 
         This method does one of two things:
@@ -1493,7 +1371,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
             **kwargs: Entrypoint function keyword arguments.
 
         Returns:
-            The outputs of the entrypoint function call.
+            If called within another pipeline, returns the outputs of the
+            `entrypoint` method. Otherwise, returns the pipeline run or `None`
+            if running with a schedule.
         """
         if Pipeline.ACTIVE_PIPELINE:
             # Calling a pipeline inside a pipeline, we return the potential
@@ -1505,7 +1385,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             return self.entrypoint(*args, **kwargs)
 
         self.prepare(*args, **kwargs)
-        return self._run(**self._run_args)
+        return self._run()
 
     def _call_entrypoint(self, *args: Any, **kwargs: Any) -> None:
         """Calls the pipeline entrypoint function with the given arguments.
@@ -1521,14 +1401,15 @@ To avoid this consider setting pipeline parameters only in one place (config or
         try:
             validated_args = pydantic_utils.validate_function_args(
                 self.entrypoint,
-                {"arbitrary_types_allowed": False, "smart_union": True},
+                ConfigDict(arbitrary_types_allowed=False),
                 *args,
                 **kwargs,
             )
         except ValidationError as e:
             raise ValueError(
-                "Invalid or missing inputs for pipeline entrypoint function. "
+                "Invalid or missing pipeline function entrypoint arguments. "
                 "Only JSON serializable inputs are allowed as pipeline inputs."
+                "Check out the pydantic error above for more details."
             ) from e
 
         self._parameters = validated_args
@@ -1551,3 +1432,58 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 )
             else:
                 self.prepare()
+
+    def create_run_template(
+        self, name: str, **kwargs: Any
+    ) -> RunTemplateResponse:
+        """Create a run template for the pipeline.
+
+        Args:
+            name: The name of the run template.
+            **kwargs: Keyword arguments for the client method to create a run
+                template.
+
+        Returns:
+            The created run template.
+        """
+        self._prepare_if_possible()
+        deployment = self._create_deployment(
+            **self._run_args, skip_schedule_registration=True
+        )
+
+        return Client().create_run_template(
+            name=name, deployment_id=deployment.id, **kwargs
+        )
+
+    def _reconfigure_from_file_with_overrides(
+        self,
+        config_path: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Update the pipeline configuration from config file.
+
+        Accepts overrides as kwargs.
+
+        Args:
+            config_path: Path to a yaml configuration file. This file will
+                be parsed as a
+                `zenml.config.pipeline_configurations.PipelineRunConfiguration`
+                object. Options provided in this file will be overwritten by
+                options provided in code using the other arguments of this
+                method.
+            **kwargs: Pipeline configuration options. These will be passed
+                to the `pipeline.configure(...)` method.
+        """
+        self._from_config_file = {}
+        if config_path:
+            self._from_config_file = self._parse_config_file(
+                config_path=config_path,
+                matcher=inspect.getfullargspec(self.configure)[0],
+            )
+
+        _from_config_file = dict_utils.recursive_update(
+            self._from_config_file, kwargs
+        )
+
+        with self.__suppress_configure_warnings__():
+            self.configure(**_from_config_file)