zenml-nightly 0.82.0.dev20250508__tar.gz → 0.82.0.dev20250510__tar.gz

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.82.0.dev20250508
+Version: 0.82.0.dev20250510
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zenml-nightly"
-version = "0.82.0.dev20250508"
+version = "0.82.0.dev20250510"
 packages = [{ include = "zenml", from = "src" }]
 description = "ZenML: Write production-ready ML code."
 authors = ["ZenML GmbH <info@zenml.io>"]

zenml_nightly-0.82.0.dev20250510/src/zenml/VERSION

@@ -0,0 +1 @@
+0.82.0.dev20250510

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/config/compiler.py

@@ -27,6 +27,7 @@ from typing import (
 
 from zenml import __version__
 from zenml.config.base_settings import BaseSettings, ConfigurationLevel
+from zenml.config.pipeline_configurations import PipelineConfiguration
 from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.config.pipeline_spec import PipelineSpec
 from zenml.config.settings_resolver import SettingsResolver
@@ -43,7 +44,6 @@ from zenml.pipelines.run_utils import get_default_run_name
 from zenml.utils import pydantic_utils, settings_utils
 
 if TYPE_CHECKING:
-    from zenml.config.source import Source
     from zenml.pipelines.pipeline_definition import Pipeline
     from zenml.stack import Stack, StackComponent
     from zenml.steps.step_invocation import StepInvocation
@@ -112,21 +112,12 @@ class Compiler:
         with pipeline.__suppress_configure_warnings__():
             pipeline.configure(settings=pipeline_settings, merge=False)
 
-        settings_to_passdown = {
-            key: settings
-            for key, settings in pipeline_settings.items()
-            if ConfigurationLevel.STEP in settings.LEVEL
-        }
-
         steps = {
             invocation_id: self._compile_step_invocation(
                 invocation=invocation,
-                pipeline_settings=settings_to_passdown,
-                pipeline_extra=pipeline.configuration.extra,
                 stack=stack,
                 step_config=run_configuration.steps.get(invocation_id),
-                pipeline_failure_hook_source=pipeline.configuration.failure_hook_source,
-                pipeline_success_hook_source=pipeline.configuration.success_hook_source,
+                pipeline_configuration=pipeline.configuration,
             )
             for invocation_id, invocation in self._get_sorted_invocations(
                 pipeline=pipeline
@@ -447,24 +438,17 @@ class Compiler:
     def _compile_step_invocation(
         self,
         invocation: "StepInvocation",
-        pipeline_settings: Dict[str, "BaseSettings"],
-        pipeline_extra: Dict[str, Any],
         stack: "Stack",
         step_config: Optional["StepConfigurationUpdate"],
-        pipeline_failure_hook_source: Optional["Source"] = None,
-        pipeline_success_hook_source: Optional["Source"] = None,
+        pipeline_configuration: "PipelineConfiguration",
     ) -> Step:
         """Compiles a ZenML step.
 
         Args:
             invocation: The step invocation to compile.
-            pipeline_settings: settings configured on the
-                pipeline of the step.
-            pipeline_extra: Extra values configured on the pipeline of the step.
             stack: The stack on which the pipeline will be run.
             step_config: Run configuration for the step.
-            pipeline_failure_hook_source: Source for the failure hook.
-            pipeline_success_hook_source: Source for the success hook.
+            pipeline_configuration: Configuration for the pipeline.
 
         Returns:
             The compiled step.
@@ -488,32 +472,27 @@ class Compiler:
             configuration_level=ConfigurationLevel.STEP,
             stack=stack,
         )
-        step_extra = step.configuration.extra
-        step_on_failure_hook_source = step.configuration.failure_hook_source
-        step_on_success_hook_source = step.configuration.success_hook_source
-
-        step.configure(
-            settings=pipeline_settings,
-            extra=pipeline_extra,
-            on_failure=pipeline_failure_hook_source,
-            on_success=pipeline_success_hook_source,
-            merge=False,
-        )
         step.configure(
             settings=step_settings,
-            extra=step_extra,
-            on_failure=step_on_failure_hook_source,
-            on_success=step_on_success_hook_source,
-            merge=True,
+            merge=False,
         )
 
         parameters_to_ignore = (
             set(step_config.parameters) if step_config else set()
         )
-        complete_step_configuration = invocation.finalize(
+        step_configuration_overrides = invocation.finalize(
             parameters_to_ignore=parameters_to_ignore
         )
-        return Step(spec=step_spec, config=complete_step_configuration)
+        full_step_config = (
+            step_configuration_overrides.apply_pipeline_configuration(
+                pipeline_configuration=pipeline_configuration
+            )
+        )
+        return Step(
+            spec=step_spec,
+            config=full_step_config,
+            step_config_overrides=step_configuration_overrides,
+        )
 
     def _get_sorted_invocations(
         self,

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/config/pipeline_configurations.py

@@ -52,23 +52,30 @@ class PipelineConfigurationUpdate(StrictBaseModel):
     retry: Optional[StepRetryConfig] = None
     substitutions: Dict[str, str] = {}
 
-    def _get_full_substitutions(
-        self, start_time: Optional[datetime]
+    def finalize_substitutions(
+        self, start_time: Optional[datetime] = None, inplace: bool = False
     ) -> Dict[str, str]:
         """Returns the full substitutions dict.
 
         Args:
             start_time: Start time of the pipeline run.
+            inplace: Whether to update the substitutions in place.
 
         Returns:
             The full substitutions dict including date and time.
         """
         if start_time is None:
             start_time = utc_now()
-        ret = self.substitutions.copy()
-        ret.setdefault("date", start_time.strftime("%Y_%m_%d"))
-        ret.setdefault("time", start_time.strftime("%H_%M_%S_%f"))
-        return ret
+
+        if inplace:
+            dict_ = self.substitutions
+        else:
+            dict_ = self.substitutions.copy()
+
+        dict_.setdefault("date", start_time.strftime("%Y_%m_%d"))
+        dict_.setdefault("time", start_time.strftime("%H_%M_%S_%f"))
+
+        return dict_
 
 
 class PipelineConfiguration(PipelineConfigurationUpdate):

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/config/step_configurations.py

@@ -13,7 +13,6 @@
 #  permissions and limitations under the License.
 """Pipeline configuration classes."""
 
-from datetime import datetime
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -46,7 +45,7 @@ from zenml.logger import get_logger
 from zenml.model.lazy_load import ModelVersionDataLazyLoader
 from zenml.model.model import Model
 from zenml.utils import deprecation_utils
-from zenml.utils.pydantic_utils import before_validator_handler
+from zenml.utils.pydantic_utils import before_validator_handler, update_model
 
 if TYPE_CHECKING:
     from zenml.config import DockerSettings, ResourceSettings
@@ -241,23 +240,43 @@ class StepConfiguration(PartialStepConfiguration):
             model_or_dict = model_or_dict.model_dump()
         return DockerSettings.model_validate(model_or_dict)
 
-    def _get_full_substitutions(
-        self,
-        pipeline_config: "PipelineConfiguration",
-        start_time: Optional[datetime],
-    ) -> Dict[str, str]:
-        """Get the full set of substitutions for this step configuration.
+    def apply_pipeline_configuration(
+        self, pipeline_configuration: "PipelineConfiguration"
+    ) -> "StepConfiguration":
+        """Apply the pipeline configuration to this step configuration.
 
         Args:
-            pipeline_config: The pipeline configuration.
-            start_time: The start time of the pipeline run.
+            pipeline_configuration: The pipeline configuration to apply.
 
         Returns:
-            The full set of substitutions for this step configuration.
+            The updated step configuration.
         """
-        ret = pipeline_config._get_full_substitutions(start_time)
-        ret.update(self.substitutions)
-        return ret
+        pipeline_values = pipeline_configuration.model_dump(
+            include={
+                "settings",
+                "extra",
+                "failure_hook_source",
+                "success_hook_source",
+                "substitutions",
+            },
+            exclude_none=True,
+        )
+        if pipeline_values:
+            original_values = self.model_dump(
+                include={
+                    "settings",
+                    "extra",
+                    "failure_hook_source",
+                    "success_hook_source",
+                    "substitutions",
+                },
+                exclude_none=True,
+            )
+
+            updated_config = self.model_copy(update=pipeline_values, deep=True)
+            return update_model(updated_config, original_values)
+        else:
+            return self.model_copy(deep=True)
 
 
 class InputSpec(StrictBaseModel):
@@ -311,3 +330,59 @@ class Step(StrictBaseModel):
 
     spec: StepSpec
     config: StepConfiguration
+    step_config_overrides: StepConfiguration
+
+    @model_validator(mode="before")
+    @classmethod
+    @before_validator_handler
+    def _add_step_config_overrides_if_missing(cls, data: Any) -> Any:
+        """Add step config overrides if missing.
+
+        This is to ensure backwards compatibility with data stored in the DB
+        before the `step_config_overrides` field was added. In that case, only
+        the `config` field, which contains the merged pipeline and step configs,
+        existed. We have no way to figure out which of those values were defined
+        on the step vs the pipeline level, so we just use the entire `config`
+        object as the `step_config_overrides`.
+
+        Args:
+            data: The values dict used to instantiate the model.
+
+        Returns:
+            The values dict with the step config overrides added if missing.
+        """
+        if "step_config_overrides" not in data:
+            data["step_config_overrides"] = data["config"]
+
+        return data
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: Dict[str, Any],
+        pipeline_configuration: "PipelineConfiguration",
+    ) -> "Step":
+        """Create a step from a dictionary.
+
+        This method can create a step from data stored without the merged
+        `config` attribute, by merging the `step_config_overrides` with the
+        pipeline configuration.
+
+        Args:
+            data: The dictionary to create the `Step` object from.
+            pipeline_configuration: The pipeline configuration to apply to the
+                step configuration.
+
+        Returns:
+            The instantiated object.
+        """
+        if "config" not in data:
+            config = StepConfiguration.model_validate(
+                data["step_config_overrides"]
+            )
+            config = config.apply_pipeline_configuration(
+                pipeline_configuration
+            )
+            data["config"] = config
+
+        return cls.model_validate(data)

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/constants.py

@@ -199,6 +199,7 @@ ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK = (
 )
 ENV_ZENML_PREVENT_CLIENT_SIDE_CACHING = "ZENML_PREVENT_CLIENT_SIDE_CACHING"
 ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING = "DISABLE_CREDENTIALS_DISK_CACHING"
+ENV_ZENML_RUNNER_PARENT_IMAGE = "ZENML_RUNNER_PARENT_IMAGE"
 ENV_ZENML_RUNNER_IMAGE_DISABLE_UV = "ZENML_RUNNER_IMAGE_DISABLE_UV"
 ENV_ZENML_RUNNER_POD_TIMEOUT = "ZENML_RUNNER_POD_TIMEOUT"
 ENV_ZENML_WORKLOAD_TOKEN_EXPIRATION_LEEWAY = (

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/models/v2/base/scoped.py

@@ -318,6 +318,8 @@ class ProjectScopedResponse(
     Used as a base class for all domain models that are project-scoped.
     """
 
+    project_id: UUID = Field(title="The project id.")
+
     # Analytics
     def get_analytics_metadata(self) -> Dict[str, Any]:
         """Fetches the analytics metadata for project scoped models.

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/models/v2/core/artifact_version.py

@@ -710,6 +710,7 @@ class LazyArtifactVersionResponse(ArtifactVersionResponse):
     """
 
     id: Optional[UUID] = None  # type: ignore[assignment]
+    project_id: Optional[UUID] = None  # type: ignore[assignment]
     lazy_load_name: Optional[str] = None
     lazy_load_version: Optional[str] = None
     lazy_load_model_name: str

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/orchestrators/step_launcher.py

@@ -308,8 +308,8 @@ class StepLauncher:
         start_time = utc_now()
         run_name = string_utils.format_name_template(
             name_template=self._deployment.run_name_template,
-            substitutions=self._deployment.pipeline_configuration._get_full_substitutions(
-                start_time
+            substitutions=self._deployment.pipeline_configuration.finalize_substitutions(
+                start_time=start_time,
             ),
         )
 

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/pipelines/run_utils.py

@@ -72,8 +72,8 @@ def create_placeholder_run(
     run_request = PipelineRunRequest(
         name=string_utils.format_name_template(
             name_template=deployment.run_name_template,
-            substitutions=deployment.pipeline_configuration._get_full_substitutions(
-                start_time
+            substitutions=deployment.pipeline_configuration.finalize_substitutions(
+                start_time=start_time,
             ),
         ),
         # We set the start time on the placeholder run already to
@@ -113,6 +113,7 @@ def get_placeholder_run(
         size=1,
         deployment_id=deployment_id,
         status=ExecutionStatus.INITIALIZING,
+        hydrate=True,
     )
     if len(runs.items) == 0:
         return None

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/stack/stack_component.py

@@ -529,10 +529,13 @@ class StackComponent:
             else container.pipeline_configuration.settings
         )
 
+        # Use the current config as a base
+        settings_dict = self.config.model_dump()
+
         if key in all_settings:
-            return self.settings_class.model_validate(dict(all_settings[key]))
-        else:
-            return self.settings_class()
+            settings_dict.update(dict(all_settings[key]))
+
+        return self.settings_class.model_validate(settings_dict)
 
     def connector_has_expired(self) -> bool:
         """Checks whether the connector linked to this stack component has expired.

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/zen_server/rbac/utils.py

@@ -373,8 +373,7 @@ def get_resource_for_model(model: AnyModel) -> Optional[Resource]:
 
     project_id: Optional[UUID] = None
     if isinstance(model, ProjectScopedResponse):
-        # A project scoped response is always scoped to a specific project
-        project_id = model.project.id
+        project_id = model.project_id
     elif isinstance(model, ProjectScopedRequest):
         # A project scoped request is always scoped to a specific project
         project_id = model.project

{zenml_nightly-0.82.0.dev20250508 → zenml_nightly-0.82.0.dev20250510}/src/zenml/zen_server/routers/pipeline_deployments_endpoints.py

@@ -13,18 +13,16 @@
 #  permissions and limitations under the License.
 """Endpoint definitions for deployments."""
 
-from typing import Optional, Union
+from typing import Any, Optional, Union
 from uuid import UUID
 
-from fastapi import APIRouter, Depends, Security
+from fastapi import APIRouter, Depends, Request, Security
 
 from zenml.constants import API, PIPELINE_DEPLOYMENTS, VERSION_1
 from zenml.logging.step_logging import fetch_logs
 from zenml.models import (
-    Page,
     PipelineDeploymentFilter,
     PipelineDeploymentRequest,
-    PipelineDeploymentResponse,
     PipelineRunFilter,
 )
 from zenml.zen_server.auth import AuthContext, authorize
@@ -45,6 +43,37 @@ from zenml.zen_server.utils import (
     zen_store,
 )
 
+
+# TODO: Remove this as soon as there is only a low number of users running
+# versions < 0.82.0. Also change back the return type to
+# `PipelineDeploymentResponse` once we have removed the `exclude` logic.
+def _should_remove_step_config_overrides(
+    request: Request,
+) -> bool:
+    """Check if the step config overrides should be removed from the response.
+
+    Args:
+        request: The request object.
+
+    Returns:
+        If the step config overrides should be removed from the response.
+    """
+    from packaging import version
+
+    user_agent = request.headers.get("User-Agent", "")
+
+    if not user_agent.startswith("zenml/"):
+        # This request is not coming from a ZenML client
+        return False
+
+    client_version = version.parse(user_agent.removeprefix("zenml/"))
+
+    # Versions before 0.82.0 did have `extra="forbid"` in the pydantic model
+    # that stores the step configurations. This means it would crash if we
+    # included the `step_config_overrides` in the response.
+    return client_version < version.parse("0.82.0")
+
+
 router = APIRouter(
     prefix=API + VERSION_1 + PIPELINE_DEPLOYMENTS,
     tags=["deployments"],
@@ -66,13 +95,15 @@ router = APIRouter(
 )
 @handle_exceptions
 def create_deployment(
+    request: Request,
     deployment: PipelineDeploymentRequest,
     project_name_or_id: Optional[Union[str, UUID]] = None,
     _: AuthContext = Security(authorize),
-) -> PipelineDeploymentResponse:
+) -> Any:
     """Creates a deployment.
 
     Args:
+        request: The request object.
         deployment: Deployment to create.
         project_name_or_id: Optional name or ID of the project.
 
@@ -83,11 +114,21 @@ def create_deployment(
         project = zen_store().get_project(project_name_or_id)
         deployment.project = project.id
 
-    return verify_permissions_and_create_entity(
+    deployment_response = verify_permissions_and_create_entity(
         request_model=deployment,
         create_method=zen_store().create_deployment,
     )
 
+    exclude = None
+    if _should_remove_step_config_overrides(request):
+        exclude = {
+            "metadata": {
+                "step_configurations": {"__all__": {"step_config_overrides"}}
+            }
+        }
+
+    return deployment_response.model_dump(mode="json", exclude=exclude)
+
 
 @router.get(
     "",
@@ -103,16 +144,18 @@ def create_deployment(
 )
 @handle_exceptions
 def list_deployments(
+    request: Request,
     deployment_filter_model: PipelineDeploymentFilter = Depends(
         make_dependable(PipelineDeploymentFilter)
     ),
     project_name_or_id: Optional[Union[str, UUID]] = None,
     hydrate: bool = False,
     _: AuthContext = Security(authorize),
-) -> Page[PipelineDeploymentResponse]:
+) -> Any:
     """Gets a list of deployments.
 
     Args:
+        request: The request object.
         deployment_filter_model: Filter model used for pagination, sorting,
             filtering.
         project_name_or_id: Optional name or ID of the project to filter by.
@@ -125,13 +168,29 @@ def list_deployments(
     if project_name_or_id:
         deployment_filter_model.project = project_name_or_id
 
-    return verify_permissions_and_list_entities(
+    page = verify_permissions_and_list_entities(
         filter_model=deployment_filter_model,
         resource_type=ResourceType.PIPELINE_DEPLOYMENT,
         list_method=zen_store().list_deployments,
         hydrate=hydrate,
     )
 
+    exclude = None
+    if _should_remove_step_config_overrides(request):
+        exclude = {
+            "items": {
+                "__all__": {
+                    "metadata": {
+                        "step_configurations": {
+                            "__all__": {"step_config_overrides"}
+                        }
+                    }
+                }
+            }
+        }
+
+    return page.model_dump(mode="json", exclude=exclude)
+
 
 @router.get(
     "/{deployment_id}",
@@ -139,13 +198,15 @@ def list_deployments(
 )
 @handle_exceptions
 def get_deployment(
+    request: Request,
     deployment_id: UUID,
     hydrate: bool = True,
     _: AuthContext = Security(authorize),
-) -> PipelineDeploymentResponse:
+) -> Any:
     """Gets a specific deployment using its unique id.
 
     Args:
+        request: The request object.
         deployment_id: ID of the deployment to get.
         hydrate: Flag deciding whether to hydrate the output model(s)
             by including metadata fields in the response.
@@ -153,12 +214,22 @@ def get_deployment(
     Returns:
         A specific deployment object.
     """
-    return verify_permissions_and_get_entity(
+    deployment = verify_permissions_and_get_entity(
         id=deployment_id,
         get_method=zen_store().get_deployment,
         hydrate=hydrate,
     )
 
+    exclude = None
+    if _should_remove_step_config_overrides(request):
+        exclude = {
+            "metadata": {
+                "step_configurations": {"__all__": {"step_config_overrides"}}
+            }
+        }
+
+    return deployment.model_dump(mode="json", exclude=exclude)
+
 
 @router.delete(
     "/{deployment_id}",