zenml-nightly 0.80.1.dev20250408__py3-none-any.whl → 0.80.1.dev20250409__py3-none-any.whl

zenml/integrations/lightning/orchestrators/lightning_orchestrator.py

@@ -46,7 +46,7 @@ from zenml.stack import StackValidator
 from zenml.utils import code_utils, io_utils, source_utils
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 
@@ -199,6 +199,7 @@ class LightningOrchestrator(WheeledOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Creates a wheel and uploads the pipeline to Lightning.
 
@@ -223,6 +224,7 @@ class LightningOrchestrator(WheeledOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             ValueError: If the schedule is not set or if the cron expression

zenml/integrations/mlflow/__init__.py

@@ -61,39 +61,12 @@ class MlflowIntegration(Integration):
         from zenml.integrations.numpy import NumpyIntegration
         from zenml.integrations.pandas import PandasIntegration
         
-
         reqs = [
             "mlflow>=2.1.1,<3",
             # TODO: remove this requirement once rapidjson is fixed
             "python-rapidjson<1.15",
-            # When you do:
-            #       pip install zenml
-            # You get all our required dependencies. However, if you follow it
-            # with:
-            #       zenml integration install mlflow
-            # This downgrades pydantic to v1 even though mlflow does not have
-            # any issues with v2. This is why we have to pin it here so a
-            # downgrade will not happen.
-            "pydantic>=2.8.0,<2.9.0",
         ]
 
-        if python_version:
-            version_minor = version.parse(python_version).minor
-        else:
-            version_minor = sys.version_info.minor
-
-        if version_minor >= 12:
-            logger.debug(
-                "The MLflow integration on Python 3.12 and above is not yet "
-                "fully supported: The extra dependencies 'mlserver' and "
-                "'mlserver-mlflow' will be skipped."
-            )
-        else:
-            reqs.extend([
-                "mlserver>=1.3.3",
-                "mlserver-mlflow>=1.3.3",
-            ])
-
         reqs.extend(NumpyIntegration.get_requirements(target_os=target_os, python_version=python_version))
         reqs.extend(PandasIntegration.get_requirements(target_os=target_os, python_version=python_version))
         return reqs

zenml/integrations/mlflow/services/mlflow_deployment.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Implementation of the MLflow deployment functionality."""
 
+import importlib
 import os
 import sys
 from typing import TYPE_CHECKING, Any, Dict, Optional, Union
@@ -126,14 +127,31 @@ class MLFlowDeploymentConfig(LocalDaemonServiceConfig):
             the validated value
 
         Raises:
-            ValueError: if mlserver is set to true on Python 3.12 as it is not
-                yet supported.
+            ValueError: if mlserver packages are not installed
         """
-        if mlserver is True and sys.version_info.minor >= 12:
-            raise ValueError(
-                "The mlserver deployment is not yet supported on Python 3.12 "
-                "or above."
-            )
+        if mlserver is True:
+            # For the mlserver deployment, the mlserver and
+            # mlserver-mlflow packages need to be installed separately
+            # because they rely on an older version of Pydantic.
+
+            # Check if the mlserver and mlserver-mlflow packages are installed
+            try:
+                importlib.import_module("mlserver")
+                importlib.import_module("mlserver_mlflow")
+            except ModuleNotFoundError:
+                if sys.version_info.minor >= 12:
+                    raise ValueError(
+                        "The mlserver deployment is not yet supported on "
+                        "Python 3.12 or above."
+                    )
+
+                raise ValueError(
+                    "The MLflow MLServer backend requires the `mlserver` and "
+                    "`mlserver-mlflow` packages to be installed. These "
+                    "packages are not included in the ZenML MLflow integration "
+                    "requirements. Please install them manually."
+                )
+
         return mlserver
 
 

zenml/integrations/seldon/services/seldon_deployment.py

@@ -72,7 +72,7 @@ class SeldonDeploymentConfig(ServiceConfig):
     model_metadata: Dict[str, Any] = Field(default_factory=dict)
     extra_args: Dict[str, Any] = Field(default_factory=dict)
     is_custom_deployment: Optional[bool] = False
-    spec: Optional[Dict[Any, Any]] = Field(default_factory=dict)
+    spec: Optional[Dict[Any, Any]] = Field(default_factory=dict)  # type: ignore[arg-type]
     serviceAccountName: Optional[str] = None
 
     def get_seldon_deployment_labels(self) -> Dict[str, str]:

zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py

@@ -39,7 +39,7 @@ from zenml.orchestrators.utils import get_orchestrator_run_name
 from zenml.stack import StackValidator
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 
@@ -149,6 +149,7 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Runs each pipeline step in a separate Skypilot container.
 
@@ -157,6 +158,7 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             Exception: If the pipeline run fails.

zenml/integrations/tekton/orchestrators/tekton_orchestrator.py

@@ -53,7 +53,7 @@ from zenml.utils import io_utils, yaml_utils
 
 if TYPE_CHECKING:
     from zenml.config.base_settings import BaseSettings
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 
@@ -460,6 +460,7 @@ class TektonOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Runs the pipeline on Tekton.
 
@@ -471,6 +472,7 @@ class TektonOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If you try to run the pipelines in a notebook

zenml/integrations/vllm/services/vllm_deployment.py

@@ -150,7 +150,7 @@ class VLLMDeploymentService(LocalDaemonService, BaseDeploymentService):
 
         try:
             parser: argparse.ArgumentParser = make_arg_parser(
-                FlexibleArgumentParser()
+                FlexibleArgumentParser()  # type: ignore[no-untyped-call]
             )
             # pass in empty list to get default args
             # otherwise it will try to get the args from sys.argv

zenml/model/model.py

@@ -81,7 +81,7 @@ class Model(BaseModel):
     # technical attributes
     model_version_id: Optional[UUID] = None
     suppress_class_validation_warnings: bool = False
-    _model_id: UUID = PrivateAttr(None)
+    _model_id: Optional[UUID] = PrivateAttr(None)
     _number: Optional[int] = PrivateAttr(None)
     _created_model_version: bool = PrivateAttr(False)
 
@@ -130,6 +130,8 @@ class Model(BaseModel):
         """
         if self._model_id is None:
             self._get_or_create_model()
+
+        assert self._model_id is not None
         return self._model_id
 
     @property

zenml/models/__init__.py

@@ -418,7 +418,11 @@ from zenml.models.v2.misc.info_models import (
 
 # V2
 ActionResponseResources.model_rebuild()
+ActionResponseMetadata.model_rebuild()
 APIKeyResponseBody.model_rebuild()
+ArtifactResponse.model_rebuild()
+ArtifactResponseBody.model_rebuild()
+ArtifactResponseMetadata.model_rebuild()
 ArtifactVersionRequest.model_rebuild()
 ArtifactVersionResponseBody.model_rebuild()
 ArtifactVersionResponseMetadata.model_rebuild()
@@ -477,6 +481,7 @@ StepRunRequest.model_rebuild()
 StepRunResponseBody.model_rebuild()
 StepRunResponseMetadata.model_rebuild()
 StepRunResponseResources.model_rebuild()
+TagResponseBody.model_rebuild()
 TriggerExecutionResponseResources.model_rebuild()
 TriggerResponseBody.model_rebuild()
 TriggerResponseMetadata.model_rebuild()

zenml/models/v2/core/run_template.py

@@ -79,6 +79,10 @@ class RunTemplateRequest(ProjectScopedRequest):
     source_deployment_id: UUID = Field(
         title="The deployment that should be the base of the created template."
     )
+    hidden: bool = Field(
+        default=False,
+        title="Whether the run template is hidden.",
+    )
     tags: Optional[List[str]] = Field(
         default=None,
         title="Tags of the run template.",
@@ -101,6 +105,10 @@ class RunTemplateUpdate(BaseUpdate):
         title="The description of the run template.",
         max_length=TEXT_FIELD_MAX_LENGTH,
     )
+    hidden: Optional[bool] = Field(
+        default=None,
+        title="Whether the run template is hidden.",
+    )
     add_tags: Optional[List[str]] = Field(
         default=None, title="New tags to add to the run template."
     )
@@ -118,6 +126,10 @@ class RunTemplateResponseBody(ProjectScopedResponseBody):
     runnable: bool = Field(
         title="If a run can be started from the template.",
     )
+    hidden: bool = Field(
+        default=False,
+        title="Whether the run template is hidden.",
+    )
     latest_run_id: Optional[UUID] = Field(
         default=None,
         title="The ID of the latest run of the run template.",
@@ -205,6 +217,15 @@ class RunTemplateResponse(
         """
         return self.get_body().runnable
 
+    @property
+    def hidden(self) -> bool:
+        """The `hidden` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_body().hidden
+
     @property
     def latest_run_id(self) -> Optional[UUID]:
         """The `latest_run_id` property.
@@ -320,6 +341,7 @@ class RunTemplateFilter(ProjectScopedFilter, TaggableFilter):
         "pipeline_id",
         "pipeline",
         "stack",
+        "hidden",
     ]
     CUSTOM_SORTING_OPTIONS = [
         *ProjectScopedFilter.CUSTOM_SORTING_OPTIONS,
@@ -334,6 +356,10 @@ class RunTemplateFilter(ProjectScopedFilter, TaggableFilter):
         default=None,
         description="Name of the run template.",
     )
+    hidden: Optional[bool] = Field(
+        default=None,
+        description="Whether the run template is hidden.",
+    )
     pipeline_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Pipeline associated with the template.",
@@ -376,7 +402,7 @@ class RunTemplateFilter(ProjectScopedFilter, TaggableFilter):
         """
         custom_filters = super().get_custom_filters(table)
 
-        from sqlmodel import and_
+        from sqlmodel import and_, col
 
         from zenml.zen_stores.schemas import (
             CodeReferenceSchema,
@@ -386,6 +412,11 @@ class RunTemplateFilter(ProjectScopedFilter, TaggableFilter):
             StackSchema,
         )
 
+        if self.hidden is not None:
+            custom_filters.append(
+                col(RunTemplateSchema.hidden).is_(self.hidden)
+            )
+
         if self.code_repository_id:
             code_repo_filter = and_(
                 RunTemplateSchema.source_deployment_id

zenml/orchestrators/base_orchestrator.py

@@ -140,6 +140,7 @@ class BaseOrchestrator(StackComponent, ABC):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Optional[Iterator[Dict[str, MetadataType]]]:
         """The method needs to be implemented by the respective orchestrator.
 
@@ -175,6 +176,7 @@ class BaseOrchestrator(StackComponent, ABC):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment. These don't need to be set if running locally.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Yields:
             Metadata for the pipeline run.
@@ -250,6 +252,7 @@ class BaseOrchestrator(StackComponent, ABC):
                 deployment=deployment,
                 stack=stack,
                 environment=environment,
+                placeholder_run=placeholder_run,
             ):
                 for metadata_dict in metadata_iterator:
                     try:

zenml/orchestrators/dag_runner.py

@@ -17,7 +17,7 @@ import threading
 import time
 from collections import defaultdict
 from enum import Enum
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 
 from zenml.logger import get_logger
 
@@ -54,6 +54,7 @@ class NodeStatus(Enum):
     WAITING = "Waiting"
     RUNNING = "Running"
     COMPLETED = "Completed"
+    FAILED = "Failed"
 
 
 class ThreadedDagRunner:
@@ -70,6 +71,7 @@ class ThreadedDagRunner:
         self,
         dag: Dict[str, List[str]],
         run_fn: Callable[[str], Any],
+        finalize_fn: Optional[Callable[[Dict[str, NodeStatus]], None]] = None,
         parallel_node_startup_waiting_period: float = 0.0,
     ) -> None:
         """Define attributes and initialize all nodes in waiting state.
@@ -79,6 +81,8 @@ class ThreadedDagRunner:
                 E.g.: [(1->2), (1->3), (2->4), (3->4)] should be represented as
                 `dag={2: [1], 3: [1], 4: [2, 3]}`
             run_fn: A function `run_fn(node)` that runs a single node
+            finalize_fn: A function `finalize_fn(node_states)` that is called
+                when all nodes have completed.
             parallel_node_startup_waiting_period: Delay in seconds to wait in
                 between starting parallel nodes.
         """
@@ -88,6 +92,7 @@ class ThreadedDagRunner:
         self.dag = dag
         self.reversed_dag = reverse_dag(dag)
         self.run_fn = run_fn
+        self.finalize_fn = finalize_fn
         self.nodes = dag.keys()
         self.node_states = {node: NodeStatus.WAITING for node in self.nodes}
         self._lock = threading.Lock()
@@ -123,8 +128,12 @@ class ThreadedDagRunner:
         Args:
             node: The node.
         """
-        self.run_fn(node)
-        self._finish_node(node)
+        try:
+            self.run_fn(node)
+            self._finish_node(node)
+        except Exception as e:
+            self._finish_node(node, failed=True)
+            logger.exception(f"Node `{node}` failed: {e}")
 
     def _run_node_in_thread(self, node: str) -> threading.Thread:
         """Run a single node in a separate thread.
@@ -148,7 +157,7 @@ class ThreadedDagRunner:
         thread.start()
         return thread
 
-    def _finish_node(self, node: str) -> None:
+    def _finish_node(self, node: str, failed: bool = False) -> None:
         """Finish a node run.
 
         First updates the node status to completed.
@@ -156,20 +165,28 @@ class ThreadedDagRunner:
 
         Args:
             node: The node.
+            failed: Whether the node failed.
         """
         # Update node status to completed.
         assert self.node_states[node] == NodeStatus.RUNNING
         with self._lock:
-            self.node_states[node] = NodeStatus.COMPLETED
+            if failed:
+                self.node_states[node] = NodeStatus.FAILED
+            else:
+                self.node_states[node] = NodeStatus.COMPLETED
+
+        if failed:
+            # If the node failed, we don't need to run any downstream nodes.
+            return
 
         # Run downstream nodes.
         threads: List[threading.Thread] = []
-        for downstram_node in self.reversed_dag[node]:
-            if self._can_run(downstram_node):
+        for downstream_node in self.reversed_dag[node]:
+            if self._can_run(downstream_node):
                 if threads and self.parallel_node_startup_waiting_period > 0:
                     time.sleep(self.parallel_node_startup_waiting_period)
 
-                thread = self._run_node_in_thread(downstram_node)
+                thread = self._run_node_in_thread(downstream_node)
                 threads.append(thread)
 
         # Wait for all downstream nodes to complete.
@@ -198,11 +215,27 @@ class ThreadedDagRunner:
         for thread in threads:
             thread.join()
 
-        # Make sure all nodes were run, otherwise print a warning.
+        # Call the finalize function.
+        if self.finalize_fn:
+            self.finalize_fn(self.node_states)
+
+        # Print a status report.
+        failed_nodes = []
+        skipped_nodes = []
         for node in self.nodes:
-            if self.node_states[node] == NodeStatus.WAITING:
-                upstream_nodes = self.dag[node]
-                logger.warning(
-                    f"Node `{node}` was never run, because it was still"
-                    f" waiting for the following nodes: `{upstream_nodes}`."
-                )
+            if self.node_states[node] == NodeStatus.FAILED:
+                failed_nodes.append(node)
+            elif self.node_states[node] == NodeStatus.WAITING:
+                skipped_nodes.append(node)
+
+        if failed_nodes:
+            logger.error(
+                "The following nodes failed: " + ", ".join(failed_nodes)
+            )
+        if skipped_nodes:
+            logger.warning(
+                "The following nodes were not run because they depend on other "
+                "nodes that didn't complete: " + ", ".join(skipped_nodes)
+            )
+        if not failed_nodes and not skipped_nodes:
+            logger.info("All nodes completed successfully.")

zenml/orchestrators/local/local_orchestrator.py

@@ -27,7 +27,7 @@ from zenml.stack import Stack
 from zenml.utils import string_utils
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
 
 logger = get_logger(__name__)
 
@@ -46,6 +46,7 @@ class LocalOrchestrator(BaseOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Iterates through all steps and executes them sequentially.
 
@@ -54,6 +55,7 @@ class LocalOrchestrator(BaseOrchestrator):
             stack: The stack on which the pipeline is deployed.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
         """
         if deployment.schedule:
             logger.warning(

zenml/orchestrators/local_docker/local_docker_orchestrator.py

@@ -39,7 +39,7 @@ from zenml.stack import Stack, StackValidator
 from zenml.utils import docker_utils, string_utils
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
 
 logger = get_logger(__name__)
 
@@ -96,6 +96,7 @@ class LocalDockerOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Sequentially runs all pipeline steps in local Docker containers.
 
@@ -104,6 +105,7 @@ class LocalDockerOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If a step fails.

zenml/utils/secret_utils.py

@@ -102,7 +102,7 @@ def SecretField(*args: Any, **kwargs: Any) -> Any:
     """
     json_schema_extra = kwargs.get("json_schema_extra", {})
     json_schema_extra.update({PYDANTIC_SENSITIVE_FIELD_MARKER: True})
-    return Field(json_schema_extra=json_schema_extra, *args, **kwargs)  # type: ignore[pydantic-field]
+    return Field(json_schema_extra=json_schema_extra, *args, **kwargs)
 
 
 def ClearTextField(*args: Any, **kwargs: Any) -> Any:
@@ -119,7 +119,7 @@ def ClearTextField(*args: Any, **kwargs: Any) -> Any:
     """
     json_schema_extra = kwargs.get("json_schema_extra", {})
     json_schema_extra.update({PYDANTIC_CLEAR_TEXT_FIELD_MARKER: True})
-    return Field(json_schema_extra=json_schema_extra, *args, **kwargs)  # type: ignore[pydantic-field]
+    return Field(json_schema_extra=json_schema_extra, *args, **kwargs)
 
 
 def is_secret_field(field: "FieldInfo") -> bool:

zenml/zen_server/dashboard/assets/{404-2I8egBQu.js → 404-CYPi9d8E.js}

@@ -1 +1 @@
-import{j as e}from"./@radix-AvWw-1nd.js";import{h as s,r as t}from"./index-DPjvk73v.js";import{E as r}from"./EmptyState-DpbfQBDE.js";import{S as a}from"./help-CfT0tY2I.js";import{L as o}from"./@react-router-BUo5vhN4.js";import"./@tanstack-CcI3lvwB.js";import"./@reactflow-BHoFKFSZ.js";function d(){return e.jsx("div",{className:"flex min-h-screen w-full flex-col",children:e.jsx(r,{icon:e.jsx(a,{className:"h-[120px] w-[120px] fill-neutral-300"}),children:e.jsxs("div",{className:"text-center",children:[e.jsx("h1",{className:"mb-2 text-display-xs font-semibold",children:"We can't find the page you are looking for"}),e.jsx("p",{className:"text-lg text-theme-text-secondary",children:"You can try typing a different URL or we can bring you back to your Homepage."}),e.jsx("div",{className:"mt-5 flex justify-center",children:e.jsx(s,{size:"md",asChild:!0,children:e.jsx(o,{className:"w-min self-center whitespace-nowrap",to:t.home,children:e.jsx("span",{className:"px-0.5",children:"Go to Home"})})})})]})})})}export{d as default};
+import{j as e}from"./@radix-C7hRs6Kx.js";import{h as s,r as t}from"./index-Be2jLIYM.js";import{E as r}from"./EmptyState-M1jafpg6.js";import{S as a}from"./help-DyMolRxD.js";import{L as o}from"./@react-router-BGgzhn5D.js";import"./@tanstack-CSxjHCME.js";import"./@reactflow-CQi1Z1Wq.js";function d(){return e.jsx("div",{className:"flex min-h-screen w-full flex-col",children:e.jsx(r,{icon:e.jsx(a,{className:"h-[120px] w-[120px] fill-neutral-300"}),children:e.jsxs("div",{className:"text-center",children:[e.jsx("h1",{className:"mb-2 text-display-xs font-semibold",children:"We can't find the page you are looking for"}),e.jsx("p",{className:"text-lg text-theme-text-secondary",children:"You can try typing a different URL or we can bring you back to your Homepage."}),e.jsx("div",{className:"mt-5 flex justify-center",children:e.jsx(s,{size:"md",asChild:!0,children:e.jsx(o,{className:"w-min self-center whitespace-nowrap",to:t.home,children:e.jsx("span",{className:"px-0.5",children:"Go to Home"})})})})]})})})}export{d as default};