zenml-nightly 0.66.0.dev20240928__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl

zenml/{new/pipelines → pipelines}/run_utils.py

@@ -22,6 +22,7 @@ from zenml.models import (
     PipelineRunResponse,
     StackResponse,
 )
+from zenml.orchestrators.publish_utils import publish_failed_pipeline_run
 from zenml.orchestrators.utils import get_run_name
 from zenml.stack import Flavor, Stack
 from zenml.utils import code_utils, notebook_utils, source_utils
@@ -81,6 +82,7 @@ def create_placeholder_run(
         deployment=deployment.id,
         pipeline=deployment.pipeline.id if deployment.pipeline else None,
         status=ExecutionStatus.INITIALIZING,
+        tags=deployment.pipeline_configuration.tags,
     )
     return Client().zen_store.create_run(run_request)
 
@@ -124,20 +126,18 @@ def deploy_pipeline(
     Args:
         deployment: The deployment to run.
         stack: The stack on which to run the deployment.
-        placeholder_run: An optional placeholder run for the deployment. This
-            will be deleted in case the pipeline deployment failed.
+        placeholder_run: An optional placeholder run for the deployment.
 
     Raises:
         Exception: Any exception that happened while deploying or running
             (in case it happens synchronously) the pipeline.
     """
-    stack.prepare_pipeline_deployment(deployment=deployment)
-
     # Prevent execution of nested pipelines which might lead to
     # unexpected behavior
     previous_value = constants.SHOULD_PREVENT_PIPELINE_EXECUTION
     constants.SHOULD_PREVENT_PIPELINE_EXECUTION = True
     try:
+        stack.prepare_pipeline_deployment(deployment=deployment)
         stack.deploy_pipeline(
             deployment=deployment,
             placeholder_run=placeholder_run,
@@ -145,13 +145,14 @@ def deploy_pipeline(
     except Exception as e:
         if (
             placeholder_run
-            and Client().get_pipeline_run(placeholder_run.id).status
+            and Client()
+            .get_pipeline_run(placeholder_run.id, hydrate=False)
+            .status
             == ExecutionStatus.INITIALIZING
         ):
-            # The run hasn't actually started yet, which means that we
-            # failed during initialization -> We don't want the
-            # placeholder run to stay in the database
-            Client().delete_pipeline_run(placeholder_run.id)
+            # The run failed during the initialization phase -> We change it's
+            # status to `Failed`
+            publish_failed_pipeline_run(placeholder_run.id)
 
         raise e
     finally:
@@ -203,7 +204,7 @@ def validate_stack_is_runnable_from_server(
         assert len(component_list) == 1
         component = component_list[0]
         flavors = zen_store.list_flavors(
-            FlavorFilter(name=component.flavor, type=component.type)
+            FlavorFilter(name=component.flavor_name, type=component.type)
         )
         assert len(flavors) == 1
         flavor_model = flavors[0]

zenml/services/local/local_daemon_entrypoint.py

@@ -24,13 +24,13 @@ import click
 
 from zenml.utils.daemon import daemonize
 
-# Try to import the LocalZenServer here because it needs to be registered in the
+# Try to import the DaemonZenServer here because it needs to be registered in the
 # service registry early on in order to be available for use in other modules.
-# If the LocalZenServer dependencies aren't installed, there is no need to register
+# If the DaemonZenServer dependencies aren't installed, there is no need to register
 # it anywhere so we simply pass.
 try:
-    from zenml.zen_server.deploy.local.local_zen_server import (  # noqa
-        LocalZenServer,
+    from zenml.zen_server.deploy.daemon.daemon_zen_server import (  # noqa
+        DaemonZenServer,
     )
 except ImportError:
     pass

zenml/services/service.py

@@ -78,9 +78,9 @@ def update_service_status(
             if pre_status:
                 self.status.update_state(pre_status, "")
             try:
-                logger.info(f"Calling {func.__name__} method...")
+                logger.debug(f"Calling {func.__name__} method...")
                 result = func(self, *args, **kwargs)
-                logger.info(f"{func.__name__} method executed successfully.")
+                logger.debug(f"{func.__name__} method executed successfully.")
                 if post_status:
                     self.status.update_state(post_status, "")
                 return result

zenml/stack/stack.py

@@ -40,7 +40,7 @@ from zenml.constants import (
     handle_bool_env_var,
 )
 from zenml.enums import SecretValidationLevel, StackComponentType
-from zenml.exceptions import ProvisioningError, StackValidationError
+from zenml.exceptions import StackValidationError
 from zenml.logger import get_logger
 from zenml.metadata.metadata_types import MetadataType
 from zenml.models import StackResponse
@@ -768,32 +768,11 @@ class Stack:
             deployment: The pipeline deployment
 
         Raises:
-            StackValidationError: If the stack component is not running.
             RuntimeError: If trying to deploy a pipeline that requires a remote
                 ZenML server with a local one.
         """
         self.validate(fail_if_secrets_missing=True)
 
-        for component in self.components.values():
-            if not component.is_running:
-                raise StackValidationError(
-                    f"The '{component.name}' {component.type} stack component "
-                    f"is not currently running. Please run the following "
-                    f"command to provision and start the component:\n\n"
-                    f"    `zenml stack up`\n"
-                    f"It is worth noting that the provision command will "
-                    f" be deprecated in the future. ZenML will no longer "
-                    f"be responsible for provisioning infrastructure, "
-                    f"or port-forwarding directly. Instead of managing "
-                    f"the state of the components, ZenML will be utilizing "
-                    f"the already running stack or stack components directly. "
-                    f"Additionally, we are also providing a variety of "
-                    f" deployment recipes for popular Kubernetes-based "
-                    f"integrations such as Kubeflow, Tekton, and Seldon etc."
-                    f"Check out https://docs.zenml.io/how-to/stack-deployment/deploy-a-stack-using-mlstacks"
-                    f"for more information."
-                )
-
         if self.requires_remote_server and Client().zen_store.is_local_store():
             raise RuntimeError(
                 "Stacks with remote components such as remote orchestrators "
@@ -831,18 +810,14 @@ class Stack:
         self,
         deployment: "PipelineDeploymentResponse",
         placeholder_run: Optional["PipelineRunResponse"] = None,
-    ) -> Any:
+    ) -> None:
         """Deploys a pipeline on this stack.
 
         Args:
             deployment: The pipeline deployment.
             placeholder_run: An optional placeholder run for the deployment.
-                This will be deleted in case the pipeline deployment failed.
-
-        Returns:
-            The return value of the call to `orchestrator.run_pipeline(...)`.
         """
-        return self.orchestrator.run(
+        self.orchestrator.run(
             deployment=deployment, stack=self, placeholder_run=placeholder_run
         )
 
@@ -956,83 +931,3 @@ class Stack:
             info.config
         ).values():
             component.cleanup_step_run(info=info, step_failed=step_failed)
-
-    @property
-    def is_provisioned(self) -> bool:
-        """If the stack provisioned resources to run locally.
-
-        Returns:
-            True if the stack provisioned resources to run locally.
-        """
-        return all(
-            component.is_provisioned for component in self.components.values()
-        )
-
-    @property
-    def is_running(self) -> bool:
-        """If the stack is running locally.
-
-        Returns:
-            True if the stack is running locally, False otherwise.
-        """
-        return all(
-            component.is_running for component in self.components.values()
-        )
-
-    def provision(self) -> None:
-        """Provisions resources to run the stack locally."""
-        self.validate(fail_if_secrets_missing=True)
-        logger.info("Provisioning resources for stack '%s'.", self.name)
-        for component in self.components.values():
-            if not component.is_provisioned:
-                component.provision()
-                logger.info("Provisioned resources for %s.", component)
-
-    def deprovision(self) -> None:
-        """Deprovisions all local resources of the stack."""
-        logger.info("Deprovisioning resources for stack '%s'.", self.name)
-        for component in self.components.values():
-            if component.is_provisioned:
-                try:
-                    component.deprovision()
-                    logger.info("Deprovisioned resources for %s.", component)
-                except NotImplementedError as e:
-                    logger.warning(e)
-
-    def resume(self) -> None:
-        """Resumes the provisioned local resources of the stack.
-
-        Raises:
-            ProvisioningError: If any stack component is missing provisioned
-                resources.
-        """
-        logger.info("Resuming provisioned resources for stack %s.", self.name)
-        for component in self.components.values():
-            if component.is_running:
-                # the component is already running, no need to resume anything
-                pass
-            elif component.is_provisioned:
-                component.resume()
-                logger.info("Resumed resources for %s.", component)
-            else:
-                raise ProvisioningError(
-                    f"Unable to resume resources for {component}: No "
-                    f"resources have been provisioned for this component."
-                )
-
-    def suspend(self) -> None:
-        """Suspends the provisioned local resources of the stack."""
-        logger.info(
-            "Suspending provisioned resources for stack '%s'.", self.name
-        )
-        for component in self.components.values():
-            if not component.is_suspended:
-                try:
-                    component.suspend()
-                    logger.info("Suspended resources for %s.", component)
-                except NotImplementedError:
-                    logger.warning(
-                        "Suspending provisioned resources not implemented "
-                        "for %s. Continuing without suspending resources...",
-                        component,
-                    )

zenml/stack/stack_component.py

@@ -398,12 +398,7 @@ class StackComponent:
         Raises:
             ImportError: If the flavor can't be imported.
         """
-        from zenml.client import Client
-
-        flavor_model = Client().get_flavor_by_name_and_type(
-            name=component_model.flavor,
-            component_type=component_model.type,
-        )
+        flavor_model = component_model.flavor
 
         try:
             from zenml.stack import Flavor
@@ -429,7 +424,7 @@ class StackComponent:
                 id=component_model.id,
                 config=configuration,
                 labels=component_model.labels,
-                flavor=component_model.flavor,
+                flavor=component_model.flavor_name,
                 type=component_model.type,
                 created=component_model.created,
                 updated=component_model.updated,
@@ -789,77 +784,6 @@ class StackComponent:
         """
         return None
 
-    @property
-    def is_provisioned(self) -> bool:
-        """If the component provisioned resources to run.
-
-        Returns:
-            True if the component provisioned resources to run.
-        """
-        return True
-
-    @property
-    def is_running(self) -> bool:
-        """If the component is running.
-
-        Returns:
-            True if the component is running.
-        """
-        return True
-
-    @property
-    def is_suspended(self) -> bool:
-        """If the component is suspended.
-
-        Returns:
-            True if the component is suspended.
-        """
-        return not self.is_running
-
-    def provision(self) -> None:
-        """Provisions resources to run the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Provisioning resources not implemented for {self}."
-        )
-
-    def deprovision(self) -> None:
-        """Deprovisions all resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Deprovisioning resource not implemented for {self}."
-        )
-
-    def resume(self) -> None:
-        """Resumes the provisioned resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Resuming provisioned resources not implemented for {self}."
-        )
-
-    def suspend(self) -> None:
-        """Suspends the provisioned resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Suspending provisioned resources not implemented for {self}."
-        )
-
     def cleanup(self) -> None:
         """Cleans up the component after it has been used."""
         pass

zenml/stack/utils.py

@@ -13,7 +13,7 @@
 #  permissions and limitations under the License.
 """Util functions for handling stacks, components, and flavors."""
 
-from typing import Any, Dict, Optional, Type
+from typing import Any, Dict, Optional, Type, Union
 
 from zenml.client import Client
 from zenml.enums import StackComponentType, StoreType
@@ -28,7 +28,7 @@ logger = get_logger(__name__)
 
 def validate_stack_component_config(
     configuration_dict: Dict[str, Any],
-    flavor_name: str,
+    flavor: Union[FlavorResponse, str],
     component_type: StackComponentType,
     zen_store: Optional[BaseZenStore] = None,
     validate_custom_flavors: bool = True,
@@ -37,7 +37,7 @@ def validate_stack_component_config(
 
     Args:
         configuration_dict: The stack component configuration to validate.
-        flavor_name: The name of the flavor of the stack component.
+        flavor: The model or name of the flavor of the stack component.
         component_type: The type of the stack component.
         zen_store: An optional ZenStore in which to look for the flavor. If not
             provided, the flavor will be fetched via the regular ZenML Client.
@@ -56,17 +56,21 @@ def validate_stack_component_config(
         ImportError: If the flavor class could not be imported.
         ModuleNotFoundError: If the flavor class could not be imported.
     """
-    if zen_store:
-        flavor_model = get_flavor_by_name_and_type_from_zen_store(
-            zen_store=zen_store,
-            flavor_name=flavor_name,
-            component_type=component_type,
-        )
+    if isinstance(flavor, FlavorResponse):
+        flavor_model = flavor
     else:
-        flavor_model = Client().get_flavor_by_name_and_type(
-            name=flavor_name,
-            component_type=component_type,
-        )
+        if zen_store:
+            flavor_model = get_flavor_by_name_and_type_from_zen_store(
+                zen_store=zen_store,
+                flavor_name=flavor,
+                component_type=component_type,
+            )
+        else:
+            flavor_model = Client().get_flavor_by_name_and_type(
+                name=flavor,
+                component_type=component_type,
+            )
+
     try:
         flavor_class = Flavor.from_model(flavor_model)
     except (ImportError, ModuleNotFoundError):
@@ -103,7 +107,7 @@ def warn_if_config_server_mismatch(
     """
     zen_store = Client().zen_store
     if configuration.is_remote and zen_store.is_local_store():
-        if zen_store.type != StoreType.REST:
+        if zen_store.type == StoreType.REST:
             logger.warning(
                 "You are configuring a stack component that is running "
                 "remotely while using a local ZenML server. The component "
@@ -111,6 +115,14 @@ def warn_if_config_server_mismatch(
                 "therefore not be functional. Please consider deploying "
                 "and/or using a remote ZenML server instead."
             )
+        else:
+            logger.warning(
+                "You are configuring a stack component that is running "
+                "remotely while connected to the local database. The component "
+                "will not be able to reach the local database and will "
+                "therefore not be functional. Please consider deploying "
+                "and/or using a remote ZenML server instead."
+            )
     elif configuration.is_local and not zen_store.is_local_store():
         logger.warning(
             "You are configuring a stack component that is using "

zenml/stack_deployments/aws_stack_deployment.py

@@ -139,6 +139,9 @@ console.
                 "s3:GetObject",
                 "s3:PutObject",
                 "s3:DeleteObject",
+                "s3:GetBucketVersioning",
+                "s3:ListBucketVersions",
+                "s3:DeleteObjectVersion",
             ],
             "ECR Repository": [
                 "ecr:DescribeRepositories",

zenml/step_operators/step_operator_entrypoint_configuration.py

@@ -91,7 +91,7 @@ class StepOperatorEntrypointConfiguration(StepEntrypointConfiguration):
 
         stack = Client().active_stack
         input_artifacts, _ = input_utils.resolve_step_inputs(
-            step=step, run_id=pipeline_run.id
+            step=step, pipeline_run=pipeline_run
         )
         output_artifact_uris = output_utils.prepare_output_artifact_uris(
             step_run=step_run, stack=stack, step=step

zenml/steps/__init__.py

@@ -27,20 +27,14 @@ decorator.
 """
 
 from zenml.config import ResourceSettings
-from zenml.steps.base_parameters import BaseParameters
 from zenml.steps.base_step import BaseStep
-from zenml.new.steps.step_context import StepContext
+from zenml.steps.step_context import StepContext, get_step_context
 from zenml.steps.step_decorator import step
-from zenml.steps.step_environment import STEP_ENVIRONMENT_NAME, StepEnvironment
-from zenml.steps.step_output import Output
 
 __all__ = [
-    "step",
     "BaseStep",
-    "BaseParameters",
-    "Output",
     "ResourceSettings",
     "StepContext",
-    "StepEnvironment",
-    "STEP_ENVIRONMENT_NAME",
+    "step",
+    "get_step_context"
 ]