zenml-nightly 0.67.0.dev20241002__py3-none-any.whl → 0.68.0.dev20241027__py3-none-any.whl

zenml/pipelines/pipeline_decorator.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
-"""Legacy ZenML pipeline decorator definition."""
+"""ZenML pipeline decorator definition."""
 
 from typing import (
     TYPE_CHECKING,
@@ -19,42 +19,26 @@ from typing import (
     Callable,
     Dict,
     Optional,
-    Type,
     TypeVar,
     Union,
     overload,
 )
 
 from zenml.logger import get_logger
-from zenml.model.model import Model
-from zenml.pipelines.base_pipeline import (
-    CLASS_CONFIGURATION,
-    PARAM_ENABLE_ARTIFACT_METADATA,
-    PARAM_ENABLE_ARTIFACT_VISUALIZATION,
-    PARAM_ENABLE_CACHE,
-    PARAM_ENABLE_STEP_LOGS,
-    PARAM_EXTRA_OPTIONS,
-    PARAM_MODEL,
-    PARAM_ON_FAILURE,
-    PARAM_ON_SUCCESS,
-    PARAM_PIPELINE_NAME,
-    PARAM_SETTINGS,
-    PIPELINE_INNER_FUNC_NAME,
-    BasePipeline,
-)
 
 if TYPE_CHECKING:
     from zenml.config.base_settings import SettingsOrDict
+    from zenml.model.model import Model
+    from zenml.pipelines.pipeline_definition import Pipeline
     from zenml.types import HookSpecification
 
+    F = TypeVar("F", bound=Callable[..., None])
 
 logger = get_logger(__name__)
 
-F = TypeVar("F", bound=Callable[..., None])
-
 
 @overload
-def pipeline(_func: F) -> Type[BasePipeline]: ...
+def pipeline(_func: "F") -> "Pipeline": ...
 
 
 @overload
@@ -63,29 +47,29 @@ def pipeline(
     name: Optional[str] = None,
     enable_cache: Optional[bool] = None,
     enable_artifact_metadata: Optional[bool] = None,
-    enable_artifact_visualization: Optional[bool] = None,
     enable_step_logs: Optional[bool] = None,
     settings: Optional[Dict[str, "SettingsOrDict"]] = None,
     extra: Optional[Dict[str, Any]] = None,
+    on_failure: Optional["HookSpecification"] = None,
+    on_success: Optional["HookSpecification"] = None,
     model: Optional["Model"] = None,
-) -> Callable[[F], Type[BasePipeline]]: ...
+) -> Callable[["F"], "Pipeline"]: ...
 
 
 def pipeline(
-    _func: Optional[F] = None,
+    _func: Optional["F"] = None,
     *,
     name: Optional[str] = None,
     enable_cache: Optional[bool] = None,
     enable_artifact_metadata: Optional[bool] = None,
-    enable_artifact_visualization: Optional[bool] = None,
     enable_step_logs: Optional[bool] = None,
     settings: Optional[Dict[str, "SettingsOrDict"]] = None,
     extra: Optional[Dict[str, Any]] = None,
     on_failure: Optional["HookSpecification"] = None,
     on_success: Optional["HookSpecification"] = None,
     model: Optional["Model"] = None,
-) -> Union[Type[BasePipeline], Callable[[F], Type[BasePipeline]]]:
-    """Outer decorator function for the creation of a ZenML pipeline.
+) -> Union["Pipeline", Callable[["F"], "Pipeline"]]:
+    """Decorator to create a pipeline.
 
     Args:
         _func: The decorated function.
@@ -93,8 +77,7 @@ def pipeline(
             decorated function will be used as a fallback.
         enable_cache: Whether to use caching or not.
         enable_artifact_metadata: Whether to enable artifact metadata or not.
-        enable_artifact_visualization: Whether to enable artifact visualization.
-        enable_step_logs: Whether to enable step logs.
+        enable_step_logs: If step logs should be enabled for this pipeline.
         settings: Settings for this pipeline.
         extra: Extra configurations for this pipeline.
         on_failure: Callback function in event of failure of the step. Can be a
@@ -106,40 +89,26 @@ def pipeline(
         model: configuration of the model in the Model Control Plane.
 
     Returns:
-        the inner decorator which creates the pipeline class based on the
-        ZenML BasePipeline
+        A pipeline instance.
     """
 
-    def inner_decorator(func: F) -> Type[BasePipeline]:
-        pipeline_name = name or func.__name__
-        logger.warning(
-            "The `@pipeline` decorator that you used to define your "
-            f"{pipeline_name} pipeline is deprecated. Check out the 0.40.0 "
-            "migration guide for more information on how to migrate your "
-            "pipelines to the new syntax: "
-            "https://docs.zenml.io/reference/migration-guide/migration-zero-forty.html"
+    def inner_decorator(func: "F") -> "Pipeline":
+        from zenml.pipelines.pipeline_definition import Pipeline
+
+        p = Pipeline(
+            name=name or func.__name__,
+            enable_cache=enable_cache,
+            enable_artifact_metadata=enable_artifact_metadata,
+            enable_step_logs=enable_step_logs,
+            settings=settings,
+            extra=extra,
+            on_failure=on_failure,
+            on_success=on_success,
+            model=model,
+            entrypoint=func,
         )
 
-        return type(
-            name or func.__name__,
-            (BasePipeline,),
-            {
-                PIPELINE_INNER_FUNC_NAME: staticmethod(func),
-                CLASS_CONFIGURATION: {
-                    PARAM_PIPELINE_NAME: name,
-                    PARAM_ENABLE_CACHE: enable_cache,
-                    PARAM_ENABLE_ARTIFACT_METADATA: enable_artifact_metadata,
-                    PARAM_ENABLE_ARTIFACT_VISUALIZATION: enable_artifact_visualization,
-                    PARAM_ENABLE_STEP_LOGS: enable_step_logs,
-                    PARAM_SETTINGS: settings,
-                    PARAM_EXTRA_OPTIONS: extra,
-                    PARAM_ON_FAILURE: on_failure,
-                    PARAM_ON_SUCCESS: on_success,
-                    PARAM_MODEL: model,
-                },
-                "__module__": func.__module__,
-                "__doc__": func.__doc__,
-            },
-        )
+        p.__doc__ = func.__doc__
+        return p
 
     return inner_decorator if _func is None else inner_decorator(_func)

zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py}

@@ -67,8 +67,8 @@ from zenml.models import (
     PipelineRunResponse,
     ScheduleRequest,
 )
-from zenml.new.pipelines import build_utils
-from zenml.new.pipelines.run_utils import (
+from zenml.pipelines import build_utils
+from zenml.pipelines.run_utils import (
     create_placeholder_run,
     deploy_pipeline,
     upload_notebook_cell_code_if_necessary,
@@ -786,15 +786,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
             deployment_model: The model for the pipeline deployment
         """
         try:
-            # Log about the schedule/run
-            if deployment_model.schedule:
-                logger.info(
-                    "Scheduling a run with the schedule: "
-                    f"`{deployment_model.schedule.name}`"
-                )
-            else:
-                logger.info("Executing a new run.")
-
             # Log about the caching status
             if deployment_model.pipeline_configuration.enable_cache is False:
                 logger.info(
@@ -847,22 +838,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
         except Exception as e:
             logger.debug(f"Logging pipeline deployment metadata failed: {e}")
 
-    def get_runs(self, **kwargs: Any) -> List["PipelineRunResponse"]:
-        """(Deprecated) Get runs of this pipeline.
-
-        Args:
-            **kwargs: Further arguments for filtering or pagination that are
-                passed to `client.list_pipeline_runs()`.
-
-        Returns:
-            List of runs of this pipeline.
-        """
-        logger.warning(
-            "`Pipeline.get_runs()` is deprecated and will be removed in a "
-            "future version. Please use `Pipeline.model.get_runs()` instead."
-        )
-        return self.model.get_runs(**kwargs)
-
     def write_run_configuration_template(
         self, path: str, stack: Optional["Stack"] = None
     ) -> None:
@@ -897,19 +872,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
         steps = {}
         for step_name, invocation in self.invocations.items():
             step = invocation.step
-            parameters = (
-                pydantic_utils.TemplateGenerator(
-                    step.entrypoint_definition.legacy_params.annotation
-                ).run()
-                if step.entrypoint_definition.legacy_params
-                else {}
-            )
             outputs = {
                 name: PartialArtifactConfiguration()
                 for name in step.entrypoint_definition.outputs
             }
             step_template = StepConfigurationUpdate(
-                parameters=parameters,
+                parameters={},
                 settings=step_settings,
                 outputs=outputs,
             )
@@ -1260,14 +1228,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 {k: v for k, v in _from_config_file.items() if k in matcher}
             )
 
-            # TODO: deprecate me
-            if "model_version" in _from_config_file:
-                logger.warning(
-                    "YAML config option `model_version` is deprecated. Please use `model`."
-                )
-                _from_config_file["model"] = _from_config_file["model_version"]
-                del _from_config_file["model_version"]
-
             if "model" in _from_config_file:
                 if "model" in self._from_config_file:
                     _from_config_file["model"] = self._from_config_file[
@@ -1454,10 +1414,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
         if config_path:
             self._from_config_file = self._parse_config_file(
                 config_path=config_path,
-                matcher=inspect.getfullargspec(self.configure)[0]
-                + [
-                    "model_version"
-                ],  # TODO: deprecate `model_version` later on
+                matcher=inspect.getfullargspec(self.configure)[0],
             )
 
         _from_config_file = dict_utils.recursive_update(

zenml/stack/stack.py

@@ -40,7 +40,7 @@ from zenml.constants import (
     handle_bool_env_var,
 )
 from zenml.enums import SecretValidationLevel, StackComponentType
-from zenml.exceptions import ProvisioningError, StackValidationError
+from zenml.exceptions import StackValidationError
 from zenml.logger import get_logger
 from zenml.metadata.metadata_types import MetadataType
 from zenml.models import StackResponse
@@ -768,32 +768,11 @@ class Stack:
             deployment: The pipeline deployment
 
         Raises:
-            StackValidationError: If the stack component is not running.
             RuntimeError: If trying to deploy a pipeline that requires a remote
                 ZenML server with a local one.
         """
         self.validate(fail_if_secrets_missing=True)
 
-        for component in self.components.values():
-            if not component.is_running:
-                raise StackValidationError(
-                    f"The '{component.name}' {component.type} stack component "
-                    f"is not currently running. Please run the following "
-                    f"command to provision and start the component:\n\n"
-                    f"    `zenml stack up`\n"
-                    f"It is worth noting that the provision command will "
-                    f" be deprecated in the future. ZenML will no longer "
-                    f"be responsible for provisioning infrastructure, "
-                    f"or port-forwarding directly. Instead of managing "
-                    f"the state of the components, ZenML will be utilizing "
-                    f"the already running stack or stack components directly. "
-                    f"Additionally, we are also providing a variety of "
-                    f" deployment recipes for popular Kubernetes-based "
-                    f"integrations such as Kubeflow, Tekton, and Seldon etc."
-                    f"Check out https://docs.zenml.io/how-to/stack-deployment/deploy-a-stack-using-mlstacks"
-                    f"for more information."
-                )
-
         if self.requires_remote_server and Client().zen_store.is_local_store():
             raise RuntimeError(
                 "Stacks with remote components such as remote orchestrators "
@@ -956,83 +935,3 @@ class Stack:
             info.config
         ).values():
             component.cleanup_step_run(info=info, step_failed=step_failed)
-
-    @property
-    def is_provisioned(self) -> bool:
-        """If the stack provisioned resources to run locally.
-
-        Returns:
-            True if the stack provisioned resources to run locally.
-        """
-        return all(
-            component.is_provisioned for component in self.components.values()
-        )
-
-    @property
-    def is_running(self) -> bool:
-        """If the stack is running locally.
-
-        Returns:
-            True if the stack is running locally, False otherwise.
-        """
-        return all(
-            component.is_running for component in self.components.values()
-        )
-
-    def provision(self) -> None:
-        """Provisions resources to run the stack locally."""
-        self.validate(fail_if_secrets_missing=True)
-        logger.info("Provisioning resources for stack '%s'.", self.name)
-        for component in self.components.values():
-            if not component.is_provisioned:
-                component.provision()
-                logger.info("Provisioned resources for %s.", component)
-
-    def deprovision(self) -> None:
-        """Deprovisions all local resources of the stack."""
-        logger.info("Deprovisioning resources for stack '%s'.", self.name)
-        for component in self.components.values():
-            if component.is_provisioned:
-                try:
-                    component.deprovision()
-                    logger.info("Deprovisioned resources for %s.", component)
-                except NotImplementedError as e:
-                    logger.warning(e)
-
-    def resume(self) -> None:
-        """Resumes the provisioned local resources of the stack.
-
-        Raises:
-            ProvisioningError: If any stack component is missing provisioned
-                resources.
-        """
-        logger.info("Resuming provisioned resources for stack %s.", self.name)
-        for component in self.components.values():
-            if component.is_running:
-                # the component is already running, no need to resume anything
-                pass
-            elif component.is_provisioned:
-                component.resume()
-                logger.info("Resumed resources for %s.", component)
-            else:
-                raise ProvisioningError(
-                    f"Unable to resume resources for {component}: No "
-                    f"resources have been provisioned for this component."
-                )
-
-    def suspend(self) -> None:
-        """Suspends the provisioned local resources of the stack."""
-        logger.info(
-            "Suspending provisioned resources for stack '%s'.", self.name
-        )
-        for component in self.components.values():
-            if not component.is_suspended:
-                try:
-                    component.suspend()
-                    logger.info("Suspended resources for %s.", component)
-                except NotImplementedError:
-                    logger.warning(
-                        "Suspending provisioned resources not implemented "
-                        "for %s. Continuing without suspending resources...",
-                        component,
-                    )

zenml/stack/stack_component.py

@@ -789,77 +789,6 @@ class StackComponent:
         """
         return None
 
-    @property
-    def is_provisioned(self) -> bool:
-        """If the component provisioned resources to run.
-
-        Returns:
-            True if the component provisioned resources to run.
-        """
-        return True
-
-    @property
-    def is_running(self) -> bool:
-        """If the component is running.
-
-        Returns:
-            True if the component is running.
-        """
-        return True
-
-    @property
-    def is_suspended(self) -> bool:
-        """If the component is suspended.
-
-        Returns:
-            True if the component is suspended.
-        """
-        return not self.is_running
-
-    def provision(self) -> None:
-        """Provisions resources to run the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Provisioning resources not implemented for {self}."
-        )
-
-    def deprovision(self) -> None:
-        """Deprovisions all resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Deprovisioning resource not implemented for {self}."
-        )
-
-    def resume(self) -> None:
-        """Resumes the provisioned resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Resuming provisioned resources not implemented for {self}."
-        )
-
-    def suspend(self) -> None:
-        """Suspends the provisioned resources of the component.
-
-        Raises:
-            NotImplementedError: If the component does not implement this
-                method.
-        """
-        raise NotImplementedError(
-            f"Suspending provisioned resources not implemented for {self}."
-        )
-
     def cleanup(self) -> None:
         """Cleans up the component after it has been used."""
         pass

zenml/stack_deployments/aws_stack_deployment.py

@@ -139,6 +139,9 @@ console.
                 "s3:GetObject",
                 "s3:PutObject",
                 "s3:DeleteObject",
+                "s3:GetBucketVersioning",
+                "s3:ListBucketVersions",
+                "s3:DeleteObjectVersion",
             ],
             "ECR Repository": [
                 "ecr:DescribeRepositories",

zenml/step_operators/step_operator_entrypoint_configuration.py

@@ -91,7 +91,7 @@ class StepOperatorEntrypointConfiguration(StepEntrypointConfiguration):
 
         stack = Client().active_stack
         input_artifacts, _ = input_utils.resolve_step_inputs(
-            step=step, run_id=pipeline_run.id
+            step=step, pipeline_run=pipeline_run
         )
         output_artifact_uris = output_utils.prepare_output_artifact_uris(
             step_run=step_run, stack=stack, step=step

zenml/steps/__init__.py

@@ -27,20 +27,14 @@ decorator.
 """
 
 from zenml.config import ResourceSettings
-from zenml.steps.base_parameters import BaseParameters
 from zenml.steps.base_step import BaseStep
-from zenml.new.steps.step_context import StepContext
+from zenml.steps.step_context import StepContext, get_step_context
 from zenml.steps.step_decorator import step
-from zenml.steps.step_environment import STEP_ENVIRONMENT_NAME, StepEnvironment
-from zenml.steps.step_output import Output
 
 __all__ = [
-    "step",
     "BaseStep",
-    "BaseParameters",
-    "Output",
     "ResourceSettings",
     "StepContext",
-    "StepEnvironment",
-    "STEP_ENVIRONMENT_NAME",
+    "step",
+    "get_step_context"
 ]