zenml-nightly 0.66.0.dev20240928__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl

zenml/new/steps/step_decorator.py

@@ -1,176 +0,0 @@
-#  Copyright (c) ZenML GmbH 2021. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Step decorator function."""
-
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Dict,
-    Mapping,
-    Optional,
-    Sequence,
-    Type,
-    TypeVar,
-    Union,
-    overload,
-)
-
-from zenml.logger import get_logger
-
-if TYPE_CHECKING:
-    from zenml.config.base_settings import SettingsOrDict
-    from zenml.config.retry_config import StepRetryConfig
-    from zenml.config.source import Source
-    from zenml.materializers.base_materializer import BaseMaterializer
-    from zenml.model.model import Model
-    from zenml.steps import BaseStep
-    from zenml.types import HookSpecification
-
-    MaterializerClassOrSource = Union[str, Source, Type[BaseMaterializer]]
-
-    OutputMaterializersSpecification = Union[
-        MaterializerClassOrSource,
-        Sequence[MaterializerClassOrSource],
-        Mapping[str, MaterializerClassOrSource],
-        Mapping[str, Sequence[MaterializerClassOrSource]],
-    ]
-    F = TypeVar("F", bound=Callable[..., Any])
-
-
-logger = get_logger(__name__)
-
-
-@overload
-def step(_func: "F") -> "BaseStep": ...
-
-
-@overload
-def step(
-    *,
-    name: Optional[str] = None,
-    enable_cache: Optional[bool] = None,
-    enable_artifact_metadata: Optional[bool] = None,
-    enable_artifact_visualization: Optional[bool] = None,
-    enable_step_logs: Optional[bool] = None,
-    experiment_tracker: Optional[str] = None,
-    step_operator: Optional[str] = None,
-    output_materializers: Optional["OutputMaterializersSpecification"] = None,
-    settings: Optional[Dict[str, "SettingsOrDict"]] = None,
-    extra: Optional[Dict[str, Any]] = None,
-    on_failure: Optional["HookSpecification"] = None,
-    on_success: Optional["HookSpecification"] = None,
-    model: Optional["Model"] = None,
-    retry: Optional["StepRetryConfig"] = None,
-    model_version: Optional["Model"] = None,  # TODO: deprecate me
-) -> Callable[["F"], "BaseStep"]: ...
-
-
-def step(
-    _func: Optional["F"] = None,
-    *,
-    name: Optional[str] = None,
-    enable_cache: Optional[bool] = None,
-    enable_artifact_metadata: Optional[bool] = None,
-    enable_artifact_visualization: Optional[bool] = None,
-    enable_step_logs: Optional[bool] = None,
-    experiment_tracker: Optional[str] = None,
-    step_operator: Optional[str] = None,
-    output_materializers: Optional["OutputMaterializersSpecification"] = None,
-    settings: Optional[Dict[str, "SettingsOrDict"]] = None,
-    extra: Optional[Dict[str, Any]] = None,
-    on_failure: Optional["HookSpecification"] = None,
-    on_success: Optional["HookSpecification"] = None,
-    model: Optional["Model"] = None,
-    retry: Optional["StepRetryConfig"] = None,
-    model_version: Optional["Model"] = None,  # TODO: deprecate me
-) -> Union["BaseStep", Callable[["F"], "BaseStep"]]:
-    """Decorator to create a ZenML step.
-
-    Args:
-        _func: The decorated function.
-        name: The name of the step. If left empty, the name of the decorated
-            function will be used as a fallback.
-        enable_cache: Specify whether caching is enabled for this step. If no
-            value is passed, caching is enabled by default.
-        enable_artifact_metadata: Specify whether metadata is enabled for this
-            step. If no value is passed, metadata is enabled by default.
-        enable_artifact_visualization: Specify whether visualization is enabled
-            for this step. If no value is passed, visualization is enabled by
-            default.
-        enable_step_logs: Specify whether step logs are enabled for this step.
-        experiment_tracker: The experiment tracker to use for this step.
-        step_operator: The step operator to use for this step.
-        output_materializers: Output materializers for this step. If
-            given as a dict, the keys must be a subset of the output names
-            of this step. If a single value (type or string) is given, the
-            materializer will be used for all outputs.
-        settings: Settings for this step.
-        extra: Extra configurations for this step.
-        on_failure: Callback function in event of failure of the step. Can be a
-            function with a single argument of type `BaseException`, or a source
-            path to such a function (e.g. `module.my_function`).
-        on_success: Callback function in event of success of the step. Can be a
-            function with no arguments, or a source path to such a function
-            (e.g. `module.my_function`).
-        model: configuration of the model in the Model Control Plane.
-        retry: configuration of step retry in case of step failure.
-        model_version: DEPRECATED, please use `model` instead.
-
-    Returns:
-        The step instance.
-    """
-
-    def inner_decorator(func: "F") -> "BaseStep":
-        from zenml.new.steps.decorated_step import _DecoratedStep
-
-        # TODO: deprecate me
-        if model_version:
-            logger.warning(
-                "Step decorator argument `model_version` is deprecated. Please use `model` instead."
-            )
-
-        class_: Type["BaseStep"] = type(
-            func.__name__,
-            (_DecoratedStep,),
-            {
-                "entrypoint": staticmethod(func),
-                "__module__": func.__module__,
-                "__doc__": func.__doc__,
-            },
-        )
-
-        step_instance = class_(
-            name=name or func.__name__,
-            enable_cache=enable_cache,
-            enable_artifact_metadata=enable_artifact_metadata,
-            enable_artifact_visualization=enable_artifact_visualization,
-            enable_step_logs=enable_step_logs,
-            experiment_tracker=experiment_tracker,
-            step_operator=step_operator,
-            output_materializers=output_materializers,
-            settings=settings,
-            extra=extra,
-            on_failure=on_failure,
-            on_success=on_success,
-            model=model or model_version,
-            retry=retry,
-        )
-
-        return step_instance
-
-    if _func is None:
-        return inner_decorator
-    else:
-        return inner_decorator(_func)

zenml/pipelines/base_pipeline.py

@@ -1,273 +0,0 @@
-#  Copyright (c) ZenML GmbH 2023. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Legacy ZenML pipeline class definition."""
-
-import inspect
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Mapping, Optional, Union
-from uuid import UUID
-
-from zenml.config.schedule import Schedule
-from zenml.exceptions import PipelineInterfaceError
-from zenml.logger import get_logger
-from zenml.new.pipelines.pipeline import Pipeline
-from zenml.steps import BaseStep
-from zenml.utils import dict_utils, source_utils
-
-if TYPE_CHECKING:
-    from zenml.config.base_settings import SettingsOrDict
-    from zenml.config.source import Source
-    from zenml.config.step_configurations import StepConfigurationUpdate
-    from zenml.models import PipelineBuildBase
-
-    StepConfigurationUpdateOrDict = Union[
-        Dict[str, Any], StepConfigurationUpdate
-    ]
-
-logger = get_logger(__name__)
-
-PIPELINE_INNER_FUNC_NAME = "connect"
-CLASS_CONFIGURATION = "_CLASS_CONFIGURATION"
-PARAM_PIPELINE_NAME = "name"
-PARAM_ENABLE_CACHE = "enable_cache"
-PARAM_ENABLE_ARTIFACT_METADATA = "enable_artifact_metadata"
-PARAM_ENABLE_ARTIFACT_VISUALIZATION = "enable_artifact_visualization"
-PARAM_ENABLE_STEP_LOGS = "enable_step_logs"
-PARAM_SETTINGS = "settings"
-PARAM_EXTRA_OPTIONS = "extra"
-PARAM_ON_FAILURE = "on_failure"
-PARAM_ON_SUCCESS = "on_success"
-PARAM_MODEL = "model"
-
-TEMPLATE_NAME_ATTRIBUTE = "_template_name"
-
-
-class BasePipeline(Pipeline, ABC):
-    """Legacy pipeline class."""
-
-    _CLASS_CONFIGURATION: ClassVar[Optional[Dict[str, Any]]] = None
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Initializes a pipeline.
-
-        Args:
-            *args: Initialization arguments.
-            **kwargs: Initialization keyword arguments.
-        """
-        config = self._CLASS_CONFIGURATION or {}
-        pipeline_name = (
-            config.pop(PARAM_PIPELINE_NAME, None) or self.__class__.__name__
-        )
-        self._steps = self._verify_steps(
-            *args, __name__=pipeline_name, **kwargs
-        )
-
-        def entrypoint() -> None:
-            self.connect(**self._steps)
-
-        super().__init__(
-            name=pipeline_name,
-            entrypoint=entrypoint,
-            **config,
-        )
-
-    @property
-    def steps(self) -> Dict[str, BaseStep]:
-        """Returns the steps of the pipeline.
-
-        Returns:
-            The steps of the pipeline.
-        """
-        return self._steps
-
-    @abstractmethod
-    def connect(self, *args: BaseStep, **kwargs: BaseStep) -> None:
-        """Abstract method that connects the pipeline steps.
-
-        Args:
-            *args: Connect method arguments.
-            **kwargs: Connect method keyword arguments.
-        """
-        raise NotImplementedError
-
-    def resolve(self) -> "Source":
-        """Resolves the pipeline.
-
-        Returns:
-            The pipeline source.
-        """
-        return source_utils.resolve(self.__class__)
-
-    @property
-    def source_object(self) -> Any:
-        """The source object of this pipeline.
-
-        Returns:
-            The source object of this pipeline.
-        """
-        return self.connect
-
-    def run(
-        self,
-        *,
-        run_name: Optional[str] = None,
-        enable_cache: Optional[bool] = None,
-        enable_artifact_metadata: Optional[bool] = None,
-        enable_artifact_visualization: Optional[bool] = None,
-        enable_step_logs: Optional[bool] = None,
-        schedule: Optional[Schedule] = None,
-        build: Union[str, "UUID", "PipelineBuildBase", None] = None,
-        settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
-        step_configurations: Optional[
-            Mapping[str, "StepConfigurationUpdateOrDict"]
-        ] = None,
-        extra: Optional[Dict[str, Any]] = None,
-        config_path: Optional[str] = None,
-        unlisted: bool = False,
-        prevent_build_reuse: bool = False,
-    ) -> None:
-        """Runs the pipeline on the active stack.
-
-        Args:
-            run_name: Name of the pipeline run.
-            enable_cache: If caching should be enabled for this pipeline run.
-            enable_artifact_metadata: If artifact metadata should be enabled
-                for this pipeline run.
-            enable_artifact_visualization: If artifact visualization should be
-                enabled for this pipeline run.
-            enable_step_logs: If step logs should be enabled for this pipeline
-                run.
-            schedule: Optional schedule to use for the run.
-            build: Optional build to use for the run.
-            settings: Settings for this pipeline run.
-            step_configurations: Configurations for steps of the pipeline.
-            extra: Extra configurations for this pipeline run.
-            config_path: Path to a yaml configuration file. This file will
-                be parsed as a
-                `zenml.config.pipeline_configurations.PipelineRunConfiguration`
-                object. Options provided in this file will be overwritten by
-                options provided in code using the other arguments of this
-                method.
-            unlisted: Whether the pipeline run should be unlisted (not assigned
-                to any pipeline).
-            prevent_build_reuse: Whether to prevent the reuse of a build.
-        """
-        pipeline_copy = self.with_options(
-            run_name=run_name,
-            schedule=schedule,
-            build=build,
-            step_configurations=step_configurations,
-            config_path=config_path,
-            unlisted=unlisted,
-            prevent_build_reuse=prevent_build_reuse,
-        )
-        new_run_args = dict_utils.remove_none_values(
-            {
-                "enable_cache": enable_cache,
-                "enable_artifact_metadata": enable_artifact_metadata,
-                "enable_artifact_visualization": enable_artifact_visualization,
-                "enable_step_logs": enable_step_logs,
-                "settings": settings,
-                "extra": extra,
-            }
-        )
-        pipeline_copy._run_args.update(new_run_args)
-
-        pipeline_copy()
-
-    def _compute_invocation_id(
-        self,
-        step: "BaseStep",
-        custom_id: Optional[str] = None,
-        allow_suffix: bool = True,
-    ) -> str:
-        """Compute the invocation ID.
-
-        Args:
-            step: The step for which to compute the ID.
-            custom_id: Custom ID to use for the invocation.
-            allow_suffix: Whether a suffix can be appended to the invocation
-                ID.
-
-        Returns:
-            The invocation ID.
-        """
-        custom_id = getattr(step, TEMPLATE_NAME_ATTRIBUTE, None)
-
-        return super()._compute_invocation_id(
-            step=step, custom_id=custom_id, allow_suffix=False
-        )
-
-    def _verify_steps(
-        self, *args: Any, __name__: str, **kwargs: Any
-    ) -> Dict[str, "BaseStep"]:
-        """Verifies the initialization args and kwargs of this pipeline.
-
-        This method makes sure that no missing/unexpected arguments or
-        arguments of a wrong type are passed when creating a pipeline.
-
-        Args:
-            *args: The args passed to the init method of this pipeline.
-            __name__: The pipeline name. The naming of this argument is to avoid
-                conflicts with other arguments.
-            **kwargs: The kwargs passed to the init method of this pipeline.
-
-        Raises:
-            PipelineInterfaceError: If there are too many/few arguments or
-                arguments with a wrong name/type.
-
-        Returns:
-            The verified steps.
-        """
-        signature = inspect.signature(self.connect, follow_wrapped=True)
-
-        try:
-            bound_args = signature.bind(*args, **kwargs)
-        except TypeError as e:
-            raise PipelineInterfaceError(
-                f"Wrong arguments when initializing pipeline '{__name__}': {e}"
-            ) from e
-
-        steps = {}
-
-        for key, potential_step in bound_args.arguments.items():
-            step_class = type(potential_step)
-
-            if inspect.isclass(potential_step) and issubclass(
-                potential_step, BaseStep
-            ):
-                raise PipelineInterfaceError(
-                    f"Wrong argument type (`{step_class}`) for argument "
-                    f"'{key}' of pipeline '{__name__}'. "
-                    f"A `BaseStep` subclass was provided instead of an "
-                    f"instance. "
-                    f"This might have been caused due to missing brackets of "
-                    f"your steps when creating a pipeline with `@step` "
-                    f"decorated functions, "
-                    f"for which the correct syntax is `pipeline(step=step())`."
-                )
-
-            if not isinstance(potential_step, BaseStep):
-                raise PipelineInterfaceError(
-                    f"Wrong argument type (`{step_class}`) for argument "
-                    f"'{key}' of pipeline '{__name__}'. Only "
-                    f"`@step` decorated functions or instances of `BaseStep` "
-                    f"subclasses can be used as arguments when creating "
-                    f"a pipeline."
-                )
-
-            steps[key] = potential_step
-            setattr(potential_step, TEMPLATE_NAME_ATTRIBUTE, key)
-
-        return steps

zenml/post_execution/__init__.py

@@ -1,30 +0,0 @@
-#  Copyright (c) ZenML GmbH 2021. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Deprecated post-execution utility functions."""
-
-from zenml.post_execution.pipeline import (
-    get_pipeline,
-    get_pipelines,
-)
-from zenml.post_execution.pipeline_run import (
-    get_run,
-    get_unlisted_runs,
-)
-
-__all__ = [
-    "get_pipeline",
-    "get_pipelines",
-    "get_run",
-    "get_unlisted_runs",
-]

zenml/post_execution/pipeline.py

@@ -1,55 +0,0 @@
-#  Copyright (c) ZenML GmbH 2023. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Implementation of the post-execution pipeline."""
-
-from typing import List, Optional
-
-from zenml.client import Client
-from zenml.logger import get_logger
-from zenml.models import PipelineResponse
-
-logger = get_logger(__name__)
-
-
-def get_pipelines() -> List["PipelineResponse"]:
-    """(Deprecated) Fetches all pipelines in the active workspace.
-
-    Returns:
-        A list of pipeline models.
-    """
-    logger.warning(
-        "`zenml.post_execution.get_pipelines()` is deprecated and will be "
-        "removed in a future release. Please use "
-        "`zenml.client.Client().list_pipelines()` instead."
-    )
-    return Client().list_pipelines().items
-
-
-def get_pipeline(
-    pipeline: str,
-) -> Optional["PipelineResponse"]:
-    """(Deprecated) Fetches a pipeline model.
-
-    Args:
-        pipeline: The name of the pipeline.
-
-    Returns:
-        The pipeline model.
-    """
-    logger.warning(
-        "`zenml.post_execution.get_pipeline()` is deprecated and will be "
-        "removed in a future release. Please use "
-        "`zenml.client.Client().get_pipeline()` instead."
-    )
-    return Client().get_pipeline(name_id_or_prefix=pipeline)

zenml/post_execution/pipeline_run.py

@@ -1,55 +0,0 @@
-#  Copyright (c) ZenML GmbH 2023. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Implementation of the post-execution pipeline run class."""
-
-from typing import List
-
-from zenml.client import Client
-from zenml.logger import get_logger
-from zenml.models import PipelineRunResponse
-
-logger = get_logger(__name__)
-
-
-def get_run(name: str) -> "PipelineRunResponse":
-    """(Deprecated) Fetches the run with the given name.
-
-    Args:
-        name: The name of the run to fetch.
-
-    Returns:
-        The run with the given name.
-    """
-    logger.warning(
-        "`zenml.post_execution.get_run(<name>)` is deprecated and will be "
-        "removed in a future release. Please use "
-        "`zenml.client.Client().get_pipeline_run(<name>)` instead."
-    )
-    return Client().get_pipeline_run(name)
-
-
-def get_unlisted_runs() -> List["PipelineRunResponse"]:
-    """(Deprecated) Fetches the 50 most recent unlisted runs.
-
-    Unlisted runs are runs that are not associated with any pipeline.
-
-    Returns:
-        A list of the 50 most recent unlisted runs.
-    """
-    logger.warning(
-        "`zenml.post_execution.get_unlisted_runs()` is deprecated and will be "
-        "removed in a future release. Please use "
-        "`zenml.client.Client().list_pipeline_runs(unlisted=True)` instead."
-    )
-    return Client().list_pipeline_runs(unlisted=True).items

zenml/services/terraform/__init__.py

@@ -1,14 +0,0 @@
-#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Initialization of a Terraform ZenML service."""