zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.63.0.dev20240731__py3-none-any.whl

README.md

@@ -327,7 +327,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.62.0 is out. Check out the release notes
+    🎉 Version 0.63.0 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.

RELEASE_NOTES.md

@@ -1,4 +1,45 @@
 <!-- markdown-link-check-disable -->
+
+# 0.63.0
+
+Moving forward from the last two releases, we have further improved the 
+1-click deployment tool and the stack wizard by adding support for Azure.
+
+Moreover, we implemented a new step operator that allows you to run individual 
+steps of your pipeline in Kubernetes pods.
+
+Lastly, we have simplified our pipeline models by removing their versions.
+
+## What's Changed
+* Enable cloud build service in GCP stack deployment by @stefannica in https://github.com/zenml-io/zenml/pull/2864
+* Adding a `logo_url` and the of the `integration` to component responses by @bcdurak in https://github.com/zenml-io/zenml/pull/2866
+* Use REST in Model tests by @avishniakov in https://github.com/zenml-io/zenml/pull/2834
+* Add Azure stack wizard by @avishniakov in https://github.com/zenml-io/zenml/pull/2841
+* Migration testing for 0.62.0 by @schustmi in https://github.com/zenml-io/zenml/pull/2860
+* Fix RBAC in combination with lazy loaders by @schustmi in https://github.com/zenml-io/zenml/pull/2869
+* Misc cleanup after release by @schustmi in https://github.com/zenml-io/zenml/pull/2861
+* Disable notebook error for Kubernetes orchestrator by @strickvl in https://github.com/zenml-io/zenml/pull/2870
+* Added ability to add labels to k8s pod by @htahir1 in https://github.com/zenml-io/zenml/pull/2872
+* Fix zenml pro links by @schustmi in https://github.com/zenml-io/zenml/pull/2875
+* Fix mlstacks docs typo by @begoechavarren in https://github.com/zenml-io/zenml/pull/2878
+* Fix requests vulnerability by @stefannica in https://github.com/zenml-io/zenml/pull/2843
+* Fixed some minor docs things i noticed by @htahir1 in https://github.com/zenml-io/zenml/pull/2881
+* Serialize source as Any to keep subclass attributes by @schustmi in https://github.com/zenml-io/zenml/pull/2880
+* Fix node selectors for Vertex orchestrator by @schustmi in https://github.com/zenml-io/zenml/pull/2876
+* Kubernetes step operator by @schustmi in https://github.com/zenml-io/zenml/pull/2883
+* Automatically populate GCP/azure path when using wizard from the frontend by @schustmi in https://github.com/zenml-io/zenml/pull/2886
+* Remove pipeline versioning and add run templates by @schustmi in https://github.com/zenml-io/zenml/pull/2830
+* Implement the Azure 1-click stack deployment by @stefannica in https://github.com/zenml-io/zenml/pull/2887
+* Better error message sagemaker, better documentation server env vars by @AlexejPenner in https://github.com/zenml-io/zenml/pull/2885
+* Azure Stack Wizard docs by @bcdurak in https://github.com/zenml-io/zenml/pull/2890
+* Docs update mlflow deploy function call by @safoinme in https://github.com/zenml-io/zenml/pull/2863
+* Fix databricks resource setting by @safoinme in https://github.com/zenml-io/zenml/pull/2889
+
+## New Contributors
+* @begoechavarren made their first contribution in https://github.com/zenml-io/zenml/pull/2878
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.62.0...0.63.0
+
 # 0.62.0
 
 Building on top of the last release, this release adds a new and easy way to deploy a GCP ZenML stack from the dashboard and the CLI. Give it a try by going to the `Stacks` section in the dashboard or running the `zenml stack deploy` command! For more information on this new feature, please do check out [the video and blog](https://www.zenml.io/blog/easy-mlops-pipelines) from our previous release.

zenml/VERSION

@@ -1 +1 @@
-0.62.0.dev20240729
+0.63.0.dev20240731

zenml/actions/pipeline_run/pipeline_run_action.py

@@ -33,8 +33,8 @@ from zenml.models import (
 )
 from zenml.models.v2.base.base import BaseResponse
 from zenml.zen_server.auth import AuthContext
-from zenml.zen_server.pipeline_deployment.utils import run_pipeline
 from zenml.zen_server.rbac.models import ResourceType
+from zenml.zen_server.template_execution.utils import run_template
 from zenml.zen_server.utils import server_config
 
 logger = get_logger(__name__)
@@ -45,7 +45,7 @@ logger = get_logger(__name__)
 class PipelineRunActionConfiguration(ActionConfig):
     """Configuration class to configure a pipeline run action."""
 
-    pipeline_deployment_id: UUID
+    template_id: UUID
     run_config: Optional[PipelineRunConfiguration] = None
 
 
@@ -95,10 +95,10 @@ class PipelineRunActionHandler(BaseActionHandler):
 
         assert isinstance(config, PipelineRunActionConfiguration)
 
-        deployment = zen_store().get_deployment(config.pipeline_deployment_id)
-        logger.debug("Running deployment:", deployment)
-        run_pipeline(
-            deployment=deployment,
+        template = zen_store().get_run_template(config.template_id)
+        logger.debug("Running template:", template)
+        run_template(
+            template=template,
             run_config=config.run_config,
             auth_context=auth_context,
         )
@@ -114,13 +114,14 @@ class PipelineRunActionHandler(BaseActionHandler):
         Raises:
             ValueError: In case no deployment can be found with the deployment_id
         """
-        deployment_id = config.pipeline_deployment_id
         zen_store = GlobalConfiguration().zen_store
 
         try:
-            zen_store.get_deployment(deployment_id=deployment_id)
+            zen_store.get_run_template(template_id=config.template_id)
         except KeyError:
-            raise ValueError(f"No deployment found with id {deployment_id}.")
+            raise ValueError(
+                f"No template found with id {config.template_id}."
+            )
 
     def _validate_action_request(
         self, action: ActionRequest, config: ActionConfig
@@ -176,27 +177,28 @@ class PipelineRunActionHandler(BaseActionHandler):
             List of resources related to the action.
 
         Raises:
-            ValueError: In case the deployment_id does not exist.
+            ValueError: In case the specified template does not exist.
         """
         assert isinstance(action_config, PipelineRunActionConfiguration)
 
-        deployment_id = action_config.pipeline_deployment_id
         zen_store = GlobalConfiguration().zen_store
 
         try:
-            deployment = zen_store.get_deployment(
-                deployment_id=deployment_id, hydrate=hydrate
+            template = zen_store.get_run_template(
+                template_id=action_config.template_id, hydrate=hydrate
             )
         except KeyError:
-            raise ValueError(f"No deployment found with id {deployment_id}.")
+            raise ValueError(
+                f"No template found with id {action_config.template_id}."
+            )
 
         resources: Dict[ResourceType, BaseResponse[Any, Any, Any]] = {
-            ResourceType.PIPELINE_DEPLOYMENT: deployment
+            ResourceType.RUN_TEMPLATE: template
         }
 
-        if deployment.pipeline is not None:
+        if template.pipeline is not None:
             pipeline = zen_store.get_pipeline(
-                pipeline_id=deployment.pipeline.id, hydrate=hydrate
+                pipeline_id=template.pipeline.id, hydrate=hydrate
             )
             resources[ResourceType.PIPELINE] = pipeline
 

zenml/analytics/enums.py

@@ -48,6 +48,10 @@ class AnalyticsEvent(str, Enum):
     CREATED_TRIGGER = "Trigger created"
     UPDATED_TRIGGER = "Trigger updated"
 
+    # Templates
+    CREATED_RUN_TEMPLATE = "Run template created"
+    EXECUTED_RUN_TEMPLATE = "Run templated executed"
+
     # Model Control Plane
     MODEL_DEPLOYED = "Model deployed"
     CREATED_MODEL = "Model created"

zenml/cli/__init__.py

@@ -1684,7 +1684,7 @@ def my_pipeline(...):
 
 You can register your pipeline like this:
 ```bash
-zenml pipeline register my_pipeline
+zenml pipeline register run.my_pipeline
 ```
 
 To list all registered pipelines, use:
@@ -1693,21 +1693,15 @@ To list all registered pipelines, use:
 zenml pipeline list
 ```
 
-Since every pipeline run creates a new pipeline by default, you might
-occasionally want to delete a pipeline, which you can do via:
+To delete a pipeline, run:
 
 ```bash
 zenml pipeline delete <PIPELINE_NAME>
 ```
 
-This will delete the latest pipeline version and change all corresponding
+This will delete the pipeline and change all corresponding
 pipeline runs to become unlisted (not linked to any pipeline).
 
-If you want to delete all versions of a pipeline, you can do so as follows:
-
-```bash
-zenml pipeline delete <PIPELINE_NAME> --all-versions
-```
 
 To list all pipeline runs that you have executed, use:
 
@@ -2378,11 +2372,30 @@ zenml code-repository delete <REPOSITORY_NAME_OR_ID>
 Building an image without Runs
 ------------------------------
 
+To build or run a pipeline from the CLI, you need to know the source path of
+your pipeline. Let's imagine you have defined your pipeline in a python file
+called `run.py` like this:
+
+```python
+from zenml import pipeline
+
+@pipeline
+def my_pipeline(...):
+   # Connect your pipeline steps here
+   pass
+```
+
+The source path of your pipeline will be `run.my_pipeline`. In a generalized
+way, this will be `<MODULE_PATH>.<PIPELINE_FUNCTION_NAME>`. If the python file
+defining the pipeline is not in your current directory, the module path consists
+of the full path to the file, separated by dots, e.g.
+`some_directory.some_file.my_pipeline`.
+
 To [build Docker images for your pipeline](https://docs.zenml.io/how-to/customize-docker-builds)
 without actually running the pipeline, use:
 
 ```bash
-zenml pipeline build <PIPELINE_ID_OR_NAME>
+zenml pipeline build <PIPELINE_SOURCE_PATH>
 ```
 
 To specify settings for the Docker builds, use the `--config/-c` option of the
@@ -2390,20 +2403,20 @@ command. For more information about the structure of this configuration file,
 check out the `zenml.pipelines.base_pipeline.BasePipeline.build(...)` method.
 
 ```bash
-zenml pipeline build <PIPELINE_ID_OR_NAME> --config=<PATH_TO_CONFIG_YAML>
+zenml pipeline build <PIPELINE_SOURCE_PATH> --config=<PATH_TO_CONFIG_YAML>
 ```
 
 If you want to build the pipeline for a stack other than your current active
 stack, use the `--stack` option.
 
 ```bash
-zenml pipeline build <PIPELINE_ID_OR_NAME> --stack=<STACK_ID_OR_NAME>
+zenml pipeline build <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
 ```
 
 To run a pipeline that was previously registered, use:
 
 ```bash
-zenml pipeline run  <PIPELINE_ID_OR_NAME>
+zenml pipeline run <PIPELINE_SOURCE_PATH>
 ```
 
 To specify settings for the pipeline, use the `--config/-c` option of the
@@ -2411,14 +2424,14 @@ command. For more information about the structure of this configuration file,
 check out the `zenml.pipelines.base_pipeline.BasePipeline.run(...)` method.
 
 ```bash
-zenml pipeline run <PIPELINE_ID_OR_NAME> --config=<PATH_TO_CONFIG_YAML>
+zenml pipeline run <PIPELINE_SOURCE_PATH> --config=<PATH_TO_CONFIG_YAML>
 ```
 
 If you want to run the pipeline on a stack different than your current active
 stack, use the `--stack` option.
 
 ```bash
-zenml pipeline run <PIPELINE_ID_OR_NAME> --stack=<STACK_ID_OR_NAME>
+zenml pipeline run <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
 ```
 
 Tagging your resources with ZenML

zenml/cli/base.py

@@ -83,7 +83,7 @@ ZENML_PROJECT_TEMPLATES = dict(
     ),
     starter=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-starter",
-        github_tag="2024.06.06",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.07.26",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",

zenml/cli/pipeline.py

@@ -121,15 +121,13 @@ def register_pipeline(
     pipeline_instance.register()
 
 
-@pipeline.command("build", help="Build Docker images for a pipeline.")
-@click.argument("pipeline_name_or_id")
-@click.option(
-    "--version",
-    "-v",
-    type=str,
-    required=False,
-    help="Optional version of the pipeline.",
+@pipeline.command(
+    "build",
+    help="Build Docker images for a pipeline. The SOURCE argument needs to be "
+    " an importable source path resolving to a ZenML pipeline instance, e.g. "
+    "`my_module.my_pipeline_instance`.",
 )
+@click.argument("source")
 @click.option(
     "--config",
     "-c",
@@ -155,8 +153,7 @@ def register_pipeline(
     help="Output path for the build information.",
 )
 def build_pipeline(
-    pipeline_name_or_id: str,
-    version: Optional[str] = None,
+    source: str,
     config_path: Optional[str] = None,
     stack_name_or_id: Optional[str] = None,
     output_path: Optional[str] = None,
@@ -164,8 +161,7 @@ def build_pipeline(
     """Build Docker images for a pipeline.
 
     Args:
-        pipeline_name_or_id: Name or ID of the pipeline.
-        version: Version of the pipeline.
+        source: Importable source resolving to a pipeline instance.
         config_path: Path to pipeline configuration file.
         stack_name_or_id: Name or ID of the stack for which the images should
             be built.
@@ -180,13 +176,28 @@ def build_pipeline(
             "your source code root."
         )
 
-    pipeline_model = Client().get_pipeline(
-        name_id_or_prefix=pipeline_name_or_id, version=version
-    )
+    try:
+        pipeline_instance = source_utils.load(source)
+    except ModuleNotFoundError as e:
+        source_root = source_utils.get_source_root()
+        cli_utils.error(
+            f"Unable to import module `{e.name}`. Make sure the source path is "
+            f"relative to your source root `{source_root}`."
+        )
+    except AttributeError as e:
+        cli_utils.error("Unable to load attribute from module: " + str(e))
+
+    if not isinstance(pipeline_instance, Pipeline):
+        cli_utils.error(
+            f"The given source path `{source}` does not resolve to a pipeline "
+            "object."
+        )
 
     with cli_utils.temporary_active_stack(stack_name_or_id=stack_name_or_id):
-        pipeline_instance = Pipeline.from_model(pipeline_model)
-        build = pipeline_instance.build(config_path=config_path)
+        pipeline_instance = pipeline_instance.with_options(
+            config_path=config_path
+        )
+        build = pipeline_instance.build()
 
     if build:
         cli_utils.declare(f"Created pipeline build `{build.id}`.")
@@ -200,15 +211,13 @@ def build_pipeline(
         cli_utils.declare("No docker builds required.")
 
 
-@pipeline.command("run", help="Run a pipeline.")
-@click.argument("pipeline_name_or_id")
-@click.option(
-    "--version",
-    "-v",
-    type=str,
-    required=False,
-    help="Optional version of the pipeline.",
+@pipeline.command(
+    "run",
+    help="Run a pipeline. The SOURCE argument needs to be an "
+    "importable source path resolving to a ZenML pipeline instance, e.g. "
+    "`my_module.my_pipeline_instance`.",
 )
+@click.argument("source")
 @click.option(
     "--config",
     "-c",
@@ -241,8 +250,7 @@ def build_pipeline(
     help="Prevent automatic build reusing.",
 )
 def run_pipeline(
-    pipeline_name_or_id: str,
-    version: Optional[str] = None,
+    source: str,
     config_path: Optional[str] = None,
     stack_name_or_id: Optional[str] = None,
     build_path_or_id: Optional[str] = None,
@@ -251,8 +259,7 @@ def run_pipeline(
     """Run a pipeline.
 
     Args:
-        pipeline_name_or_id: Name or ID of the pipeline.
-        version: Version of the pipeline.
+        source: Importable source resolving to a pipeline instance.
         config_path: Path to pipeline configuration file.
         stack_name_or_id: Name or ID of the stack on which the pipeline should
             run.
@@ -270,9 +277,22 @@ def run_pipeline(
             "your source code root."
         )
 
-    pipeline_model = Client().get_pipeline(
-        name_id_or_prefix=pipeline_name_or_id, version=version
-    )
+    try:
+        pipeline_instance = source_utils.load(source)
+    except ModuleNotFoundError as e:
+        source_root = source_utils.get_source_root()
+        cli_utils.error(
+            f"Unable to import module `{e.name}`. Make sure the source path is "
+            f"relative to your source root `{source_root}`."
+        )
+    except AttributeError as e:
+        cli_utils.error("Unable to load attribute from module: " + str(e))
+
+    if not isinstance(pipeline_instance, Pipeline):
+        cli_utils.error(
+            f"The given source path `{source}` does not resolve to a pipeline "
+            "object."
+        )
 
     build: Union[str, PipelineBuildBase, None] = None
     if build_path_or_id:
@@ -287,7 +307,6 @@ def run_pipeline(
             )
 
     with cli_utils.temporary_active_stack(stack_name_or_id=stack_name_or_id):
-        pipeline_instance = Pipeline.from_model(pipeline_model)
         pipeline_instance = pipeline_instance.with_options(
             config_path=config_path,
             build=build,
@@ -320,18 +339,6 @@ def list_pipelines(**kwargs: Any) -> None:
 
 @pipeline.command("delete")
 @click.argument("pipeline_name_or_id", type=str, required=True)
-@click.option(
-    "--version",
-    "-v",
-    help="Optional pipeline version.",
-    type=str,
-    required=False,
-)
-@click.option(
-    "--all-versions",
-    help="Delete all versions of the pipeline..",
-    is_flag=True,
-)
 @click.option(
     "--yes",
     "-y",
@@ -340,28 +347,18 @@ def list_pipelines(**kwargs: Any) -> None:
 )
 def delete_pipeline(
     pipeline_name_or_id: str,
-    version: Optional[str] = None,
-    all_versions: bool = False,
     yes: bool = False,
 ) -> None:
     """Delete a pipeline.
 
     Args:
         pipeline_name_or_id: The name or ID of the pipeline to delete.
-        version: The version of the pipeline to delete.
-        all_versions: If set, delete all versions of the pipeline.
         yes: If set, don't ask for confirmation.
     """
-    version_suffix = ""
-    if all_versions:
-        version_suffix = " (all versions)"
-    elif version:
-        version_suffix = f" (version {version})"
-
     if not yes:
         confirmation = cli_utils.confirmation(
             f"Are you sure you want to delete pipeline "
-            f"`{pipeline_name_or_id}{version_suffix}`? This will change all "
+            f"`{pipeline_name_or_id}`? This will change all "
             "existing runs of this pipeline to become unlisted."
         )
         if not confirmation:
@@ -371,15 +368,11 @@ def delete_pipeline(
     try:
         Client().delete_pipeline(
             name_id_or_prefix=pipeline_name_or_id,
-            version=version,
-            all_versions=all_versions,
         )
     except KeyError as e:
         cli_utils.error(str(e))
     else:
-        cli_utils.declare(
-            f"Deleted pipeline `{pipeline_name_or_id}{version_suffix}`."
-        )
+        cli_utils.declare(f"Deleted pipeline `{pipeline_name_or_id}`.")
 
 
 @pipeline.group()

zenml/cli/stack.py

@@ -1720,7 +1720,7 @@ def deploy(
                 + deployment.description
             )
         )
-        console.print(Markdown("## Instructions\n" + deployment.instructions))
+        console.print(Markdown("## Details\n" + deployment.instructions))
 
         deployment_config = client.zen_store.get_stack_deployment_config(
             provider=StackDeploymentProvider(provider),
@@ -1728,16 +1728,13 @@ def deploy(
             location=location,
         )
 
-        if deployment_config.configuration:
+        if deployment_config.instructions:
             console.print(
-                Markdown(
-                    "## Configuration\n"
-                    "You will be asked to provide the following configuration "
-                    "values during the deployment process:"
-                ),
+                Markdown("## Instructions\n" + deployment_config.instructions),
                 "\n",
             )
 
+        if deployment_config.configuration:
             console.print(
                 deployment_config.configuration,
                 no_wrap=True,
@@ -1748,7 +1745,8 @@ def deploy(
 
         if not cli_utils.confirmation(
             "\n\nProceed to continue with the deployment. You will be "
-            f"automatically redirected to {provider.upper()} in your browser.",
+            f"automatically redirected to "
+            f"{deployment_config.deployment_url_text} in your browser.",
         ):
             raise click.Abort()