zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl

zenml/zen_server/deploy/helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zenml
-version: "0.62.0"
+version: "0.64.0"
 description: Open source MLOps framework for portable production ready ML pipelines
 keywords:
 - mlops

zenml/zen_server/deploy/helm/README.md

@@ -20,8 +20,8 @@ ZenML is an open-source MLOps framework designed to help you create robust, main
 To install the ZenML chart directly from Amazon ECR, use the following command:
 
 ```bash
-# example command for version 0.62.0
-helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.62.0
+# example command for version 0.64.0
+helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.64.0
 ```
 
 Note: Ensure you have OCI support enabled in your Helm client and that you are authenticated with Amazon ECR.

zenml/zen_server/rbac/models.py

@@ -58,6 +58,7 @@ class ResourceType(StrEnum):
     PIPELINE_RUN = "pipeline_run"
     PIPELINE_DEPLOYMENT = "pipeline_deployment"
     PIPELINE_BUILD = "pipeline_build"
+    RUN_TEMPLATE = "run_template"
     USER = "user"
     SERVICE = "service"
     RUN_METADATA = "run_metadata"

zenml/zen_server/rbac/utils.py

@@ -405,6 +405,7 @@ def get_resource_type_for_model(
         PipelineResponse,
         PipelineRunResponse,
         RunMetadataResponse,
+        RunTemplateResponse,
         SecretResponse,
         ServiceAccountResponse,
         ServiceConnectorResponse,
@@ -440,6 +441,7 @@ def get_resource_type_for_model(
         PipelineDeploymentResponse: ResourceType.PIPELINE_DEPLOYMENT,
         PipelineBuildResponse: ResourceType.PIPELINE_BUILD,
         PipelineRunResponse: ResourceType.PIPELINE_RUN,
+        RunTemplateResponse: ResourceType.RUN_TEMPLATE,
         TagResponse: ResourceType.TAG,
         TriggerResponse: ResourceType.TRIGGER,
         TriggerExecutionResponse: ResourceType.TRIGGER_EXECUTION,
@@ -556,6 +558,7 @@ def get_schema_for_resource_type(
         PipelineRunSchema,
         PipelineSchema,
         RunMetadataSchema,
+        RunTemplateSchema,
         SecretSchema,
         ServiceConnectorSchema,
         ServiceSchema,
@@ -587,6 +590,7 @@ def get_schema_for_resource_type(
         ResourceType.PIPELINE_RUN: PipelineRunSchema,
         ResourceType.PIPELINE_DEPLOYMENT: PipelineDeploymentSchema,
         ResourceType.PIPELINE_BUILD: PipelineBuildSchema,
+        ResourceType.RUN_TEMPLATE: RunTemplateSchema,
         ResourceType.RUN_METADATA: RunMetadataSchema,
         ResourceType.USER: UserSchema,
         ResourceType.ACTION: ActionSchema,

zenml/zen_server/routers/pipeline_builds_endpoints.py

@@ -13,18 +13,15 @@
 #  permissions and limitations under the License.
 """Endpoint definitions for builds."""
 
-from typing import Optional
 from uuid import UUID
 
-from fastapi import APIRouter, BackgroundTasks, Depends, Security
+from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.constants import API, PIPELINE_BUILDS, VERSION_1
 from zenml.models import (
     Page,
     PipelineBuildFilter,
     PipelineBuildResponse,
-    PipelineRunResponse,
 )
 from zenml.zen_server.auth import AuthContext, authorize
 from zenml.zen_server.exceptions import error_response
@@ -33,12 +30,10 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_get_entity,
     verify_permissions_and_list_entities,
 )
-from zenml.zen_server.rbac.models import Action, ResourceType
-from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.rbac.models import ResourceType
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
-    server_config,
     zen_store,
 )
 
@@ -126,62 +121,3 @@ def delete_build(
         get_method=zen_store().get_build,
         delete_method=zen_store().delete_build,
     )
-
-
-if server_config().workload_manager_enabled:
-
-    @router.post(
-        "/{build_id}/runs",
-        responses={
-            401: error_response,
-            404: error_response,
-            422: error_response,
-        },
-    )
-    @handle_exceptions
-    def run_build(
-        build_id: UUID,
-        background_tasks: BackgroundTasks,
-        config: Optional[PipelineRunConfiguration] = None,
-        auth_context: AuthContext = Security(authorize),
-    ) -> PipelineRunResponse:
-        """Run a pipeline from a pipeline build.
-
-        Args:
-            build_id: The ID of the build.
-            background_tasks: Background tasks.
-            config: Configuration for the pipeline run.
-            auth_context: Authentication context.
-
-        Raises:
-            ValueError: If the build does not have an associated deployment.
-
-        Returns:
-            The created run.
-        """
-        from zenml.zen_server.pipeline_deployment.utils import (
-            run_pipeline,
-        )
-
-        build = verify_permissions_and_get_entity(
-            id=build_id, get_method=zen_store().get_build, hydrate=True
-        )
-
-        verify_permission(
-            resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
-        )
-
-        if not build.template_deployment_id:
-            raise ValueError("Build does not have template deployment.")
-
-        deployment = zen_store().get_deployment(
-            deployment_id=build.template_deployment_id, hydrate=True
-        )
-        deployment.get_metadata().build = build
-
-        return run_pipeline(
-            deployment=deployment,
-            run_config=config,
-            background_tasks=background_tasks,
-            auth_context=auth_context,
-        )

zenml/zen_server/routers/pipeline_deployments_endpoints.py

@@ -13,18 +13,15 @@
 #  permissions and limitations under the License.
 """Endpoint definitions for deployments."""
 
-from typing import Optional
 from uuid import UUID
 
-from fastapi import APIRouter, BackgroundTasks, Depends, Security
+from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.constants import API, PIPELINE_DEPLOYMENTS, VERSION_1
 from zenml.models import (
     Page,
     PipelineDeploymentFilter,
     PipelineDeploymentResponse,
-    PipelineRunResponse,
 )
 from zenml.zen_server.auth import AuthContext, authorize
 from zenml.zen_server.exceptions import error_response
@@ -33,8 +30,7 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_get_entity,
     verify_permissions_and_list_entities,
 )
-from zenml.zen_server.rbac.models import Action, ResourceType
-from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.rbac.models import ResourceType
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
@@ -133,53 +129,6 @@ def delete_deployment(
 
 if server_config().workload_manager_enabled:
 
-    @router.post(
-        "/{deployment_id}/runs",
-        responses={
-            401: error_response,
-            404: error_response,
-            422: error_response,
-        },
-    )
-    @handle_exceptions
-    def create_deployment_run(
-        deployment_id: UUID,
-        background_tasks: BackgroundTasks,
-        config: Optional[PipelineRunConfiguration] = None,
-        auth_context: AuthContext = Security(authorize),
-    ) -> PipelineRunResponse:
-        """Run a pipeline from a pipeline deployment.
-
-        Args:
-            deployment_id: The ID of the deployment.
-            background_tasks: Background tasks.
-            config: Configuration for the pipeline run.
-            auth_context: Authentication context.
-
-        Returns:
-            The created run.
-        """
-        from zenml.zen_server.pipeline_deployment.utils import (
-            run_pipeline,
-        )
-
-        deployment = verify_permissions_and_get_entity(
-            id=deployment_id,
-            get_method=zen_store().get_deployment,
-            hydrate=True,
-        )
-
-        verify_permission(
-            resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
-        )
-
-        return run_pipeline(
-            deployment=deployment,
-            run_config=config,
-            background_tasks=background_tasks,
-            auth_context=auth_context,
-        )
-
     @router.get(
         "/{deployment_id}/logs",
         responses={

zenml/zen_server/routers/pipelines_endpoints.py

@@ -17,10 +17,8 @@ from uuid import UUID
 
 from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_spec import PipelineSpec
 from zenml.constants import (
     API,
-    PIPELINE_SPEC,
     PIPELINES,
     REPORTABLE_RESOURCES,
     RUNS,
@@ -29,8 +27,6 @@ from zenml.constants import (
 from zenml.models import (
     Page,
     PipelineFilter,
-    PipelineNamespaceFilter,
-    PipelineNamespaceResponse,
     PipelineResponse,
     PipelineRunFilter,
     PipelineRunResponse,
@@ -46,9 +42,6 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_update_entity,
 )
 from zenml.zen_server.rbac.models import ResourceType
-from zenml.zen_server.rbac.utils import (
-    get_allowed_resource_ids,
-)
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
@@ -64,7 +57,6 @@ router = APIRouter(
 
 @router.get(
     "",
-    response_model=Page[PipelineResponse],
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -96,7 +88,6 @@ def list_pipelines(
 
 @router.get(
     "/{pipeline_id}",
-    response_model=PipelineResponse,
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -122,7 +113,6 @@ def get_pipeline(
 
 @router.put(
     "/{pipeline_id}",
-    response_model=PipelineResponse,
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -179,7 +169,6 @@ def delete_pipeline(
 
 @router.get(
     "/{pipeline_id}" + RUNS,
-    response_model=Page[PipelineRunResponse],
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -201,66 +190,4 @@ def list_pipeline_runs(
     Returns:
         The pipeline runs according to query filters.
     """
-    return zen_store().list_runs(pipeline_run_filter_model)
-
-
-@router.get(
-    "/{pipeline_id}" + PIPELINE_SPEC,
-    response_model=PipelineSpec,
-    responses={401: error_response, 404: error_response, 422: error_response},
-)
-@handle_exceptions
-def get_pipeline_spec(
-    pipeline_id: UUID,
-    _: AuthContext = Security(authorize),
-) -> PipelineSpec:
-    """Gets the spec of a specific pipeline using its unique id.
-
-    Args:
-        pipeline_id: ID of the pipeline to get.
-
-    Returns:
-        The spec of the pipeline.
-    """
-    pipeline = verify_permissions_and_get_entity(
-        id=pipeline_id, get_method=zen_store().get_pipeline
-    )
-    return pipeline.spec
-
-
-namespace_router = APIRouter(
-    prefix=API + VERSION_1 + "/pipeline_namespaces",
-    tags=["pipeline_namespaces"],
-    responses={401: error_response, 403: error_response},
-)
-
-
-@namespace_router.get(
-    "",
-    responses={401: error_response, 404: error_response, 422: error_response},
-)
-@handle_exceptions
-def list_pipeline_namespaces(
-    filter_model: PipelineNamespaceFilter = Depends(
-        make_dependable(PipelineNamespaceFilter)
-    ),
-    hydrate: bool = False,
-    auth_context: AuthContext = Security(authorize),
-) -> Page[PipelineNamespaceResponse]:
-    """Gets a list of pipeline namespaces.
-
-    Args:
-        filter_model: Filter model used for pagination, sorting,
-            filtering.
-        hydrate: Flag deciding whether to hydrate the output model(s)
-            by including metadata fields in the response.
-        auth_context: Authentication context.
-
-    Returns:
-        List of pipeline namespace objects.
-    """
-    allowed_ids = get_allowed_resource_ids(resource_type=ResourceType.PIPELINE)
-    filter_model.configure_rbac(
-        authenticated_user_id=auth_context.user.id, id=allowed_ids
-    )
-    return zen_store().list_pipeline_namespaces(filter_model, hydrate=hydrate)
+    return zen_store().list_runs(pipeline_run_filter_model, hydrate=hydrate)

zenml/zen_server/routers/run_templates_endpoints.py

@@ -0,0 +1,212 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Endpoint definitions for run templates."""
+
+from typing import Optional
+from uuid import UUID
+
+from fastapi import APIRouter, BackgroundTasks, Depends, Security
+
+from zenml.analytics.enums import AnalyticsEvent
+from zenml.analytics.utils import track_handler
+from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
+from zenml.constants import API, RUN_TEMPLATES, VERSION_1
+from zenml.models import (
+    Page,
+    PipelineRunResponse,
+    RunTemplateFilter,
+    RunTemplateResponse,
+    RunTemplateUpdate,
+)
+from zenml.zen_server.auth import AuthContext, authorize
+from zenml.zen_server.exceptions import error_response
+from zenml.zen_server.rbac.endpoint_utils import (
+    verify_permissions_and_delete_entity,
+    verify_permissions_and_get_entity,
+    verify_permissions_and_list_entities,
+    verify_permissions_and_update_entity,
+)
+from zenml.zen_server.rbac.models import Action, ResourceType
+from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.utils import (
+    handle_exceptions,
+    make_dependable,
+    server_config,
+    zen_store,
+)
+
+router = APIRouter(
+    prefix=API + VERSION_1 + RUN_TEMPLATES,
+    tags=["run_templates"],
+    responses={401: error_response, 403: error_response},
+)
+
+
+@router.get(
+    "",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def list_run_templates(
+    filter_model: RunTemplateFilter = Depends(
+        make_dependable(RunTemplateFilter)
+    ),
+    hydrate: bool = False,
+    _: AuthContext = Security(authorize),
+) -> Page[RunTemplateResponse]:
+    """Get a page of run templates.
+
+    Args:
+        filter_model: Filter model used for pagination, sorting,
+            filtering.
+        hydrate: Flag deciding whether to hydrate the output model(s)
+            by including metadata fields in the response.
+
+    Returns:
+        Page of run templates.
+    """
+    return verify_permissions_and_list_entities(
+        filter_model=filter_model,
+        resource_type=ResourceType.RUN_TEMPLATE,
+        list_method=zen_store().list_run_templates,
+        hydrate=hydrate,
+    )
+
+
+@router.get(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def get_run_template(
+    template_id: UUID,
+    hydrate: bool = True,
+    _: AuthContext = Security(authorize),
+) -> RunTemplateResponse:
+    """Get a run template.
+
+    Args:
+        template_id: ID of the run template to get.
+        hydrate: Flag deciding whether to hydrate the output model(s)
+            by including metadata fields in the response.
+
+    Returns:
+        The run template.
+    """
+    return verify_permissions_and_get_entity(
+        id=template_id,
+        get_method=zen_store().get_run_template,
+        hydrate=hydrate,
+    )
+
+
+@router.put(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def update_run_template(
+    template_id: UUID,
+    update: RunTemplateUpdate,
+    _: AuthContext = Security(authorize),
+) -> RunTemplateResponse:
+    """Update a run template.
+
+    Args:
+        template_id: ID of the run template to get.
+        update: The updates to apply.
+
+    Returns:
+        The updated run template.
+    """
+    return verify_permissions_and_update_entity(
+        id=template_id,
+        update_model=update,
+        get_method=zen_store().get_run_template,
+        update_method=zen_store().update_run_template,
+    )
+
+
+@router.delete(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def delete_run_template(
+    template_id: UUID,
+    _: AuthContext = Security(authorize),
+) -> None:
+    """Delete a run template.
+
+    Args:
+        template_id: ID of the run template to delete.
+    """
+    verify_permissions_and_delete_entity(
+        id=template_id,
+        get_method=zen_store().get_run_template,
+        delete_method=zen_store().delete_run_template,
+    )
+
+
+if server_config().workload_manager_enabled:
+
+    @router.post(
+        "/{template_id}/runs",
+        responses={
+            401: error_response,
+            404: error_response,
+            422: error_response,
+        },
+    )
+    @handle_exceptions
+    def create_template_run(
+        template_id: UUID,
+        background_tasks: BackgroundTasks,
+        config: Optional[PipelineRunConfiguration] = None,
+        auth_context: AuthContext = Security(authorize),
+    ) -> PipelineRunResponse:
+        """Run a pipeline from a template.
+
+        Args:
+            template_id: The ID of the template.
+            background_tasks: Background tasks.
+            config: Configuration for the pipeline run.
+            auth_context: Authentication context.
+
+        Returns:
+            The created pipeline run.
+        """
+        from zenml.zen_server.template_execution.utils import run_template
+
+        with track_handler(event=AnalyticsEvent.EXECUTED_RUN_TEMPLATE):
+            template = verify_permissions_and_get_entity(
+                id=template_id,
+                get_method=zen_store().get_run_template,
+                hydrate=True,
+            )
+
+            verify_permission(
+                resource_type=ResourceType.PIPELINE_DEPLOYMENT,
+                action=Action.CREATE,
+            )
+            verify_permission(
+                resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
+            )
+
+            return run_template(
+                template=template,
+                auth_context=auth_context,
+                background_tasks=background_tasks,
+                run_config=config,
+            )

zenml/zen_server/routers/stack_deployment_endpoints.py

@@ -78,6 +78,7 @@ def get_stack_deployment_config(
     provider: StackDeploymentProvider,
     stack_name: str,
     location: Optional[str] = None,
+    terraform: bool = False,
     auth_context: AuthContext = Security(authorize),
 ) -> StackDeploymentConfig:
     """Return the URL to deploy the ZenML stack to the specified cloud provider.
@@ -87,6 +88,7 @@ def get_stack_deployment_config(
         provider: The stack deployment provider.
         stack_name: The name of the stack.
         location: The location where the stack should be deployed.
+        terraform: Whether the stack should be deployed using Terraform.
         auth_context: The authentication context.
 
     Returns:
@@ -118,6 +120,7 @@ def get_stack_deployment_config(
     api_token = token.encode(expires=expires)
 
     return stack_deployment_class(
+        terraform=terraform,
         stack_name=stack_name,
         location=location,
         zenml_server_url=str(url),
@@ -134,6 +137,7 @@ def get_deployed_stack(
     stack_name: str,
     location: Optional[str] = None,
     date_start: Optional[datetime.datetime] = None,
+    terraform: bool = False,
     _: AuthContext = Security(authorize),
 ) -> Optional[DeployedStack]:
     """Return a matching ZenML stack that was deployed and registered.
@@ -143,6 +147,7 @@ def get_deployed_stack(
         stack_name: The name of the stack.
         location: The location where the stack should be deployed.
         date_start: The date when the deployment started.
+        terraform: Whether the stack was deployed using Terraform.
 
     Returns:
         The ZenML stack that was deployed and registered or None if the stack
@@ -150,6 +155,7 @@ def get_deployed_stack(
     """
     stack_deployment_class = get_stack_deployment_class(provider)
     return stack_deployment_class(
+        terraform=terraform,
         stack_name=stack_name,
         location=location,
         # These fields are not needed for this operation

zenml/zen_server/routers/users_endpoints.py

@@ -286,7 +286,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         # - active
         # - password
         # - email_opted_in + email
-        # - hub_token
         #
         safe_user_update = user_update.create_copy(
             exclude={
@@ -298,7 +297,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
                 "old_password",
                 "email_opted_in",
                 "email",
-                "hub_token",
             },
         )
 
@@ -387,7 +385,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         if (
             user_update.email_opted_in is not None
             or user_update.email is not None
-            or user_update.hub_token is not None
         ):
             if user.id != auth_context.user.id:
                 raise IllegalOperationError(
@@ -399,8 +396,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
             if safe_user_update.email_opted_in is not None:
                 safe_user_update.email_opted_in = user_update.email_opted_in
                 safe_user_update.email = user_update.email
-            if safe_user_update.hub_token is not None:
-                safe_user_update.hub_token = user_update.hub_token
 
         updated_user = zen_store().update_user(
             user_id=user.id,
@@ -444,7 +439,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         # - is_admin
         # - active
         # - old_password
-        # - hub_token
         #
         safe_user_update = user_update.create_copy(
             exclude={
@@ -453,7 +447,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
                 "is_admin",
                 "active",
                 "old_password",
-                "hub_token",
             },
         )