zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.62.0.dev20240730__py3-none-any.whl

zenml/steps/utils.py

@@ -413,7 +413,6 @@ def log_step_metadata(
     metadata: Dict[str, "MetadataType"],
     step_name: Optional[str] = None,
     pipeline_name_id_or_prefix: Optional[Union[str, UUID]] = None,
-    pipeline_version: Optional[str] = None,
     run_id: Optional[str] = None,
 ) -> None:
     """Logs step metadata.
@@ -424,8 +423,6 @@ def log_step_metadata(
             when being called inside a step.
         pipeline_name_id_or_prefix: The name of the pipeline to log metadata
             for. Can be omitted when being called inside a step.
-        pipeline_version: The version of the pipeline to log metadata for.
-            Can be omitted when being called inside a step.
         run_id: The ID of the run to log metadata for. Can be omitted when
             being called inside a step.
 
@@ -460,7 +457,6 @@ def log_step_metadata(
             )
         pipeline_run = client.get_pipeline(
             name_id_or_prefix=pipeline_name_id_or_prefix,
-            version=pipeline_version,
         ).last_run
         step_run_id = pipeline_run.steps[step_name].id
     client.create_run_metadata(

zenml/utils/package_utils.py

@@ -13,6 +13,8 @@
 #  permissions and limitations under the License.
 """Utility functions for the package."""
 
+from typing import List
+
 import requests
 from packaging import version
 
@@ -48,3 +50,40 @@ def is_latest_zenml_version() -> bool:
         return False
     else:
         return True
+
+
+def clean_requirements(requirements: List[str]) -> List[str]:
+    """Clean requirements list from redundant requirements.
+
+    Args:
+        requirements: List of requirements.
+
+    Returns:
+        Cleaned list of requirements
+
+    Raises:
+        TypeError: If input is not a list
+        ValueError: If any element in the list is not a string
+    """
+    if not isinstance(requirements, list):
+        raise TypeError("Input must be a list")
+
+    if not all(isinstance(req, str) for req in requirements):
+        raise ValueError("All elements in the list must be strings")
+
+    cleaned = {}
+    for req in requirements:
+        package = (
+            req.split(">=")[0]
+            .split("==")[0]
+            .split("<")[0]
+            .split("~=")[0]
+            .split("^=")[0]
+            .split("[")[0]
+            .strip()
+        )
+        if package not in cleaned or any(
+            op in req for op in ["=", ">", "<", "~", "^"]
+        ):
+            cleaned[package] = req
+    return sorted(cleaned.values())

zenml/zen_server/rbac/models.py

@@ -58,6 +58,7 @@ class ResourceType(StrEnum):
     PIPELINE_RUN = "pipeline_run"
     PIPELINE_DEPLOYMENT = "pipeline_deployment"
     PIPELINE_BUILD = "pipeline_build"
+    RUN_TEMPLATE = "run_template"
     USER = "user"
     SERVICE = "service"
     RUN_METADATA = "run_metadata"

zenml/zen_server/rbac/utils.py

@@ -405,6 +405,7 @@ def get_resource_type_for_model(
         PipelineResponse,
         PipelineRunResponse,
         RunMetadataResponse,
+        RunTemplateResponse,
         SecretResponse,
         ServiceAccountResponse,
         ServiceConnectorResponse,
@@ -440,6 +441,7 @@ def get_resource_type_for_model(
         PipelineDeploymentResponse: ResourceType.PIPELINE_DEPLOYMENT,
         PipelineBuildResponse: ResourceType.PIPELINE_BUILD,
         PipelineRunResponse: ResourceType.PIPELINE_RUN,
+        RunTemplateResponse: ResourceType.RUN_TEMPLATE,
         TagResponse: ResourceType.TAG,
         TriggerResponse: ResourceType.TRIGGER,
         TriggerExecutionResponse: ResourceType.TRIGGER_EXECUTION,
@@ -556,6 +558,7 @@ def get_schema_for_resource_type(
         PipelineRunSchema,
         PipelineSchema,
         RunMetadataSchema,
+        RunTemplateSchema,
         SecretSchema,
         ServiceConnectorSchema,
         ServiceSchema,
@@ -587,6 +590,7 @@ def get_schema_for_resource_type(
         ResourceType.PIPELINE_RUN: PipelineRunSchema,
         ResourceType.PIPELINE_DEPLOYMENT: PipelineDeploymentSchema,
         ResourceType.PIPELINE_BUILD: PipelineBuildSchema,
+        ResourceType.RUN_TEMPLATE: RunTemplateSchema,
         ResourceType.RUN_METADATA: RunMetadataSchema,
         ResourceType.USER: UserSchema,
         ResourceType.ACTION: ActionSchema,

zenml/zen_server/routers/pipeline_builds_endpoints.py

@@ -13,18 +13,15 @@
 #  permissions and limitations under the License.
 """Endpoint definitions for builds."""
 
-from typing import Optional
 from uuid import UUID
 
-from fastapi import APIRouter, BackgroundTasks, Depends, Security
+from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.constants import API, PIPELINE_BUILDS, VERSION_1
 from zenml.models import (
     Page,
     PipelineBuildFilter,
     PipelineBuildResponse,
-    PipelineRunResponse,
 )
 from zenml.zen_server.auth import AuthContext, authorize
 from zenml.zen_server.exceptions import error_response
@@ -33,12 +30,10 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_get_entity,
     verify_permissions_and_list_entities,
 )
-from zenml.zen_server.rbac.models import Action, ResourceType
-from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.rbac.models import ResourceType
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
-    server_config,
     zen_store,
 )
 
@@ -126,62 +121,3 @@ def delete_build(
         get_method=zen_store().get_build,
         delete_method=zen_store().delete_build,
     )
-
-
-if server_config().workload_manager_enabled:
-
-    @router.post(
-        "/{build_id}/runs",
-        responses={
-            401: error_response,
-            404: error_response,
-            422: error_response,
-        },
-    )
-    @handle_exceptions
-    def run_build(
-        build_id: UUID,
-        background_tasks: BackgroundTasks,
-        config: Optional[PipelineRunConfiguration] = None,
-        auth_context: AuthContext = Security(authorize),
-    ) -> PipelineRunResponse:
-        """Run a pipeline from a pipeline build.
-
-        Args:
-            build_id: The ID of the build.
-            background_tasks: Background tasks.
-            config: Configuration for the pipeline run.
-            auth_context: Authentication context.
-
-        Raises:
-            ValueError: If the build does not have an associated deployment.
-
-        Returns:
-            The created run.
-        """
-        from zenml.zen_server.pipeline_deployment.utils import (
-            run_pipeline,
-        )
-
-        build = verify_permissions_and_get_entity(
-            id=build_id, get_method=zen_store().get_build, hydrate=True
-        )
-
-        verify_permission(
-            resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
-        )
-
-        if not build.template_deployment_id:
-            raise ValueError("Build does not have template deployment.")
-
-        deployment = zen_store().get_deployment(
-            deployment_id=build.template_deployment_id, hydrate=True
-        )
-        deployment.get_metadata().build = build
-
-        return run_pipeline(
-            deployment=deployment,
-            run_config=config,
-            background_tasks=background_tasks,
-            auth_context=auth_context,
-        )

zenml/zen_server/routers/pipeline_deployments_endpoints.py

@@ -13,18 +13,15 @@
 #  permissions and limitations under the License.
 """Endpoint definitions for deployments."""
 
-from typing import Optional
 from uuid import UUID
 
-from fastapi import APIRouter, BackgroundTasks, Depends, Security
+from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
 from zenml.constants import API, PIPELINE_DEPLOYMENTS, VERSION_1
 from zenml.models import (
     Page,
     PipelineDeploymentFilter,
     PipelineDeploymentResponse,
-    PipelineRunResponse,
 )
 from zenml.zen_server.auth import AuthContext, authorize
 from zenml.zen_server.exceptions import error_response
@@ -33,8 +30,7 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_get_entity,
     verify_permissions_and_list_entities,
 )
-from zenml.zen_server.rbac.models import Action, ResourceType
-from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.rbac.models import ResourceType
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
@@ -133,53 +129,6 @@ def delete_deployment(
 
 if server_config().workload_manager_enabled:
 
-    @router.post(
-        "/{deployment_id}/runs",
-        responses={
-            401: error_response,
-            404: error_response,
-            422: error_response,
-        },
-    )
-    @handle_exceptions
-    def create_deployment_run(
-        deployment_id: UUID,
-        background_tasks: BackgroundTasks,
-        config: Optional[PipelineRunConfiguration] = None,
-        auth_context: AuthContext = Security(authorize),
-    ) -> PipelineRunResponse:
-        """Run a pipeline from a pipeline deployment.
-
-        Args:
-            deployment_id: The ID of the deployment.
-            background_tasks: Background tasks.
-            config: Configuration for the pipeline run.
-            auth_context: Authentication context.
-
-        Returns:
-            The created run.
-        """
-        from zenml.zen_server.pipeline_deployment.utils import (
-            run_pipeline,
-        )
-
-        deployment = verify_permissions_and_get_entity(
-            id=deployment_id,
-            get_method=zen_store().get_deployment,
-            hydrate=True,
-        )
-
-        verify_permission(
-            resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
-        )
-
-        return run_pipeline(
-            deployment=deployment,
-            run_config=config,
-            background_tasks=background_tasks,
-            auth_context=auth_context,
-        )
-
     @router.get(
         "/{deployment_id}/logs",
         responses={

zenml/zen_server/routers/pipelines_endpoints.py

@@ -17,10 +17,8 @@ from uuid import UUID
 
 from fastapi import APIRouter, Depends, Security
 
-from zenml.config.pipeline_spec import PipelineSpec
 from zenml.constants import (
     API,
-    PIPELINE_SPEC,
     PIPELINES,
     REPORTABLE_RESOURCES,
     RUNS,
@@ -29,8 +27,6 @@ from zenml.constants import (
 from zenml.models import (
     Page,
     PipelineFilter,
-    PipelineNamespaceFilter,
-    PipelineNamespaceResponse,
     PipelineResponse,
     PipelineRunFilter,
     PipelineRunResponse,
@@ -46,9 +42,6 @@ from zenml.zen_server.rbac.endpoint_utils import (
     verify_permissions_and_update_entity,
 )
 from zenml.zen_server.rbac.models import ResourceType
-from zenml.zen_server.rbac.utils import (
-    get_allowed_resource_ids,
-)
 from zenml.zen_server.utils import (
     handle_exceptions,
     make_dependable,
@@ -64,7 +57,6 @@ router = APIRouter(
 
 @router.get(
     "",
-    response_model=Page[PipelineResponse],
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -96,7 +88,6 @@ def list_pipelines(
 
 @router.get(
     "/{pipeline_id}",
-    response_model=PipelineResponse,
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -122,7 +113,6 @@ def get_pipeline(
 
 @router.put(
     "/{pipeline_id}",
-    response_model=PipelineResponse,
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -179,7 +169,6 @@ def delete_pipeline(
 
 @router.get(
     "/{pipeline_id}" + RUNS,
-    response_model=Page[PipelineRunResponse],
     responses={401: error_response, 404: error_response, 422: error_response},
 )
 @handle_exceptions
@@ -201,66 +190,4 @@ def list_pipeline_runs(
     Returns:
         The pipeline runs according to query filters.
     """
-    return zen_store().list_runs(pipeline_run_filter_model)
-
-
-@router.get(
-    "/{pipeline_id}" + PIPELINE_SPEC,
-    response_model=PipelineSpec,
-    responses={401: error_response, 404: error_response, 422: error_response},
-)
-@handle_exceptions
-def get_pipeline_spec(
-    pipeline_id: UUID,
-    _: AuthContext = Security(authorize),
-) -> PipelineSpec:
-    """Gets the spec of a specific pipeline using its unique id.
-
-    Args:
-        pipeline_id: ID of the pipeline to get.
-
-    Returns:
-        The spec of the pipeline.
-    """
-    pipeline = verify_permissions_and_get_entity(
-        id=pipeline_id, get_method=zen_store().get_pipeline
-    )
-    return pipeline.spec
-
-
-namespace_router = APIRouter(
-    prefix=API + VERSION_1 + "/pipeline_namespaces",
-    tags=["pipeline_namespaces"],
-    responses={401: error_response, 403: error_response},
-)
-
-
-@namespace_router.get(
-    "",
-    responses={401: error_response, 404: error_response, 422: error_response},
-)
-@handle_exceptions
-def list_pipeline_namespaces(
-    filter_model: PipelineNamespaceFilter = Depends(
-        make_dependable(PipelineNamespaceFilter)
-    ),
-    hydrate: bool = False,
-    auth_context: AuthContext = Security(authorize),
-) -> Page[PipelineNamespaceResponse]:
-    """Gets a list of pipeline namespaces.
-
-    Args:
-        filter_model: Filter model used for pagination, sorting,
-            filtering.
-        hydrate: Flag deciding whether to hydrate the output model(s)
-            by including metadata fields in the response.
-        auth_context: Authentication context.
-
-    Returns:
-        List of pipeline namespace objects.
-    """
-    allowed_ids = get_allowed_resource_ids(resource_type=ResourceType.PIPELINE)
-    filter_model.configure_rbac(
-        authenticated_user_id=auth_context.user.id, id=allowed_ids
-    )
-    return zen_store().list_pipeline_namespaces(filter_model, hydrate=hydrate)
+    return zen_store().list_runs(pipeline_run_filter_model, hydrate=hydrate)

zenml/zen_server/routers/run_templates_endpoints.py

@@ -0,0 +1,212 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Endpoint definitions for run templates."""
+
+from typing import Optional
+from uuid import UUID
+
+from fastapi import APIRouter, BackgroundTasks, Depends, Security
+
+from zenml.analytics.enums import AnalyticsEvent
+from zenml.analytics.utils import track_handler
+from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
+from zenml.constants import API, RUN_TEMPLATES, VERSION_1
+from zenml.models import (
+    Page,
+    PipelineRunResponse,
+    RunTemplateFilter,
+    RunTemplateResponse,
+    RunTemplateUpdate,
+)
+from zenml.zen_server.auth import AuthContext, authorize
+from zenml.zen_server.exceptions import error_response
+from zenml.zen_server.rbac.endpoint_utils import (
+    verify_permissions_and_delete_entity,
+    verify_permissions_and_get_entity,
+    verify_permissions_and_list_entities,
+    verify_permissions_and_update_entity,
+)
+from zenml.zen_server.rbac.models import Action, ResourceType
+from zenml.zen_server.rbac.utils import verify_permission
+from zenml.zen_server.utils import (
+    handle_exceptions,
+    make_dependable,
+    server_config,
+    zen_store,
+)
+
+router = APIRouter(
+    prefix=API + VERSION_1 + RUN_TEMPLATES,
+    tags=["run_templates"],
+    responses={401: error_response, 403: error_response},
+)
+
+
+@router.get(
+    "",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def list_run_templates(
+    filter_model: RunTemplateFilter = Depends(
+        make_dependable(RunTemplateFilter)
+    ),
+    hydrate: bool = False,
+    _: AuthContext = Security(authorize),
+) -> Page[RunTemplateResponse]:
+    """Get a page of run templates.
+
+    Args:
+        filter_model: Filter model used for pagination, sorting,
+            filtering.
+        hydrate: Flag deciding whether to hydrate the output model(s)
+            by including metadata fields in the response.
+
+    Returns:
+        Page of run templates.
+    """
+    return verify_permissions_and_list_entities(
+        filter_model=filter_model,
+        resource_type=ResourceType.RUN_TEMPLATE,
+        list_method=zen_store().list_run_templates,
+        hydrate=hydrate,
+    )
+
+
+@router.get(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def get_run_template(
+    template_id: UUID,
+    hydrate: bool = True,
+    _: AuthContext = Security(authorize),
+) -> RunTemplateResponse:
+    """Get a run template.
+
+    Args:
+        template_id: ID of the run template to get.
+        hydrate: Flag deciding whether to hydrate the output model(s)
+            by including metadata fields in the response.
+
+    Returns:
+        The run template.
+    """
+    return verify_permissions_and_get_entity(
+        id=template_id,
+        get_method=zen_store().get_run_template,
+        hydrate=hydrate,
+    )
+
+
+@router.put(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def update_run_template(
+    template_id: UUID,
+    update: RunTemplateUpdate,
+    _: AuthContext = Security(authorize),
+) -> RunTemplateResponse:
+    """Update a run template.
+
+    Args:
+        template_id: ID of the run template to get.
+        update: The updates to apply.
+
+    Returns:
+        The updated run template.
+    """
+    return verify_permissions_and_update_entity(
+        id=template_id,
+        update_model=update,
+        get_method=zen_store().get_run_template,
+        update_method=zen_store().update_run_template,
+    )
+
+
+@router.delete(
+    "/{template_id}",
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def delete_run_template(
+    template_id: UUID,
+    _: AuthContext = Security(authorize),
+) -> None:
+    """Delete a run template.
+
+    Args:
+        template_id: ID of the run template to delete.
+    """
+    verify_permissions_and_delete_entity(
+        id=template_id,
+        get_method=zen_store().get_run_template,
+        delete_method=zen_store().delete_run_template,
+    )
+
+
+if server_config().workload_manager_enabled:
+
+    @router.post(
+        "/{template_id}/runs",
+        responses={
+            401: error_response,
+            404: error_response,
+            422: error_response,
+        },
+    )
+    @handle_exceptions
+    def create_template_run(
+        template_id: UUID,
+        background_tasks: BackgroundTasks,
+        config: Optional[PipelineRunConfiguration] = None,
+        auth_context: AuthContext = Security(authorize),
+    ) -> PipelineRunResponse:
+        """Run a pipeline from a template.
+
+        Args:
+            template_id: The ID of the template.
+            background_tasks: Background tasks.
+            config: Configuration for the pipeline run.
+            auth_context: Authentication context.
+
+        Returns:
+            The created pipeline run.
+        """
+        from zenml.zen_server.template_execution.utils import run_template
+
+        with track_handler(event=AnalyticsEvent.EXECUTED_RUN_TEMPLATE):
+            template = verify_permissions_and_get_entity(
+                id=template_id,
+                get_method=zen_store().get_run_template,
+                hydrate=True,
+            )
+
+            verify_permission(
+                resource_type=ResourceType.PIPELINE_DEPLOYMENT,
+                action=Action.CREATE,
+            )
+            verify_permission(
+                resource_type=ResourceType.PIPELINE_RUN, action=Action.CREATE
+            )
+
+            return run_template(
+                template=template,
+                auth_context=auth_context,
+                background_tasks=background_tasks,
+                run_config=config,
+            )