zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl

zenml/client.py

@@ -139,6 +139,10 @@ from zenml.models import (
     RunMetadataFilter,
     RunMetadataRequest,
     RunMetadataResponse,
+    RunTemplateFilter,
+    RunTemplateRequest,
+    RunTemplateResponse,
+    RunTemplateUpdate,
     ScheduleFilter,
     ScheduleResponse,
     SecretFilter,
@@ -864,7 +868,6 @@ class Client(metaclass=ClientMetaClass):
         updated_full_name: Optional[str] = None,
         updated_email: Optional[str] = None,
         updated_email_opt_in: Optional[bool] = None,
-        updated_hub_token: Optional[str] = None,
         updated_password: Optional[str] = None,
         old_password: Optional[str] = None,
         updated_is_admin: Optional[bool] = None,
@@ -879,7 +882,6 @@ class Client(metaclass=ClientMetaClass):
             updated_full_name: The new full name of the user.
             updated_email: The new email of the user.
             updated_email_opt_in: The new email opt-in status of the user.
-            updated_hub_token: Update the hub token
             updated_password: The new password of the user.
             old_password: The old password of the user. Required for password
                 update.
@@ -907,8 +909,6 @@ class Client(metaclass=ClientMetaClass):
             )
         if updated_email_opt_in is not None:
             user_update.email_opted_in = updated_email_opt_in
-        if updated_hub_token is not None:
-            user_update.hub_token = updated_hub_token
         if updated_password is not None:
             user_update.password = updated_password
             if old_password is None:
@@ -2350,11 +2350,9 @@ class Client(metaclass=ClientMetaClass):
         created: Optional[Union[datetime, str]] = None,
         updated: Optional[Union[datetime, str]] = None,
         name: Optional[str] = None,
-        version: Optional[str] = None,
-        version_hash: Optional[str] = None,
-        docstring: Optional[str] = None,
         workspace_id: Optional[Union[str, UUID]] = None,
         user_id: Optional[Union[str, UUID]] = None,
+        tag: Optional[str] = None,
         hydrate: bool = False,
     ) -> Page[PipelineResponse]:
         """List all pipelines.
@@ -2368,11 +2366,9 @@ class Client(metaclass=ClientMetaClass):
             created: Use to filter by time of creation
             updated: Use the last updated date for filtering
             name: The name of the pipeline to filter by.
-            version: The version of the pipeline to filter by.
-            version_hash: The version hash of the pipeline to filter by.
-            docstring: The docstring of the pipeline to filter by.
             workspace_id: The id of the workspace to filter by.
             user_id: The id of the user to filter by.
+            tag: Tag to filter by.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
 
@@ -2388,11 +2384,9 @@ class Client(metaclass=ClientMetaClass):
             created=created,
             updated=updated,
             name=name,
-            version=version,
-            version_hash=version_hash,
-            docstring=docstring,
             workspace_id=workspace_id,
             user_id=user_id,
+            tag=tag,
         )
         pipeline_filter_model.set_scope_workspace(self.active_workspace.id)
         return self.zen_store.list_pipelines(
@@ -2403,124 +2397,83 @@ class Client(metaclass=ClientMetaClass):
     def get_pipeline(
         self,
         name_id_or_prefix: Union[str, UUID],
-        version: Optional[str] = None,
         hydrate: bool = True,
     ) -> PipelineResponse:
         """Get a pipeline by name, id or prefix.
 
         Args:
             name_id_or_prefix: The name, ID or ID prefix of the pipeline.
-            version: The pipeline version. If not specified, the latest
-                version is returned.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
 
         Returns:
             The pipeline.
         """
-        return self._get_entity_version_by_id_or_name_or_prefix(
+        return self._get_entity_by_id_or_name_or_prefix(
             get_method=self.zen_store.get_pipeline,
             list_method=self.list_pipelines,
             name_id_or_prefix=name_id_or_prefix,
-            version=version,
             hydrate=hydrate,
         )
 
     def delete_pipeline(
         self,
         name_id_or_prefix: Union[str, UUID],
-        version: Optional[str] = None,
-        all_versions: bool = False,
     ) -> None:
         """Delete a pipeline.
 
         Args:
             name_id_or_prefix: The name, ID or ID prefix of the pipeline.
-            version: The pipeline version. If left empty, will delete
-                the latest version.
-            all_versions: If `True`, delete all versions of the pipeline.
-
-        Raises:
-            ValueError: If an ID is supplied when trying to delete all versions
-                of a pipeline.
         """
-        if all_versions:
-            if is_valid_uuid(name_id_or_prefix):
-                raise ValueError(
-                    "You need to supply a name (not an ID) when trying to "
-                    "delete all versions of a pipeline."
-                )
-
-            for pipeline in depaginate(
-                Client().list_pipelines, name=name_id_or_prefix
-            ):
-                Client().delete_pipeline(pipeline.id)
-        else:
-            pipeline = self.get_pipeline(
-                name_id_or_prefix=name_id_or_prefix, version=version
-            )
-            self.zen_store.delete_pipeline(pipeline_id=pipeline.id)
+        pipeline = self.get_pipeline(name_id_or_prefix=name_id_or_prefix)
+        self.zen_store.delete_pipeline(pipeline_id=pipeline.id)
 
     @_fail_for_sql_zen_store
     def trigger_pipeline(
         self,
         pipeline_name_or_id: Union[str, UUID, None] = None,
-        pipeline_version: Optional[str] = None,
         run_configuration: Optional[PipelineRunConfiguration] = None,
         config_path: Optional[str] = None,
-        deployment_id: Optional[UUID] = None,
-        build_id: Optional[UUID] = None,
+        template_id: Optional[UUID] = None,
         stack_name_or_id: Union[str, UUID, None] = None,
         synchronous: bool = False,
     ) -> PipelineRunResponse:
         """Trigger a pipeline from the server.
 
         Usage examples:
-        * Run the latest runnable build for the latest version of a pipeline:
+        * Run the latest runnable template for a pipeline:
         ```python
         Client().trigger_pipeline(pipeline_name_or_id=<NAME>)
         ```
-        * Run the latest runnable build for a specific version of a pipeline:
+        * Run the latest runnable template for a pipeline on a specific stack:
         ```python
         Client().trigger_pipeline(
             pipeline_name_or_id=<NAME>,
-            pipeline_version=<VERSION>
-        )
-        ```
-        * Run a specific pipeline version on a specific stack:
-        ```python
-        Client().trigger_pipeline(
-            pipeline_name_or_id=<ID>,
-            stack_name_or_id=<ID>
+            stack_name_or_id=<STACK_NAME_OR_ID>
         )
         ```
-        * Run a specific deployment:
+        * Run a specific template:
         ```python
-        Client().trigger_pipeline(deployment_id=<ID>)
-        ```
-        * Run a specific build:
-        ```python
-        Client().trigger_pipeline(build_id=<ID>)
+        Client().trigger_pipeline(template_id=<ID>)
         ```
 
         Args:
-            pipeline_name_or_id: Name or ID of the pipeline. If not given,
-                either the build or deployment that should be run needs to be
-                specified.
-            pipeline_version: Version of the pipeline. This is only used if a
-                pipeline name is given.
+            pipeline_name_or_id: Name or ID of the pipeline. If this is
+                specified, the latest runnable template for this pipeline will
+                be used for the run (Runnable here means that the build
+                associated with the template is for a remote stack without any
+                custom flavor stack components). If not given, a template ID
+                that should be run needs to be specified.
             run_configuration: Configuration for the run. Either this or a
                 path to a config file can be specified.
             config_path: Path to a YAML configuration file. This file will be
                 parsed as a `PipelineRunConfiguration` object. Either this or
                 the configuration in code can be specified.
-            deployment_id: ID of the deployment to run. Either this or a build
-                to run can be specified.
-            build_id: ID of the build to run. Either this or a deployment to
-                run can be specified.
+            template_id: ID of the template to run. Either this or a pipeline
+                can be specified.
             stack_name_or_id: Name or ID of the stack on which to run the
                 pipeline. If not specified, this method will try to find a
-                runnable build on any stack.
+                runnable template on any stack.
             synchronous: If `True`, this method will wait until the triggered
                 run is finished.
 
@@ -2536,10 +2489,10 @@ class Client(metaclass=ClientMetaClass):
             wait_for_pipeline_run_to_finish,
         )
 
-        if Counter([build_id, deployment_id, pipeline_name_or_id])[None] != 2:
+        if Counter([template_id, pipeline_name_or_id])[None] != 1:
             raise RuntimeError(
-                "You need to specify exactly one of pipeline, build or "
-                "deployment to trigger."
+                "You need to specify exactly one of pipeline or template "
+                "to trigger."
             )
 
         if run_configuration and config_path:
@@ -2553,32 +2506,20 @@ class Client(metaclass=ClientMetaClass):
         if run_configuration:
             validate_run_config_is_runnable_from_server(run_configuration)
 
-        if deployment_id:
+        if template_id:
             if stack_name_or_id:
                 logger.warning(
-                    "Deployment ID and stack specified, ignoring the stack and "
-                    "using stack from deployment instead."
+                    "Template ID and stack specified, ignoring the stack and "
+                    "using stack associated with the template instead."
                 )
 
-            run = self.zen_store.run_deployment(
-                deployment_id=deployment_id,
+            run = self.zen_store.run_template(
+                template_id=template_id,
                 run_configuration=run_configuration,
             )
-        elif build_id:
-            if stack_name_or_id:
-                logger.warning(
-                    "Build ID and stack specified, ignoring the stack and "
-                    "using stack from build instead."
-                )
-
-            run = self.zen_store.run_build(
-                build_id=build_id, run_configuration=run_configuration
-            )
         else:
             assert pipeline_name_or_id
-            pipeline = self.get_pipeline(
-                name_id_or_prefix=pipeline_name_or_id, version=pipeline_version
-            )
+            pipeline = self.get_pipeline(name_id_or_prefix=pipeline_name_or_id)
 
             stack = None
             if stack_name_or_id:
@@ -2589,34 +2530,36 @@ class Client(metaclass=ClientMetaClass):
                     zen_store=self.zen_store, stack=stack
                 )
 
-            builds = depaginate(
-                self.list_builds,
+            templates = depaginate(
+                self.list_run_templates,
                 pipeline_id=pipeline.id,
                 stack_id=stack.id if stack else None,
             )
 
-            for build in builds:
-                if not build.template_deployment_id:
+            for template in templates:
+                if not template.build:
                     continue
 
-                if not build.stack:
+                stack = template.build.stack
+                if not stack:
                     continue
 
                 try:
                     validate_stack_is_runnable_from_server(
-                        zen_store=self.zen_store, stack=build.stack
+                        zen_store=self.zen_store, stack=stack
                     )
                 except ValueError:
                     continue
 
-                run = self.zen_store.run_build(
-                    build_id=build.id, run_configuration=run_configuration
+                run = self.zen_store.run_template(
+                    template_id=template.id,
+                    run_configuration=run_configuration,
                 )
                 break
             else:
                 raise RuntimeError(
-                    "Unable to find a runnable build for the given stack and "
-                    "pipeline."
+                    "Unable to find a runnable template for the given stack "
+                    "and pipeline."
                 )
 
         if synchronous:
@@ -3400,6 +3343,7 @@ class Client(metaclass=ClientMetaClass):
         pipeline_id: Optional[Union[str, UUID]] = None,
         stack_id: Optional[Union[str, UUID]] = None,
         build_id: Optional[Union[str, UUID]] = None,
+        template_id: Optional[Union[str, UUID]] = None,
         hydrate: bool = False,
     ) -> Page[PipelineDeploymentResponse]:
         """List all deployments.
@@ -3417,6 +3361,7 @@ class Client(metaclass=ClientMetaClass):
             pipeline_id: The id of the pipeline to filter by.
             stack_id: The id of the stack to filter by.
             build_id: The id of the build to filter by.
+            template_id: The ID of the template to filter by.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
 
@@ -3436,6 +3381,7 @@ class Client(metaclass=ClientMetaClass):
             pipeline_id=pipeline_id,
             stack_id=stack_id,
             build_id=build_id,
+            template_id=template_id,
         )
         deployment_filter_model.set_scope_workspace(self.active_workspace.id)
         return self.zen_store.list_deployments(
@@ -3452,6 +3398,183 @@ class Client(metaclass=ClientMetaClass):
         deployment = self.get_deployment(id_or_prefix=id_or_prefix)
         self.zen_store.delete_deployment(deployment_id=deployment.id)
 
+    # ------------------------------ Run templates -----------------------------
+
+    def create_run_template(
+        self,
+        name: str,
+        deployment_id: UUID,
+        description: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+    ) -> RunTemplateResponse:
+        """Create a run template.
+
+        Args:
+            name: The name of the run template.
+            deployment_id: ID of the deployment which this template should be
+                based off of.
+            description: The description of the run template.
+            tags: Tags associated with the run template.
+
+        Returns:
+            The created run template.
+        """
+        return self.zen_store.create_run_template(
+            template=RunTemplateRequest(
+                name=name,
+                description=description,
+                source_deployment_id=deployment_id,
+                tags=tags,
+                user=self.active_user.id,
+                workspace=self.active_workspace.id,
+            )
+        )
+
+    def get_run_template(
+        self,
+        name_id_or_prefix: Union[str, UUID],
+        hydrate: bool = True,
+    ) -> RunTemplateResponse:
+        """Get a run template.
+
+        Args:
+            name_id_or_prefix: Name/ID/ID prefix of the template to get.
+            hydrate: Flag deciding whether to hydrate the output model(s)
+                by including metadata fields in the response.
+
+        Returns:
+            The run template.
+        """
+        return self._get_entity_by_id_or_name_or_prefix(
+            get_method=self.zen_store.get_run_template,
+            list_method=self.list_run_templates,
+            name_id_or_prefix=name_id_or_prefix,
+            allow_name_prefix_match=False,
+            hydrate=hydrate,
+        )
+
+    def list_run_templates(
+        self,
+        sort_by: str = "created",
+        page: int = PAGINATION_STARTING_PAGE,
+        size: int = PAGE_SIZE_DEFAULT,
+        logical_operator: LogicalOperators = LogicalOperators.AND,
+        created: Optional[Union[datetime, str]] = None,
+        updated: Optional[Union[datetime, str]] = None,
+        name: Optional[str] = None,
+        tag: Optional[str] = None,
+        workspace_id: Optional[Union[str, UUID]] = None,
+        user_id: Optional[Union[str, UUID]] = None,
+        pipeline_id: Optional[Union[str, UUID]] = None,
+        build_id: Optional[Union[str, UUID]] = None,
+        stack_id: Optional[Union[str, UUID]] = None,
+        code_repository_id: Optional[Union[str, UUID]] = None,
+        hydrate: bool = False,
+    ) -> Page[RunTemplateResponse]:
+        """Get a page of run templates.
+
+        Args:
+            sort_by: The column to sort by.
+            page: The page of items.
+            size: The maximum size of all pages.
+            logical_operator: Which logical operator to use [and, or].
+            created: Filter by the creation date.
+            updated: Filter by the last updated date.
+            name: Filter by run template name.
+            tag: Filter by run template tags.
+            workspace_id: Filter by workspace ID.
+            user_id: Filter by user ID.
+            pipeline_id: Filter by pipeline ID.
+            build_id: Filter by build ID.
+            stack_id: Filter by stack ID.
+            code_repository_id: Filter by code repository ID.
+            hydrate: Flag deciding whether to hydrate the output model(s)
+                by including metadata fields in the response.
+
+        Returns:
+            A page of run templates.
+        """
+        filter = RunTemplateFilter(
+            sort_by=sort_by,
+            page=page,
+            size=size,
+            logical_operator=logical_operator,
+            created=created,
+            updated=updated,
+            name=name,
+            tag=tag,
+            workspace_id=workspace_id,
+            user_id=user_id,
+            pipeline_id=pipeline_id,
+            build_id=build_id,
+            stack_id=stack_id,
+            code_repository_id=code_repository_id,
+        )
+
+        return self.zen_store.list_run_templates(
+            template_filter_model=filter, hydrate=hydrate
+        )
+
+    def update_run_template(
+        self,
+        name_id_or_prefix: Union[str, UUID],
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        add_tags: Optional[List[str]] = None,
+        remove_tags: Optional[List[str]] = None,
+    ) -> RunTemplateResponse:
+        """Update a run template.
+
+        Args:
+            name_id_or_prefix: Name/ID/ID prefix of the template to update.
+            name: The new name of the run template.
+            description: The new description of the run template.
+            add_tags: Tags to add to the run template.
+            remove_tags: Tags to remove from the run template.
+
+        Returns:
+            The updated run template.
+        """
+        if is_valid_uuid(name_id_or_prefix):
+            template_id = (
+                UUID(name_id_or_prefix)
+                if isinstance(name_id_or_prefix, str)
+                else name_id_or_prefix
+            )
+        else:
+            template_id = self.get_run_template(
+                name_id_or_prefix, hydrate=False
+            ).id
+
+        return self.zen_store.update_run_template(
+            template_id=template_id,
+            template_update=RunTemplateUpdate(
+                name=name,
+                description=description,
+                add_tags=add_tags,
+                remove_tags=remove_tags,
+            ),
+        )
+
+    def delete_run_template(self, name_id_or_prefix: Union[str, UUID]) -> None:
+        """Delete a run template.
+
+        Args:
+            name_id_or_prefix: Name/ID/ID prefix of the template to delete.
+        """
+        if is_valid_uuid(name_id_or_prefix):
+            template_id = (
+                UUID(name_id_or_prefix)
+                if isinstance(name_id_or_prefix, str)
+                else name_id_or_prefix
+            )
+        else:
+            template_id = self.get_run_template(
+                name_id_or_prefix, hydrate=False
+            ).id
+
+        self.zen_store.delete_run_template(template_id=template_id)
+
     # ------------------------------- Schedules --------------------------------
 
     def get_schedule(
@@ -3620,12 +3743,14 @@ class Client(metaclass=ClientMetaClass):
         build_id: Optional[Union[str, UUID]] = None,
         deployment_id: Optional[Union[str, UUID]] = None,
         code_repository_id: Optional[Union[str, UUID]] = None,
+        template_id: Optional[Union[str, UUID]] = None,
         orchestrator_run_id: Optional[str] = None,
         status: Optional[str] = None,
         start_time: Optional[Union[datetime, str]] = None,
         end_time: Optional[Union[datetime, str]] = None,
         num_steps: Optional[Union[int, str]] = None,
         unlisted: Optional[bool] = None,
+        tag: Optional[str] = None,
         hydrate: bool = False,
     ) -> Page[PipelineRunResponse]:
         """List all pipeline runs.
@@ -3647,6 +3772,7 @@ class Client(metaclass=ClientMetaClass):
             build_id: The id of the build to filter by.
             deployment_id: The id of the deployment to filter by.
             code_repository_id: The id of the code repository to filter by.
+            template_id: The ID of the template to filter by.
             orchestrator_run_id: The run id of the orchestrator to filter by.
             name: The name of the run to filter by.
             status: The status of the pipeline run
@@ -3654,6 +3780,7 @@ class Client(metaclass=ClientMetaClass):
             end_time: The end_time for the pipeline run
             num_steps: The number of steps for the pipeline run
             unlisted: If the runs should be unlisted or not.
+            tag: Tag to filter by.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
 
@@ -3676,6 +3803,7 @@ class Client(metaclass=ClientMetaClass):
             build_id=build_id,
             deployment_id=deployment_id,
             code_repository_id=code_repository_id,
+            template_id=template_id,
             orchestrator_run_id=orchestrator_run_id,
             user_id=user_id,
             stack_id=stack_id,
@@ -3683,6 +3811,7 @@ class Client(metaclass=ClientMetaClass):
             start_time=start_time,
             end_time=end_time,
             num_steps=num_steps,
+            tag=tag,
             unlisted=unlisted,
         )
         runs_filter_model.set_scope_workspace(self.active_workspace.id)

zenml/config/build_configuration.py

@@ -14,11 +14,13 @@
 """Build configuration class."""
 
 import hashlib
+import json
 from typing import TYPE_CHECKING, Dict, Optional
 
 from pydantic import BaseModel
 
-from zenml.config.docker_settings import DockerSettings, SourceFileMode
+from zenml.config.docker_settings import DockerSettings
+from zenml.utils import json_utils
 
 if TYPE_CHECKING:
     from zenml.code_repositories import BaseCodeRepository
@@ -60,7 +62,14 @@ class BuildConfiguration(BaseModel):
             The checksum.
         """
         hash_ = hashlib.md5()  # nosec
-        hash_.update(self.settings.model_dump_json().encode())
+        settings_json = json.dumps(
+            self.settings.model_dump(
+                mode="json", exclude={"prevent_build_reuse"}
+            ),
+            sort_keys=True,
+            default=json_utils.pydantic_encoder,
+        )
+        hash_.update(settings_json.encode())
         if self.entrypoint:
             hash_.update(self.entrypoint.encode())
 
@@ -72,7 +81,7 @@ class BuildConfiguration(BaseModel):
             PipelineDockerImageBuilder,
         )
 
-        pass_code_repo = self.should_download_files(
+        pass_code_repo = self.should_download_files_from_code_repository(
             code_repository=code_repository
         )
         requirements_files = (
@@ -101,34 +110,51 @@ class BuildConfiguration(BaseModel):
         Returns:
             Whether files should be included in the image.
         """
-        if self.settings.source_files == SourceFileMode.INCLUDE:
-            return True
+        if self.should_download_files(code_repository=code_repository):
+            return False
 
-        if (
-            self.settings.source_files == SourceFileMode.DOWNLOAD_OR_INCLUDE
-            and not code_repository
+        return self.settings.allow_including_files_in_images
+
+    def should_download_files(
+        self,
+        code_repository: Optional["BaseCodeRepository"],
+    ) -> bool:
+        """Whether files should be downloaded in the image.
+
+        Args:
+            code_repository: Code repository that can be used to download files
+                inside the image.
+
+        Returns:
+            Whether files should be downloaded in the image.
+        """
+        if self.should_download_files_from_code_repository(
+            code_repository=code_repository
         ):
             return True
 
+        if self.settings.allow_download_from_artifact_store:
+            return True
+
         return False
 
-    def should_download_files(
+    def should_download_files_from_code_repository(
         self,
         code_repository: Optional["BaseCodeRepository"],
     ) -> bool:
-        """Whether files should be downloaded in the image.
+        """Whether files should be downloaded from the code repository.
 
         Args:
             code_repository: Code repository that can be used to download files
                 inside the image.
 
         Returns:
-            Whether files should be downloaded in the image.
+            Whether files should be downloaded from the code repository.
         """
-        if not code_repository:
-            return False
+        if (
+            code_repository
+            and self.settings.allow_download_from_code_repository
+        ):
+            return True
 
-        return self.settings.source_files in {
-            SourceFileMode.DOWNLOAD,
-            SourceFileMode.DOWNLOAD_OR_INCLUDE,
-        }
+        return False