zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.63.0.dev20240731__py3-none-any.whl

zenml/new/pipelines/build_utils.py

@@ -15,7 +15,6 @@
 
 import hashlib
 import platform
-from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Dict,
@@ -31,12 +30,11 @@ from zenml.code_repositories import BaseCodeRepository
 from zenml.logger import get_logger
 from zenml.models import (
     BuildItem,
-    CodeReferenceRequest,
     PipelineBuildBase,
     PipelineBuildRequest,
     PipelineBuildResponse,
     PipelineDeploymentBase,
-    PipelineDeploymentRequest,
+    StackResponse,
 )
 from zenml.stack import Stack
 from zenml.utils import (
@@ -53,53 +51,6 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)
 
 
-def _create_deployment(
-    deployment: "PipelineDeploymentBase",
-    pipeline_id: Optional[UUID] = None,
-    code_repository: Optional["BaseCodeRepository"] = None,
-) -> UUID:
-    """Creates a deployment in the ZenStore.
-
-    Args:
-        deployment: Base of the deployment to create.
-        pipeline_id: Pipeline ID to use for the deployment.
-        code_repository: Code repository to use for the deployment.
-
-    Returns:
-        The ID of the deployment.
-    """
-    source_root = source_utils.get_source_root()
-
-    code_reference = None
-    local_repo_context = (
-        code_repository.get_local_context(source_root)
-        if code_repository
-        else None
-    )
-    if local_repo_context and not local_repo_context.is_dirty:
-        subdirectory = (
-            Path(source_root).resolve().relative_to(local_repo_context.root)
-        )
-
-        code_reference = CodeReferenceRequest(
-            commit=local_repo_context.current_commit,
-            subdirectory=subdirectory.as_posix(),
-            code_repository=local_repo_context.code_repository_id,
-        )
-
-    deployment_request = PipelineDeploymentRequest(
-        user=Client().active_user.id,
-        workspace=Client().active_workspace.id,
-        stack=Client().active_stack.id,
-        pipeline=pipeline_id,
-        code_reference=code_reference,
-        **deployment.model_dump(),
-    )
-    return (
-        Client().zen_store.create_deployment(deployment=deployment_request).id
-    )
-
-
 def build_required(deployment: "PipelineDeploymentBase") -> bool:
     """Checks whether a build is required for the deployment and active stack.
 
@@ -269,6 +220,7 @@ def create_pipeline_build(
             settings were specified.
     """
     client = Client()
+    stack_model = Client().active_stack_model
     stack = client.active_stack
     required_builds = stack.get_docker_builds(deployment=deployment)
 
@@ -362,16 +314,11 @@ def create_pipeline_build(
     build_checksum = compute_build_checksum(
         required_builds, stack=stack, code_repository=code_repository
     )
-    template_deployment_id = _create_deployment(
-        deployment=deployment,
-        pipeline_id=pipeline_id,
-        code_repository=code_repository,
-    )
-
+    stack_checksum = compute_stack_checksum(stack=stack_model)
     build_request = PipelineBuildRequest(
         user=client.active_user.id,
         workspace=client.active_workspace.id,
-        stack=client.active_stack_model.id,
+        stack=stack_model.id,
         pipeline=pipeline_id,
         is_local=is_local,
         contains_code=contains_code,
@@ -379,7 +326,7 @@ def create_pipeline_build(
         zenml_version=zenml.__version__,
         python_version=platform.python_version(),
         checksum=build_checksum,
-        template_deployment_id=template_deployment_id,
+        stack_checksum=stack_checksum,
     )
     return client.zen_store.create_build(build_request)
 
@@ -585,3 +532,32 @@ def verify_custom_build(
             "your local machine or the image tags have been "
             "overwritten since the original build happened."
         )
+
+
+def compute_stack_checksum(stack: StackResponse) -> str:
+    """Compute a stack checksum.
+
+    Args:
+        stack: The stack for which to compute the checksum.
+
+    Returns:
+        The checksum.
+    """
+    hash_ = hashlib.md5()  # nosec
+
+    # This checksum is used to see if the stack has been updated since a build
+    # was created for it. We create this checksum not with specific requirements
+    # as these might change with new ZenML releases, but they don't actually
+    # invalidate those Docker images.
+    required_integrations = sorted(
+        {
+            component.integration
+            for components in stack.components.values()
+            for component in components
+            if component.integration and component.integration != "built-in"
+        }
+    )
+    for integration in required_integrations:
+        hash_.update(integration.encode())
+
+    return hash_.hexdigest()

zenml/new/pipelines/pipeline.py

@@ -241,20 +241,6 @@ class Pipeline:
         """
         return inspect.getsource(self.source_object)
 
-    @classmethod
-    def from_model(cls, model: "PipelineResponse") -> "Pipeline":
-        """Creates a pipeline instance from a model.
-
-        Args:
-            model: The model to load the pipeline instance from.
-
-        Returns:
-            The pipeline instance.
-        """
-        from zenml.new.pipelines.deserialization_utils import load_pipeline
-
-        return load_pipeline(model=model)
-
     @property
     def model(self) -> "PipelineResponse":
         """Gets the registered pipeline model for this instance.
@@ -267,14 +253,7 @@ class Pipeline:
         """
         self._prepare_if_possible()
 
-        pipeline_spec = Compiler().compile_spec(self)
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
-        pipelines = Client().list_pipelines(
-            name=self.name, version_hash=version_hash
-        )
+        pipelines = Client().list_pipelines(name=self.name)
         if len(pipelines) == 1:
             return pipelines.items[0]
 
@@ -513,8 +492,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 "pipeline build/run` commands."
             )
 
-        pipeline_spec = Compiler().compile_spec(self)
-        return self._register(pipeline_spec=pipeline_spec)
+        return self._register()
 
     def build(
         self,
@@ -541,12 +519,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
         """
         with track_handler(event=AnalyticsEvent.BUILD_PIPELINE):
             self._prepare_if_possible()
-            deployment, pipeline_spec, _, _ = self._compile(
+            deployment, _, _ = self._compile(
                 config_path=config_path,
                 steps=step_configurations,
                 settings=settings,
             )
-            pipeline_id = self._register(pipeline_spec=pipeline_spec).id
+            pipeline_id = self._register().id
 
             local_repo = code_repository_utils.find_active_code_repository()
             code_repository = build_utils.verify_local_repository_context(
@@ -623,7 +601,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
         logger.info(f"Initiating a new run for the pipeline: `{self.name}`.")
 
         with track_handler(AnalyticsEvent.RUN_PIPELINE) as analytics_handler:
-            deployment, pipeline_spec, schedule, build = self._compile(
+            deployment, schedule, build = self._compile(
                 config_path=config_path,
                 run_name=run_name,
                 enable_cache=enable_cache,
@@ -646,7 +624,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
             pipeline_id = None
             if register_pipeline:
-                pipeline_id = self._register(pipeline_spec=pipeline_spec).id
+                pipeline_id = self._register().id
 
             else:
                 logger.debug(f"Pipeline {self.name} is unlisted.")
@@ -987,7 +965,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
         self, config_path: Optional[str] = None, **run_configuration_args: Any
     ) -> Tuple[
         "PipelineDeploymentBase",
-        "PipelineSpec",
         Optional["Schedule"],
         Union["PipelineBuildBase", UUID, None],
     ]:
@@ -998,7 +975,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             **run_configuration_args: Configurations for the pipeline run.
 
         Returns:
-            A tuple containing the deployment, spec, schedule and build of
+            A tuple containing the deployment, schedule and build of
             the compiled pipeline.
         """
         # Activating the built-in integrations to load all materializers
@@ -1019,61 +996,41 @@ To avoid this consider setting pipeline parameters only in one place (config or
         # Update with the values in code so they take precedence
         run_config = pydantic_utils.update_model(run_config, update=update)
 
-        deployment, pipeline_spec = Compiler().compile(
+        deployment = Compiler().compile(
             pipeline=self,
             stack=Client().active_stack,
             run_configuration=run_config,
         )
 
-        return deployment, pipeline_spec, run_config.schedule, run_config.build
+        return deployment, run_config.schedule, run_config.build
 
-    def _register(self, pipeline_spec: "PipelineSpec") -> "PipelineResponse":
+    def _register(self) -> "PipelineResponse":
         """Register the pipeline in the server.
 
-        Args:
-            pipeline_spec: The pipeline spec to register.
-
         Returns:
             The registered pipeline model.
         """
+        client = Client()
 
-        def _get(version_hash: str) -> PipelineResponse:
-            client = Client()
-
+        def _get() -> PipelineResponse:
             matching_pipelines = client.list_pipelines(
                 name=self.name,
-                version_hash=version_hash,
                 size=1,
                 sort_by="desc:created",
             )
+
             if matching_pipelines.total:
                 registered_pipeline = matching_pipelines.items[0]
-                logger.info(
-                    "Reusing registered pipeline version: `(version: %s)`.",
-                    registered_pipeline.version,
-                )
                 return registered_pipeline
             raise RuntimeError("No matching pipelines found.")
 
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
-        client = Client()
         try:
-            return _get(version_hash)
+            return _get()
         except RuntimeError:
-            latest_version = self._get_latest_version() or 0
-            version = str(latest_version + 1)
-
             request = PipelineRequest(
                 workspace=client.active_workspace.id,
                 user=client.active_user.id,
                 name=self.name,
-                version=version,
-                version_hash=version_hash,
-                spec=pipeline_spec,
-                docstring=self.__doc__,
             )
 
             try:
@@ -1081,12 +1038,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
                     pipeline=request
                 )
                 logger.info(
-                    "Registered new version: `(version %s)`.",
-                    registered_pipeline.version,
+                    "Registered new pipeline: `%s`.",
+                    registered_pipeline.name,
                 )
                 return registered_pipeline
             except EntityExistsError:
-                return _get(version_hash)
+                return _get()
 
     def _compute_unique_identifier(self, pipeline_spec: PipelineSpec) -> str:
         """Computes a unique identifier from the pipeline spec and steps.
@@ -1113,22 +1070,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         return hash_.hexdigest()
 
-    def _get_latest_version(self) -> Optional[int]:
-        """Gets the latest version of this pipeline.
-
-        Returns:
-            The latest version or `None` if no version exists.
-        """
-        all_pipelines = Client().list_pipelines(
-            name=self.name, sort_by="desc:created", size=1
-        )
-        if not all_pipelines.total:
-            return None
-        pipeline = all_pipelines.items[0]
-        if pipeline.version == "UNVERSIONED":
-            return None
-        return int(all_pipelines.items[0].version)
-
     def add_step_invocation(
         self,
         step: "BaseStep",

zenml/new/pipelines/run_utils.py

@@ -36,6 +36,18 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)
 
 
+def get_default_run_name(pipeline_name: str) -> str:
+    """Gets the default name for a pipeline run.
+
+    Args:
+        pipeline_name: Name of the pipeline which will be run.
+
+    Returns:
+        Run name.
+    """
+    return f"{pipeline_name}-{{date}}-{{time}}"
+
+
 def create_placeholder_run(
     deployment: "PipelineDeploymentResponse",
 ) -> Optional["PipelineRunResponse"]:

zenml/post_execution/pipeline.py

@@ -38,14 +38,11 @@ def get_pipelines() -> List["PipelineResponse"]:
 
 def get_pipeline(
     pipeline: str,
-    version: Optional[str] = None,
 ) -> Optional["PipelineResponse"]:
     """(Deprecated) Fetches a pipeline model.
 
     Args:
         pipeline: The name of the pipeline.
-        version: Optional pipeline version. Specifies the version of the
-            pipeline to return. If not given, returns the latest version.
 
     Returns:
         The pipeline model.
@@ -55,4 +52,4 @@ def get_pipeline(
         "removed in a future release. Please use "
         "`zenml.client.Client().get_pipeline()` instead."
     )
-    return Client().get_pipeline(name_id_or_prefix=pipeline, version=version)
+    return Client().get_pipeline(name_id_or_prefix=pipeline)

zenml/service_connectors/service_connector_utils.py

@@ -290,7 +290,8 @@ def get_resources_options_from_resource_model_for_full_stack(
                             resource_ids=each.resource_ids,
                             stack_component_type=StackComponentType.ARTIFACT_STORE,
                             flavor="gcp",
-                            required_configuration={},
+                            required_configuration={"path": "Path"},
+                            use_resource_value_as_fixed_config=True,
                             flavor_display_name="GCS Bucket",
                         )
                     )
@@ -350,7 +351,8 @@ def get_resources_options_from_resource_model_for_full_stack(
                             resource_ids=each.resource_ids,
                             stack_component_type=StackComponentType.ARTIFACT_STORE,
                             flavor="azure",
-                            required_configuration={},
+                            required_configuration={"path": "Path"},
+                            use_resource_value_as_fixed_config=True,
                             flavor_display_name="Blob container",
                         )
                     )

zenml/stack_deployments/aws_stack_deployment.py

@@ -218,15 +218,16 @@ console.
         URL query parameters as possible.
         * a textual description of the URL
         * some deployment providers may require additional configuration
-        parameters to be passed to the cloud provider in addition to the
-        deployment URL query parameters. Where that is the case, this method
+        parameters or scripts to be passed to the cloud provider in addition to
+        the deployment URL query parameters. Where that is the case, this method
         should also return a string that the user can copy and paste into the
         cloud provider console to deploy the ZenML stack (e.g. a set of
-        environment variables, or YAML configuration snippet etc.).
+        environment variables, YAML configuration snippet, bash or Terraform
+        script etc.).
 
         Returns:
-            The configuration to deploy the ZenML stack to the specified cloud
-            provider.
+            The configuration or script to deploy the ZenML stack to the
+            specified cloud provider.
         """
         params = dict(
             stackName=self.stack_name,

zenml/stack_deployments/azure_stack_deployment.py

@@ -17,15 +17,15 @@ import re
 from typing import ClassVar, Dict, List
 
 from zenml.enums import StackDeploymentProvider
+from zenml.models import StackDeploymentConfig
 from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
 
 
-# TODO: this class just implements the regions list, and is not suitable for other
-# deployment tasks.
 class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
     """Azure ZenML Cloud Stack Deployment."""
 
     provider: ClassVar[StackDeploymentProvider] = StackDeploymentProvider.AZURE
+    deployment: ClassVar[str] = "azure-cloud-shell"
 
     @classmethod
     def description(cls) -> str:
@@ -37,8 +37,11 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
         Returns:
             A MarkDown description of the ZenML Cloud Stack Deployment.
         """
-        # TODO: Implement this
-        return ""
+        return """
+Provision and register a basic Azure ZenML stack authenticated and connected to
+all the necessary cloud infrastructure resources required to run pipelines in
+Azure.
+"""
 
     @classmethod
     def instructions(cls) -> str:
@@ -51,8 +54,44 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
             MarkDown instructions on how to deploy the ZenML stack to the
             specified cloud provider.
         """
-        # TODO: Implement this
-        return ""
+        return """
+You will be redirected to an Azure Cloud Shell console in your browser where
+you'll be asked to log into your Azure project and then use
+[the Azure ZenML Stack Terraform module](https://registry.terraform.io/modules/zenml-io/zenml-stack/azure)
+to provision the necessary cloud resources for ZenML.
+
+**NOTE**: The Azure ZenML Stack Terraform module will create the following new
+resources in your Azure subscription. Please ensure you have the necessary
+permissions and are aware of any potential costs:
+
+- An Azure Resource Group to contain all the resources required for the ZenML stack
+- An Azure Storage Account and Blob Storage Container registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/azure).
+- An Azure Container Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/azure).
+- SkyPilot will be registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/skypilot-vm) and used to run pipelines in your Azure subscription.
+- An Azure Service Principal with the minimum necessary permissions to access
+the above resources.
+- An Azure Service Principal client secret used to give access to ZenML to
+connect to the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/azure-service-connector).
+
+The Azure ZenML Stack Terraform module will automatically create an Azure
+Service Principal client secret and will share it with ZenML to give it
+permission to access the resources created by the stack. You can revoke these
+permissions at any time by deleting the Service Principal in your Azure
+subscription.
+
+**Estimated costs**
+
+A small training job would cost around: $0.60
+
+These are rough estimates and actual costs may vary based on your usage and specific Azure pricing. 
+Some services may be eligible for the Azure Free Tier. Use [the Azure Pricing Calculator](https://azure.microsoft.com/en-us/pricing/calculator)
+for a detailed estimate based on your usage.
+
+
+💡 **After the Terraform deployment is complete, you can close the Cloud
+Shell session and return to the CLI to view details about the associated ZenML
+stack automatically registered with ZenML.**
+"""
 
     @classmethod
     def post_deploy_instructions(cls) -> str:
@@ -64,8 +103,11 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
             MarkDown instructions on what to do after the deployment is
             complete.
         """
-        # TODO: Implement this
-        return ""
+        return """
+The ZenML stack has been successfully deployed and registered. You can delete
+the provisioned Service Principal and Resource Group at any time to revoke
+ZenML's access to your Azure subscription.
+"""
 
     @classmethod
     def integrations(cls) -> List[str]:
@@ -75,7 +117,7 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
             The list of ZenML integrations that need to be installed for the
             stack to be usable.
         """
-        return ["azure"]
+        return ["azure", "skypilot_azure"]
 
     @classmethod
     def permissions(cls) -> Dict[str, List[str]]:
@@ -85,8 +127,19 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
             The permissions granted to ZenML to access the cloud resources, as
             a dictionary grouping permissions by resource.
         """
-        # TODO: Implement this
-        return {}
+        return {
+            "Storage Account": [
+                "Storage Blob Data Contributor",
+            ],
+            "Container Registry": [
+                "AcrPull",
+                "AcrPush",
+                "Contributor",
+            ],
+            "Subscription": [
+                "Owner (required by SkyPilot)",
+            ],
+        }
 
     @classmethod
     def locations(cls) -> Dict[str, str]:
@@ -177,3 +230,57 @@ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
             for k, v in cls.locations().items()
             if "(US)" in k and matcher.match(v)
         }
+
+    def get_deployment_config(
+        self,
+    ) -> StackDeploymentConfig:
+        """Return the configuration to deploy the ZenML stack to the specified cloud provider.
+
+        The configuration should include:
+
+        * a cloud provider console URL where the user will be redirected to
+        deploy the ZenML stack. The URL should include as many pre-filled
+        URL query parameters as possible.
+        * a textual description of the URL
+        * some deployment providers may require additional configuration
+        parameters or scripts to be passed to the cloud provider in addition to
+        the deployment URL query parameters. Where that is the case, this method
+        should also return a string that the user can copy and paste into the
+        cloud provider console to deploy the ZenML stack (e.g. a set of
+        environment variables, YAML configuration snippet, bash or Terraform
+        script etc.).
+
+        Returns:
+            The configuration or script to deploy the ZenML stack to the
+            specified cloud provider.
+        """
+        config = f"""module "zenml_stack" {{
+    source  = "zenml-io/zenml-stack/azure"
+
+    location = "{self.location or "eastus"}"
+    zenml_server_url = "{self.zenml_server_url}"
+    zenml_api_key = ""
+    zenml_api_token = "{self.zenml_server_api_token}"
+    zenml_stack_name = "{self.stack_name}"
+    zenml_stack_deployment = "{self.deployment}"
+}}
+output "zenml_stack_id" {{
+    value = module.zenml_stack.zenml_stack_id
+}}
+output "zenml_stack_name" {{
+    value = module.zenml_stack.zenml_stack_name
+}}"""
+        instructions = """
+1. The Azure Cloud Shell console will open in your browser.
+2. Create a file named `main.tf` in the Cloud Shell and copy and paste the
+Terraform configuration below into it.
+3. Run `terraform init --upgrade` to initialize the Terraform configuration.
+4. Run `terraform apply` to deploy the ZenML stack to Azure.
+"""
+
+        return StackDeploymentConfig(
+            deployment_url="https://shell.azure.com",
+            deployment_url_text="Azure Cloud Shell Console",
+            configuration=config,
+            instructions=instructions,
+        )

zenml/stack_deployments/gcp_stack_deployment.py

@@ -231,15 +231,16 @@ GCP project and to clean up the resources created by the stack by using
         URL query parameters as possible.
         * a textual description of the URL
         * some deployment providers may require additional configuration
-        parameters to be passed to the cloud provider in addition to the
-        deployment URL query parameters. Where that is the case, this method
+        parameters or scripts to be passed to the cloud provider in addition to
+        the deployment URL query parameters. Where that is the case, this method
         should also return a string that the user can copy and paste into the
         cloud provider console to deploy the ZenML stack (e.g. a set of
-        environment variables, or YAML configuration snippet etc.).
+        environment variables, YAML configuration snippet, bash or Terraform
+        script etc.).
 
         Returns:
-            The configuration to deploy the ZenML stack to the specified cloud
-            provider.
+            The configuration or script to deploy the ZenML stack to the
+            specified cloud provider.
         """
         params = dict(
             cloudshell_git_repo="https://github.com/zenml-io/zenml",
@@ -262,8 +263,14 @@ ZENML_SERVER_URL={self.zenml_server_url}
 ZENML_SERVER_API_TOKEN={self.zenml_server_api_token}
 ### END CONFIGURATION ###"""
 
+        instructions = (
+            "You will be asked to provide the following configuration values "
+            "during the deployment process:"
+        )
+
         return StackDeploymentConfig(
             deployment_url=url,
             deployment_url_text="GCP Cloud Shell Console",
             configuration=config,
+            instructions=instructions,
         )

zenml/stack_deployments/stack_deployment.py

@@ -145,15 +145,16 @@ class ZenMLCloudStackDeployment(BaseModel):
         URL query parameters as possible.
         * a textual description of the URL
         * some deployment providers may require additional configuration
-        parameters to be passed to the cloud provider in addition to the
-        deployment URL query parameters. Where that is the case, this method
+        parameters or scripts to be passed to the cloud provider in addition to
+        the deployment URL query parameters. Where that is the case, this method
         should also return a string that the user can copy and paste into the
         cloud provider console to deploy the ZenML stack (e.g. a set of
-        environment variables, or YAML configuration snippet etc.).
+        environment variables, YAML configuration snippet, bash or Terraform
+        script etc.).
 
         Returns:
-            The configuration to deploy the ZenML stack to the specified cloud
-            provider.
+            The configuration or script to deploy the ZenML stack to the
+            specified cloud provider.
         """
 
     def get_stack(