zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl

zenml/new/pipelines/pipeline.py

@@ -73,6 +73,7 @@ from zenml.new.pipelines.run_utils import (
     create_placeholder_run,
     deploy_pipeline,
     prepare_model_versions,
+    upload_notebook_cell_code_if_necessary,
 )
 from zenml.stack import Stack
 from zenml.steps import BaseStep
@@ -82,6 +83,7 @@ from zenml.steps.entrypoint_function_utils import (
 from zenml.steps.step_invocation import StepInvocation
 from zenml.utils import (
     code_repository_utils,
+    code_utils,
     dashboard_utils,
     dict_utils,
     pydantic_utils,
@@ -241,20 +243,6 @@ class Pipeline:
         """
         return inspect.getsource(self.source_object)
 
-    @classmethod
-    def from_model(cls, model: "PipelineResponse") -> "Pipeline":
-        """Creates a pipeline instance from a model.
-
-        Args:
-            model: The model to load the pipeline instance from.
-
-        Returns:
-            The pipeline instance.
-        """
-        from zenml.new.pipelines.deserialization_utils import load_pipeline
-
-        return load_pipeline(model=model)
-
     @property
     def model(self) -> "PipelineResponse":
         """Gets the registered pipeline model for this instance.
@@ -267,14 +255,7 @@ class Pipeline:
         """
         self._prepare_if_possible()
 
-        pipeline_spec = Compiler().compile_spec(self)
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
-        pipelines = Client().list_pipelines(
-            name=self.name, version_hash=version_hash
-        )
+        pipelines = Client().list_pipelines(name=self.name)
         if len(pipelines) == 1:
             return pipelines.items[0]
 
@@ -513,8 +494,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 "pipeline build/run` commands."
             )
 
-        pipeline_spec = Compiler().compile_spec(self)
-        return self._register(pipeline_spec=pipeline_spec)
+        return self._register()
 
     def build(
         self,
@@ -541,12 +521,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
         """
         with track_handler(event=AnalyticsEvent.BUILD_PIPELINE):
             self._prepare_if_possible()
-            deployment, pipeline_spec, _, _ = self._compile(
+            deployment, _, _ = self._compile(
                 config_path=config_path,
                 steps=step_configurations,
                 settings=settings,
             )
-            pipeline_id = self._register(pipeline_spec=pipeline_spec).id
+            pipeline_id = self._register().id
 
             local_repo = code_repository_utils.find_active_code_repository()
             code_repository = build_utils.verify_local_repository_context(
@@ -601,7 +581,8 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 method.
             unlisted: Whether the pipeline run should be unlisted (not assigned
                 to any pipeline).
-            prevent_build_reuse: Whether to prevent the reuse of a build.
+            prevent_build_reuse: DEPRECATED: Use
+                `DockerSettings.prevent_build_reuse` instead.
 
         Returns:
             Model of the pipeline run if running without a schedule, `None` if
@@ -623,7 +604,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
         logger.info(f"Initiating a new run for the pipeline: `{self.name}`.")
 
         with track_handler(AnalyticsEvent.RUN_PIPELINE) as analytics_handler:
-            deployment, pipeline_spec, schedule, build = self._compile(
+            deployment, schedule, build = self._compile(
                 config_path=config_path,
                 run_name=run_name,
                 enable_cache=enable_cache,
@@ -646,7 +627,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
             pipeline_id = None
             if register_pipeline:
-                pipeline_id = self._register(pipeline_spec=pipeline_spec).id
+                pipeline_id = self._register().id
 
             else:
                 logger.debug(f"Pipeline {self.name} is unlisted.")
@@ -689,6 +670,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
             stack = Client().active_stack
             stack.validate()
+            upload_notebook_cell_code_if_necessary(
+                deployment=deployment, stack=stack
+            )
 
             prepare_model_versions(deployment)
 
@@ -699,6 +683,13 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 deployment=deployment, local_repo_context=local_repo_context
             )
 
+            if prevent_build_reuse:
+                logger.warning(
+                    "Passing `prevent_build_reuse=True` to "
+                    "`pipeline.with_opitions(...)` is deprecated. Use "
+                    "`DockerSettings.prevent_build_reuse` instead."
+                )
+
             build_model = build_utils.reuse_or_create_pipeline_build(
                 deployment=deployment,
                 pipeline_id=pipeline_id,
@@ -723,6 +714,18 @@ To avoid this consider setting pipeline parameters only in one place (config or
                     code_repository=local_repo_context.code_repository_id,
                 )
 
+            code_path = None
+            if build_utils.should_upload_code(
+                deployment=deployment,
+                build=build_model,
+                code_reference=code_reference,
+            ):
+                code_archive = code_utils.CodeArchive(
+                    root=source_utils.get_source_root()
+                )
+                logger.info("Archiving pipeline code...")
+                code_path = code_utils.upload_code_if_necessary(code_archive)
+
             deployment_request = PipelineDeploymentRequest(
                 user=Client().active_user.id,
                 workspace=Client().active_workspace.id,
@@ -731,6 +734,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 build=build_id,
                 schedule=schedule_id,
                 code_reference=code_reference,
+                code_path=code_path,
                 **deployment.model_dump(),
             )
             deployment_model = Client().zen_store.create_deployment(
@@ -987,7 +991,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
         self, config_path: Optional[str] = None, **run_configuration_args: Any
     ) -> Tuple[
         "PipelineDeploymentBase",
-        "PipelineSpec",
         Optional["Schedule"],
         Union["PipelineBuildBase", UUID, None],
     ]:
@@ -998,7 +1001,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             **run_configuration_args: Configurations for the pipeline run.
 
         Returns:
-            A tuple containing the deployment, spec, schedule and build of
+            A tuple containing the deployment, schedule and build of
             the compiled pipeline.
         """
         # Activating the built-in integrations to load all materializers
@@ -1019,61 +1022,41 @@ To avoid this consider setting pipeline parameters only in one place (config or
         # Update with the values in code so they take precedence
         run_config = pydantic_utils.update_model(run_config, update=update)
 
-        deployment, pipeline_spec = Compiler().compile(
+        deployment = Compiler().compile(
             pipeline=self,
             stack=Client().active_stack,
             run_configuration=run_config,
         )
 
-        return deployment, pipeline_spec, run_config.schedule, run_config.build
+        return deployment, run_config.schedule, run_config.build
 
-    def _register(self, pipeline_spec: "PipelineSpec") -> "PipelineResponse":
+    def _register(self) -> "PipelineResponse":
         """Register the pipeline in the server.
 
-        Args:
-            pipeline_spec: The pipeline spec to register.
-
         Returns:
             The registered pipeline model.
         """
+        client = Client()
 
-        def _get(version_hash: str) -> PipelineResponse:
-            client = Client()
-
+        def _get() -> PipelineResponse:
             matching_pipelines = client.list_pipelines(
                 name=self.name,
-                version_hash=version_hash,
                 size=1,
                 sort_by="desc:created",
             )
+
             if matching_pipelines.total:
                 registered_pipeline = matching_pipelines.items[0]
-                logger.info(
-                    "Reusing registered pipeline version: `(version: %s)`.",
-                    registered_pipeline.version,
-                )
                 return registered_pipeline
             raise RuntimeError("No matching pipelines found.")
 
-        version_hash = self._compute_unique_identifier(
-            pipeline_spec=pipeline_spec
-        )
-
-        client = Client()
         try:
-            return _get(version_hash)
+            return _get()
         except RuntimeError:
-            latest_version = self._get_latest_version() or 0
-            version = str(latest_version + 1)
-
             request = PipelineRequest(
                 workspace=client.active_workspace.id,
                 user=client.active_user.id,
                 name=self.name,
-                version=version,
-                version_hash=version_hash,
-                spec=pipeline_spec,
-                docstring=self.__doc__,
             )
 
             try:
@@ -1081,12 +1064,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
                     pipeline=request
                 )
                 logger.info(
-                    "Registered new version: `(version %s)`.",
-                    registered_pipeline.version,
+                    "Registered new pipeline: `%s`.",
+                    registered_pipeline.name,
                 )
                 return registered_pipeline
             except EntityExistsError:
-                return _get(version_hash)
+                return _get()
 
     def _compute_unique_identifier(self, pipeline_spec: PipelineSpec) -> str:
         """Computes a unique identifier from the pipeline spec and steps.
@@ -1113,22 +1096,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         return hash_.hexdigest()
 
-    def _get_latest_version(self) -> Optional[int]:
-        """Gets the latest version of this pipeline.
-
-        Returns:
-            The latest version or `None` if no version exists.
-        """
-        all_pipelines = Client().list_pipelines(
-            name=self.name, sort_by="desc:created", size=1
-        )
-        if not all_pipelines.total:
-            return None
-        pipeline = all_pipelines.items[0]
-        if pipeline.version == "UNVERSIONED":
-            return None
-        return int(all_pipelines.items[0].version)
-
     def add_step_invocation(
         self,
         step: "BaseStep",
@@ -1330,7 +1297,8 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 method.
             unlisted: Whether the pipeline run should be unlisted (not assigned
                 to any pipeline).
-            prevent_build_reuse: Whether to prevent the reuse of a build.
+            prevent_build_reuse: DEPRECATED: Use
+                `DockerSettings.prevent_build_reuse` instead.
             **kwargs: Pipeline configuration options. These will be passed
                 to the `pipeline.configure(...)` method.
 

zenml/new/pipelines/run_utils.py

@@ -1,5 +1,6 @@
 """Utility functions for running pipelines."""
 
+import hashlib
 import time
 from collections import defaultdict
 from datetime import datetime
@@ -9,6 +10,7 @@ from uuid import UUID
 from zenml import constants
 from zenml.client import Client
 from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
+from zenml.config.source import SourceType
 from zenml.config.step_configurations import StepConfigurationUpdate
 from zenml.enums import ExecutionStatus, ModelStages
 from zenml.logger import get_logger
@@ -23,7 +25,7 @@ from zenml.models import (
 from zenml.new.pipelines.model_utils import NewModelRequest
 from zenml.orchestrators.utils import get_run_name
 from zenml.stack import Flavor, Stack
-from zenml.utils import cloud_utils
+from zenml.utils import cloud_utils, code_utils, notebook_utils
 from zenml.zen_stores.base_zen_store import BaseZenStore
 
 if TYPE_CHECKING:
@@ -36,6 +38,18 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)
 
 
+def get_default_run_name(pipeline_name: str) -> str:
+    """Gets the default name for a pipeline run.
+
+    Args:
+        pipeline_name: Name of the pipeline which will be run.
+
+    Returns:
+        Run name.
+    """
+    return f"{pipeline_name}-{{date}}-{{time}}"
+
+
 def create_placeholder_run(
     deployment: "PipelineDeploymentResponse",
 ) -> Optional["PipelineRunResponse"]:
@@ -349,3 +363,67 @@ def validate_run_config_is_runnable_from_server(
             raise ValueError(
                 "Can't set DockerSettings when running pipeline via Rest API."
             )
+
+
+def upload_notebook_cell_code_if_necessary(
+    deployment: "PipelineDeploymentBase", stack: "Stack"
+) -> None:
+    """Upload notebook cell code if necessary.
+
+    This function checks if any of the steps of the pipeline that will be
+    executed in a different process are defined in a notebook. If that is the
+    case, it will extract that notebook cell code into python files and upload
+    an archive of all the necessary files to the artifact store.
+
+    Args:
+        deployment: The deployment.
+        stack: The stack on which the deployment will happen.
+
+    Raises:
+        RuntimeError: If the code for one of the steps that will run out of
+            process cannot be extracted into a python file.
+    """
+    code_archive = code_utils.CodeArchive(root=None)
+    should_upload = False
+    sources_that_require_upload = []
+
+    for step in deployment.step_configurations.values():
+        source = step.spec.source
+
+        if source.type == SourceType.NOTEBOOK:
+            if (
+                stack.orchestrator.flavor != "local"
+                or step.config.step_operator
+            ):
+                should_upload = True
+                cell_code = getattr(step.spec.source, "_cell_code", None)
+
+                # Code does not run in-process, which means we need to
+                # extract the step code into a python file
+                if not cell_code:
+                    raise RuntimeError(
+                        f"Unable to run step {step.config.name}. This step is "
+                        "defined in a notebook and you're trying to run it "
+                        "in a remote environment, but ZenML was not able to "
+                        "detect the step code in the notebook. To fix "
+                        "this error, define your step in a python file instead "
+                        "of a notebook."
+                    )
+
+                notebook_utils.warn_about_notebook_cell_magic_commands(
+                    cell_code=cell_code
+                )
+
+                code_hash = hashlib.sha1(cell_code.encode()).hexdigest()  # nosec
+                module_name = f"extracted_notebook_code_{code_hash}"
+                file_name = f"{module_name}.py"
+                code_archive.add_file(source=cell_code, destination=file_name)
+
+                setattr(step.spec.source, "replacement_module", module_name)
+                sources_that_require_upload.append(source)
+
+    if should_upload:
+        logger.info("Archiving notebook code...")
+        code_path = code_utils.upload_code_if_necessary(code_archive)
+        for source in sources_that_require_upload:
+            setattr(source, "code_path", code_path)

zenml/post_execution/pipeline.py

@@ -38,14 +38,11 @@ def get_pipelines() -> List["PipelineResponse"]:
 
 def get_pipeline(
     pipeline: str,
-    version: Optional[str] = None,
 ) -> Optional["PipelineResponse"]:
     """(Deprecated) Fetches a pipeline model.
 
     Args:
         pipeline: The name of the pipeline.
-        version: Optional pipeline version. Specifies the version of the
-            pipeline to return. If not given, returns the latest version.
 
     Returns:
         The pipeline model.
@@ -55,4 +52,4 @@ def get_pipeline(
         "removed in a future release. Please use "
         "`zenml.client.Client().get_pipeline()` instead."
     )
-    return Client().get_pipeline(name_id_or_prefix=pipeline, version=version)
+    return Client().get_pipeline(name_id_or_prefix=pipeline)

zenml/service_connectors/service_connector_utils.py

@@ -290,7 +290,8 @@ def get_resources_options_from_resource_model_for_full_stack(
                             resource_ids=each.resource_ids,
                             stack_component_type=StackComponentType.ARTIFACT_STORE,
                             flavor="gcp",
-                            required_configuration={},
+                            required_configuration={"path": "Path"},
+                            use_resource_value_as_fixed_config=True,
                             flavor_display_name="GCS Bucket",
                         )
                     )
@@ -350,7 +351,8 @@ def get_resources_options_from_resource_model_for_full_stack(
                             resource_ids=each.resource_ids,
                             stack_component_type=StackComponentType.ARTIFACT_STORE,
                             flavor="azure",
-                            required_configuration={},
+                            required_configuration={"path": "Path"},
+                            use_resource_value_as_fixed_config=True,
                             flavor_display_name="Blob container",
                         )
                     )
@@ -366,6 +368,20 @@ def get_resources_options_from_resource_model_for_full_stack(
                             flavor_display_name="Skypilot (VM)",
                         )
                     )
+                    orchestrators.append(
+                        _prepare_resource_info(
+                            connector_details=connector_details,
+                            resource_ids=each.resource_ids,
+                            stack_component_type=StackComponentType.ORCHESTRATOR,
+                            flavor="azureml",
+                            required_configuration={
+                                "subscription_id": "subscription ID",
+                                "resource_group": "resource group",
+                                "workspace": "workspace",
+                            },
+                            flavor_display_name="AzureML",
+                        )
+                    )
 
                 if each.resource_type == "kubernetes-cluster":
                     orchestrators.append(

zenml/stack_deployments/aws_stack_deployment.py

@@ -13,11 +13,14 @@
 #  permissions and limitations under the License.
 """Functionality to deploy a ZenML stack to AWS."""
 
-from typing import ClassVar, Dict, List
+from typing import ClassVar, Dict, List, Optional
 
 from zenml.enums import StackDeploymentProvider
 from zenml.models import StackDeploymentConfig
-from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
+from zenml.stack_deployments.stack_deployment import (
+    STACK_DEPLOYMENT_TERRAFORM,
+    ZenMLCloudStackDeployment,
+)
 from zenml.utils.string_utils import random_str
 
 AWS_DEPLOYMENT_TYPE = "cloud-formation"
@@ -217,16 +220,18 @@ console.
         deploy the ZenML stack. The URL should include as many pre-filled
         URL query parameters as possible.
         * a textual description of the URL
+        * a Terraform script used to deploy the ZenML stack
         * some deployment providers may require additional configuration
-        parameters to be passed to the cloud provider in addition to the
-        deployment URL query parameters. Where that is the case, this method
+        parameters or scripts to be passed to the cloud provider in addition to
+        the deployment URL query parameters. Where that is the case, this method
         should also return a string that the user can copy and paste into the
         cloud provider console to deploy the ZenML stack (e.g. a set of
-        environment variables, or YAML configuration snippet etc.).
+        environment variables, YAML configuration snippet, bash or Terraform
+        script etc.).
 
         Returns:
-            The configuration to deploy the ZenML stack to the specified cloud
-            provider.
+            The configuration or script to deploy the ZenML stack to the
+            specified cloud provider.
         """
         params = dict(
             stackName=self.stack_name,
@@ -247,8 +252,27 @@ console.
             f"{region}#/stacks/create/review?{query_params}"
         )
 
+        config: Optional[str] = None
+        if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
+            config = f"""module "zenml_stack" {{
+    source  = "zenml-io/zenml-stack/aws"
+
+    region = "{self.location or "eu-central-1"}"
+    zenml_server_url = "{self.zenml_server_url}"
+    zenml_api_key = ""
+    zenml_api_token = "{self.zenml_server_api_token}"
+    zenml_stack_name = "{self.stack_name}"
+    zenml_stack_deployment = "{self.deployment_type}"
+}}
+output "zenml_stack_id" {{
+    value = module.zenml_stack.zenml_stack_id
+}}
+output "zenml_stack_name" {{
+    value = module.zenml_stack.zenml_stack_name
+}}"""
+
         return StackDeploymentConfig(
             deployment_url=url,
             deployment_url_text="AWS CloudFormation Console",
-            configuration=None,
+            configuration=config,
         )