zenml-nightly 0.68.1.dev20241111__py3-none-any.whl → 0.68.1.dev20241112__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.68.1.dev20241111
+0.68.1.dev20241112

zenml/cli/base.py

@@ -91,7 +91,7 @@ ZENML_PROJECT_TEMPLATES = dict(
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",
-        github_tag="2024.10.30",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.08",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
 )
 

zenml/client.py

@@ -3426,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
         Args:
             id_or_prefix: The id or id prefix of the deployment.
         """
-        deployment = self.get_deployment(id_or_prefix=id_or_prefix)
+        deployment = self.get_deployment(
+            id_or_prefix=id_or_prefix, hydrate=False
+        )
         self.zen_store.delete_deployment(deployment_id=deployment.id)
 
     # ------------------------------ Run templates -----------------------------

zenml/exceptions.py

@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
     """Raised when trying to register an entity that already exists."""
 
 
+class EntityCreationError(ZenMLBaseException, RuntimeError):
+    """Raised when failing to create an entity."""
+
+
 class ActionExistsError(EntityExistsError):
     """Raised when registering an action with a name that already exists."""
 

zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py

@@ -518,19 +518,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
             "pipeline_execution_arn": pipeline_execution_arn,
         }
 
-        aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
-
-        region_name, _, _ = dissect_pipeline_execution_arn(
-            pipeline_execution_arn=pipeline_execution_arn
-        )
-
-        orchestrator_logs_url = (
-            f"https://{region_name}.console.aws.amazon.com/"
-            f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
-            f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
-            f"$3Dpipelines-{aws_run_id}-"
-        )
-        run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
         return run_metadata
 
     def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
@@ -673,7 +660,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
             return (
                 f"https://{region_name}.console.aws.amazon.com/"
                 f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
-                f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
+                f"/$252Faws$252Fsagemaker$252FTrainingJobs$3FlogStreamNameFilter"
                 f"$3Dpipelines-{execution_id}-"
             )
         except Exception as e:

zenml/integrations/lightning/orchestrators/lightning_orchestrator.py

@@ -390,10 +390,13 @@ class LightningOrchestrator(WheeledOrchestrator):
             studio.run(
                 f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
             )
-            studio.upload_file(
-                env_file_path,
-                remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
+            studio.upload_file(env_file_path)
+            time.sleep(6)
+            studio.run(
+                f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
             )
+            studio.run(f"rm {env_file_path.split('/')[-1]}")
+
             studio.run("pip install uv")
             logger.info(
                 f"Installing requirements: {pipeline_requirements_to_string}"
@@ -481,16 +484,25 @@ class LightningOrchestrator(WheeledOrchestrator):
                 code_path,
                 remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
             )
+            time.sleep(6)
             studio.run(
                 f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
             )
             logger.info(
                 "Uploading wheel package and installing dependencies on main studio"
             )
-            studio.upload_file(
-                env_file_path,
-                remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
+            # for some reason uploading file directly to /teamspace/studios/this_studio/.lightning_studio/.studiorc
+            # doesn't work, while other file names can be uploaded just fine.
+            # below is a workaround to copy the file to the correct location.
+            # we first upload it under a different name and then copy it to the
+            # right location.
+            studio.upload_file(env_file_path)
+            time.sleep(6)
+            studio.run(
+                f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
             )
+            studio.run(f"rm {env_file_path.split('/')[-1]}")
+
             studio.run("pip install uv")
             studio.run(f"uv pip install {requirements}")
             studio.run("pip install zenml")
@@ -557,13 +569,21 @@ class LightningOrchestrator(WheeledOrchestrator):
         studio.run(
             f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
         )
-        studio.upload_file(code_path, remote_path=f"/zenml_codes/{filename}")
+        studio.upload_file(
+            code_path,
+            remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
+        )
+        time.sleep(6)
         studio.run(
             f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
         )
-        studio.upload_file(
-            env_file_path, remote_path=".lightning_studio/.studiorc"
+        studio.upload_file(env_file_path)
+        time.sleep(6)
+        studio.run(
+            f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
         )
+        studio.run(f"rm {env_file_path.split('/')[-1]}")
+
         studio.run("pip install uv")
         studio.run(f"uv pip install {details['requirements']}")
         studio.run("pip install zenml")

zenml/model/model.py

@@ -14,7 +14,6 @@
 """Model user facing interface to pass into pipeline or step."""
 
 import datetime
-import time
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -28,7 +27,6 @@ from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
 
-from zenml.constants import MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION
 from zenml.enums import MetadataResourceTypes, ModelStages
 from zenml.exceptions import EntityExistsError
 from zenml.logger import get_logger
@@ -527,14 +525,6 @@ class Model(BaseModel):
         data["suppress_class_validation_warnings"] = True
         return data
 
-    def _validate_config_in_runtime(self) -> "ModelVersionResponse":
-        """Validate that config doesn't conflict with runtime environment.
-
-        Returns:
-            The model version based on configuration.
-        """
-        return self._get_or_create_model_version()
-
     def _get_or_create_model(self) -> "ModelResponse":
         """This method should get or create a model from Model Control Plane.
 
@@ -678,16 +668,6 @@ class Model(BaseModel):
         if isinstance(self.version, str):
             self.version = format_name_template(self.version)
 
-        zenml_client = Client()
-        model_version_request = ModelVersionRequest(
-            user=zenml_client.active_user.id,
-            workspace=zenml_client.active_workspace.id,
-            name=str(self.version) if self.version else None,
-            description=self.description,
-            model=model.id,
-            tags=self.tags,
-        )
-        mv_request = ModelVersionRequest.model_validate(model_version_request)
         try:
             if self.version or self.model_version_id:
                 model_version = self._get_model_version()
@@ -717,60 +697,34 @@ class Model(BaseModel):
                     " as an example. You can explore model versions using "
                     f"`zenml model version list -n {self.name}` CLI command."
                 )
-            retries_made = 0
-            for i in range(MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION):
-                try:
-                    model_version = (
-                        zenml_client.zen_store.create_model_version(
-                            model_version=mv_request
-                        )
-                    )
-                    break
-                except EntityExistsError as e:
-                    if i == MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION - 1:
-                        raise RuntimeError(
-                            f"Failed to create model version "
-                            f"`{self.version if self.version else 'new'}` "
-                            f"in model `{self.name}`. Retried {retries_made} times. "
-                            "This could be driven by exceptionally high concurrency of "
-                            "pipeline runs. Please, reach out to us on ZenML Slack for support."
-                        ) from e
-                    # smoothed exponential back-off, it will go as 0.2, 0.3,
-                    # 0.45, 0.68, 1.01, 1.52, 2.28, 3.42, 5.13, 7.69, ...
-                    sleep = 0.2 * 1.5**i
-                    logger.debug(
-                        f"Failed to create new model version for "
-                        f"model `{self.name}`. Retrying in {sleep}..."
-                    )
-                    time.sleep(sleep)
-                    retries_made += 1
-            self.version = model_version.name
+
+            client = Client()
+            model_version_request = ModelVersionRequest(
+                user=client.active_user.id,
+                workspace=client.active_workspace.id,
+                name=str(self.version) if self.version else None,
+                description=self.description,
+                model=model.id,
+                tags=self.tags,
+            )
+            model_version = client.zen_store.create_model_version(
+                model_version=model_version_request
+            )
+
             self._created_model_version = True
 
             logger.info(
                 "Created new model version `%s` for model `%s`.",
-                self.version,
+                model_version.name,
                 self.name,
             )
 
+        self.version = model_version.name
         self.model_version_id = model_version.id
         self._model_id = model_version.model.id
         self._number = model_version.number
         return model_version
 
-    def _merge(self, model: "Model") -> None:
-        self.license = self.license or model.license
-        self.description = self.description or model.description
-        self.audience = self.audience or model.audience
-        self.use_cases = self.use_cases or model.use_cases
-        self.limitations = self.limitations or model.limitations
-        self.trade_offs = self.trade_offs or model.trade_offs
-        self.ethics = self.ethics or model.ethics
-        if model.tags is not None:
-            self.tags = list(
-                {t for t in self.tags or []}.union(set(model.tags))
-            )
-
     def __hash__(self) -> int:
         """Get hash of the `Model`.
 

zenml/stack_deployments/aws_stack_deployment.py

@@ -71,7 +71,8 @@ of any potential costs:
 
 - An S3 bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/s3).
 - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
-- Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker).
+- Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
+as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
 - An IAM user and IAM role with the minimum necessary permissions to access the
 above resources.
 - An AWS access key used to give access to ZenML to connect to the above
@@ -257,13 +258,29 @@ console.
 
         config: Optional[str] = None
         if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
-            config = f"""module "zenml_stack" {{
-    source  = "zenml-io/zenml-stack/aws"
+            config = f"""terraform {{
+    required_providers {{
+        aws = {{
+            source  = "hashicorp/aws"
+        }}
+        zenml = {{
+            source = "zenml-io/zenml"
+        }}
+    }}
+}}
 
+provider "aws" {{
     region = "{self.location or "eu-central-1"}"
-    zenml_server_url = "{self.zenml_server_url}"
-    zenml_api_key = ""
-    zenml_api_token = "{self.zenml_server_api_token}"
+}}
+
+provider "zenml" {{
+    server_url = "{self.zenml_server_url}"
+    api_token = "{self.zenml_server_api_token}"
+}}
+
+module "zenml_stack" {{
+    source  = "zenml-io/zenml-stack/aws"
+
     zenml_stack_name = "{self.stack_name}"
     zenml_stack_deployment = "{self.deployment_type}"
 }}

zenml/stack_deployments/azure_stack_deployment.py

@@ -259,14 +259,37 @@ ZenML's access to your Azure subscription.
             The configuration or script to deploy the ZenML stack to the
             specified cloud provider.
         """
-        config = f"""module "zenml_stack" {{
+        config = f"""terraform {{
+    required_providers {{
+        azurerm = {{
+            source  = "hashicorp/azurerm"
+        }}
+        azuread = {{
+            source  = "hashicorp/azuread"
+        }}
+        zenml = {{
+            source = "zenml-io/zenml"
+        }}
+    }}
+}}
+
+provider "azurerm" {{
+    features {{
+        resource_group {{
+            prevent_deletion_if_contains_resources = false
+        }}
+    }}
+}}
+
+provider "zenml" {{
+    server_url = "{self.zenml_server_url}"
+    api_token = "{self.zenml_server_api_token}"
+}}
+
+module "zenml_stack" {{
     source  = "zenml-io/zenml-stack/azure"
 
     location = "{self.location or "eastus"}"
-    orchestrator = "azureml"
-    zenml_server_url = "{self.zenml_server_url}"
-    zenml_api_key = ""
-    zenml_api_token = "{self.zenml_server_api_token}"
     zenml_stack_name = "{self.stack_name}"
     zenml_stack_deployment = "{self.deployment_type}"
 }}

zenml/stack_deployments/gcp_stack_deployment.py

@@ -70,11 +70,12 @@ and are aware of any potential costs:
 
 - A GCS bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/gcp).
 - A Google Artifact Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/gcp).
-- Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex).
+- Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
+and as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/vertex).
 - GCP Cloud Build registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builders/gcp).
 - A GCP Service Account with the minimum necessary permissions to access the
 above resources.
-- An GCP Service Account access key used to give access to ZenML to connect to
+- A GCP Service Account access key used to give access to ZenML to connect to
 the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/gcp-service-connector).
 
 The Deployment Manager deployment will automatically create a GCP Service
@@ -259,14 +260,30 @@ GCP project and to clean up the resources created by the stack by using
         )
 
         if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
-            config = f"""module "zenml_stack" {{
+            config = f"""terraform {{
+    required_providers {{
+        google = {{
+            source  = "hashicorp/google"
+        }}
+        zenml = {{
+            source = "zenml-io/zenml"
+        }}
+    }}
+}}
+
+provider "google" {{
+    region  = "{self.location or "europe-west3"}"
+    project = your GCP project name
+}}
+
+provider "zenml" {{
+    server_url = "{self.zenml_server_url}"
+    api_token = "{self.zenml_server_api_token}"
+}}
+
+module "zenml_stack" {{
     source  = "zenml-io/zenml-stack/gcp"
 
-    project_id = "my-gcp-project"
-    region = "{self.location or "europe-west3"}"
-    zenml_server_url = "{self.zenml_server_url}"
-    zenml_api_key = ""
-    zenml_api_token = "{self.zenml_server_api_token}"
     zenml_stack_name = "{self.stack_name}"
     zenml_stack_deployment = "{self.deployment_type}"
 }}

zenml/stack_deployments/stack_deployment.py

@@ -219,16 +219,14 @@ class ZenMLCloudStackDeployment(BaseModel):
             if stack.labels.get("zenml:deployment") != self.deployment_type:
                 continue
 
-            artifact_store = stack.components[
-                StackComponentType.ARTIFACT_STORE
-            ][0]
+            orchestrator = stack.components[StackComponentType.ORCHESTRATOR][0]
 
-            if not artifact_store.connector:
+            if not orchestrator.connector:
                 continue
 
             return DeployedStack(
                 stack=stack,
-                service_connector=artifact_store.connector,
+                service_connector=orchestrator.connector,
             )
 
         return None

zenml/zen_server/exceptions.py

@@ -23,6 +23,7 @@ from zenml.exceptions import (
     CredentialsNotValid,
     DoesNotExistException,
     DuplicateRunNameError,
+    EntityCreationError,
     EntityExistsError,
     IllegalOperationError,
     MethodNotAllowedError,
@@ -93,6 +94,7 @@ REST_API_EXCEPTIONS: List[Tuple[Type[Exception], int]] = [
     # 422 Unprocessable Entity
     (ValueError, 422),
     # 500 Internal Server Error
+    (EntityCreationError, 500),
     (RuntimeError, 500),
     # 501 Not Implemented,
     (NotImplementedError, 501),

zenml/zen_server/routers/logs_endpoints.py

@@ -0,0 +1,66 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Endpoint definitions for logs."""
+
+from uuid import UUID
+
+from fastapi import APIRouter, Security
+
+from zenml.constants import (
+    API,
+    LOGS,
+    VERSION_1,
+)
+from zenml.models.v2.core.logs import LogsResponse
+from zenml.zen_server.auth import AuthContext, authorize
+from zenml.zen_server.exceptions import error_response
+from zenml.zen_server.rbac.endpoint_utils import (
+    verify_permissions_and_get_entity,
+)
+from zenml.zen_server.utils import (
+    handle_exceptions,
+    zen_store,
+)
+
+router = APIRouter(
+    prefix=API + VERSION_1 + LOGS,
+    tags=["logs"],
+    responses={401: error_response, 403: error_response},
+)
+
+
+@router.get(
+    "/{logs_id}",
+    response_model=LogsResponse,
+    responses={401: error_response, 404: error_response, 422: error_response},
+)
+@handle_exceptions
+def get_logs(
+    logs_id: UUID,
+    hydrate: bool = True,
+    _: AuthContext = Security(authorize),
+) -> LogsResponse:
+    """Returns the requested logs.
+
+    Args:
+        logs_id: ID of the logs.
+        hydrate: Flag deciding whether to hydrate the output model(s)
+            by including metadata fields in the response.
+
+    Returns:
+        The requested logs.
+    """
+    return verify_permissions_and_get_entity(
+        id=logs_id, get_method=zen_store().get_logs, hydrate=hydrate
+    )

zenml/zen_server/zen_server_api.py

@@ -64,6 +64,7 @@ from zenml.zen_server.routers import (
     devices_endpoints,
     event_source_endpoints,
     flavors_endpoints,
+    logs_endpoints,
     model_versions_endpoints,
     models_endpoints,
     pipeline_builds_endpoints,
@@ -414,6 +415,7 @@ app.include_router(code_repositories_endpoints.router)
 app.include_router(plugin_endpoints.plugin_router)
 app.include_router(event_source_endpoints.event_source_router)
 app.include_router(flavors_endpoints.router)
+app.include_router(logs_endpoints.router)
 app.include_router(models_endpoints.router)
 app.include_router(model_versions_endpoints.router)
 app.include_router(model_versions_endpoints.model_version_artifacts_router)