zenml-nightly 0.68.0.dev20241027__py3-none-any.whl → 0.68.1.dev20241101__py3-none-any.whl

README.md

@@ -228,22 +228,28 @@ And finally, here are some other examples and use cases for inspiration:
 5. [LLMops](https://github.com/zenml-io/zenml-projects/tree/main/llm-complete-guide): Complete guide to do LLM with ZenML
 
 
+## 📚 Learn from Books
+
+<div align="center">
+  <a href="https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072">
+    <img src="docs/book/.gitbook/assets/llm_engineering_handbook_cover.jpg" alt="LLM Engineer's Handbook Cover" width="200"/></img>
+  </a>&nbsp;&nbsp;&nbsp;&nbsp;
+  <a href="https://www.amazon.com/-/en/Andrew-McMahon/dp/1837631964">
+    <img src="docs/book/.gitbook/assets/ml_engineering_with_python.jpg" alt="Machine Learning Engineering with Python Cover" width="200"/></img>
+  </a>
+  </br></br>
+</div>
+
+ZenML is featured in these comprehensive guides to modern MLOps and LLM engineering. Learn how to build production-ready machine learning systems with real-world examples and best practices.
+
 ## 🔋 Deploy ZenML
 
 For full functionality ZenML should be deployed on the cloud to
 enable collaborative features as the central MLOps interface for teams.
 
-Currently, there are two main ways to deploy ZenML:
+Read more about various deployment options [here](https://docs.zenml.io/getting-started/deploying-zenml).
 
-- **ZenML Pro**: [ZenML Pro](cloud.zenml.io/?utm_source=readme&utm_medium=referral_link&utm_campaign=cloud_promotion&utm_content=signup_link),
-offers a SaaS version which comes with a control plane to create and manage multiple ZenML servers.
-These servers are managed and maintained by ZenML’s dedicated team, alleviating
-the burden of server management from your end. They also come with additional features like RBAC, a Model Control Plane
-and [much more](https://zenml-io.gitbook.io/zenml-documentation/getting-started/zenml-pro.
-- **Self-hosted deployment**: Alternatively, you have the flexibility to [deploy
-ZenML on your own self-hosted environment](https://docs.zenml.io/getting-started/deploying-zenml#deploying-a-zenml-server).
-This can be achieved through various methods, including using our CLI, Docker,
-Helm, or HuggingFace Spaces.
+Or, sign up for [ZenML Pro to get a fully managed server on a free trial](https://cloud.zenml.io/?utm_source=readme&utm_medium=referral_link&utm_campaign=cloud_promotion&utm_content=signup_link).
 
 ## Use ZenML with VS Code
 
@@ -327,7 +333,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.68.0 is out. Check out the release notes
+    🎉 Version 0.68.1 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.

RELEASE_NOTES.md

@@ -1,5 +1,14 @@
 <!-- markdown-link-check-disable -->
 
+# 0.68.1
+
+Fixes an issue with some partially cached pipelines running on remote orchestrators.
+
+## What's Changed
+* Remove unavailable upstream steps during cache precomputation by @schustmi in https://github.com/zenml-io/zenml/pull/3146
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.68.0...0.68.1
+
 # 0.68.0
 
 ## Highlights

zenml/VERSION

@@ -1 +1 @@
-0.68.0.dev20241027
+0.68.1.dev20241101

zenml/__init__.py

@@ -31,7 +31,7 @@ init_logging()
 from zenml.models import *  # noqa: F401
 
 # Define public Python API
-from zenml.api import show
+from zenml.zen_server.utils import show_dashboard as show
 from zenml.artifacts.utils import (
     log_artifact_metadata,
     save_artifact,

zenml/analytics/context.py

@@ -25,6 +25,7 @@ from uuid import UUID
 
 from zenml import __version__
 from zenml.analytics.client import default_client
+from zenml.analytics.utils import is_analytics_disabled_internally
 from zenml.constants import (
     ENV_ZENML_SERVER,
     handle_bool_env_var,
@@ -73,14 +74,28 @@ class AnalyticsContext:
         Returns:
             The analytics context.
         """
+        if is_analytics_disabled_internally():
+            self.analytics_opt_in = False
+            return self
+
         # Fetch the analytics opt-in setting
         from zenml.config.global_config import GlobalConfiguration
 
         try:
             gc = GlobalConfiguration()
+
+            if not gc.is_initialized:
+                # If the global configuration is not initialized, using the
+                # zen store can lead to multiple initialization issues, because
+                # the analytics are triggered during the initialization of the
+                # zen store.
+                return self
+
             store_info = gc.zen_store.get_store_info()
 
-            if self.in_server:
+            # For local ZenML servers, we always use the client's analytics
+            # opt-in configuration.
+            if self.in_server and not store_info.is_local():
                 self.analytics_opt_in = store_info.analytics_enabled
             else:
                 self.analytics_opt_in = gc.analytics_opt_in

zenml/analytics/utils.py

@@ -14,7 +14,6 @@
 """Utility functions and classes for ZenML analytics."""
 
 import json
-import os
 from functools import wraps
 from typing import Any, Callable, Dict, Optional, TypeVar, cast
 from uuid import UUID
@@ -22,13 +21,15 @@ from uuid import UUID
 from zenml.analytics import identify, track
 from zenml.analytics.enums import AnalyticsEvent
 from zenml.analytics.models import AnalyticsTrackedModelMixin
-from zenml.constants import ENV_ZENML_ANALYTICS_OPT_IN
 from zenml.logger import get_logger
 
 logger = get_logger(__name__)
 F = TypeVar("F", bound=Callable[..., Any])
 
 
+_INTERNAL_DISABLE_ANALYTICS = False
+
+
 class AnalyticsEncoder(json.JSONEncoder):
     """Helper encoder class for JSON serialization."""
 
@@ -101,11 +102,13 @@ class analytics_disabler:
 
     def __init__(self) -> None:
         """Initialization of the context manager."""
-        self.original_value = os.environ.get(ENV_ZENML_ANALYTICS_OPT_IN)
+        self.original_value: Optional[bool] = None
 
     def __enter__(self) -> None:
         """Disable the analytics."""
-        os.environ[ENV_ZENML_ANALYTICS_OPT_IN] = str(False)
+        global _INTERNAL_DISABLE_ANALYTICS
+        self.original_value = _INTERNAL_DISABLE_ANALYTICS
+        _INTERNAL_DISABLE_ANALYTICS = True
 
     def __exit__(
         self,
@@ -120,10 +123,18 @@ class analytics_disabler:
             exc_value: The instance of the exception
             traceback: The traceback of the exception
         """
+        global _INTERNAL_DISABLE_ANALYTICS
         if self.original_value is not None:
-            os.environ[ENV_ZENML_ANALYTICS_OPT_IN] = self.original_value
-        else:
-            del os.environ[ENV_ZENML_ANALYTICS_OPT_IN]
+            _INTERNAL_DISABLE_ANALYTICS = self.original_value
+
+
+def is_analytics_disabled_internally() -> bool:
+    """Whether analytics are disabled by an internal helper function.
+
+    Returns:
+        Whether analytics are disabled by an internal helper function.
+    """
+    return _INTERNAL_DISABLE_ANALYTICS
 
 
 def track_decorator(event: AnalyticsEvent) -> Callable[[F], F]:

zenml/artifacts/utils.py

@@ -16,13 +16,11 @@
 import base64
 import os
 import tempfile
-import time
 import zipfile
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
     Dict,
     List,
     Optional,
@@ -38,7 +36,6 @@ from zenml.artifacts.load_directory_materializer import (
 )
 from zenml.client import Client
 from zenml.constants import (
-    MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION,
     MODEL_METADATA_YAML_FILE_NAME,
 )
 from zenml.enums import (
@@ -50,13 +47,12 @@ from zenml.enums import (
 )
 from zenml.exceptions import (
     DoesNotExistException,
-    EntityExistsError,
     StepContextError,
 )
 from zenml.io import fileio
 from zenml.logger import get_logger
+from zenml.metadata.metadata_types import validate_metadata
 from zenml.models import (
-    ArtifactRequest,
     ArtifactVersionRequest,
     ArtifactVersionResponse,
     ArtifactVisualizationRequest,
@@ -65,7 +61,6 @@ from zenml.models import (
     StepRunResponse,
     StepRunUpdate,
 )
-from zenml.models.v2.core.artifact import ArtifactResponse
 from zenml.stack import StackComponent
 from zenml.steps.step_context import get_step_context
 from zenml.utils import source_utils
@@ -135,12 +130,6 @@ def save_artifact(
 
     client = Client()
 
-    artifact = _get_or_create_artifact(
-        name=name,
-        has_custom_name=has_custom_name,
-        tags=tags,
-    )
-
     # Get the current artifact store
     artifact_store = client.active_stack.artifact_store
 
@@ -176,7 +165,7 @@ def save_artifact(
 
     # Save the artifact to the artifact store
     data_type = type(data)
-    materializer_object.validate_type_compatibility(data_type)
+    materializer_object.validate_save_type_compatibility(data_type)
     materializer_object.save(data)
 
     # Save visualizations of the artifact
@@ -207,52 +196,35 @@ def save_artifact(
                 f"Failed to extract metadata for output artifact '{name}': {e}"
             )
 
-    # Create the artifact version
-    def _create_version(
-        version: Union[int, str],
-    ) -> Optional[ArtifactVersionResponse]:
-        artifact_version = ArtifactVersionRequest(
-            artifact_id=artifact.id,
-            version=version,
-            tags=tags,
-            type=materializer_object.ASSOCIATED_ARTIFACT_TYPE,
-            uri=materializer_object.uri,
-            materializer=source_utils.resolve(materializer_object.__class__),
-            data_type=source_utils.resolve(data_type),
-            user=Client().active_user.id,
-            workspace=Client().active_workspace.id,
-            artifact_store_id=artifact_store.id,
-            visualizations=visualizations,
-            has_custom_name=has_custom_name,
-        )
-        try:
-            return client.zen_store.create_artifact_version(
-                artifact_version=artifact_version
-            )
-        except EntityExistsError:
-            return None
-
-    response = _create_artifact_version_with_retries(
-        name=name,
+    artifact_version_request = ArtifactVersionRequest(
+        artifact_name=name,
         version=version,
-        create_version_fn=_create_version,
+        tags=tags,
+        type=materializer_object.ASSOCIATED_ARTIFACT_TYPE,
+        uri=materializer_object.uri,
+        materializer=source_utils.resolve(materializer_object.__class__),
+        data_type=source_utils.resolve(data_type),
+        user=Client().active_user.id,
+        workspace=Client().active_workspace.id,
+        artifact_store_id=artifact_store.id,
+        visualizations=visualizations,
+        has_custom_name=has_custom_name,
+        metadata=validate_metadata(artifact_metadata)
+        if artifact_metadata
+        else None,
+    )
+    artifact_version = client.zen_store.create_artifact_version(
+        artifact_version=artifact_version_request
     )
-
-    if artifact_metadata:
-        client.create_run_metadata(
-            metadata=artifact_metadata,
-            resource_id=response.id,
-            resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
-        )
 
     if manual_save:
         _link_artifact_version_to_the_step_and_model(
-            artifact_version=response,
+            artifact_version=artifact_version,
             is_model_artifact=is_model_artifact,
             is_deployment_artifact=is_deployment_artifact,
         )
 
-    return response
+    return artifact_version
 
 
 def register_artifact(
@@ -304,56 +276,33 @@ def register_artifact(
         name=name,
     )
 
-    artifact = _get_or_create_artifact(
-        name=name,
-        has_custom_name=has_custom_name,
+    artifact_version_request = ArtifactVersionRequest(
+        artifact_name=name,
+        version=version,
         tags=tags,
+        type=ArtifactType.DATA,
+        uri=folder_or_file_uri,
+        materializer=source_utils.resolve(PreexistingDataMaterializer),
+        data_type=source_utils.resolve(Path),
+        user=Client().active_user.id,
+        workspace=Client().active_workspace.id,
+        artifact_store_id=artifact_store.id,
+        has_custom_name=has_custom_name,
+        metadata=validate_metadata(artifact_metadata)
+        if artifact_metadata
+        else None,
     )
-
-    # Create the artifact version
-    def _create_version(
-        version: Union[int, str],
-    ) -> Optional[ArtifactVersionResponse]:
-        artifact_version = ArtifactVersionRequest(
-            artifact_id=artifact.id,
-            version=version,
-            tags=tags,
-            type=ArtifactType.DATA,
-            uri=folder_or_file_uri,
-            materializer=source_utils.resolve(PreexistingDataMaterializer),
-            data_type=source_utils.resolve(Path),
-            user=Client().active_user.id,
-            workspace=Client().active_workspace.id,
-            artifact_store_id=artifact_store.id,
-            has_custom_name=has_custom_name,
-        )
-        try:
-            return client.zen_store.create_artifact_version(
-                artifact_version=artifact_version
-            )
-        except EntityExistsError:
-            return None
-
-    response = _create_artifact_version_with_retries(
-        name=name,
-        version=version,
-        create_version_fn=_create_version,
+    artifact_version = client.zen_store.create_artifact_version(
+        artifact_version=artifact_version_request
     )
 
-    if artifact_metadata:
-        client.create_run_metadata(
-            metadata=artifact_metadata,
-            resource_id=response.id,
-            resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
-        )
-
     _link_artifact_version_to_the_step_and_model(
-        artifact_version=response,
+        artifact_version=artifact_version,
         is_model_artifact=is_model_artifact,
         is_deployment_artifact=is_deployment_artifact,
     )
 
-    return response
+    return artifact_version
 
 
 def load_artifact(
@@ -662,107 +611,6 @@ def _check_if_artifact_with_given_uri_already_registered(
             )
 
 
-def _get_or_create_artifact(
-    name: str, has_custom_name: bool, tags: Optional[List[str]] = None
-) -> ArtifactResponse:
-    """Get or create an artifact with the given name.
-
-    Args:
-        name: The name of the artifact.
-        has_custom_name: If the artifact name is custom and should be listed in
-            the dashboard "Artifacts" tab.
-        tags: Tags to associate with the artifact.
-
-    Returns:
-        The artifact.
-    """
-    client = Client()
-    # Get or create the artifact
-    try:
-        artifact = client.list_artifacts(name=name)[0]
-        if artifact.has_custom_name != has_custom_name:
-            client.update_artifact(
-                name_id_or_prefix=artifact.id, has_custom_name=has_custom_name
-            )
-    except IndexError:
-        try:
-            artifact = client.zen_store.create_artifact(
-                ArtifactRequest(
-                    name=name,
-                    has_custom_name=has_custom_name,
-                    tags=tags,
-                )
-            )
-        except EntityExistsError:
-            artifact = client.list_artifacts(name=name)[0]
-    return artifact
-
-
-def _create_artifact_version_with_retries(
-    name: str,
-    version: Optional[Union[int, str]],
-    create_version_fn: Callable[
-        [
-            Union[int, str],
-        ],
-        Optional[ArtifactVersionResponse],
-    ],
-) -> ArtifactVersionResponse:
-    """Create an artifact version with some retries.
-
-    This function will retry the creation of an artifact version up to
-    MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION times if it fails.
-    It can fail in high-concurrency environments.
-
-    Args:
-        name: The name of the artifact.
-        version: The version of the artifact. If not provided, a new
-            auto-incremented version will be used.
-        create_version_fn: The function to create the artifact version.
-
-    Returns:
-        The created artifact version.
-
-    Raises:
-        EntityExistsError: If the artifact version could not be created
-            after MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION attempts due
-            to collisions.
-
-    """
-    response = None
-    if not version:
-        retries_made = 0
-        for i in range(MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION):
-            # Get new artifact version
-            version = _get_new_artifact_version(name)
-            if response := create_version_fn(version):
-                break
-            # smoothed exponential back-off, it will go as 0.2, 0.3,
-            # 0.45, 0.68, 1.01, 1.52, 2.28, 3.42, 5.13, 7.69, ...
-            sleep = 0.2 * 1.5**i
-            logger.debug(
-                f"Failed to create artifact version `{version}` for "
-                f"artifact `{name}`. Retrying in {sleep}..."
-            )
-            time.sleep(sleep)
-            retries_made += 1
-        if not response:
-            raise EntityExistsError(
-                f"Failed to create new artifact version for artifact "
-                f"`{name}`. Retried {retries_made} times. "
-                "This could be driven by exceptionally high concurrency of "
-                "pipeline runs. Please, reach out to us on ZenML Slack for support."
-            )
-    else:
-        response = create_version_fn(version)
-        if not response:
-            raise EntityExistsError(
-                f"Failed to create artifact version `{version}` for artifact "
-                f"`{name}`. Given version already exists."
-            )
-    return response
-
-
 def _link_artifact_version_to_the_step_and_model(
     artifact_version: ArtifactVersionResponse,
     is_model_artifact: bool,
@@ -962,30 +810,6 @@ def _get_artifact_store_from_response_or_from_active_stack(
     return Client().active_stack.artifact_store
 
 
-def _get_new_artifact_version(artifact_name: str) -> int:
-    """Get the next auto-incremented version for an artifact name.
-
-    Args:
-        artifact_name: The name of the artifact.
-
-    Returns:
-        The next auto-incremented version.
-    """
-    artifact_versions = Client().list_artifact_versions(
-        name=artifact_name,
-        sort_by="desc:version_number",
-        size=1,
-    )
-
-    # If a numbered version exists, increment it
-    try:
-        return int(artifact_versions[0].version) + 1
-
-    # If no numbered versions exist yet, start at 1
-    except (IndexError, ValueError):
-        return 1
-
-
 def _load_file_from_artifact_store(
     uri: str,
     artifact_store: "BaseArtifactStore",