zenml-nightly 0.66.0.dev20240927__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl

zenml/artifacts/external_artifact.py

@@ -40,19 +40,11 @@ class ExternalArtifact(ExternalArtifactConfiguration):
     can be used to provide any value as input to a step without needing to
     write an additional step that returns this value.
 
-    The external artifact needs to have either a value associated with it
-    that will be uploaded to the artifact store, or reference an artifact
-    that is already registered in ZenML.
-
-    There are several ways to reference an existing artifact:
-    - By providing an artifact ID.
-    - By providing an artifact name and version. If no version is provided,
-        the latest version of that artifact will be used.
+    The external artifact needs to have a value associated with it
+    that will be uploaded to the artifact store.
 
     Args:
         value: The artifact value.
-        id: The ID of an artifact that should be referenced by this external
-            artifact.
         materializer: The materializer to use for saving the artifact value
             to the artifact store. Only used when `value` is provided.
         store_artifact_metadata: Whether metadata for the artifact should
@@ -90,51 +82,16 @@ class ExternalArtifact(ExternalArtifactConfiguration):
         """Model validator for the external artifact.
 
         Raises:
-            ValueError: if the value, id and name fields are set incorrectly.
+            ValueError: If an ID was set.
 
         Returns:
-            the validated instance.
+            The validated instance.
         """
-        deprecation_msg = (
-            "Parameter `{param}` or `ExternalArtifact` will be deprecated "
-            "in upcoming releases. Please use `{substitute}` instead."
-        )
-        for param, substitute in [
-            ["id", "Client().get_artifact_version(name_id_or_prefix=<id>)"],
-            [
-                "name",
-                "Client().get_artifact_version(name_id_or_prefix=<name>)",
-            ],
-            [
-                "version",
-                "Client().get_artifact_version(name_id_or_prefix=<name>,version=<version>)",
-            ],
-            [
-                "model",
-                "Client().get_model_version(<model_name>,<model_version>).get_artifact(name)",
-            ],
-        ]:
-            if getattr(self, param, None):
-                logger.warning(
-                    deprecation_msg.format(
-                        param=param,
-                        substitute=substitute,
-                    )
-                )
-        options = [
-            getattr(self, field, None) is not None
-            for field in ["value", "id", "name"]
-        ]
-        if sum(options) > 1:
+        if self.id:
             raise ValueError(
-                "Only one of `value`, `id`, or `name` can be provided when "
-                "creating an external artifact."
-            )
-        elif sum(options) == 0:
-            raise ValueError(
-                "Either `value`, `id`, or `name` must be provided when "
-                "creating an external artifact."
+                "External artifacts can only be initialized with a value."
             )
+
         return self
 
     def upload_by_value(self) -> UUID:
@@ -177,7 +134,4 @@ class ExternalArtifact(ExternalArtifactConfiguration):
         """
         return ExternalArtifactConfiguration(
             id=self.id,
-            name=self.name,
-            version=self.version,
-            model=self.model,
         )

zenml/artifacts/external_artifact_config.py

@@ -13,14 +13,13 @@
 #  permissions and limitations under the License.
 """External artifact definition."""
 
-from typing import Optional
+from typing import Any, Dict, Optional
 from uuid import UUID
 
 from pydantic import BaseModel, model_validator
 
 from zenml.logger import get_logger
-from zenml.model.model import Model
-from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
+from zenml.utils.pydantic_utils import before_validator_handler
 
 logger = get_logger(__name__)
 
@@ -32,26 +31,23 @@ class ExternalArtifactConfiguration(BaseModel):
     """
 
     id: Optional[UUID] = None
-    name: Optional[str] = None
-    version: Optional[str] = None
-    model: Optional[Model] = None
 
-    @model_validator(mode="after")
-    def external_artifact_validator(self) -> "ExternalArtifactConfiguration":
-        """Model validator for the external artifact configuration.
+    @model_validator(mode="before")
+    @classmethod
+    @before_validator_handler
+    def _remove_old_attributes(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Remove old attributes that are not used anymore.
 
-        Raises:
-            ValueError: if both version and model fields are set.
+        Args:
+            data: The model data.
 
         Returns:
-            the validated instance.
+            Model data without the removed attributes.
         """
-        if self.version and self.model:
-            raise ValueError(
-                "Cannot provide both `version` and `model` when "
-                "creating an external artifact."
-            )
-        return self
+        data.pop("name", None)
+        data.pop("version", None)
+        data.pop("model", None)
+        return data
 
     def get_artifact_version_id(self) -> UUID:
         """Get the artifact.
@@ -71,29 +67,16 @@ class ExternalArtifactConfiguration(BaseModel):
 
         if self.id:
             response = client.get_artifact_version(self.id)
-        elif self.name:
-            if self.version:
-                response = client.get_artifact_version(
-                    self.name, version=self.version
-                )
-            elif self.model:
-                response_ = self.model.get_artifact(self.name)
-                if not isinstance(response_, ArtifactVersionResponse):
-                    raise RuntimeError(
-                        f"Failed to pull artifact `{self.name}` from the Model "
-                        f"(name=`{self.model.name}`, version="
-                        f"`{self.model.version}`). Please validate the "
-                        "input and try again."
-                    )
-                response = response_
-            else:
-                response = client.get_artifact_version(self.name)
         else:
             raise RuntimeError(
-                "Either the ID or name of the artifact must be provided. "
-                "If you created this ExternalArtifact from a value, please "
+                "The ID of the artifact must be provided. "
+                "- If you created this ExternalArtifact from a value, please "
                 "ensure that `upload_by_value` was called before trying to "
-                "fetch the artifact ID."
+                "fetch the artifact ID.\n- If you specified an artifact name "
+                "or model name for this external artifact, this functionality "
+                "was removed from the ExternalArtifact class. Use Client "
+                "methods instead to dynamically fetch an artifact via name or "
+                "from a model instead."
             )
 
         artifact_store_id = client.active_stack.artifact_store.id
@@ -107,6 +90,4 @@ class ExternalArtifactConfiguration(BaseModel):
                 "store."
             )
 
-        self.id = response.id
-
         return self.id

zenml/artifacts/load_directory_materializer.py

@@ -0,0 +1,91 @@
+#  Copyright (c) ZenML GmbH 2023. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Only-load materializer for directories."""
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, ClassVar, Tuple, Type
+
+from zenml.enums import ArtifactType
+from zenml.io import fileio
+from zenml.materializers.base_materializer import BaseMaterializer
+
+
+class PreexistingDataMaterializer(BaseMaterializer):
+    """Materializer to load directories from the artifact store.
+
+    This materializer is very special, since it do not implement save
+    logic at all. The save of the data to some URI inside the artifact store
+    shall happen outside and is in user's responsibility.
+
+    This materializer solely supports the `register_artifact` function.
+    """
+
+    ASSOCIATED_TYPES: ClassVar[Tuple[Type[Any], ...]] = (Path,)
+    ASSOCIATED_ARTIFACT_TYPE: ClassVar[ArtifactType] = ArtifactType.DATA
+    SKIP_REGISTRATION: ClassVar[bool] = True
+
+    def load(self, data_type: Type[Any]) -> Any:
+        """Copy the artifact file(s) to a local temp directory.
+
+        Args:
+            data_type: Unused.
+
+        Returns:
+            Path to the local directory that contains the artifact files.
+        """
+        directory = tempfile.mkdtemp(prefix="zenml-artifact")
+        if fileio.isdir(self.uri):
+            self._copy_directory(src=self.uri, dst=directory)
+            return Path(directory)
+        else:
+            dst = os.path.join(directory, os.path.split(self.uri)[-1])
+            fileio.copy(src=self.uri, dst=dst)
+            return Path(dst)
+
+    def save(self, data: Any) -> None:
+        """Store the directory in the artifact store.
+
+        Args:
+            data: Path to a local directory to store.
+
+        Raises:
+            NotImplementedError: Always
+        """
+        raise NotImplementedError(
+            "`PreexistingDataMaterializer` can only be used in the "
+            "context of `register_artifact` function, "
+            "which expects the data to be already properly saved in "
+            "the Artifact Store, thus `save` logic makes no sense here."
+        )
+
+    @staticmethod
+    def _copy_directory(src: str, dst: str) -> None:
+        """Recursively copy a directory.
+
+        Args:
+            src: The directory to copy.
+            dst: Where to copy the directory to.
+        """
+        for src_dir, _, files in fileio.walk(src):
+            src_dir_ = str(src_dir)
+            dst_dir = str(os.path.join(dst, os.path.relpath(src_dir_, src)))
+            fileio.makedirs(dst_dir)
+
+            for file in files:
+                file_ = str(file)
+                src_file = os.path.join(src_dir_, file_)
+                dst_file = os.path.join(dst_dir, file_)
+                fileio.copy(src_file, dst_file)