zenml-nightly 0.70.0.dev20241122__py3-none-any.whl → 0.70.0.dev20241201__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.70.0.dev20241122
+0.70.0.dev20241201

zenml/artifact_stores/base_artifact_store.py

@@ -267,11 +267,11 @@ class BaseArtifactStore(StackComponent):
 
     # --- User interface ---
     @abstractmethod
-    def open(self, name: PathType, mode: str = "r") -> Any:
+    def open(self, path: PathType, mode: str = "r") -> Any:
         """Open a file at the given path.
 
         Args:
-            name: The path of the file to open.
+            path: The path of the file to open.
             mode: The mode to open the file.
 
         Returns:

zenml/artifacts/artifact_config.py

@@ -45,7 +45,13 @@ class ArtifactConfig(BaseModel):
     ```
 
     Attributes:
-        name: The name of the artifact.
+        name: The name of the artifact:
+            - static string e.g. "name"
+            - dynamic string e.g. "name_{date}_{time}_{custom_placeholder}"
+            If you use any placeholders besides `date` and `time`,
+            you need to provide the values for them in the `substitutions`
+            argument of the step decorator or the `substitutions` argument
+            of `with_options` of the step.
         version: The version of the artifact.
         tags: The tags of the artifact.
         run_metadata: Metadata to add to the artifact.

zenml/artifacts/utils.py

@@ -14,6 +14,7 @@
 """Utility functions for handling artifacts."""
 
 import base64
+import contextlib
 import os
 import tempfile
 import zipfile
@@ -41,7 +42,6 @@ from zenml.enums import (
     ArtifactSaveType,
     ArtifactType,
     ExecutionStatus,
-    MetadataResourceTypes,
     StackComponentType,
     VisualizationType,
 )
@@ -404,50 +404,71 @@ def log_artifact_metadata(
         artifact_version: The version of the artifact to log metadata for. If
             not provided, when being called inside a step that produces an
             artifact named `artifact_name`, the metadata will be associated to
-            the corresponding newly created artifact. Or, if not provided when
-            being called outside a step, or in a step that does not produce
-            any artifact named `artifact_name`, the metadata will be associated
-            to the latest version of that artifact.
+            the corresponding newly created artifact.
 
     Raises:
         ValueError: If no artifact name is provided and the function is not
             called inside a step with a single output, or, if neither an
             artifact nor an output with the given name exists.
+
     """
     logger.warning(
         "The `log_artifact_metadata` function is deprecated and will soon be "
         "removed. Please use `log_metadata` instead."
     )
-    try:
+
+    from zenml import log_metadata
+
+    if artifact_name and artifact_version:
+        assert artifact_name is not None
+
+        log_metadata(
+            metadata=metadata,
+            artifact_name=artifact_name,
+            artifact_version=artifact_version,
+        )
+
+    step_context = None
+    with contextlib.suppress(RuntimeError):
         step_context = get_step_context()
-        in_step_outputs = (artifact_name in step_context._outputs) or (
-            not artifact_name and len(step_context._outputs) == 1
+
+    if step_context and artifact_name in step_context._outputs.keys():
+        log_metadata(
+            metadata=metadata,
+            artifact_name=artifact_name,
+            infer_artifact=True,
         )
-    except RuntimeError:
-        step_context = None
-        in_step_outputs = False
-
-    if not step_context or not in_step_outputs or artifact_version:
-        if not artifact_name:
-            raise ValueError(
-                "Artifact name must be provided unless the function is called "
-                "inside a step with a single output."
-            )
+    elif step_context and len(step_context._outputs) == 1:
+        single_output_name = list(step_context._outputs.keys())[0]
+
+        log_metadata(
+            metadata=metadata,
+            artifact_name=single_output_name,
+            infer_artifact=True,
+        )
+    elif artifact_name:
         client = Client()
-        response = client.get_artifact_version(artifact_name, artifact_version)
-        client.create_run_metadata(
+        logger.warning(
+            "Deprecation warning! Currently, you are calling "
+            "`log_artifact_metadata` from a context, where we use the "
+            "`artifact_name` to fetch it and link the metadata to its "
+            "latest version. This behavior is deprecated and will be "
+            "removed in the future. To circumvent this, please check"
+            "the `log_metadata` function."
+        )
+        artifact_version_model = client.get_artifact_version(
+            name_id_or_prefix=artifact_name
+        )
+        log_metadata(
             metadata=metadata,
-            resource_id=response.id,
-            resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
+            artifact_version_id=artifact_version_model.id,
         )
-
     else:
-        try:
-            step_context.add_output_metadata(
-                metadata=metadata, output_name=artifact_name
-            )
-        except StepContextError as e:
-            raise ValueError(e)
+        raise ValueError(
+            "You need to call `log_artifact_metadata` either within a step "
+            "(potentially with an artifact name) or outside of a step with an "
+            "artifact name (and/or version)."
+        )
 
 
 # -----------------
@@ -575,7 +596,7 @@ def download_artifact_files_from_response(
                     )
                     file_path = str(Path(artifact.uri) / file_str)
                     with artifact_store.open(
-                        name=file_path, mode="rb"
+                        file_path, mode="rb"
                     ) as store_file:
                         # Use a loop to read and write chunks of the file
                         # instead of reading the entire file into memory
@@ -689,7 +710,11 @@ def _link_artifact_version_to_the_step_and_model(
         client.zen_store.update_run_step(
             step_run_id=step_run.id,
             step_run_update=StepRunUpdate(
-                outputs={artifact_version.artifact.name: artifact_version.id}
+                outputs={
+                    artifact_version.artifact.name: [
+                        artifact_version.id,
+                    ]
+                }
             ),
         )
         error_message = "model"

zenml/cli/__init__.py

@@ -2242,6 +2242,9 @@ export ZENML_STORE_URL=https://...
 export ZENML_STORE_API_KEY=<API_KEY>
 ```
 
+You don't need to run `zenml login` after setting these two environment
+variables and can start interacting with your server right away.
+
 To see all the service accounts you've created and their API keys, use the
 following commands:
 
@@ -2407,6 +2410,21 @@ stack, use the `--stack` option.
 zenml pipeline run <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
 ```
 
+If you want to create a run template based on your pipeline that can later be used to trigger a run either from the dashboard or through an HTTP request:
+
+```bash
+zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
+   --name=<TEMPLATE_NAME>
+
+To specify a config file, use the `--config/-c` option. If you would like to use a different stack than the active one, use the `--stack` option.
+
+```bash
+zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
+   --name=<TEMPLATE_NAME> \
+   --config=<PATH_TO_CONFIG_YAML> \
+   --stack=<STACK_ID_OR_NAME>
+```
+
 Tagging your resources with ZenML
 ---------------------------------
 

zenml/cli/base.py

@@ -79,19 +79,19 @@ class ZenMLProjectTemplateLocation(BaseModel):
 ZENML_PROJECT_TEMPLATES = dict(
     e2e_batch=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-e2e-batch",
-        github_tag="2024.11.20",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     starter=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-starter",
-        github_tag="2024.10.30",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",
-        github_tag="2024.10.30",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",
-        github_tag="2024.11.08",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
 )
 

zenml/cli/login.py

@@ -14,6 +14,7 @@
 """CLI for managing ZenML server deployments."""
 
 import ipaddress
+import os
 import re
 import sys
 import time
@@ -413,6 +414,27 @@ def connect_to_pro_server(
     cli_utils.declare(f"Connected to ZenML Pro server: {server.name}.")
 
 
+def _fail_if_authentication_environment_variables_set() -> None:
+    """Fail if any of the authentication environment variables are set."""
+    environment_variables = [
+        "ZENML_STORE_URL",
+        "ZENML_STORE_API_KEY",
+        "ZENML_STORE_USERNAME",
+        "ZENML_STORE_PASSWORD",
+    ]
+
+    if any(env_var in os.environ for env_var in environment_variables):
+        cli_utils.error(
+            "You're running to login/logout while having one of the "
+            f"{environment_variables} environment variables set. "
+            "If you want to use those environment variables to authenticate "
+            "to your ZenML server, there is no need to login/logout, you can "
+            "start interacting with your server right away. If you want to use "
+            "the `zenml login` command for authentication, please unset these "
+            "environment variables first."
+        )
+
+
 @cli.command(
     "login",
     help=(
@@ -670,6 +692,8 @@ def login(
             dashboard on a public domain. Primarily used for accessing the
             dashboard in Colab.
     """
+    _fail_if_authentication_environment_variables_set()
+
     if local:
         if api_key:
             cli_utils.error(
@@ -849,6 +873,8 @@ def logout(
     """
     from zenml.login.credentials_store import get_credentials_store
 
+    _fail_if_authentication_environment_variables_set()
+
     credentials_store = get_credentials_store()
     gc = GlobalConfiguration()
     store_cfg = gc.store_configuration

zenml/cli/pipeline.py

@@ -315,6 +315,86 @@ def run_pipeline(
         pipeline_instance()
 
 
+@pipeline.command(
+    "create-run-template",
+    help="Create a run template for a pipeline. The SOURCE argument needs to "
+    "be an importable source path resolving to a ZenML pipeline instance, e.g. "
+    "`my_module.my_pipeline_instance`.",
+)
+@click.argument("source")
+@click.option(
+    "--name",
+    "-n",
+    type=str,
+    required=True,
+    help="Name for the template",
+)
+@click.option(
+    "--config",
+    "-c",
+    "config_path",
+    type=click.Path(exists=True, dir_okay=False),
+    required=False,
+    help="Path to configuration file for the build.",
+)
+@click.option(
+    "--stack",
+    "-s",
+    "stack_name_or_id",
+    type=str,
+    required=False,
+    help="Name or ID of the stack to use for the build.",
+)
+def create_run_template(
+    source: str,
+    name: str,
+    config_path: Optional[str] = None,
+    stack_name_or_id: Optional[str] = None,
+) -> None:
+    """Create a run template for a pipeline.
+
+    Args:
+        source: Importable source resolving to a pipeline instance.
+        name: Name of the run template.
+        config_path: Path to pipeline configuration file.
+        stack_name_or_id: Name or ID of the stack for which the template should
+            be created.
+    """
+    if not Client().root:
+        cli_utils.warning(
+            "You're running the `zenml pipeline create-run-template` command "
+            "without a ZenML repository. Your current working directory will "
+            "be used as the source root relative to which the registered step "
+            "classes will be resolved. To silence this warning, run `zenml "
+            "init` at your source code root."
+        )
+
+    try:
+        pipeline_instance = source_utils.load(source)
+    except ModuleNotFoundError as e:
+        source_root = source_utils.get_source_root()
+        cli_utils.error(
+            f"Unable to import module `{e.name}`. Make sure the source path is "
+            f"relative to your source root `{source_root}`."
+        )
+    except AttributeError as e:
+        cli_utils.error("Unable to load attribute from module: " + str(e))
+
+    if not isinstance(pipeline_instance, Pipeline):
+        cli_utils.error(
+            f"The given source path `{source}` does not resolve to a pipeline "
+            "object."
+        )
+
+    with cli_utils.temporary_active_stack(stack_name_or_id=stack_name_or_id):
+        pipeline_instance = pipeline_instance.with_options(
+            config_path=config_path
+        )
+        template = pipeline_instance.create_run_template(name=name)
+
+    cli_utils.declare(f"Created run template `{template.id}`.")
+
+
 @pipeline.command("list", help="List all registered pipelines.")
 @list_options(PipelineFilter)
 def list_pipelines(**kwargs: Any) -> None:

zenml/cli/server.py

@@ -469,7 +469,7 @@ def logs(
     if server is None:
         cli_utils.error(
             "The local ZenML dashboard is not running. Please call `zenml "
-            "up` first to start the ZenML dashboard locally."
+            "login --local` first to start the ZenML dashboard locally."
         )
 
     from zenml.zen_server.deploy.deployer import LocalServerDeployer

zenml/cli/service_connectors.py

@@ -1194,9 +1194,9 @@ to provide access to:
 use from an STS token to an AWS secret key. This involves updating some config
 values and deleting others:
 
-    $ zenml service-connector update aws-auto-multi \\                       
---aws-access-key-id=<aws-key-id> \\                                 
---aws_secret_access_key=<aws-secret-key>  \\                      
+    $ zenml service-connector update aws-auto-multi \\
+--aws_access_key_id=<aws-key-id> \\
+--aws_secret_access_key=<aws-secret-key>  \\
 --remove-attr aws-sts-token
 
 - update the foo label to a new value and delete the baz label from a connector:

zenml/cli/stack.py

@@ -1129,14 +1129,12 @@ def export_stack(
 def _import_stack_component(
     component_type: StackComponentType,
     component_dict: Dict[str, Any],
-    component_spec_path: Optional[str] = None,
 ) -> UUID:
     """Import a single stack component with given type/config.
 
     Args:
         component_type: The type of component to import.
         component_dict: Dict representation of the component to import.
-        component_spec_path: Path to the component spec file.
 
     Returns:
         The ID of the imported component.
@@ -1172,7 +1170,6 @@ def _import_stack_component(
         component_type=component_type,
         flavor=flavor,
         configuration=config,
-        component_spec_path=component_spec_path,
     )
     return component.id
 

zenml/cli/stack_components.py

@@ -573,7 +573,6 @@ def generate_stack_component_copy_command(
                 component_type=component_to_copy.type,
                 configuration=component_to_copy.configuration,
                 labels=component_to_copy.labels,
-                component_spec_path=component_to_copy.component_spec_path,
             )
             print_model_url(get_component_url(copied_component))
 

zenml/cli/utils.py

@@ -711,11 +711,6 @@ def print_stack_component_configuration(
 
         console.print(rich_table)
 
-    if component.component_spec_path:
-        declare(
-            f"Component spec path for `mlstacks`: {component.component_spec_path}"
-        )
-
 
 def expand_argument_value_from_file(name: str, value: str) -> str:
     """Expands the value of an argument pointing to a file into the contents of that file.

zenml/client.py

@@ -60,7 +60,6 @@ from zenml.constants import (
 from zenml.enums import (
     ArtifactType,
     LogicalOperators,
-    MetadataResourceTypes,
     ModelStages,
     OAuthDeviceStatus,
     PluginSubType,
@@ -137,6 +136,7 @@ from zenml.models import (
     PipelineRunFilter,
     PipelineRunResponse,
     RunMetadataRequest,
+    RunMetadataResource,
     RunTemplateFilter,
     RunTemplateRequest,
     RunTemplateResponse,
@@ -1979,7 +1979,6 @@ class Client(metaclass=ClientMetaClass):
         flavor: str,
         component_type: StackComponentType,
         configuration: Dict[str, str],
-        component_spec_path: Optional[str] = None,
         labels: Optional[Dict[str, Any]] = None,
     ) -> "ComponentResponse":
         """Registers a stack component.
@@ -1987,7 +1986,6 @@ class Client(metaclass=ClientMetaClass):
         Args:
             name: The name of the stack component.
             flavor: The flavor of the stack component.
-            component_spec_path: The path to the stack spec file.
             component_type: The type of the stack component.
             configuration: The configuration of the stack component.
             labels: The labels of the stack component.
@@ -2016,7 +2014,6 @@ class Client(metaclass=ClientMetaClass):
             name=name,
             type=component_type,
             flavor=flavor,
-            component_spec_path=component_spec_path,
             configuration=configuration,
             user=self.active_user.id,
             workspace=self.active_workspace.id,
@@ -2033,7 +2030,6 @@ class Client(metaclass=ClientMetaClass):
         name_id_or_prefix: Optional[Union[UUID, str]],
         component_type: StackComponentType,
         name: Optional[str] = None,
-        component_spec_path: Optional[str] = None,
         configuration: Optional[Dict[str, Any]] = None,
         labels: Optional[Dict[str, Any]] = None,
         disconnect: Optional[bool] = None,
@@ -2047,7 +2043,6 @@ class Client(metaclass=ClientMetaClass):
                 update.
             component_type: The type of the stack component to update.
             name: The new name of the stack component.
-            component_spec_path: The new path to the stack spec file.
             configuration: The new configuration of the stack component.
             labels: The new labels of the stack component.
             disconnect: Whether to disconnect the stack component from its
@@ -2072,7 +2067,6 @@ class Client(metaclass=ClientMetaClass):
         update_model = ComponentUpdate(
             workspace=self.active_workspace.id,
             user=self.active_user.id,
-            component_spec_path=component_spec_path,
         )
 
         if name is not None:
@@ -4438,23 +4432,20 @@ class Client(metaclass=ClientMetaClass):
     def create_run_metadata(
         self,
         metadata: Dict[str, "MetadataType"],
-        resource_id: UUID,
-        resource_type: MetadataResourceTypes,
+        resources: List[RunMetadataResource],
         stack_component_id: Optional[UUID] = None,
+        publisher_step_id: Optional[UUID] = None,
     ) -> None:
         """Create run metadata.
 
         Args:
             metadata: The metadata to create as a dictionary of key-value pairs.
-            resource_id: The ID of the resource for which the
-                metadata was produced.
-            resource_type: The type of the resource for which the
+            resources: The list of IDs and types of the resources for that the
                 metadata was produced.
             stack_component_id: The ID of the stack component that produced
                 the metadata.
-
-        Returns:
-            None
+            publisher_step_id: The ID of the step execution that publishes
+                this metadata automatically.
         """
         from zenml.metadata.metadata_types import get_metadata_type
 
@@ -4483,14 +4474,13 @@ class Client(metaclass=ClientMetaClass):
         run_metadata = RunMetadataRequest(
             workspace=self.active_workspace.id,
             user=self.active_user.id,
-            resource_id=resource_id,
-            resource_type=resource_type,
+            resources=resources,
             stack_component_id=stack_component_id,
+            publisher_step_id=publisher_step_id,
             values=values,
             types=types,
         )
         self.zen_store.create_run_metadata(run_metadata)
-        return None
 
     # -------------------------------- Secrets ---------------------------------
 

zenml/config/compiler.py

@@ -99,7 +99,10 @@ class Compiler:
 
         self._apply_stack_default_settings(pipeline=pipeline, stack=stack)
         if run_configuration.run_name:
-            self._verify_run_name(run_configuration.run_name)
+            self._verify_run_name(
+                run_configuration.run_name,
+                pipeline.configuration.substitutions,
+            )
 
         pipeline_settings = self._filter_and_validate_settings(
             settings=pipeline.configuration.settings,
@@ -305,16 +308,22 @@ class Compiler:
         return default_settings
 
     @staticmethod
-    def _verify_run_name(run_name: str) -> None:
+    def _verify_run_name(
+        run_name: str,
+        substitutions: Dict[str, str],
+    ) -> None:
         """Verifies that the run name contains only valid placeholders.
 
         Args:
             run_name: The run name to verify.
+            substitutions: The substitutions to be used in the run name.
 
         Raises:
             ValueError: If the run name contains invalid placeholders.
         """
-        valid_placeholder_names = {"date", "time"}
+        valid_placeholder_names = {"date", "time"}.union(
+            set(substitutions.keys())
+        )
         placeholders = {
             v[1] for v in string.Formatter().parse(run_name) if v[1]
         }

zenml/config/pipeline_configurations.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Pipeline configuration classes."""
 
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from pydantic import SerializeAsAny, field_validator
@@ -46,6 +47,25 @@ class PipelineConfigurationUpdate(StrictBaseModel):
     model: Optional[Model] = None
     parameters: Optional[Dict[str, Any]] = None
     retry: Optional[StepRetryConfig] = None
+    substitutions: Dict[str, str] = {}
+
+    def _get_full_substitutions(
+        self, start_time: Optional[datetime]
+    ) -> Dict[str, str]:
+        """Returns the full substitutions dict.
+
+        Args:
+            start_time: Start time of the pipeline run.
+
+        Returns:
+            The full substitutions dict including date and time.
+        """
+        if start_time is None:
+            start_time = datetime.utcnow()
+        ret = self.substitutions.copy()
+        ret.setdefault("date", start_time.strftime("%Y_%m_%d"))
+        ret.setdefault("time", start_time.strftime("%H_%M_%S_%f"))
+        return ret
 
 
 class PipelineConfiguration(PipelineConfigurationUpdate):

zenml/config/pipeline_run_configuration.py

@@ -52,3 +52,4 @@ class PipelineRunConfiguration(
     retry: Optional[StepRetryConfig] = None
     failure_hook_source: Optional[SourceWithValidator] = None
     success_hook_source: Optional[SourceWithValidator] = None
+    substitutions: Dict[str, str] = {}

zenml/config/step_configurations.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Pipeline configuration classes."""
 
+from datetime import datetime
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -49,6 +50,7 @@ from zenml.utils.pydantic_utils import before_validator_handler
 
 if TYPE_CHECKING:
     from zenml.config import DockerSettings, ResourceSettings
+    from zenml.config.pipeline_configurations import PipelineConfiguration
 
 logger = get_logger(__name__)
 
@@ -152,6 +154,7 @@ class StepConfigurationUpdate(StrictBaseModel):
     success_hook_source: Optional[SourceWithValidator] = None
     model: Optional[Model] = None
     retry: Optional[StepRetryConfig] = None
+    substitutions: Dict[str, str] = {}
 
     outputs: Mapping[str, PartialArtifactConfiguration] = {}
 
@@ -237,6 +240,24 @@ class StepConfiguration(PartialStepConfiguration):
             model_or_dict = model_or_dict.model_dump()
         return DockerSettings.model_validate(model_or_dict)
 
+    def _get_full_substitutions(
+        self,
+        pipeline_config: "PipelineConfiguration",
+        start_time: Optional[datetime],
+    ) -> Dict[str, str]:
+        """Get the full set of substitutions for this step configuration.
+
+        Args:
+            pipeline_config: The pipeline configuration.
+            start_time: The start time of the pipeline run.
+
+        Returns:
+            The full set of substitutions for this step configuration.
+        """
+        ret = pipeline_config._get_full_substitutions(start_time)
+        ret.update(self.substitutions)
+        return ret
+
 
 class InputSpec(StrictBaseModel):
     """Step input specification."""