zenml-nightly 0.70.0.dev20241127__py3-none-any.whl → 0.70.0.dev20241129__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.70.0.dev20241127
+0.70.0.dev20241129

zenml/artifacts/artifact_config.py

@@ -21,6 +21,7 @@ from zenml.enums import ArtifactType
 from zenml.logger import get_logger
 from zenml.metadata.metadata_types import MetadataType
 from zenml.utils.pydantic_utils import before_validator_handler
+from zenml.utils.string_utils import format_name_template
 
 logger = get_logger(__name__)
 
@@ -45,7 +46,13 @@ class ArtifactConfig(BaseModel):
     ```
 
     Attributes:
-        name: The name of the artifact.
+        name: The name of the artifact:
+            - static string e.g. "name"
+            - dynamic string e.g. "name_{date}_{time}_{custom_placeholder}"
+            If you use any placeholders besides `date` and `time`,
+            you need to provide the values for them in the `substitutions`
+            argument of the step decorator or the `substitutions` argument
+            of `with_options` of the step.
         version: The version of the artifact.
         tags: The tags of the artifact.
         run_metadata: Metadata to add to the artifact.
@@ -111,3 +118,16 @@ class ArtifactConfig(BaseModel):
             data.setdefault("artifact_type", ArtifactType.SERVICE)
 
         return data
+
+    def _evaluated_name(self, substitutions: Dict[str, str]) -> Optional[str]:
+        """Evaluated name of the artifact.
+
+        Args:
+            substitutions: Extra placeholders to use in the name template.
+
+        Returns:
+            The evaluated name of the artifact.
+        """
+        if self.name:
+            return format_name_template(self.name, substitutions=substitutions)
+        return self.name

zenml/artifacts/utils.py

@@ -689,7 +689,11 @@ def _link_artifact_version_to_the_step_and_model(
         client.zen_store.update_run_step(
             step_run_id=step_run.id,
             step_run_update=StepRunUpdate(
-                outputs={artifact_version.artifact.name: artifact_version.id}
+                outputs={
+                    artifact_version.artifact.name: [
+                        artifact_version.id,
+                    ]
+                }
             ),
         )
         error_message = "model"

zenml/cli/pipeline.py

@@ -315,6 +315,86 @@ def run_pipeline(
         pipeline_instance()
 
 
+@pipeline.command(
+    "create-run-template",
+    help="Create a run template for a pipeline. The SOURCE argument needs to "
+    "be an importable source path resolving to a ZenML pipeline instance, e.g. "
+    "`my_module.my_pipeline_instance`.",
+)
+@click.argument("source")
+@click.option(
+    "--name",
+    "-n",
+    type=str,
+    required=True,
+    help="Name for the template",
+)
+@click.option(
+    "--config",
+    "-c",
+    "config_path",
+    type=click.Path(exists=True, dir_okay=False),
+    required=False,
+    help="Path to configuration file for the build.",
+)
+@click.option(
+    "--stack",
+    "-s",
+    "stack_name_or_id",
+    type=str,
+    required=False,
+    help="Name or ID of the stack to use for the build.",
+)
+def create_run_template(
+    source: str,
+    name: str,
+    config_path: Optional[str] = None,
+    stack_name_or_id: Optional[str] = None,
+) -> None:
+    """Create a run template for a pipeline.
+
+    Args:
+        source: Importable source resolving to a pipeline instance.
+        name: Name of the run template.
+        config_path: Path to pipeline configuration file.
+        stack_name_or_id: Name or ID of the stack for which the template should
+            be created.
+    """
+    if not Client().root:
+        cli_utils.warning(
+            "You're running the `zenml pipeline create-run-template` command "
+            "without a ZenML repository. Your current working directory will "
+            "be used as the source root relative to which the registered step "
+            "classes will be resolved. To silence this warning, run `zenml "
+            "init` at your source code root."
+        )
+
+    try:
+        pipeline_instance = source_utils.load(source)
+    except ModuleNotFoundError as e:
+        source_root = source_utils.get_source_root()
+        cli_utils.error(
+            f"Unable to import module `{e.name}`. Make sure the source path is "
+            f"relative to your source root `{source_root}`."
+        )
+    except AttributeError as e:
+        cli_utils.error("Unable to load attribute from module: " + str(e))
+
+    if not isinstance(pipeline_instance, Pipeline):
+        cli_utils.error(
+            f"The given source path `{source}` does not resolve to a pipeline "
+            "object."
+        )
+
+    with cli_utils.temporary_active_stack(stack_name_or_id=stack_name_or_id):
+        pipeline_instance = pipeline_instance.with_options(
+            config_path=config_path
+        )
+        template = pipeline_instance.create_run_template(name=name)
+
+    cli_utils.declare(f"Created run template `{template.id}`.")
+
+
 @pipeline.command("list", help="List all registered pipelines.")
 @list_options(PipelineFilter)
 def list_pipelines(**kwargs: Any) -> None:

zenml/config/compiler.py

@@ -99,7 +99,10 @@ class Compiler:
 
         self._apply_stack_default_settings(pipeline=pipeline, stack=stack)
         if run_configuration.run_name:
-            self._verify_run_name(run_configuration.run_name)
+            self._verify_run_name(
+                run_configuration.run_name,
+                pipeline.configuration.substitutions,
+            )
 
         pipeline_settings = self._filter_and_validate_settings(
             settings=pipeline.configuration.settings,
@@ -305,16 +308,22 @@ class Compiler:
         return default_settings
 
     @staticmethod
-    def _verify_run_name(run_name: str) -> None:
+    def _verify_run_name(
+        run_name: str,
+        substitutions: Dict[str, str],
+    ) -> None:
         """Verifies that the run name contains only valid placeholders.
 
         Args:
             run_name: The run name to verify.
+            substitutions: The substitutions to be used in the run name.
 
         Raises:
             ValueError: If the run name contains invalid placeholders.
         """
-        valid_placeholder_names = {"date", "time"}
+        valid_placeholder_names = {"date", "time"}.union(
+            set(substitutions.keys())
+        )
         placeholders = {
             v[1] for v in string.Formatter().parse(run_name) if v[1]
         }

zenml/config/pipeline_configurations.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Pipeline configuration classes."""
 
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from pydantic import SerializeAsAny, field_validator
@@ -46,6 +47,25 @@ class PipelineConfigurationUpdate(StrictBaseModel):
     model: Optional[Model] = None
     parameters: Optional[Dict[str, Any]] = None
     retry: Optional[StepRetryConfig] = None
+    substitutions: Dict[str, str] = {}
+
+    def _get_full_substitutions(
+        self, start_time: Optional[datetime]
+    ) -> Dict[str, str]:
+        """Returns the full substitutions dict.
+
+        Args:
+            start_time: Start time of the pipeline run.
+
+        Returns:
+            The full substitutions dict including date and time.
+        """
+        if start_time is None:
+            start_time = datetime.utcnow()
+        ret = self.substitutions.copy()
+        ret.setdefault("date", start_time.strftime("%Y_%m_%d"))
+        ret.setdefault("time", start_time.strftime("%H_%M_%S_%f"))
+        return ret
 
 
 class PipelineConfiguration(PipelineConfigurationUpdate):

zenml/config/pipeline_run_configuration.py

@@ -52,3 +52,4 @@ class PipelineRunConfiguration(
     retry: Optional[StepRetryConfig] = None
     failure_hook_source: Optional[SourceWithValidator] = None
     success_hook_source: Optional[SourceWithValidator] = None
+    substitutions: Dict[str, str] = {}

zenml/config/step_configurations.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Pipeline configuration classes."""
 
+from datetime import datetime
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -49,6 +50,7 @@ from zenml.utils.pydantic_utils import before_validator_handler
 
 if TYPE_CHECKING:
     from zenml.config import DockerSettings, ResourceSettings
+    from zenml.config.pipeline_configurations import PipelineConfiguration
 
 logger = get_logger(__name__)
 
@@ -152,6 +154,7 @@ class StepConfigurationUpdate(StrictBaseModel):
     success_hook_source: Optional[SourceWithValidator] = None
     model: Optional[Model] = None
     retry: Optional[StepRetryConfig] = None
+    substitutions: Dict[str, str] = {}
 
     outputs: Mapping[str, PartialArtifactConfiguration] = {}
 
@@ -237,6 +240,24 @@ class StepConfiguration(PartialStepConfiguration):
             model_or_dict = model_or_dict.model_dump()
         return DockerSettings.model_validate(model_or_dict)
 
+    def _get_full_substitutions(
+        self,
+        pipeline_config: "PipelineConfiguration",
+        start_time: Optional[datetime],
+    ) -> Dict[str, str]:
+        """Get the full set of substitutions for this step configuration.
+
+        Args:
+            pipeline_config: The pipeline configuration.
+            start_time: The start time of the pipeline run.
+
+        Returns:
+            The full set of substitutions for this step configuration.
+        """
+        ret = pipeline_config._get_full_substitutions(start_time)
+        ret.update(self.substitutions)
+        return ret
+
 
 class InputSpec(StrictBaseModel):
     """Step input specification."""

zenml/enums.py

@@ -60,6 +60,7 @@ class VisualizationType(StrEnum):
     HTML = "html"
     IMAGE = "image"
     MARKDOWN = "markdown"
+    JSON = "json"
 
 
 class ZenMLServiceType(StrEnum):

zenml/integrations/__init__.py

@@ -48,6 +48,7 @@ from zenml.integrations.lightgbm import LightGBMIntegration  # noqa
 
 # from zenml.integrations.llama_index import LlamaIndexIntegration  # noqa
 from zenml.integrations.mlflow import MlflowIntegration  # noqa
+from zenml.integrations.modal import ModalIntegration  # noqa
 from zenml.integrations.neptune import NeptuneIntegration  # noqa
 from zenml.integrations.neural_prophet import NeuralProphetIntegration  # noqa
 from zenml.integrations.numpy import NumpyIntegration  # noqa

zenml/integrations/constants.py

@@ -42,6 +42,7 @@ LANGCHAIN = "langchain"
 LIGHTGBM = "lightgbm"
 # LLAMA_INDEX = "llama_index"
 MLFLOW = "mlflow"
+MODAL = "modal"
 NEPTUNE = "neptune"
 NEURAL_PROPHET = "neural_prophet"
 NUMPY = "numpy"

zenml/integrations/feast/__init__.py

@@ -31,7 +31,7 @@ class FeastIntegration(Integration):
 
     NAME = FEAST
     # click is added to keep the feast click version in sync with ZenML's click
-    REQUIREMENTS = ["feast", "click>=8.0.1,<8.1.4"]
+    REQUIREMENTS = ["feast>=0.12.0", "click>=8.0.1,<8.1.4"]
     REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click", "pandas"]
 
     @classmethod

zenml/integrations/feast/feature_stores/feast_feature_store.py

@@ -16,7 +16,7 @@
 from typing import Any, Dict, List, Union, cast
 
 import pandas as pd
-from feast import FeatureStore  # type: ignore
+from feast import FeatureService, FeatureStore  # type: ignore
 from feast.infra.registry.base_registry import BaseRegistry  # type: ignore
 
 from zenml.feature_stores.base_feature_store import BaseFeatureStore
@@ -43,14 +43,14 @@ class FeastFeatureStore(BaseFeatureStore):
     def get_historical_features(
         self,
         entity_df: Union[pd.DataFrame, str],
-        features: List[str],
+        features: Union[List[str], FeatureService],
         full_feature_names: bool = False,
     ) -> pd.DataFrame:
         """Returns the historical features for training or batch scoring.
 
         Args:
             entity_df: The entity DataFrame or entity name.
-            features: The features to retrieve.
+            features: The features to retrieve or a FeatureService.
             full_feature_names: Whether to return the full feature names.
 
         Raise:
@@ -70,14 +70,14 @@ class FeastFeatureStore(BaseFeatureStore):
     def get_online_features(
         self,
         entity_rows: List[Dict[str, Any]],
-        features: List[str],
+        features: Union[List[str], FeatureService],
         full_feature_names: bool = False,
     ) -> Dict[str, Any]:
         """Returns the latest online feature data.
 
         Args:
             entity_rows: The entity rows to retrieve.
-            features: The features to retrieve.
+            features: The features to retrieve or a FeatureService.
             full_feature_names: Whether to return the full feature names.
 
         Raise:
@@ -118,17 +118,21 @@ class FeastFeatureStore(BaseFeatureStore):
         fs = FeatureStore(repo_path=self.config.feast_repo)
         return [ds.name for ds in fs.list_entities()]
 
-    def get_feature_services(self) -> List[str]:
-        """Returns the feature service names.
+    def get_feature_services(self) -> List[FeatureService]:
+        """Returns the feature services.
 
         Raise:
             ConnectionError: If the online component (Redis) is not available.
 
         Returns:
-            The feature service names.
+            The feature services.
         """
         fs = FeatureStore(repo_path=self.config.feast_repo)
-        return [ds.name for ds in fs.list_feature_services()]
+        feature_services: List[FeatureService] = list(
+            fs.list_feature_services()
+        )
+
+        return feature_services
 
     def get_feature_views(self) -> List[str]:
         """Returns the feature view names.

zenml/integrations/kubernetes/orchestrators/kube_utils.py

@@ -94,18 +94,62 @@ def load_kube_config(
         k8s_config.load_kube_config(context=context)
 
 
-def sanitize_pod_name(pod_name: str) -> str:
+def calculate_max_pod_name_length_for_namespace(namespace: str) -> int:
+    """Calculate the max pod length for a certain namespace.
+
+    Args:
+        namespace: The namespace in which the pod will be created.
+
+    Returns:
+        The maximum pod name length.
+    """
+    # Kubernetes allows Pod names to have 253 characters. However, when
+    # creating a pod they try to create a log file which is called
+    # <NAMESPACE>_<POD_NAME>_<UUID>, which adds additional characters and
+    # runs into filesystem limitations for filename lengths (255). We therefore
+    # subtract the length of a UUID (36), the two underscores and the
+    # namespace length from the max filename length.
+    return 255 - 38 - len(namespace)
+
+
+def sanitize_pod_name(pod_name: str, namespace: str) -> str:
     """Sanitize pod names so they conform to Kubernetes pod naming convention.
 
     Args:
         pod_name: Arbitrary input pod name.
+        namespace: Namespace in which the Pod will be created.
 
     Returns:
         Sanitized pod name.
     """
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
     pod_name = re.sub(r"[^a-z0-9-]", "-", pod_name.lower())
     pod_name = re.sub(r"^[-]+", "", pod_name)
-    return re.sub(r"[-]+", "-", pod_name)
+    pod_name = re.sub(r"[-]+$", "", pod_name)
+    pod_name = re.sub(r"[-]+", "-", pod_name)
+
+    allowed_length = calculate_max_pod_name_length_for_namespace(
+        namespace=namespace
+    )
+    return pod_name[:allowed_length]
+
+
+def sanitize_label(label: str) -> str:
+    """Sanitize a label for a Kubernetes resource.
+
+    Args:
+        label: The label to sanitize.
+
+    Returns:
+        The sanitized label.
+    """
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#rfc-1035-label-names
+    label = re.sub(r"[^a-z0-9-]", "-", label.lower())
+    label = re.sub(r"^[-]+", "", label)
+    label = re.sub(r"[-]+$", "", label)
+    label = re.sub(r"[-]+", "-", label)
+
+    return label[:63]
 
 
 def pod_is_not_pending(pod: k8s_client.V1Pod) -> bool:

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py

@@ -395,8 +395,19 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 )
 
         pipeline_name = deployment.pipeline_configuration.name
-        orchestrator_run_name = get_orchestrator_run_name(pipeline_name)
-        pod_name = kube_utils.sanitize_pod_name(orchestrator_run_name)
+
+        # We already make sure the orchestrator run name has the correct length
+        # to make sure we don't cut off the randomized suffix later when
+        # sanitizing the pod name. This avoids any pod naming collisions.
+        max_length = kube_utils.calculate_max_pod_name_length_for_namespace(
+            namespace=self.config.kubernetes_namespace
+        )
+        orchestrator_run_name = get_orchestrator_run_name(
+            pipeline_name, max_length=max_length
+        )
+        pod_name = kube_utils.sanitize_pod_name(
+            orchestrator_run_name, namespace=self.config.kubernetes_namespace
+        )
 
         assert stack.container_registry
 

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py

@@ -90,7 +90,9 @@ def main() -> None:
         """
         # Define Kubernetes pod name.
         pod_name = f"{orchestrator_run_id}-{step_name}"
-        pod_name = kube_utils.sanitize_pod_name(pod_name)
+        pod_name = kube_utils.sanitize_pod_name(
+            pod_name, namespace=args.kubernetes_namespace
+        )
 
         image = KubernetesOrchestrator.get_image(
             deployment=deployment_config, step_name=step_name

zenml/integrations/kubernetes/orchestrators/manifest_utils.py

@@ -25,6 +25,7 @@ from zenml.constants import ENV_ZENML_ENABLE_REPO_INIT_WARNINGS
 from zenml.integrations.airflow.orchestrators.dag_generator import (
     ENV_ZENML_LOCAL_STORES_PATH,
 )
+from zenml.integrations.kubernetes.orchestrators import kube_utils
 from zenml.integrations.kubernetes.pod_settings import KubernetesPodSettings
 
 
@@ -167,8 +168,8 @@ def build_pod_manifest(
     # Add run_name and pipeline_name to the labels
     labels.update(
         {
-            "run": run_name,
-            "pipeline": pipeline_name,
+            "run": kube_utils.sanitize_label(run_name),
+            "pipeline": kube_utils.sanitize_label(pipeline_name),
         }
     )
 

zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py

@@ -197,7 +197,9 @@ class KubernetesStepOperator(BaseStepOperator):
         )
 
         pod_name = f"{info.run_name}_{info.pipeline_step_name}"
-        pod_name = kube_utils.sanitize_pod_name(pod_name)
+        pod_name = kube_utils.sanitize_pod_name(
+            pod_name, namespace=self.config.kubernetes_namespace
+        )
 
         command = entrypoint_command[:3]
         args = entrypoint_command[3:]

zenml/integrations/modal/__init__.py

@@ -0,0 +1,46 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Modal integration for cloud-native step execution.
+
+The Modal integration sub-module provides a step operator flavor that allows
+executing steps on Modal's cloud infrastructure.
+"""
+from typing import List, Type
+
+from zenml.integrations.constants import MODAL
+from zenml.integrations.integration import Integration
+from zenml.stack import Flavor
+
+MODAL_STEP_OPERATOR_FLAVOR = "modal"
+
+
+class ModalIntegration(Integration):
+    """Definition of Modal integration for ZenML."""
+
+    NAME = MODAL
+    REQUIREMENTS = ["modal>=0.64.49,<1"]
+
+    @classmethod
+    def flavors(cls) -> List[Type[Flavor]]:
+        """Declare the stack component flavors for the Modal integration.
+
+        Returns:
+            List of new stack component flavors.
+        """
+        from zenml.integrations.modal.flavors import ModalStepOperatorFlavor
+
+        return [ModalStepOperatorFlavor]
+
+
+ModalIntegration.check_installation()

zenml/integrations/modal/flavors/__init__.py

@@ -0,0 +1,26 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Modal integration flavors."""
+
+from zenml.integrations.modal.flavors.modal_step_operator_flavor import (
+    ModalStepOperatorConfig,
+    ModalStepOperatorFlavor,
+    ModalStepOperatorSettings,
+)
+
+__all__ = [
+    "ModalStepOperatorConfig",
+    "ModalStepOperatorFlavor",
+    "ModalStepOperatorSettings",
+]