zenml-nightly 0.70.0.dev20241203__py3-none-any.whl → 0.70.0.dev20241204__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.70.0.dev20241203
+0.70.0.dev20241204

zenml/image_builders/base_image_builder.py

@@ -25,6 +25,7 @@ from zenml.io import fileio
 from zenml.logger import get_logger
 from zenml.stack import Flavor, StackComponent
 from zenml.stack.stack_component import StackComponentConfig
+from zenml.utils.archivable import ArchiveType
 
 if TYPE_CHECKING:
     from zenml.container_registries import BaseContainerRegistry
@@ -100,6 +101,7 @@ class BaseImageBuilder(StackComponent, ABC):
     def _upload_build_context(
         build_context: "BuildContext",
         parent_path_directory_name: str,
+        archive_type: ArchiveType = ArchiveType.TAR_GZ,
     ) -> str:
         """Uploads a Docker image build context to a remote location.
 
@@ -109,6 +111,7 @@ class BaseImageBuilder(StackComponent, ABC):
                 the build context to. It will be appended to the artifact
                 store path to create the parent path where the build context
                 will be uploaded to.
+            archive_type: The type of archive to create.
 
         Returns:
             The path to the uploaded build context.
@@ -119,7 +122,7 @@ class BaseImageBuilder(StackComponent, ABC):
 
         hash_ = hashlib.sha1()  # nosec
         with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
-            build_context.write_archive(f, use_gzip=True)
+            build_context.write_archive(f, archive_type)
 
             while True:
                 data = f.read(64 * 1024)
@@ -127,7 +130,7 @@ class BaseImageBuilder(StackComponent, ABC):
                     break
                 hash_.update(data)
 
-            filename = f"{hash_.hexdigest()}.tar.gz"
+            filename = f"{hash_.hexdigest()}.{archive_type.value}"
             filepath = f"{parent_path}/{filename}"
             if not fileio.exists(filepath):
                 logger.info("Uploading build context to `%s`.", filepath)

zenml/image_builders/build_context.py

@@ -20,7 +20,7 @@ from zenml.constants import REPOSITORY_DIRECTORY_NAME
 from zenml.io import fileio
 from zenml.logger import get_logger
 from zenml.utils import io_utils, string_utils
-from zenml.utils.archivable import Archivable
+from zenml.utils.archivable import Archivable, ArchiveType
 
 logger = get_logger(__name__)
 
@@ -69,28 +69,19 @@ class BuildContext(Archivable):
         return None
 
     def write_archive(
-        self, output_file: IO[bytes], use_gzip: bool = True
+        self,
+        output_file: IO[bytes],
+        archive_type: ArchiveType = ArchiveType.TAR_GZ,
     ) -> None:
         """Writes an archive of the build context to the given file.
 
         Args:
             output_file: The file to write the archive to.
-            use_gzip: Whether to use `gzip` to compress the file.
+            archive_type: The type of archive to create.
         """
-        from docker.utils import build as docker_build_utils
-
-        files = self.get_files()
-        extra_files = self.get_extra_files()
-
-        context_archive = docker_build_utils.create_archive(
-            fileobj=output_file,
-            root=self._root,
-            files=sorted(files.keys()),
-            gzip=use_gzip,
-            extra_files=list(extra_files.items()),
-        )
+        super().write_archive(output_file, archive_type)
 
-        build_context_size = os.path.getsize(context_archive.name)
+        build_context_size = os.path.getsize(output_file.name)
         if (
             self._root
             and build_context_size > 50 * 1024 * 1024

zenml/integrations/aws/__init__.py

@@ -33,6 +33,7 @@ AWS_SAGEMAKER_ORCHESTRATOR_FLAVOR = "sagemaker"
 AWS_CONNECTOR_TYPE = "aws"
 AWS_RESOURCE_TYPE = "aws-generic"
 S3_RESOURCE_TYPE = "s3-bucket"
+AWS_IMAGE_BUILDER_FLAVOR = "aws"
 
 class AWSIntegration(Integration):
     """Definition of AWS integration for ZenML."""
@@ -59,12 +60,14 @@ class AWSIntegration(Integration):
         """
         from zenml.integrations.aws.flavors import (
             AWSContainerRegistryFlavor,
+            AWSImageBuilderFlavor,
             SagemakerOrchestratorFlavor,
             SagemakerStepOperatorFlavor,
         )
 
         return [
             AWSContainerRegistryFlavor,
+            AWSImageBuilderFlavor,
             SagemakerStepOperatorFlavor,
             SagemakerOrchestratorFlavor,
         ]

zenml/integrations/aws/flavors/__init__.py

@@ -17,6 +17,10 @@ from zenml.integrations.aws.flavors.aws_container_registry_flavor import (
     AWSContainerRegistryConfig,
     AWSContainerRegistryFlavor,
 )
+from zenml.integrations.aws.flavors.aws_image_builder_flavor import (
+    AWSImageBuilderConfig,
+    AWSImageBuilderFlavor,
+)
 from zenml.integrations.aws.flavors.sagemaker_orchestrator_flavor import (
     SagemakerOrchestratorConfig,
     SagemakerOrchestratorFlavor,
@@ -29,6 +33,8 @@ from zenml.integrations.aws.flavors.sagemaker_step_operator_flavor import (
 __all__ = [
     "AWSContainerRegistryFlavor",
     "AWSContainerRegistryConfig",
+    "AWSImageBuilderConfig",
+    "AWSImageBuilderFlavor",
     "SagemakerStepOperatorFlavor",
     "SagemakerStepOperatorConfig",
     "SagemakerOrchestratorFlavor",

zenml/integrations/aws/flavors/aws_image_builder_flavor.py

@@ -0,0 +1,146 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""AWS Code Build image builder flavor."""
+
+from typing import TYPE_CHECKING, Dict, Optional, Type
+
+from zenml.image_builders import BaseImageBuilderConfig, BaseImageBuilderFlavor
+from zenml.integrations.aws import (
+    AWS_CONNECTOR_TYPE,
+    AWS_IMAGE_BUILDER_FLAVOR,
+    AWS_RESOURCE_TYPE,
+)
+from zenml.models import ServiceConnectorRequirements
+
+if TYPE_CHECKING:
+    from zenml.integrations.aws.image_builders import AWSImageBuilder
+
+
+DEFAULT_CLOUDBUILD_IMAGE = "bentolor/docker-dind-awscli"
+DEFAULT_CLOUDBUILD_COMPUTE_TYPE = "BUILD_GENERAL1_SMALL"
+
+
+class AWSImageBuilderConfig(BaseImageBuilderConfig):
+    """AWS Code Build image builder configuration.
+
+    Attributes:
+        code_build_project: The name of an existing AWS CodeBuild project to use
+            to build the image. The CodeBuild project must exist in the AWS
+            account and region inferred from the AWS service connector
+            credentials or implicitly from the local AWS config.
+        build_image: The Docker image to use for the AWS CodeBuild environment.
+            The image must have Docker installed and be able to run Docker
+            commands. The default image is bentolor/docker-dind-awscli.
+            This can be customized to use a mirror, if needed, in case the
+            Dockerhub image is not accessible or rate-limited.
+        custom_env_vars: Custom environment variables to pass to the AWS
+            CodeBuild build.
+        compute_type: The compute type to use for the AWS CodeBuild build.
+            The default is BUILD_GENERAL1_SMALL.
+        implicit_container_registry_auth: Whether to use implicit authentication
+            to authenticate the AWS Code Build build to the container registry
+            when pushing container images. If set to False, the container
+            registry credentials must be explicitly configured for the container
+            registry stack component or the container registry stack component
+            must be linked to a service connector.
+            NOTE: When implicit_container_registry_auth is set to False, the
+            container registry credentials will be passed to the AWS Code Build
+            build as environment variables. This is not recommended for
+            production use unless your service connector is configured to
+            generate short-lived credentials.
+    """
+
+    code_build_project: str
+    build_image: str = DEFAULT_CLOUDBUILD_IMAGE
+    custom_env_vars: Optional[Dict[str, str]] = None
+    compute_type: str = DEFAULT_CLOUDBUILD_COMPUTE_TYPE
+    implicit_container_registry_auth: bool = True
+
+
+class AWSImageBuilderFlavor(BaseImageBuilderFlavor):
+    """AWS Code Build image builder flavor."""
+
+    @property
+    def name(self) -> str:
+        """The flavor name.
+
+        Returns:
+            The name of the flavor.
+        """
+        return AWS_IMAGE_BUILDER_FLAVOR
+
+    @property
+    def service_connector_requirements(
+        self,
+    ) -> Optional[ServiceConnectorRequirements]:
+        """Service connector resource requirements for service connectors.
+
+        Specifies resource requirements that are used to filter the available
+        service connector types that are compatible with this flavor.
+
+        Returns:
+            Requirements for compatible service connectors, if a service
+            connector is required for this flavor.
+        """
+        return ServiceConnectorRequirements(
+            connector_type=AWS_CONNECTOR_TYPE,
+            resource_type=AWS_RESOURCE_TYPE,
+        )
+
+    @property
+    def docs_url(self) -> Optional[str]:
+        """A url to point at docs explaining this flavor.
+
+        Returns:
+            A flavor docs url.
+        """
+        return self.generate_default_docs_url()
+
+    @property
+    def sdk_docs_url(self) -> Optional[str]:
+        """A url to point at SDK docs explaining this flavor.
+
+        Returns:
+            A flavor SDK docs url.
+        """
+        return self.generate_default_sdk_docs_url()
+
+    @property
+    def logo_url(self) -> str:
+        """A url to represent the flavor in the dashboard.
+
+        Returns:
+            The flavor logo.
+        """
+        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/image_builder/aws.png"
+
+    @property
+    def config_class(self) -> Type[BaseImageBuilderConfig]:
+        """The config class.
+
+        Returns:
+            The config class.
+        """
+        return AWSImageBuilderConfig
+
+    @property
+    def implementation_class(self) -> Type["AWSImageBuilder"]:
+        """Implementation class.
+
+        Returns:
+            The implementation class.
+        """
+        from zenml.integrations.aws.image_builders import AWSImageBuilder
+
+        return AWSImageBuilder

zenml/integrations/aws/image_builders/__init__.py

@@ -0,0 +1,20 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Initialization for the AWS image builder."""
+
+from zenml.integrations.aws.image_builders.aws_image_builder import (
+    AWSImageBuilder,
+)
+
+__all__ = ["AWSImageBuilder"]

zenml/integrations/aws/image_builders/aws_image_builder.py

@@ -0,0 +1,307 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""AWS Code Build image builder implementation."""
+
+import time
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast
+from urllib.parse import urlparse
+from uuid import uuid4
+
+import boto3
+
+from zenml.enums import StackComponentType
+from zenml.image_builders import BaseImageBuilder
+from zenml.integrations.aws import (
+    AWS_CONTAINER_REGISTRY_FLAVOR,
+)
+from zenml.integrations.aws.flavors import AWSImageBuilderConfig
+from zenml.logger import get_logger
+from zenml.stack import StackValidator
+from zenml.utils.archivable import ArchiveType
+
+if TYPE_CHECKING:
+    from zenml.container_registries import BaseContainerRegistry
+    from zenml.image_builders import BuildContext
+    from zenml.stack import Stack
+
+logger = get_logger(__name__)
+
+
+class AWSImageBuilder(BaseImageBuilder):
+    """AWS Code Build image builder implementation."""
+
+    _code_build_client: Optional[Any] = None
+
+    @property
+    def config(self) -> AWSImageBuilderConfig:
+        """The stack component configuration.
+
+        Returns:
+            The configuration.
+        """
+        return cast(AWSImageBuilderConfig, self._config)
+
+    @property
+    def is_building_locally(self) -> bool:
+        """Whether the image builder builds the images on the client machine.
+
+        Returns:
+            True if the image builder builds locally, False otherwise.
+        """
+        return False
+
+    @property
+    def validator(self) -> Optional["StackValidator"]:
+        """Validates the stack for the AWS Code Build Image Builder.
+
+        The AWS Code Build Image Builder requires a container registry to
+        push the image to and an S3 Artifact Store to upload the build context,
+        so AWS Code Build can access it.
+
+        Returns:
+            Stack validator.
+        """
+
+        def _validate_remote_components(stack: "Stack") -> Tuple[bool, str]:
+            if stack.artifact_store.flavor != "s3":
+                return False, (
+                    "The AWS Image Builder requires an S3 Artifact Store to "
+                    "upload the build context, so AWS Code Build can access it."
+                    "Please update your stack to include an S3 Artifact Store "
+                    "and try again."
+                )
+
+            return True, ""
+
+        return StackValidator(
+            required_components={StackComponentType.CONTAINER_REGISTRY},
+            custom_validation_function=_validate_remote_components,
+        )
+
+    @property
+    def code_build_client(self) -> Any:
+        """The authenticated AWS Code Build client to use for interacting with AWS services.
+
+        Returns:
+            The authenticated AWS Code Build client.
+
+        Raises:
+            RuntimeError: If the AWS Code Build client cannot be created.
+        """
+        if (
+            self._code_build_client is not None
+            and self.connector_has_expired()
+        ):
+            self._code_build_client = None
+        if self._code_build_client is not None:
+            return self._code_build_client
+
+        # Option 1: Service connector
+        if connector := self.get_connector():
+            boto_session = connector.connect()
+            if not isinstance(boto_session, boto3.Session):
+                raise RuntimeError(
+                    f"Expected to receive a `boto3.Session` object from the "
+                    f"linked connector, but got type `{type(boto_session)}`."
+                )
+        # Option 2: Implicit configuration
+        else:
+            boto_session = boto3.Session()
+
+        self._code_build_client = boto_session.client("codebuild")
+        return self._code_build_client
+
+    def build(
+        self,
+        image_name: str,
+        build_context: "BuildContext",
+        docker_build_options: Dict[str, Any],
+        container_registry: Optional["BaseContainerRegistry"] = None,
+    ) -> str:
+        """Builds and pushes a Docker image.
+
+        Args:
+            image_name: Name of the image to build and push.
+            build_context: The build context to use for the image.
+            docker_build_options: Docker build options.
+            container_registry: Optional container registry to push to.
+
+        Returns:
+            The Docker image name with digest.
+
+        Raises:
+            RuntimeError: If no container registry is passed.
+            RuntimeError: If the Cloud Build build fails.
+        """
+        if not container_registry:
+            raise RuntimeError(
+                "The AWS Image Builder requires a container registry to push "
+                "the image to. Please provide one and try again."
+            )
+
+        logger.info("Using AWS Code Build to build image `%s`", image_name)
+        cloud_build_context = self._upload_build_context(
+            build_context=build_context,
+            parent_path_directory_name=f"code-build-contexts/{str(self.id)}",
+            archive_type=ArchiveType.ZIP,
+        )
+
+        url_parts = urlparse(cloud_build_context)
+        bucket = url_parts.netloc
+        object_path = url_parts.path.lstrip("/")
+        logger.info(
+            "Build context located in bucket `%s` and object path `%s`",
+            bucket,
+            object_path,
+        )
+
+        # Pass authentication credentials as environment variables, if
+        # the container registry has credentials and if implicit authentication
+        # is disabled
+        environment_variables_override: Dict[str, str] = {}
+        pre_build_commands = []
+        if not self.config.implicit_container_registry_auth:
+            credentials = container_registry.credentials
+            if credentials:
+                environment_variables_override = {
+                    "CONTAINER_REGISTRY_USERNAME": credentials[0],
+                    "CONTAINER_REGISTRY_PASSWORD": credentials[1],
+                }
+                pre_build_commands = [
+                    "echo Logging in to container registry",
+                    'echo "$CONTAINER_REGISTRY_PASSWORD" | docker login --username "$CONTAINER_REGISTRY_USERNAME" --password-stdin '
+                    f"{container_registry.config.uri}",
+                ]
+        elif container_registry.flavor == AWS_CONTAINER_REGISTRY_FLAVOR:
+            pre_build_commands = [
+                "echo Logging in to EKS",
+                f"aws ecr get-login-password --region {self.code_build_client._client_config.region_name} | docker login --username AWS --password-stdin {container_registry.config.uri}",
+            ]
+
+        # Convert the docker_build_options dictionary to a list of strings
+        docker_build_args = ""
+        for key, value in docker_build_options.items():
+            option = f"--{key}"
+            if isinstance(value, list):
+                for val in value:
+                    docker_build_args += f"{option} {val} "
+            elif value is not None and not isinstance(value, bool):
+                docker_build_args += f"{option} {value} "
+            elif value is not False:
+                docker_build_args += f"{option} "
+
+        pre_build_commands_str = "\n".join(
+            [f"            - {command}" for command in pre_build_commands]
+        )
+
+        # Generate and use a unique tag for the Docker image. This is easier
+        # than trying to parse the image digest from the Code Build logs.
+        build_id = str(uuid4())
+        # Replace the tag in the image name with the unique build ID
+        repo_name = image_name.split(":")[0]
+        alt_image_name = f"{repo_name}:{build_id}"
+
+        buildspec = f"""
+version: 0.2
+phases:
+    pre_build:
+        commands:
+{pre_build_commands_str}
+    build:
+        commands:
+            - echo Build started on `date`
+            - echo Building the Docker image...
+            - docker build -t {image_name} . {docker_build_args}
+            - echo Build completed on `date`
+    post_build:
+        commands:
+            - echo Pushing the Docker image...
+            - docker push {image_name}
+            - docker tag {image_name} {alt_image_name}
+            - docker push {alt_image_name}
+            - echo Pushed the Docker image
+artifacts:
+    files:
+        - '**/*'
+"""
+
+        if self.config.custom_env_vars:
+            environment_variables_override.update(self.config.custom_env_vars)
+
+        environment_variables_override_list = [
+            {
+                "name": key,
+                "value": value,
+                "type": "PLAINTEXT",
+            }
+            for key, value in environment_variables_override.items()
+        ]
+
+        # Override the build project with the parameters needed to run a
+        # docker-in-docker build, as covered here: https://docs.aws.amazon.com/codebuild/latest/userguide/sample-docker-section.html
+        response = self.code_build_client.start_build(
+            projectName=self.config.code_build_project,
+            environmentTypeOverride="LINUX_CONTAINER",
+            imageOverride=self.config.build_image,
+            computeTypeOverride=self.config.compute_type,
+            privilegedModeOverride=False,
+            sourceTypeOverride="S3",
+            sourceLocationOverride=f"{bucket}/{object_path}",
+            buildspecOverride=buildspec,
+            environmentVariablesOverride=environment_variables_override_list,
+            # no artifacts
+            artifactsOverride={"type": "NO_ARTIFACTS"},
+        )
+
+        build_arn = response["build"]["arn"]
+
+        # Parse the AWS region, account, codebuild project and build name from the ARN
+        aws_region, aws_account, build = build_arn.split(":", maxsplit=5)[3:6]
+        codebuild_project = build.split("/")[1].split(":")[0]
+
+        logs_url = f"https://{aws_region}.console.aws.amazon.com/codesuite/codebuild/{aws_account}/projects/{codebuild_project}/{build}/log"
+        logger.info(
+            f"Running Code Build to build the Docker image. Cloud Build logs: `{logs_url}`",
+        )
+
+        # Wait for the build to complete
+        code_build_id = response["build"]["id"]
+        while True:
+            build_status = self.code_build_client.batch_get_builds(
+                ids=[code_build_id]
+            )
+            build = build_status["builds"][0]
+            status = build["buildStatus"]
+            if status in [
+                "SUCCEEDED",
+                "FAILED",
+                "FAULT",
+                "TIMED_OUT",
+                "STOPPED",
+            ]:
+                break
+            time.sleep(10)
+
+        if status != "SUCCEEDED":
+            raise RuntimeError(
+                f"The Code Build run to build the Docker image has failed. More "
+                f"information can be found in the Cloud Build logs: {logs_url}."
+            )
+
+        logger.info(
+            f"The Docker image has been built successfully. More information can "
+            f"be found in the Cloud Build logs: `{logs_url}`."
+        )
+
+        return alt_image_name

zenml/integrations/kaniko/image_builders/kaniko_image_builder.py

@@ -25,6 +25,7 @@ from zenml.image_builders import BaseImageBuilder
 from zenml.integrations.kaniko.flavors import KanikoImageBuilderConfig
 from zenml.logger import get_logger
 from zenml.stack import StackValidator
+from zenml.utils.archivable import ArchiveType
 
 if TYPE_CHECKING:
     from zenml.container_registries import BaseContainerRegistry
@@ -295,7 +296,7 @@ class KanikoImageBuilder(BaseImageBuilder):
         logger.debug("Writing build context to process stdin.")
         assert process.stdin
         with process.stdin as _, tempfile.TemporaryFile(mode="w+b") as f:
-            build_context.write_archive(f, use_gzip=True)
+            build_context.write_archive(f, archive_type=ArchiveType.TAR_GZ)
             while True:
                 data = f.read(1024)
                 if not data:

zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py

@@ -134,6 +134,17 @@ class KubernetesOrchestratorConfig(
         """
         return True
 
+    @property
+    def supports_client_side_caching(self) -> bool:
+        """Whether the orchestrator supports client side caching.
+
+        Returns:
+            Whether the orchestrator supports client side caching.
+        """
+        # The Kubernetes orchestrator starts step pods from a pipeline pod.
+        # This is currently not supported when using client-side caching.
+        return False
+
 
 class KubernetesOrchestratorFlavor(BaseOrchestratorFlavor):
     """Kubernetes orchestrator flavor."""

zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py

@@ -94,6 +94,17 @@ class LightningOrchestratorConfig(
         """
         return False
 
+    @property
+    def supports_client_side_caching(self) -> bool:
+        """Whether the orchestrator supports client side caching.
+
+        Returns:
+            Whether the orchestrator supports client side caching.
+        """
+        # The Lightning orchestrator starts step studios from a pipeline studio.
+        # This is currently not supported when using client-side caching.
+        return False
+
 
 class LightningOrchestratorFlavor(BaseOrchestratorFlavor):
     """Lightning orchestrator flavor."""

zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py

@@ -144,3 +144,15 @@ class SkypilotBaseOrchestratorConfig(
             True if this config is for a local component, False otherwise.
         """
         return False
+
+    @property
+    def supports_client_side_caching(self) -> bool:
+        """Whether the orchestrator supports client side caching.
+
+        Returns:
+            Whether the orchestrator supports client side caching.
+        """
+        # The Skypilot orchestrator runs the entire pipeline in a single VM, or
+        # starts additional VMs from the root VM. Both of those cases are
+        # currently not supported when using client-side caching.
+        return False

zenml/materializers/built_in_materializer.py

@@ -429,7 +429,7 @@ class BuiltInContainerMaterializer(BaseMaterializer):
         # doesn't work for non-serializable types as they
         # are saved as list of lists in different files
         if _is_serializable(data):
-            return {self.data_path: VisualizationType.JSON}
+            return {self.data_path.replace("\\", "/"): VisualizationType.JSON}
         return {}
 
     def extract_metadata(self, data: Any) -> Dict[str, "MetadataType"]:

zenml/orchestrators/base_orchestrator.py

@@ -84,6 +84,15 @@ class BaseOrchestratorConfig(StackComponentConfig):
         """
         return False
 
+    @property
+    def supports_client_side_caching(self) -> bool:
+        """Whether the orchestrator supports client side caching.
+
+        Returns:
+            Whether the orchestrator supports client side caching.
+        """
+        return True
+
 
 class BaseOrchestrator(StackComponent, ABC):
     """Base class for all orchestrators.
@@ -205,6 +214,7 @@ class BaseOrchestrator(StackComponent, ABC):
 
         if (
             placeholder_run
+            and self.config.supports_client_side_caching
             and not deployment.schedule
             and not prevent_client_side_caching
         ):
@@ -232,6 +242,8 @@ class BaseOrchestrator(StackComponent, ABC):
                 self._cleanup_run()
                 logger.info("All steps of the pipeline run were cached.")
                 return
+        else:
+            logger.debug("Skipping client-side caching.")
 
         try:
             if metadata_iterator := self.prepare_or_run_pipeline(

zenml/orchestrators/output_utils.py

@@ -19,6 +19,7 @@ from uuid import uuid4
 
 from zenml.client import Client
 from zenml.logger import get_logger
+from zenml.utils import string_utils
 
 if TYPE_CHECKING:
     from zenml.artifact_stores import BaseArtifactStore
@@ -75,10 +76,13 @@ def prepare_output_artifact_uris(
     artifact_store = stack.artifact_store
     output_artifact_uris: Dict[str, str] = {}
     for output_name in step.config.outputs.keys():
+        substituted_output_name = string_utils.format_name_template(
+            output_name, substitutions=step_run.config.substitutions
+        )
         artifact_uri = generate_artifact_uri(
             artifact_store=stack.artifact_store,
             step_run=step_run,
-            output_name=output_name,
+            output_name=substituted_output_name,
         )
         if artifact_store.exists(artifact_uri):
             raise RuntimeError("Artifact already exists")

zenml/service_connectors/service_connector_utils.py

@@ -60,15 +60,9 @@ def _raise_specific_cloud_exception_if_needed(
     orchestrators: List[ResourcesInfo],
     container_registries: List[ResourcesInfo],
 ) -> None:
-    AWS_DOCS = (
-        "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/aws-service-connector"
-    )
-    GCP_DOCS = (
-        "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/gcp-service-connector"
-    )
-    AZURE_DOCS = (
-        "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/azure-service-connector"
-    )
+    AWS_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/aws-service-connector"
+    GCP_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/gcp-service-connector"
+    AZURE_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/azure-service-connector"
 
     if not artifact_stores:
         error_msg = (

zenml/stack_deployments/aws_stack_deployment.py

@@ -73,6 +73,7 @@ of any potential costs:
 - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
 - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
 as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
+- A CodeBuild project registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builder/aws).
 - An IAM user and IAM role with the minimum necessary permissions to access the
 above resources.
 - An AWS access key used to give access to ZenML to connect to the above
@@ -158,6 +159,26 @@ console.
                 "ecr:PutImage",
                 "ecr:GetAuthorizationToken",
             ],
+            "CloudBuild (Client)": [
+                "codebuild:CreateProject",
+                "codebuild:BatchGetBuilds",
+            ],
+            "CloudBuild (Service)": [
+                "s3:GetObject",
+                "s3:GetObjectVersion",
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents",
+                "ecr:BatchGetImage",
+                "ecr:DescribeImages",
+                "ecr:BatchCheckLayerAvailability",
+                "ecr:GetDownloadUrlForLayer",
+                "ecr:InitiateLayerUpload",
+                "ecr:UploadLayerPart",
+                "ecr:CompleteLayerUpload",
+                "ecr:PutImage",
+                "ecr:GetAuthorizationToken",
+            ],
             "SageMaker (Client)": [
                 "sagemaker:CreatePipeline",
                 "sagemaker:StartPipelineExecution",
@@ -243,6 +264,7 @@ console.
             param_ResourceName=f"zenml-{random_str(6).lower()}",
             param_ZenMLServerURL=self.zenml_server_url,
             param_ZenMLServerAPIToken=self.zenml_server_api_token,
+            param_CodeBuild="true",
         )
         # Encode the parameters as URL query parameters
         query_params = "&".join([f"{k}={v}" for k, v in params.items()])

zenml/utils/archivable.py

@@ -15,11 +15,21 @@
 
 import io
 import tarfile
+import zipfile
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import IO, Any, Dict
+from typing import IO, Any, Dict, Optional
 
 from zenml.io import fileio
+from zenml.utils.enum_utils import StrEnum
+
+
+class ArchiveType(StrEnum):
+    """Archive types supported by the ZenML build context."""
+
+    TAR = "tar"
+    TAR_GZ = "tar.gz"
+    ZIP = "zip"
 
 
 class Archivable(ABC):
@@ -81,52 +91,71 @@ class Archivable(ABC):
                     self._extra_files[file_destination.as_posix()] = f.read()
 
     def write_archive(
-        self, output_file: IO[bytes], use_gzip: bool = True
+        self,
+        output_file: IO[bytes],
+        archive_type: ArchiveType = ArchiveType.TAR_GZ,
     ) -> None:
         """Writes an archive of the build context to the given file.
 
         Args:
             output_file: The file to write the archive to.
-            use_gzip: Whether to use `gzip` to compress the file.
+            archive_type: The type of archive to create.
         """
         files = self.get_files()
         extra_files = self.get_extra_files()
+        close_fileobj: Optional[Any] = None
+        fileobj: Any = output_file
 
-        if use_gzip:
-            from gzip import GzipFile
-
-            # We don't use the builtin gzip functionality of the `tarfile`
-            # library as that one includes the tar filename and creation
-            # timestamp in the archive which causes the hash of the resulting
-            # file to be different each time. We use this hash to avoid
-            # duplicate uploads, which is why we pass empty values for filename
-            # and mtime here.
-            fileobj: Any = GzipFile(
-                filename="", mode="wb", fileobj=output_file, mtime=0.0
-            )
+        if archive_type == ArchiveType.ZIP:
+            fileobj = zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED)
         else:
-            fileobj = output_file
-
-        with tarfile.open(mode="w", fileobj=fileobj) as tf:
-            for archive_path, file_path in files.items():
-                if archive_path in extra_files:
-                    continue
-
-                if info := tf.gettarinfo(file_path, arcname=archive_path):
-                    if info.isfile():
-                        with open(file_path, "rb") as f:
-                            tf.addfile(info, f)
+            if archive_type == ArchiveType.TAR_GZ:
+                from gzip import GzipFile
+
+                # We don't use the builtin gzip functionality of the `tarfile`
+                # library as that one includes the tar filename and creation
+                # timestamp in the archive which causes the hash of the resulting
+                # file to be different each time. We use this hash to avoid
+                # duplicate uploads, which is why we pass empty values for filename
+                # and mtime here.
+                close_fileobj = fileobj = GzipFile(
+                    filename="", mode="wb", fileobj=output_file, mtime=0.0
+                )
+            fileobj = tarfile.open(mode="w", fileobj=fileobj)
+
+        try:
+            with fileobj as af:
+                for archive_path, file_path in files.items():
+                    if archive_path in extra_files:
+                        continue
+                    if archive_type == ArchiveType.ZIP:
+                        assert isinstance(af, zipfile.ZipFile)
+                        af.write(file_path, arcname=archive_path)
                     else:
-                        tf.addfile(info, None)
-
-            for archive_path, contents in extra_files.items():
-                info = tarfile.TarInfo(archive_path)
-                contents_encoded = contents.encode("utf-8")
-                info.size = len(contents_encoded)
-                tf.addfile(info, io.BytesIO(contents_encoded))
-
-        if use_gzip:
-            fileobj.close()
+                        assert isinstance(af, tarfile.TarFile)
+                        if info := af.gettarinfo(
+                            file_path, arcname=archive_path
+                        ):
+                            if info.isfile():
+                                with open(file_path, "rb") as f:
+                                    af.addfile(info, f)
+                            else:
+                                af.addfile(info, None)
+
+                for archive_path, contents in extra_files.items():
+                    contents_encoded = contents.encode("utf-8")
+
+                    if archive_type == ArchiveType.ZIP:
+                        assert isinstance(af, zipfile.ZipFile)
+                        af.writestr(archive_path, contents_encoded)
+                    else:
+                        assert isinstance(af, tarfile.TarFile)
+                        info = tarfile.TarInfo(archive_path)
+                        info.size = len(contents_encoded)
+                        af.addfile(info, io.BytesIO(contents_encoded))
+        finally:
+            if close_fileobj:
+                close_fileobj.close()
 
         output_file.seek(0)
 

zenml/utils/code_utils.py

@@ -25,7 +25,7 @@ from zenml.client import Client
 from zenml.io import fileio
 from zenml.logger import get_logger
 from zenml.utils import source_utils, string_utils
-from zenml.utils.archivable import Archivable
+from zenml.utils.archivable import Archivable, ArchiveType
 
 if TYPE_CHECKING:
     from git.repo.base import Repo
@@ -152,15 +152,19 @@ class CodeArchive(Archivable):
         return all_files
 
     def write_archive(
-        self, output_file: IO[bytes], use_gzip: bool = True
+        self,
+        output_file: IO[bytes],
+        archive_type: ArchiveType = ArchiveType.TAR_GZ,
     ) -> None:
         """Writes an archive of the build context to the given file.
 
         Args:
             output_file: The file to write the archive to.
-            use_gzip: Whether to use `gzip` to compress the file.
+            archive_type: The type of archive to create.
         """
-        super().write_archive(output_file=output_file, use_gzip=use_gzip)
+        super().write_archive(
+            output_file=output_file, archive_type=archive_type
+        )
         archive_size = os.path.getsize(output_file.name)
         if archive_size > 20 * 1024 * 1024:
             logger.warning(

zenml/utils/docker_utils.py

@@ -266,6 +266,14 @@ def push_image(
     logger.info("Finished pushing Docker image.")
 
     image_name_without_tag, _ = image_name.rsplit(":", maxsplit=1)
+
+    image = docker_client.images.get(image_name)
+    repo_digests: List[str] = image.attrs["RepoDigests"]
+
+    for digest in repo_digests:
+        if digest.startswith(f"{image_name_without_tag}@"):
+            return digest
+
     for info in reversed(aux_info):
         try:
             repo_digest = info["Digest"]
@@ -304,6 +312,7 @@ def get_image_digest(image_name: str) -> Optional[str]:
 
     image = docker_client.images.get(image_name)
     repo_digests = image.attrs["RepoDigests"]
+
     if len(repo_digests) == 1:
         return cast(str, repo_digests[0])
     else:

{zenml_nightly-0.70.0.dev20241203.dist-info → zenml_nightly-0.70.0.dev20241204.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zenml-nightly
-Version: 0.70.0.dev20241203
+Version: 0.70.0.dev20241204
 Summary: ZenML: Write production-ready ML code.
 Home-page: https://zenml.io
 License: Apache-2.0

{zenml_nightly-0.70.0.dev20241203.dist-info → zenml_nightly-0.70.0.dev20241204.dist-info}/RECORD

@@ -6,7 +6,7 @@ RELEASE_NOTES.md,sha256=DleauURHESDrTrcVzCVLqPiSM9NIAk5vldvEFMc7qlk,389375
 ROADMAP.md,sha256=hiLSmr16BH8Dfx7SaQM4JcXCGCVl6mFZPFAwJeDTrJU,407
 SECURITY.md,sha256=9DepA8y03yvCZLHEfcXLTDH4lUyKHquAdukBsccNN7c,682
 zenml/README.md,sha256=827dekbOWAs1BpW7VF1a4d7EbwPbjwccX-2zdXBENZo,1777
-zenml/VERSION,sha256=X1k6h9fTwNJHLhvwMrbIeu0zyNdWZlpq_FOYVzRDLMg,19
+zenml/VERSION,sha256=-A-FM4tTGBeZdWuioST8Yb1XGa9Iixr9mH9R98y-uNI,19
 zenml/__init__.py,sha256=SkMObQA41ajqdZqGErN00S1Vf3KAxpLvbZ-OBy5uYoo,2130
 zenml/actions/__init__.py,sha256=mrt6wPo73iKRxK754_NqsGyJ3buW7RnVeIGXr1xEw8Y,681
 zenml/actions/base_action.py,sha256=UcaHev6BTuLDwuswnyaPjdA8AgUqB5xPZ-lRtuvf2FU,25553
@@ -126,8 +126,8 @@ zenml/hooks/__init__.py,sha256=lEifDzp1dkcatoBpCmmRNOEAh0vgrqtQQP_GH_4z1pk,1037
 zenml/hooks/alerter_hooks.py,sha256=XJPhrnVimOayD2VjYBG_5D6R6sZ08rytwDHvvMD3lNI,3042
 zenml/hooks/hook_validators.py,sha256=35VuolTruFoj6GUQf8n3PpQVMu54dp6apIGPui4FZu4,2516
 zenml/image_builders/__init__.py,sha256=WVBxXTCiyhVhvGezovS5O-ypwC8V0YRihCOacd7i5iQ,1210
-zenml/image_builders/base_image_builder.py,sha256=-Y5N3zFZsMJvVuzm1M3tU-r38fT9KCdLPFH4knCKM4o,5318
-zenml/image_builders/build_context.py,sha256=TTY5T8aG4epeKOOpLItr8PDjmDijfcGaY3zFzmGV1II,6157
+zenml/image_builders/base_image_builder.py,sha256=yFmtn8Y6PdNxie3F51tbpMf8zBhNRnNGr-ejTCEcq4Q,5491
+zenml/image_builders/build_context.py,sha256=k38xERo4IXU9KKM-Zi5A2HzIp1pUZeWnhCPuRPSjkEQ,5864
 zenml/image_builders/local_image_builder.py,sha256=R_zMpERtUCWLLDZg9kXuAQSWLtin1ve_rxpbUBiym7s,6448
 zenml/integrations/README.md,sha256=hFIZwjsAItHjvDWVBqGSF-ZAeMsFR2GKX1Axl2g1Bz0,6190
 zenml/integrations/__init__.py,sha256=GiLRzSCu1O9Jg5NSCRKVDWLIXtKuc90ozntqYjUHo08,4905
@@ -142,13 +142,16 @@ zenml/integrations/argilla/annotators/__init__.py,sha256=QjRMxIQ-skulcLN94GuHuuk
 zenml/integrations/argilla/annotators/argilla_annotator.py,sha256=aIA5rcwfOYLnUVp2c_aaTAUnVW8GV9gP8myze_jY_qY,15586
 zenml/integrations/argilla/flavors/__init__.py,sha256=MTogEeiZ1k7nLzGKok3azK_VhKUAJl0GL6OQvsvMlZo,917
 zenml/integrations/argilla/flavors/argilla_annotator_flavor.py,sha256=NAy6QKLWYXHPTCoWc9RIovsngcRGMwseu3rdNjiCpo4,4473
-zenml/integrations/aws/__init__.py,sha256=PGGTfYmnOqh7vFPiu_SFYQ2xE38zhf6TAm0Bp3hz2fo,2386
+zenml/integrations/aws/__init__.py,sha256=x-Ym8FlmeX7bAjL7zBxUp40U11IVZGyfcClRIeEydlI,2489
 zenml/integrations/aws/container_registries/__init__.py,sha256=fOwo84MiaX_7XhNQEXeyVh8AyOlcIl6nSu_ig68GkSs,827
 zenml/integrations/aws/container_registries/aws_container_registry.py,sha256=SCxKj_w79Ka0LdxYOdDmaXitketTQRDHrQ4XQ0_f4hs,6091
-zenml/integrations/aws/flavors/__init__.py,sha256=-7dzzHCVuGixi_fjm0TVUcaNT5LEJz_Roq6jHoozIcE,1296
+zenml/integrations/aws/flavors/__init__.py,sha256=XYL9fpwKzeFfHCjakU0iJ3SyHVRJk63QT7luOy9Giek,1480
 zenml/integrations/aws/flavors/aws_container_registry_flavor.py,sha256=GIDLOySz1zF08YSkmKIj89TxBqSLWueXLAHyxYwm-NI,4169
+zenml/integrations/aws/flavors/aws_image_builder_flavor.py,sha256=XobJOw5ymbY22i7YHWGYOKDQoJQAqTeMIfvkADt-DDc,5343
 zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py,sha256=UyHXFP9rn9DQ-Erd9Rtqun9GTcp3Rftbn2a9Xysh_TU,12826
 zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py,sha256=OnNokixzGvOTBoZJQ5TDG3k4FFsP1pJmxbiij2VLW4s,5978
+zenml/integrations/aws/image_builders/__init__.py,sha256=91hgH1OphG2i-vk-G8N4yKBFIzK89Wu7BK4-T5yOA7E,786
+zenml/integrations/aws/image_builders/aws_image_builder.py,sha256=UcPYYYjJjfsicY3hV4OZeJt552AVmwPZPlv-AsG1g1I,11489
 zenml/integrations/aws/orchestrators/__init__.py,sha256=Wh0Fhtt_uo6YrkvXY9kL0M478FL7XpapjoFreUZbgUg,794
 zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py,sha256=OlnXr_KTWE8fOyO6bNmsCbRdRaCPqWGrUEX4gyVbD80,26801
 zenml/integrations/aws/orchestrators/sagemaker_orchestrator_entrypoint_config.py,sha256=WXCWdVojIZxx5_3-g1T95S2vsJ-RLNGcp-V409wgme0,1555
@@ -325,7 +328,7 @@ zenml/integrations/kaniko/__init__.py,sha256=r40dq-xp9ji2xtfymwffryglCmsOZe2g1_9
 zenml/integrations/kaniko/flavors/__init__.py,sha256=PpI7yUpjgLwwAi1ma5uTGihG4zUWxURiRTpHoHEXBIw,865
 zenml/integrations/kaniko/flavors/kaniko_image_builder_flavor.py,sha256=iRICgO_f9RF1XOp7bO8qjmHm83v2NYqoJWorHZKhVzE,5080
 zenml/integrations/kaniko/image_builders/__init__.py,sha256=X-ydYoG_6t4OWXDuCgfbugtVvPeOc97Kwf8RQ0KcPiU,787
-zenml/integrations/kaniko/image_builders/kaniko_image_builder.py,sha256=30RnR2VvVQ5kkR0KQqS1fpiJf2zrJa7BCWOoOiX53rs,13656
+zenml/integrations/kaniko/image_builders/kaniko_image_builder.py,sha256=N4rLjIHYdw8v1CZvJP9G8RYpaMbERhzDyLP0UcEO3JA,13721
 zenml/integrations/kubeflow/__init__.py,sha256=ab3v3sJBwo09vbyI2UQ5SnKfL0C9j8hrxR5ji-Xfswo,1791
 zenml/integrations/kubeflow/flavors/__init__.py,sha256=l560A0oIpYeTpVVrdrVVYj-2-Y5CeyCSQMfwErZROxY,878
 zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py,sha256=b7W4oASHLYN9o2n__-W3zajb_MXhPaY0UutecsW0Xw4,10381
@@ -334,7 +337,7 @@ zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py,sha256=H4rpm3
 zenml/integrations/kubeflow/orchestrators/local_deployment_utils.py,sha256=qszoOdvBpgIp40XkncphXAr9dRKnyZzGiz2mJ56bYmw,15448
 zenml/integrations/kubernetes/__init__.py,sha256=UzU5CaogX6ud5ChCK8JSZ06eoW18eIudbgntgPijYSc,1853
 zenml/integrations/kubernetes/flavors/__init__.py,sha256=a5gU45qCj3FkLwl_uVjlIkL_2F5DHk-w1gdcZrvVjBI,1266
-zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py,sha256=HkCyDWqv1lDd8W6GeXE6PeHQiUrHPfSkfw3sB0B2xuA,7911
+zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py,sha256=al3EwEhwbG2uCHpkfC_4J-nOPCNW-LdJ568X0hPAvxE,8312
 zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py,sha256=ILN-H4cl7z3i4ltb4UBs55wbtIo871b4ib28pYkQoyQ,5605
 zenml/integrations/kubernetes/orchestrators/__init__.py,sha256=TJID3OTieZBox36WpQpzD0jdVRA_aZVcs_bNtfXS8ik,811
 zenml/integrations/kubernetes/orchestrators/kube_utils.py,sha256=0Cj1RoiuXL4oACnfyzEpOiCTnHt_YoF5ml3bhobyOEg,12195
@@ -369,7 +372,7 @@ zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py,sha25
 zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py,sha256=q1L1cke0j9hYNbruPxaWDDgp0thNPNHZA0gH4gyfugI,2856
 zenml/integrations/lightning/__init__.py,sha256=Wyj8P7ihX8DpOO4bmggM_Pq3HDwU02fjdB51s5sHJSA,1521
 zenml/integrations/lightning/flavors/__init__.py,sha256=Ma949QQTTYw1Zw1fo6zSUXomXfXQRTPHY9N2wZGweMA,883
-zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py,sha256=prKhMbvbtEXWYbprM9udc4HJltwCNTKqCRqUZwWgJ2I,4853
+zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py,sha256=U6ewra4C8cSrnRQTlEJrY2ouR4XJ3ShnAg1fDoC9fFQ,5259
 zenml/integrations/lightning/orchestrators/__init__.py,sha256=rUaCQPcbpAPy57jx8ZqEOjTJv8zigT_nCPrMdfQJ8OU,1030
 zenml/integrations/lightning/orchestrators/lightning_orchestrator.py,sha256=6XGg-cKKRNCot6DI9ub4oZWF4_K41OHs1lT90FL8VsM,24712
 zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py,sha256=VHVXZ6jZ0URLhwaIPMyDmrnrBpYvzDnqQdIoSVM9m6E,11506
@@ -478,7 +481,7 @@ zenml/integrations/sklearn/materializers/__init__.py,sha256=vHIx7njuVcP_kGbyECgJ
 zenml/integrations/sklearn/materializers/sklearn_materializer.py,sha256=IaE8DS9L2gjfRVv8RT8p7ls7cJJ5qzt6hRgdpL6pp20,1562
 zenml/integrations/skypilot/__init__.py,sha256=PzNuJJykzfu34pTC4kSD18ARyGysuhzEW8oLTm4oHuo,612
 zenml/integrations/skypilot/flavors/__init__.py,sha256=PzNuJJykzfu34pTC4kSD18ARyGysuhzEW8oLTm4oHuo,612
-zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py,sha256=_lMmSo1aBSpnfYPGKD1CbpEdrGfVVIavA_yl2ZvGiYU,6204
+zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py,sha256=UPHfsB75CDZ8ecxxlL8aTo3IGIVuCAU_USBHzhULMuc,6675
 zenml/integrations/skypilot/orchestrators/__init__.py,sha256=elo6QiImzys1m_bgu96U1HCVRepHQI1m7Jgqn6aEJ4Q,844
 zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py,sha256=Oxd_8MEDd5JFovPgHE5xWreLTBQhTJZPA1nLYpvgIa0,14290
 zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py,sha256=5sq04vu2ArsyW56wvH8CRLG8kmeaSkPYPzirGGe7mds,9737
@@ -595,7 +598,7 @@ zenml/login/pro/utils.py,sha256=PnQ90eiqijY-VKqd974HzCtAosrHhzOxL1NpqZbpE5M,5355
 zenml/login/web_login.py,sha256=NkFKnPj8qDZAb2FfDW8ZvkWrGiurf7YiYEEs5m844aM,9037
 zenml/materializers/__init__.py,sha256=maME5CxKcgOkIFwG_iARt1-tuW8u8ZhTzfw50uyv_BA,1667
 zenml/materializers/base_materializer.py,sha256=M4hwkw7PB0LskCE92r-S35011l7DlFemit-EuUCW3Nc,14002
-zenml/materializers/built_in_materializer.py,sha256=XGFZJ0ZzB4QrTe4bnayEcr7dkj5sihmdxVgv6Sgc0XA,15649
+zenml/materializers/built_in_materializer.py,sha256=AFrCW_LErey2ws6vfLTbBP5gx3IFi05YPEmT4BA-CzY,15668
 zenml/materializers/cloudpickle_materializer.py,sha256=x8a6jEMTky6N2YVHiwrnGWSfVJUpiy-4kQsD2Aqj_E0,4837
 zenml/materializers/materializer_registry.py,sha256=ic-aWhJ2Ex9F_rml2dDVAxhRfW3nd71QMxzfTPP6BIM,4002
 zenml/materializers/numpy_materializer.py,sha256=OLcHF9Z0tAqQ_U8TraA0vGmZjHoT7eT_XevncIutt0M,1715
@@ -674,7 +677,7 @@ zenml/models/v2/misc/service_connector_type.py,sha256=jGJLvIsBB87ZIEeZCLLueUf28H
 zenml/models/v2/misc/stack_deployment.py,sha256=PgQevLjvHTpsecY_rzrXesr7jiGms7rH_QbFtI2ZIaA,3768
 zenml/models/v2/misc/user_auth.py,sha256=1-yafNA9qK4wL8ToROjaklTVI7Mj9va0t80_4wm7w3U,6988
 zenml/orchestrators/__init__.py,sha256=p4Y9Ni7Lp33fZ6UrjgI7qu-rrg8LrlyafCM-K1WC81w,1961
-zenml/orchestrators/base_orchestrator.py,sha256=rKC76EVovOIRyRM5x3Y6gNOhnrCLan_vM_d9CU71i68,12267
+zenml/orchestrators/base_orchestrator.py,sha256=vQsMUhhZUX34EyJbYpXB4F0iPmrOHaVLtsKM0vuN1yE,12646
 zenml/orchestrators/cache_utils.py,sha256=J5sMmgy-qRJVtTWt5hDjSAR6vPPulOhMJu0m-Y1onO8,4327
 zenml/orchestrators/containerized_orchestrator.py,sha256=rdebgBW0Bk--JcHcT0NpLkAbyhY0VS5xO1uwWEgkLpA,3230
 zenml/orchestrators/dag_runner.py,sha256=0F0zOy90ZHad_YUiwG_47JiH7KEVFQ5tLen0kGPvppk,6940
@@ -683,7 +686,7 @@ zenml/orchestrators/local/__init__.py,sha256=qlU91hgqGKeycw-Qntrn-iMuoMTaNrq-Rgf
 zenml/orchestrators/local/local_orchestrator.py,sha256=VCVemtPtW1bXvEM80wgaqdmoXiPrVOTNITXJ2MauTaw,5505
 zenml/orchestrators/local_docker/__init__.py,sha256=k8J68ydy6HmmvE9tWo32g761H8P_Dw4AxWNf4UMpsbs,669
 zenml/orchestrators/local_docker/local_docker_orchestrator.py,sha256=kzxvmKwEi7ihXfeoopIUwipSorvpB4Lw2pdCAOcZaZ0,9190
-zenml/orchestrators/output_utils.py,sha256=Gz7SX2cbQ3w4eyfU0XuhKEKGtalQGBoc6moDRNVHN8M,3311
+zenml/orchestrators/output_utils.py,sha256=01vqke1ZfmfuLpgxNerF-QL2wA0VPv1zUdvlMw0OwUY,3508
 zenml/orchestrators/publish_utils.py,sha256=oLvcjCx_WT3_RITB86WNEaUe_qMsPMTD_warAvjS2dA,4752
 zenml/orchestrators/step_launcher.py,sha256=iSrp6X-fIfuqCBS4zCgHLKlV2P08lFPf7wwBjT69LRA,17922
 zenml/orchestrators/step_run_utils.py,sha256=N1_Sd8q9i5u-uWqhTH-gssxf-XkF3vbSDMwu1byyCSk,20424
@@ -712,7 +715,7 @@ zenml/service_connectors/__init__.py,sha256=gIBAVUPBZtdO6JtEBCixy9YG4wZRA1mnaxaG
 zenml/service_connectors/docker_service_connector.py,sha256=_6WPqYCcSUf8JPakipbkJRvaN2ghhY7zCr38WCHG0SI,13218
 zenml/service_connectors/service_connector.py,sha256=HW8dTE826ZSD1vWQlqQRm8XYq08v7tDZQ8ORnJ5rTcE,55351
 zenml/service_connectors/service_connector_registry.py,sha256=mCabyKAr7Y6bcAbIFXbir9YrHczDe1ZJ8Bks5bOQouk,9658
-zenml/service_connectors/service_connector_utils.py,sha256=JlpfqpjftTjQvUSlamg4eSoaWBhDGips0DnJnNb8lwo,18104
+zenml/service_connectors/service_connector_utils.py,sha256=x_8rxl2yQc8VvvaWYXyxUxVijo7CCdlnod0z-jd-vUk,18056
 zenml/services/__init__.py,sha256=8aSLtYlX9EVLB1Se1I4dbhZdTqrjNrAfsn1d_RCc_nk,2800
 zenml/services/container/__init__.py,sha256=dFHcmlXgNXjUaCR18oGQJ19-I_6f3UeAUURHjqldjTs,668
 zenml/services/container/container_service.py,sha256=DJIK7DD53Q_mgZByM0oeIyAgkrwN12T5E3QCWjubkLs,18383
@@ -736,7 +739,7 @@ zenml/stack/stack_component.py,sha256=4V111uBLpmsPaNffaQS-fLuz163WFEjsmIFzb_lq7i
 zenml/stack/stack_validator.py,sha256=hWbvvGIeWLj6NwSsF4GCc6RAxAWvxHXTcBZL9nJvcak,3111
 zenml/stack/utils.py,sha256=qHMFi8SVMSDFeZTFO4KkvaLUbF-l3B0_JZ5SSMxYrwI,6406
 zenml/stack_deployments/__init__.py,sha256=-7593cQ_ZgRn774Ol-8AKXXQquIU4DSiaThVEr6TfWM,644
-zenml/stack_deployments/aws_stack_deployment.py,sha256=_bB0N3meKc0kH3gtysO0sIGp-80AUC44SYRvbqZMNg0,11450
+zenml/stack_deployments/aws_stack_deployment.py,sha256=FnkiSwSMiM9O8KVTQmGfIKjsp5RkOiO-AbOsr7cd3zE,12358
 zenml/stack_deployments/azure_stack_deployment.py,sha256=E0PhlmC_2Urf_7k1U7EoOHU21CbhY_3jZG0wXO1NcWI,13032
 zenml/stack_deployments/gcp_stack_deployment.py,sha256=RD9Z0cZicZexUyK2YE0855LHu_VGOrLUnzy-QnrBRnA,12596
 zenml/stack_deployments/stack_deployment.py,sha256=hrupbDuIfI6YhzHcmxNLsfnS7Qu_Pgv_5didiyv0msk,7572
@@ -754,16 +757,16 @@ zenml/steps/step_invocation.py,sha256=ETfOaV-P4_iXGk9y1-xK54Kfg2QRYaGoj_jTyEYZfb
 zenml/steps/utils.py,sha256=hEIOhxHp69WO-9YX6sBpZrVLK6fytqvdzhW5p1aB09M,18580
 zenml/types.py,sha256=LhGWJ4t3nybBk1l9Ox3tqqHbTYSuCqhkRsL5FqO6yf4,1206
 zenml/utils/__init__.py,sha256=jaMTbjm8tLYkaRoxlZ0Em4ye_ZHOHKgP2goPTTiYGUQ,797
-zenml/utils/archivable.py,sha256=vabL1-2G7_-p-0jyKslxbvlad8fyN1ot-DvOJ19jsOA,5275
+zenml/utils/archivable.py,sha256=QuLe1IhyABTrE6Y0hAT3tKjaUCpcze5ffZ_RKoKtJwY,6549
 zenml/utils/callback_registry.py,sha256=QBWdaraLAxBxi8DKbv9X1SUpTKDhhj-XE0JomB2Ax2Y,2411
 zenml/utils/code_repository_utils.py,sha256=CobRYMYfP2yNoA0hcu_WRz5oAff_jY95oyLCHz4fDOo,4734
-zenml/utils/code_utils.py,sha256=rLRYUAgc9hvrxDc6QooZaVngmzJ_hgjUfQGF8g1n6Wc,11260
+zenml/utils/code_utils.py,sha256=y7_vmqYv_e11xekFjK7Dm4LkFu6SGl73tr2fVxBAK3s,11336
 zenml/utils/cuda_utils.py,sha256=RR21m__Zs-OnI5n-veFUzWniZjwLSbalHE5QV3jK1Hw,1624
 zenml/utils/daemon.py,sha256=GZ7Dx6GLHK04SR50wBxpKYmFhxPBfdLWxJiAWzJC6cM,11863
 zenml/utils/dashboard_utils.py,sha256=JyT-rOsqipazmImXBB-UzCg0GJ3UNzVCFdMcQisInfQ,5379
 zenml/utils/deprecation_utils.py,sha256=2cyeUeYas2OCcOcUJul2EGKd5zR_0KOjm6fKCshtFm0,6253
 zenml/utils/dict_utils.py,sha256=i7KAaKrkaWA_cG5IvVfMnr0CwWlBJ7KAsGvP2wxjZI8,2667
-zenml/utils/docker_utils.py,sha256=bJSlal3NeyQDvqXPE13X8ZJl7jcXE3LStiVX4IVAo3U,13661
+zenml/utils/docker_utils.py,sha256=QvkKnvIYSKAhW7mErXwSaQ432-q1LAsLjo2YWSXD8Bk,13889
 zenml/utils/downloaded_repository_context.py,sha256=S660PSSQ3dsNBA0qAj8ap_Thyw1n6x4VDcRWbCMDP2M,1502
 zenml/utils/enum_utils.py,sha256=0fA0B9v9Wjutuqlu_owUoOit1utIw2UH5J6YHXSqhLU,1368
 zenml/utils/env_utils.py,sha256=2--2DDUt3gPOvCNVyobBtAciikQ0OEXs5WWp7NvYuKo,5276
@@ -1276,8 +1279,8 @@ zenml/zen_stores/secrets_stores/sql_secrets_store.py,sha256=Bq1djrUP9saoD7vECjS7
 zenml/zen_stores/sql_zen_store.py,sha256=gq7hwCqXuvABUGHoo6J_fOdf4PGuN88BFWE84KK_-UM,408283
 zenml/zen_stores/template_utils.py,sha256=EKYBgmDLTS_PSMWaIO5yvHPLiQvMqHcsAe6NUCrv-i4,9068
 zenml/zen_stores/zen_store_interface.py,sha256=vf2gKBWfUUPtcGZC35oQB6pPNVzWVyQC8nWxVLjfrxM,92692
-zenml_nightly-0.70.0.dev20241203.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
-zenml_nightly-0.70.0.dev20241203.dist-info/METADATA,sha256=qZZHNrbpj8dGXfo9ZYuePjmJKt5XXOo3rsZpQKioRSA,21208
-zenml_nightly-0.70.0.dev20241203.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-zenml_nightly-0.70.0.dev20241203.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
-zenml_nightly-0.70.0.dev20241203.dist-info/RECORD,,
+zenml_nightly-0.70.0.dev20241204.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
+zenml_nightly-0.70.0.dev20241204.dist-info/METADATA,sha256=SsbWnr1H59o25vg7YOc1twHRrEODkGD23akgH9vanYU,21208
+zenml_nightly-0.70.0.dev20241204.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+zenml_nightly-0.70.0.dev20241204.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
+zenml_nightly-0.70.0.dev20241204.dist-info/RECORD,,