zenml-nightly 0.82.1.dev20250528__py3-none-any.whl → 0.83.0.dev20250530__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.82.1.dev20250528
+0.83.0.dev20250530

zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py

@@ -13,7 +13,7 @@
 #  permissions and limitations under the License.
 """Skypilot orchestrator base config and settings."""
 
-from typing import Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import Field
 
@@ -67,6 +67,14 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
         disk_size: the size of the OS disk in GiB.
         disk_tier: the disk performance tier to use. If None, defaults to
             ``'medium'``.
+        ports: Ports to expose. Could be an integer, a range, or a list of
+            integers and ranges. All ports will be exposed to the public internet.
+        labels: Labels to apply to instances as key-value pairs. These are
+            mapped to cloud-specific implementations (instance tags in AWS,
+            instance labels in GCP, etc.)
+        any_of: List of candidate resources to try in order of preference based on
+            cost (determined by the optimizer).
+        ordered: List of candidate resources to try in the specified order.
 
         cluster_name: name of the cluster to create/reuse.  If None,
             auto-generate a name.
@@ -88,6 +96,19 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
         stream_logs: if True, show the logs in the terminal.
         docker_run_args: Optional arguments to pass to the `docker run` command
             running inside the VM.
+        workdir: Working directory to sync to the VM. Synced to ~/sky_workdir.
+        task_name: Task name used for display purposes.
+        file_mounts: File and storage mounts configuration for remote cluster.
+        envs: Environment variables for the task.
+        task_settings: Dictionary of arbitrary settings to pass to sky.Task().
+            This allows passing future parameters added by SkyPilot without
+            requiring updates to ZenML.
+        resources_settings: Dictionary of arbitrary settings to pass to
+            sky.Resources(). This allows passing future parameters added
+            by SkyPilot without requiring updates to ZenML.
+        launch_settings: Dictionary of arbitrary settings to pass to
+            sky.launch(). This allows passing future parameters added
+            by SkyPilot without requiring updates to ZenML.
     """
 
     # Resources
@@ -103,14 +124,18 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
     )
     accelerator_args: Optional[Dict[str, str]] = None
     use_spot: Optional[bool] = None
-    job_recovery: Optional[str] = None
+    job_recovery: Union[None, str, Dict[str, Any]] = Field(
+        default=None, union_mode="left_to_right"
+    )
     region: Optional[str] = None
     zone: Optional[str] = None
     image_id: Union[Dict[str, str], str, None] = Field(
         default=None, union_mode="left_to_right"
     )
     disk_size: Optional[int] = None
-    disk_tier: Optional[Literal["high", "medium", "low"]] = None
+    disk_tier: Optional[Literal["high", "medium", "low", "ultra", "best"]] = (
+        None
+    )
 
     # Run settings
     cluster_name: Optional[str] = None
@@ -118,9 +143,25 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
     idle_minutes_to_autostop: Optional[int] = 30
     down: bool = True
     stream_logs: bool = True
-
     docker_run_args: List[str] = []
 
+    # Additional SkyPilot features
+    ports: Union[None, int, str, List[Union[int, str]]] = Field(
+        default=None, union_mode="left_to_right"
+    )
+    labels: Optional[Dict[str, str]] = None
+    any_of: Optional[List[Dict[str, Any]]] = None
+    ordered: Optional[List[Dict[str, Any]]] = None
+    workdir: Optional[str] = None
+    task_name: Optional[str] = None
+    file_mounts: Optional[Dict[str, Any]] = None
+    envs: Optional[Dict[str, str]] = None
+
+    # Future-proofing settings dictionaries
+    task_settings: Dict[str, Any] = {}
+    resources_settings: Dict[str, Any] = {}
+    launch_settings: Dict[str, Any] = {}
+
 
 class SkypilotBaseOrchestratorConfig(
     BaseOrchestratorConfig, SkypilotBaseOrchestratorSettings

zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py

@@ -14,12 +14,12 @@
 """Implementation of the Skypilot base VM orchestrator."""
 
 import os
-import re
 from abc import abstractmethod
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast
 from uuid import uuid4
 
 import sky
+from sky import StatusRefreshMode
 
 from zenml.entrypoints import PipelineEntrypointConfiguration
 from zenml.enums import StackComponentType
@@ -31,6 +31,15 @@ from zenml.integrations.skypilot.flavors.skypilot_orchestrator_base_vm_config im
 from zenml.integrations.skypilot.orchestrators.skypilot_orchestrator_entrypoint_configuration import (
     SkypilotOrchestratorEntrypointConfiguration,
 )
+from zenml.integrations.skypilot.utils import (
+    create_docker_run_command,
+    prepare_docker_setup,
+    prepare_launch_kwargs,
+    prepare_resources_kwargs,
+    prepare_task_kwargs,
+    sanitize_cluster_name,
+    sky_job_get,
+)
 from zenml.logger import get_logger
 from zenml.orchestrators import (
     ContainerizedOrchestrator,
@@ -252,32 +261,21 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
         entrypoint_str = " ".join(command)
         arguments_str = " ".join(args)
 
-        task_envs = environment
-        docker_environment_str = " ".join(
-            f"-e {k}={v}" for k, v in environment.items()
-        )
-        custom_run_args = " ".join(settings.docker_run_args)
-        if custom_run_args:
-            custom_run_args += " "
-
-        instance_type = settings.instance_type or self.DEFAULT_INSTANCE_TYPE
+        task_envs = environment.copy()
 
         # Set up credentials
         self.setup_credentials()
 
-        # Guaranteed by stack validation
-        assert stack is not None and stack.container_registry is not None
+        # Prepare Docker setup
+        setup, docker_creds_envs = prepare_docker_setup(
+            container_registry_uri=stack.container_registry.config.uri,
+            credentials=stack.container_registry.credentials,
+            use_sudo=True,  # Base orchestrator uses sudo
+        )
 
-        if docker_creds := stack.container_registry.credentials:
-            docker_username, docker_password = docker_creds
-            setup = (
-                f"sudo docker login --username $DOCKER_USERNAME --password "
-                f"$DOCKER_PASSWORD {stack.container_registry.config.uri}"
-            )
-            task_envs["DOCKER_USERNAME"] = docker_username
-            task_envs["DOCKER_PASSWORD"] = docker_password
-        else:
-            setup = None
+        # Update task_envs with Docker credentials
+        if docker_creds_envs:
+            task_envs.update(docker_creds_envs)
 
         # Run the entire pipeline
 
@@ -291,45 +289,49 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
                 down = False
                 idle_minutes_to_autostop = None
             else:
-                run_command = f"sudo docker run --rm {custom_run_args}{docker_environment_str} {image} {entrypoint_str} {arguments_str}"
+                run_command = create_docker_run_command(
+                    image=image,
+                    entrypoint_str=entrypoint_str,
+                    arguments_str=arguments_str,
+                    environment=task_envs,
+                    docker_run_args=settings.docker_run_args,
+                    use_sudo=True,  # Base orchestrator uses sudo
+                )
                 down = settings.down
                 idle_minutes_to_autostop = settings.idle_minutes_to_autostop
-            task = sky.Task(
-                run=run_command,
+
+            # Create the Task with all parameters and task settings
+            task_kwargs = prepare_task_kwargs(
+                settings=settings,
+                run_command=run_command,
                 setup=setup,
-                envs=task_envs,
+                task_envs=task_envs,
+                task_name=f"{orchestrator_run_name}",
             )
+
+            task = sky.Task(**task_kwargs)
             logger.debug(f"Running run: {run_command}")
 
-            task = task.set_resources(
-                sky.Resources(
-                    cloud=self.cloud,
-                    instance_type=instance_type,
-                    cpus=settings.cpus,
-                    memory=settings.memory,
-                    accelerators=settings.accelerators,
-                    accelerator_args=settings.accelerator_args,
-                    use_spot=settings.use_spot,
-                    job_recovery=settings.job_recovery,
-                    region=settings.region,
-                    zone=settings.zone,
-                    image_id=image
-                    if isinstance(self.cloud, sky.clouds.Kubernetes)
-                    else settings.image_id,
-                    disk_size=settings.disk_size,
-                    disk_tier=settings.disk_tier,
-                )
+            # Set resources with all parameters and resource settings
+            resources_kwargs = prepare_resources_kwargs(
+                cloud=self.cloud,
+                settings=settings,
+                default_instance_type=self.DEFAULT_INSTANCE_TYPE,
+                kubernetes_image=image
+                if isinstance(self.cloud, sky.clouds.Kubernetes)
+                else None,
             )
-            # Do not detach run if logs are being streamed
-            # Otherwise, the logs will not be streamed after the task is submitted
-            # Could also be a parameter in the settings to control this behavior
-            detach_run = not settings.stream_logs
+
+            task = task.set_resources(sky.Resources(**resources_kwargs))
 
             launch_new_cluster = True
             if settings.cluster_name:
-                cluster_info = sky.status(
-                    refresh=True, cluster_names=settings.cluster_name
+                status_request_id = sky.status(
+                    refresh=StatusRefreshMode.AUTO,
+                    cluster_names=[settings.cluster_name],
                 )
+                cluster_info = sky.stream_and_get(status_request_id)
+
                 if cluster_info:
                     logger.info(
                         f"Found existing cluster {settings.cluster_name}. Reusing..."
@@ -342,7 +344,7 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
                     )
                     cluster_name = settings.cluster_name
             else:
-                cluster_name = self.sanitize_cluster_name(
+                cluster_name = sanitize_cluster_name(
                     f"{orchestrator_run_name}"
                 )
                 logger.info(
@@ -350,33 +352,55 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
                 )
 
             if launch_new_cluster:
-                sky.launch(
+                # Prepare launch parameters with additional launch settings
+                launch_kwargs = prepare_launch_kwargs(
+                    settings=settings,
+                    down=down,
+                    idle_minutes_to_autostop=idle_minutes_to_autostop,
+                )
+                logger.info(
+                    f"Launching the task on a new cluster: {cluster_name}"
+                )
+                launch_job_id = sky.launch(
                     task,
                     cluster_name,
-                    retry_until_up=settings.retry_until_up,
-                    idle_minutes_to_autostop=idle_minutes_to_autostop,
-                    down=down,
-                    stream_logs=settings.stream_logs,
-                    backend=None,
-                    detach_setup=True,
-                    detach_run=detach_run,
+                    **launch_kwargs,
                 )
+                sky_job_get(launch_job_id, settings.stream_logs, cluster_name)
+
             else:
-                # Make sure the cluster is up -
-                # If the cluster is already up, this will not do anything
-                sky.start(
+                # Prepare exec parameters with additional launch settings
+                exec_kwargs = {
+                    "down": down,
+                    "backend": None,
+                    **settings.launch_settings,  # Can reuse same settings for exec
+                }
+
+                # Remove None values to avoid overriding SkyPilot defaults
+                exec_kwargs = {
+                    k: v for k, v in exec_kwargs.items() if v is not None
+                }
+
+                # Make sure the cluster is up
+                start_request_id = sky.start(
                     settings.cluster_name,
                     down=down,
                     idle_minutes_to_autostop=idle_minutes_to_autostop,
                     retry_until_up=settings.retry_until_up,
                 )
-                sky.exec(
+                sky.stream_and_get(start_request_id)
+
+                logger.info(
+                    f"Executing the task on the cluster: {settings.cluster_name}"
+                )
+                exec_job_id = sky.exec(
                     task,
-                    settings.cluster_name,
-                    down=down,
-                    stream_logs=settings.stream_logs,
-                    backend=None,
-                    detach_run=detach_run,
+                    cluster_name=settings.cluster_name,
+                    **exec_kwargs,
+                )
+                assert settings.cluster_name is not None
+                sky_job_get(
+                    exec_job_id, settings.stream_logs, settings.cluster_name
                 )
 
         except Exception as e:
@@ -386,19 +410,3 @@ class SkypilotBaseOrchestrator(ContainerizedOrchestrator):
         finally:
             # Unset the service connector AWS profile ENV variable
             self.prepare_environment_variable(set=False)
-
-    def sanitize_cluster_name(self, name: str) -> str:
-        """Sanitize the value to be used in a cluster name.
-
-        Args:
-            name: Arbitrary input cluster name.
-
-        Returns:
-            Sanitized cluster name.
-        """
-        name = re.sub(
-            r"[^a-z0-9-]", "-", name.lower()
-        )  # replaces any character that is not a lowercase letter, digit, or hyphen with a hyphen
-        name = re.sub(r"^[-]+", "", name)  # trim leading hyphens
-        name = re.sub(r"[-]+$", "", name)  # trim trailing hyphens
-        return name

zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py

@@ -32,8 +32,20 @@ from zenml.integrations.skypilot.orchestrators.skypilot_base_vm_orchestrator imp
     ENV_ZENML_SKYPILOT_ORCHESTRATOR_RUN_ID,
     SkypilotBaseOrchestrator,
 )
+from zenml.integrations.skypilot.utils import (
+    create_docker_run_command,
+    prepare_docker_setup,
+    prepare_launch_kwargs,
+    prepare_resources_kwargs,
+    prepare_task_kwargs,
+    sanitize_cluster_name,
+    sky_job_get,
+)
 from zenml.logger import get_logger
-from zenml.orchestrators.dag_runner import ThreadedDagRunner
+from zenml.orchestrators.dag_runner import NodeStatus, ThreadedDagRunner
+from zenml.orchestrators.publish_utils import (
+    publish_failed_pipeline_run,
+)
 from zenml.orchestrators.utils import get_config_environment_vars
 
 logger = get_logger(__name__)
@@ -65,212 +77,228 @@ def main() -> None:
         TypeError: If the active stack's orchestrator is not an instance of
             SkypilotBaseOrchestrator.
         ValueError: If the active stack's container registry is None.
+        Exception: If the orchestration or one of the steps fails.
     """
     # Log to the container's stdout so it can be streamed by the client.
     logger.info("Skypilot orchestrator VM started.")
 
     # Parse / extract args.
     args = parse_args()
-
     orchestrator_run_id = socket.gethostname()
 
-    deployment = Client().get_deployment(args.deployment_id)
-
-    pipeline_dag = {
-        step_name: step.spec.upstream_steps
-        for step_name, step in deployment.step_configurations.items()
-    }
-    step_command = StepEntrypointConfiguration.get_entrypoint_command()
-    entrypoint_str = " ".join(step_command)
-
-    active_stack = Client().active_stack
-
-    orchestrator = active_stack.orchestrator
-    if not isinstance(orchestrator, SkypilotBaseOrchestrator):
-        raise TypeError(
-            "The active stack's orchestrator is not an instance of SkypilotBaseOrchestrator."
-        )
-
-    # Set up credentials
-    orchestrator.setup_credentials()
-
-    # Set the service connector AWS profile ENV variable
-    orchestrator.prepare_environment_variable(set=True)
+    run = None
 
-    # get active container registry
-    container_registry = active_stack.container_registry
-    if container_registry is None:
-        raise ValueError("Container registry cannot be None.")
+    try:
+        deployment = Client().get_deployment(args.deployment_id)
 
-    if docker_creds := container_registry.credentials:
-        docker_username, docker_password = docker_creds
-        setup = (
-            f"docker login --username $DOCKER_USERNAME --password "
-            f"$DOCKER_PASSWORD {container_registry.config.uri}"
-        )
-        task_envs = {
-            "DOCKER_USERNAME": docker_username,
-            "DOCKER_PASSWORD": docker_password,
+        pipeline_dag = {
+            step_name: step.spec.upstream_steps
+            for step_name, step in deployment.step_configurations.items()
         }
-    else:
-        setup = None
-        task_envs = None
-
-    unique_resource_configs: Dict[str, str] = {}
-    for step_name, step in deployment.step_configurations.items():
-        settings = cast(
-            SkypilotBaseOrchestratorSettings,
-            orchestrator.get_settings(step),
-        )
-        # Handle both str and Dict[str, int] types for accelerators
-        if isinstance(settings.accelerators, dict):
-            accelerators_hashable = frozenset(settings.accelerators.items())
-        elif isinstance(settings.accelerators, str):
-            accelerators_hashable = frozenset({(settings.accelerators, 1)})
-        else:
-            accelerators_hashable = None
-        resource_config = (
-            settings.instance_type,
-            settings.cpus,
-            settings.memory,
-            settings.disk_size,  # Assuming disk_size is part of the settings
-            settings.disk_tier,  # Assuming disk_tier is part of the settings
-            settings.use_spot,
-            settings.job_recovery,
-            settings.region,
-            settings.zone,
-            accelerators_hashable,
-        )
-        cluster_name_parts = [
-            orchestrator.sanitize_cluster_name(str(part))
-            for part in resource_config
-            if part is not None
-        ]
-        cluster_name = f"cluster-{orchestrator_run_id}" + "-".join(
-            cluster_name_parts
-        )
-        unique_resource_configs[step_name] = cluster_name
-
-    run = Client().list_pipeline_runs(
-        sort_by="asc:created",
-        size=1,
-        deployment_id=args.deployment_id,
-        status=ExecutionStatus.INITIALIZING,
-    )[0]
+        step_command = StepEntrypointConfiguration.get_entrypoint_command()
+        entrypoint_str = " ".join(step_command)
 
-    logger.info("Fetching pipeline run: %s", run.id)
+        active_stack = Client().active_stack
 
-    def run_step_on_skypilot_vm(step_name: str) -> None:
-        """Run a pipeline step in a separate Skypilot VM.
+        orchestrator = active_stack.orchestrator
+        if not isinstance(orchestrator, SkypilotBaseOrchestrator):
+            raise TypeError(
+                "The active stack's orchestrator is not an instance of SkypilotBaseOrchestrator."
+            )
 
-        Args:
-            step_name: Name of the step.
-        """
-        cluster_name = unique_resource_configs[step_name]
+        # Set up credentials
+        orchestrator.setup_credentials()
 
-        image = SkypilotBaseOrchestrator.get_image(
-            deployment=deployment, step_name=step_name
-        )
+        # Set the service connector AWS profile ENV variable
+        orchestrator.prepare_environment_variable(set=True)
 
-        step_args = StepEntrypointConfiguration.get_entrypoint_arguments(
-            step_name=step_name, deployment_id=deployment.id
-        )
-        arguments_str = " ".join(step_args)
+        # get active container registry
+        container_registry = active_stack.container_registry
+        if container_registry is None:
+            raise ValueError("Container registry cannot be None.")
 
-        step = deployment.step_configurations[step_name]
-        settings = cast(
-            SkypilotBaseOrchestratorSettings,
-            orchestrator.get_settings(step),
+        # Prepare Docker setup
+        setup, task_envs = prepare_docker_setup(
+            container_registry_uri=container_registry.config.uri,
+            credentials=container_registry.credentials,
+            use_sudo=False,  # Entrypoint doesn't use sudo
         )
-        env = get_config_environment_vars()
-        env[ENV_ZENML_SKYPILOT_ORCHESTRATOR_RUN_ID] = orchestrator_run_id
 
-        docker_environment_str = " ".join(
-            f"-e {k}={v}" for k, v in env.items()
-        )
-        custom_run_args = " ".join(settings.docker_run_args)
-        if custom_run_args:
-            custom_run_args += " "
-
-        # Set up the task
-        run_command = f"docker run --rm {custom_run_args}{docker_environment_str} {image} {entrypoint_str} {arguments_str}"
-        task_name = f"{deployment.id}-{step_name}-{time.time()}"
-        task = sky.Task(
-            run=run_command,
-            setup=setup,
-            envs=task_envs,
-            name=task_name,
-        )
-        task = task.set_resources(
-            sky.Resources(
-                cloud=orchestrator.cloud,
-                instance_type=settings.instance_type
-                or orchestrator.DEFAULT_INSTANCE_TYPE,
-                cpus=settings.cpus,
-                memory=settings.memory,
-                disk_size=settings.disk_size,
-                disk_tier=settings.disk_tier,
-                accelerators=settings.accelerators,
-                accelerator_args=settings.accelerator_args,
-                use_spot=settings.use_spot,
-                job_recovery=settings.job_recovery,
-                region=settings.region,
-                zone=settings.zone,
-                image_id=settings.image_id,
+        unique_resource_configs: Dict[str, str] = {}
+        for step_name, step in deployment.step_configurations.items():
+            settings = cast(
+                SkypilotBaseOrchestratorSettings,
+                orchestrator.get_settings(step),
             )
-        )
+            # Handle both str and Dict[str, int] types for accelerators
+            if isinstance(settings.accelerators, dict):
+                accelerators_hashable = frozenset(
+                    settings.accelerators.items()
+                )
+            elif isinstance(settings.accelerators, str):
+                accelerators_hashable = frozenset({(settings.accelerators, 1)})
+            else:
+                accelerators_hashable = None
+            resource_config = (
+                settings.instance_type,
+                settings.cpus,
+                settings.memory,
+                settings.disk_size,  # Assuming disk_size is part of the settings
+                settings.disk_tier,  # Assuming disk_tier is part of the settings
+                settings.use_spot,
+                settings.job_recovery,
+                settings.region,
+                settings.zone,
+                accelerators_hashable,
+            )
+            cluster_name_parts = [
+                sanitize_cluster_name(str(part))
+                for part in resource_config
+                if part is not None
+            ]
+            cluster_name = f"cluster-{orchestrator_run_id}" + "-".join(
+                cluster_name_parts
+            )
+            unique_resource_configs[step_name] = cluster_name
 
-        sky.launch(
-            task,
-            cluster_name,
-            retry_until_up=settings.retry_until_up,
-            idle_minutes_to_autostop=settings.idle_minutes_to_autostop,
-            down=settings.down,
-            stream_logs=settings.stream_logs,
-            detach_setup=True,
-            detach_run=True,
-        )
+        run = Client().list_pipeline_runs(
+            sort_by="asc:created",
+            size=1,
+            deployment_id=args.deployment_id,
+            status=ExecutionStatus.INITIALIZING,
+        )[0]
 
-        # Wait for pod to finish.
-        logger.info(f"Waiting for pod of step `{step_name}` to start...")
+        logger.info("Fetching pipeline run: %s", run.id)
 
-        current_run = Client().get_pipeline_run(run.id)
+        def run_step_on_skypilot_vm(step_name: str) -> None:
+            """Run a pipeline step in a separate Skypilot VM.
 
-        step_is_finished = False
-        while not step_is_finished:
-            time.sleep(10)
-            current_run = Client().get_pipeline_run(run.id)
+            Args:
+                step_name: Name of the step.
+
+            Raises:
+                Exception: If the step execution fails.
+            """
+            logger.info(f"Running step `{step_name}` on a VM...")
             try:
-                step_is_finished = current_run.steps[
-                    step_name
-                ].status.is_finished
-            except KeyError:
-                # Step is not yet in the run, so we wait for it to appear
-                continue
-
-        # Pop the resource configuration for this step
-        unique_resource_configs.pop(step_name)
-
-        if cluster_name in unique_resource_configs.values():
-            # If there are more steps using this configuration, skip deprovisioning the cluster
-            logger.info(
-                f"Resource configuration for cluster '{cluster_name}' "
-                "is used by subsequent steps. Skipping the deprovisioning of "
-                "the cluster."
-            )
-        else:
-            # If there are no more steps using this configuration, down the cluster
-            logger.info(
-                f"Resource configuration for cluster '{cluster_name}' "
-                "is not used by subsequent steps. deprovisioning the cluster."
-            )
-            sky.down(cluster_name)
+                cluster_name = unique_resource_configs[step_name]
+
+                image = SkypilotBaseOrchestrator.get_image(
+                    deployment=deployment, step_name=step_name
+                )
+
+                step_args = (
+                    StepEntrypointConfiguration.get_entrypoint_arguments(
+                        step_name=step_name, deployment_id=deployment.id
+                    )
+                )
+                arguments_str = " ".join(step_args)
+
+                step = deployment.step_configurations[step_name]
+                settings = cast(
+                    SkypilotBaseOrchestratorSettings,
+                    orchestrator.get_settings(step),
+                )
+                env = get_config_environment_vars()
+                env[ENV_ZENML_SKYPILOT_ORCHESTRATOR_RUN_ID] = (
+                    orchestrator_run_id
+                )
+
+                # Create the Docker run command
+                run_command = create_docker_run_command(
+                    image=image,
+                    entrypoint_str=entrypoint_str,
+                    arguments_str=arguments_str,
+                    environment=env,
+                    docker_run_args=settings.docker_run_args,
+                    use_sudo=False,  # Entrypoint doesn't use sudo
+                )
+
+                task_name = f"{deployment.id}-{step_name}-{time.time()}"
+
+                # Create task kwargs
+                task_kwargs = prepare_task_kwargs(
+                    settings=settings,
+                    run_command=run_command,
+                    setup=setup,
+                    task_envs=task_envs,
+                    task_name=task_name,
+                )
+
+                task = sky.Task(**task_kwargs)
+
+                # Set resources
+                resources_kwargs = prepare_resources_kwargs(
+                    cloud=orchestrator.cloud,
+                    settings=settings,
+                    default_instance_type=orchestrator.DEFAULT_INSTANCE_TYPE,
+                )
+
+                task = task.set_resources(sky.Resources(**resources_kwargs))
+
+                # Prepare launch parameters
+                launch_kwargs = prepare_launch_kwargs(
+                    settings=settings,
+                )
+
+                # sky.launch now returns a request ID (async). Capture it so we can
+                # optionally stream logs and block until completion when desired.
+                launch_request_id = sky.launch(
+                    task,
+                    cluster_name,
+                    **launch_kwargs,
+                )
+                sky_job_get(launch_request_id, True, cluster_name)
+
+                # Pop the resource configuration for this step
+                unique_resource_configs.pop(step_name)
+
+                if cluster_name in unique_resource_configs.values():
+                    # If there are more steps using this configuration, skip deprovisioning the cluster
+                    logger.info(
+                        f"Resource configuration for cluster '{cluster_name}' "
+                        "is used by subsequent steps. Skipping the deprovisioning of "
+                        "the cluster."
+                    )
+                else:
+                    # If there are no more steps using this configuration, down the cluster
+                    logger.info(
+                        f"Resource configuration for cluster '{cluster_name}' "
+                        "is not used by subsequent steps. deprovisioning the cluster."
+                    )
+                    down_request_id = sky.down(cluster_name)
+                    # Wait for the cluster to be terminated
+                    sky.stream_and_get(down_request_id)
+
+                logger.info(
+                    f"Running step `{step_name}` on a VM is completed."
+                )
+
+            except Exception as e:
+                logger.error(f"Failed while launching step `{step_name}`: {e}")
+                raise
+
+        dag_runner = ThreadedDagRunner(
+            dag=pipeline_dag, run_fn=run_step_on_skypilot_vm
+        )
+        dag_runner.run()
+
+        failed_nodes = []
+        for node in dag_runner.nodes:
+            if dag_runner.node_states[node] == NodeStatus.FAILED:
+                failed_nodes.append(node)
 
-        logger.info(f"Running step `{step_name}` on a VM is completed.")
+        if failed_nodes:
+            raise Exception(f"One or more steps failed: {failed_nodes}")
 
-    ThreadedDagRunner(dag=pipeline_dag, run_fn=run_step_on_skypilot_vm).run()
+    except Exception as e:
+        logger.error(f"Orchestrator failed: {e}")
 
-    logger.info("Orchestration VM provisioned.")
+        # Try to mark the pipeline run as failed
+        if run:
+            publish_failed_pipeline_run(run.id)
+            logger.info("Marked pipeline run as failed in ZenML.")
+        raise
 
 
 if __name__ == "__main__":

zenml/integrations/skypilot/utils.py

@@ -0,0 +1,273 @@
+"""Utility functions for Skypilot orchestrators."""
+
+import re
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+import sky
+
+from zenml.integrations.skypilot.flavors.skypilot_orchestrator_base_vm_config import (
+    SkypilotBaseOrchestratorSettings,
+)
+from zenml.logger import get_logger
+
+logger = get_logger(__name__)
+
+if TYPE_CHECKING:
+    from sky.clouds.cloud import Cloud
+
+
+def sanitize_cluster_name(name: str) -> str:
+    """Sanitize the value to be used in a cluster name.
+
+    Args:
+        name: Arbitrary input cluster name.
+
+    Returns:
+        Sanitized cluster name.
+    """
+    name = re.sub(
+        r"[^a-z0-9-]", "-", name.lower()
+    )  # replaces any character that is not a lowercase letter, digit, or hyphen with a hyphen
+    name = re.sub(r"^[-]+", "", name)  # trim leading hyphens
+    name = re.sub(r"[-]+$", "", name)  # trim trailing hyphens
+    return name
+
+
+def prepare_docker_setup(
+    container_registry_uri: str,
+    credentials: Optional[Tuple[str, str]] = None,
+    use_sudo: bool = True,
+) -> Tuple[Optional[str], Dict[str, str]]:
+    """Prepare Docker login setup command and environment variables.
+
+    Args:
+        container_registry_uri: URI of the container registry.
+        credentials: Optional credentials (username, password) tuple.
+        use_sudo: Whether to use sudo prefix in docker commands.
+
+    Returns:
+        Tuple of (setup command, environment variables)
+    """
+    if credentials:
+        docker_username, docker_password = credentials
+        sudo_prefix = "sudo " if use_sudo else ""
+        setup = (
+            f"{sudo_prefix}docker login --username $DOCKER_USERNAME --password "
+            f"$DOCKER_PASSWORD {container_registry_uri}"
+        )
+        task_envs = {
+            "DOCKER_USERNAME": docker_username,
+            "DOCKER_PASSWORD": docker_password,
+        }
+    else:
+        setup = None
+        task_envs = {}
+
+    return setup, task_envs
+
+
+def create_docker_run_command(
+    image: str,
+    entrypoint_str: str,
+    arguments_str: str,
+    environment: Dict[str, str],
+    docker_run_args: List[str],
+    use_sudo: bool = True,
+) -> str:
+    """Create a Docker run command string.
+
+    Args:
+        image: Docker image to run.
+        entrypoint_str: Entrypoint command.
+        arguments_str: Command arguments.
+        environment: Environment variables.
+        docker_run_args: Additional Docker run arguments.
+        use_sudo: Whether to use sudo prefix in docker commands.
+
+    Returns:
+        Docker run command as string.
+    """
+    docker_environment_str = " ".join(
+        f"-e {k}={v}" for k, v in environment.items()
+    )
+    custom_run_args = " ".join(docker_run_args)
+    if custom_run_args:
+        custom_run_args += " "
+
+    sudo_prefix = "sudo " if use_sudo else ""
+    return f"{sudo_prefix}docker run --rm {custom_run_args}{docker_environment_str} {image} {entrypoint_str} {arguments_str}"
+
+
+def prepare_task_kwargs(
+    settings: SkypilotBaseOrchestratorSettings,
+    run_command: str,
+    setup: Optional[str],
+    task_envs: Dict[str, str],
+    task_name: str,
+) -> Dict[str, Any]:
+    """Prepare task keyword arguments for sky.Task.
+
+    Args:
+        settings: Skypilot orchestrator settings.
+        run_command: Command to run.
+        setup: Setup command.
+        task_envs: Task environment variables.
+        task_name: Task name.
+
+    Returns:
+        Task keyword arguments dictionary.
+    """
+    # Merge envs from settings with existing task_envs
+    merged_envs = {}
+
+    # First add user-provided envs
+    if settings.envs:
+        merged_envs.update(settings.envs)
+
+    # Then add task_envs which take precedence
+    if task_envs:
+        merged_envs.update(task_envs)
+
+    task_kwargs = {
+        "run": run_command,
+        "setup": setup,
+        "envs": merged_envs,
+        "name": settings.task_name or task_name,
+        "workdir": settings.workdir,
+        "file_mounts_mapping": settings.file_mounts,
+        **settings.task_settings,  # Add any arbitrary task settings
+    }
+
+    # Remove None values to avoid overriding SkyPilot defaults
+    return {k: v for k, v in task_kwargs.items() if v is not None}
+
+
+def prepare_resources_kwargs(
+    cloud: "Cloud",
+    settings: SkypilotBaseOrchestratorSettings,
+    default_instance_type: Optional[str] = None,
+    kubernetes_image: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Prepare resources keyword arguments for sky.Resources.
+
+    Args:
+        cloud: Skypilot cloud.
+        settings: Skypilot orchestrator settings.
+        default_instance_type: Default instance type.
+        kubernetes_image: Image to use for Kubernetes (if applicable).
+
+    Returns:
+        Resources keyword arguments dictionary.
+    """
+    resources_kwargs = {
+        "cloud": cloud,
+        "instance_type": settings.instance_type or default_instance_type,
+        "cpus": settings.cpus,
+        "memory": settings.memory,
+        "accelerators": settings.accelerators,
+        "accelerator_args": settings.accelerator_args,
+        "use_spot": settings.use_spot,
+        "job_recovery": settings.job_recovery,
+        "region": settings.region,
+        "zone": settings.zone,
+        "image_id": kubernetes_image
+        if kubernetes_image
+        else settings.image_id,
+        "disk_size": settings.disk_size,
+        "disk_tier": settings.disk_tier,
+        "ports": settings.ports,
+        "labels": settings.labels,
+        "any_of": settings.any_of,
+        "ordered": settings.ordered,
+        **settings.resources_settings,  # Add any arbitrary resource settings
+    }
+
+    # Remove None values to avoid overriding SkyPilot defaults
+    return {k: v for k, v in resources_kwargs.items() if v is not None}
+
+
+def prepare_launch_kwargs(
+    settings: SkypilotBaseOrchestratorSettings,
+    down: Optional[bool] = None,
+    idle_minutes_to_autostop: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Prepare launch keyword arguments for sky.launch.
+
+    Args:
+        settings: Skypilot orchestrator settings.
+        down: Whether to tear down the cluster after job completion.
+        idle_minutes_to_autostop: Minutes to autostop after idleness.
+
+    Returns:
+        Launch keyword arguments dictionary.
+    """
+    # Determine values falling back to settings where applicable
+    down_value = down if down is not None else settings.down
+    idle_value = (
+        idle_minutes_to_autostop
+        if idle_minutes_to_autostop is not None
+        else settings.idle_minutes_to_autostop
+    )
+
+    # The following parameters were removed from sky.launch in versions > 0.8.
+    # We therefore no longer include them in the kwargs passed to the call.
+    # • stream_logs – handled by explicitly calling sky.stream_and_get
+    # • detach_setup / detach_run – setup/run are now detached by default
+
+    launch_kwargs = {
+        "retry_until_up": settings.retry_until_up,
+        "idle_minutes_to_autostop": idle_value,
+        "down": down_value,
+        "backend": None,
+        **settings.launch_settings,  # Keep user-provided extras
+    }
+
+    # Remove keys that are no longer supported by sky.launch.
+    for _deprecated in (
+        "stream_logs",
+        "detach_setup",
+        "detach_run",
+        "num_nodes",
+    ):
+        launch_kwargs.pop(_deprecated, None)
+
+    # Remove None values to avoid overriding SkyPilot defaults
+    return {k: v for k, v in launch_kwargs.items() if v is not None}
+
+
+def sky_job_get(request_id: str, stream_logs: bool, cluster_name: str) -> Any:
+    """Handle SkyPilot request results based on stream_logs setting.
+
+    SkyPilot API exec and launch methods are asynchronous and return a request ID.
+    This method waits for the operation to complete and returns the result.
+    If stream_logs is True, it will also stream the logs and wait for the
+    job to complete.
+
+    Args:
+        request_id: The request ID returned from a SkyPilot operation.
+        stream_logs: Whether to stream logs while waiting for completion.
+        cluster_name: The name of the cluster to tail logs for.
+
+    Returns:
+        The result of the SkyPilot operation.
+
+    Raises:
+        Exception: If the SkyPilot job fails.
+    """
+    if stream_logs:
+        # Stream logs and wait for completion
+        job_id, _ = sky.stream_and_get(request_id)
+    else:
+        # Just wait for completion without streaming logs
+        job_id, _ = sky.get(request_id)
+
+    status = 0  # 0=Successful, 100=Failed
+    if stream_logs:
+        status = sky.tail_logs(
+            cluster_name=cluster_name, job_id=job_id, follow=True
+        )
+
+    if status != 0:
+        raise Exception(f"SkyPilot job {job_id} failed with status {status}")
+
+    return job_id

zenml/integrations/skypilot_aws/__init__.py

@@ -32,7 +32,7 @@ class SkypilotAWSIntegration(Integration):
 
     NAME = SKYPILOT_AWS
     # all 0.6.x versions of skypilot[aws] are compatible
-    REQUIREMENTS = ["skypilot[aws]~=0.8.0"]
+    REQUIREMENTS = ["skypilot[aws]==0.9.3"]
     APT_PACKAGES = ["openssh-client", "rsync"]
 
     @classmethod
@@ -47,4 +47,3 @@ class SkypilotAWSIntegration(Integration):
         )
 
         return [SkypilotAWSOrchestratorFlavor]
-

zenml/integrations/skypilot_azure/__init__.py

@@ -31,7 +31,7 @@ class SkypilotAzureIntegration(Integration):
     """Definition of Skypilot (Azure) Integration for ZenML."""
 
     NAME = SKYPILOT_AZURE
-    REQUIREMENTS = ["skypilot[azure]~=0.8.0"]
+    REQUIREMENTS = ["skypilot[azure]==0.9.3"]
     APT_PACKAGES = ["openssh-client", "rsync"]
 
     @classmethod
@@ -46,4 +46,3 @@ class SkypilotAzureIntegration(Integration):
         )
 
         return [SkypilotAzureOrchestratorFlavor]
-

zenml/integrations/skypilot_gcp/__init__.py

@@ -31,7 +31,15 @@ class SkypilotGCPIntegration(Integration):
     """Definition of Skypilot (GCP) Integration for ZenML."""
 
     NAME = SKYPILOT_GCP
-    REQUIREMENTS = ["skypilot[gcp]~=0.8.0"]
+    REQUIREMENTS = [
+        "skypilot[gcp]==0.9.3",
+        # TODO: Remove this once the issue is fixed:
+        # Adding the dependencies of the GCP integration on top of the
+        # requirements of the skypilot integration results in a
+        # very long resolution time for pip. This is a workaround to
+        # speed up the resolution.
+        "protobuf>=4.25.0,<5.0.0",
+    ]
     APT_PACKAGES = ["openssh-client", "rsync"]
 
     @classmethod

zenml/integrations/skypilot_kubernetes/__init__.py

@@ -31,8 +31,8 @@ class SkypilotKubernetesIntegration(Integration):
     """Definition of Skypilot Kubernetes Integration for ZenML."""
 
     NAME = SKYPILOT_KUBERNETES
-    # all 0.6.x versions of skypilot[kubernetes] are compatible
-    REQUIREMENTS = ["skypilot[kubernetes]~=0.8.0"]
+
+    REQUIREMENTS = ["skypilot[kubernetes]==0.9.3"]
     APT_PACKAGES = ["openssh-client", "rsync"]
 
     @classmethod
@@ -47,4 +47,3 @@ class SkypilotKubernetesIntegration(Integration):
         )
 
         return [SkypilotKubernetesOrchestratorFlavor]
-

zenml/integrations/skypilot_lambda/__init__.py

@@ -31,7 +31,7 @@ class SkypilotLambdaIntegration(Integration):
     """Definition of Skypilot Lambda Integration for ZenML."""
 
     NAME = SKYPILOT_LAMBDA
-    REQUIREMENTS = ["skypilot[lambda]~=0.8.0"]
+    REQUIREMENTS = ["skypilot[lambda]==0.9.3"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:
@@ -45,4 +45,3 @@ class SkypilotLambdaIntegration(Integration):
         )
 
         return [SkypilotLambdaOrchestratorFlavor]
-

zenml/zen_stores/migrations/versions/0.83.0_release.py

@@ -0,0 +1,23 @@
+"""Release [0.83.0].
+
+Revision ID: 0.83.0
+Revises: 0.82.1
+Create Date: 2025-05-28 08:23:38.860697
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.83.0"
+down_revision = "0.82.1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    pass
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    pass

{zenml_nightly-0.82.1.dev20250528.dist-info → zenml_nightly-0.83.0.dev20250530.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.82.1.dev20250528
+Version: 0.83.0.dev20250530
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops
@@ -532,7 +532,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.82.1 is out. Check out the release notes
+    🎉 Version 0.83.0 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.

{zenml_nightly-0.82.1.dev20250528.dist-info → zenml_nightly-0.83.0.dev20250530.dist-info}/RECORD

@@ -1,5 +1,5 @@
 zenml/README.md,sha256=827dekbOWAs1BpW7VF1a4d7EbwPbjwccX-2zdXBENZo,1777
-zenml/VERSION,sha256=RF8Wvwi_9I6TwR0kIAio6cRckEGPUKpRfA8KhaZGZH0,19
+zenml/VERSION,sha256=GJZSRVTbwPM2Da26xqgWxii6udSKMzbQ260OLsFphRY,19
 zenml/__init__.py,sha256=CKEyepFK-7akXYiMrNVh92Nb01Cjs23w4_YyI6sgdc8,2242
 zenml/actions/__init__.py,sha256=mrt6wPo73iKRxK754_NqsGyJ3buW7RnVeIGXr1xEw8Y,681
 zenml/actions/base_action.py,sha256=UcaHev6BTuLDwuswnyaPjdA8AgUqB5xPZ-lRtuvf2FU,25553
@@ -478,32 +478,33 @@ zenml/integrations/sklearn/materializers/__init__.py,sha256=vHIx7njuVcP_kGbyECgJ
 zenml/integrations/sklearn/materializers/sklearn_materializer.py,sha256=IaE8DS9L2gjfRVv8RT8p7ls7cJJ5qzt6hRgdpL6pp20,1562
 zenml/integrations/skypilot/__init__.py,sha256=PzNuJJykzfu34pTC4kSD18ARyGysuhzEW8oLTm4oHuo,612
 zenml/integrations/skypilot/flavors/__init__.py,sha256=PzNuJJykzfu34pTC4kSD18ARyGysuhzEW8oLTm4oHuo,612
-zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py,sha256=UPHfsB75CDZ8ecxxlL8aTo3IGIVuCAU_USBHzhULMuc,6675
+zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py,sha256=pYr_Wi2rjh6XTpHDLYBC7guPM23MzlUoIadzMDckKMI,8840
 zenml/integrations/skypilot/orchestrators/__init__.py,sha256=elo6QiImzys1m_bgu96U1HCVRepHQI1m7Jgqn6aEJ4Q,844
-zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py,sha256=yEoRnEaNoamZocFsNMKanqQqF-FA7o9sMpbs_Ib8J1I,15494
-zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py,sha256=5sq04vu2ArsyW56wvH8CRLG8kmeaSkPYPzirGGe7mds,9737
+zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py,sha256=hTLBB7_YPoDVwe9HobPPbz11fXM7w2jjk423qY5IGpc,15580
+zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py,sha256=L8ZSbJAwZfFOS0igxw1-LBr7UxspqVyvXrvMQ8AbJbk,11143
 zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint_configuration.py,sha256=wB0QKtDe6a3XyFmoFTwRRdztn7Y6PG-VSI8lHM5n47U,2243
-zenml/integrations/skypilot_aws/__init__.py,sha256=c3_i5LA-sbmGykaoFMH8QB5L8072LNFGsDA4r4Ftaw0,1723
+zenml/integrations/skypilot/utils.py,sha256=YUX9lPqEyPYDV36xEefoQGnyQwu9Kum-gen2LVIgqhs,8680
+zenml/integrations/skypilot_aws/__init__.py,sha256=U8G0PeQyGcKtQ8NR0YG3CI1K3SxD77_-3s-t4mvFZYA,1722
 zenml/integrations/skypilot_aws/flavors/__init__.py,sha256=UGL7NuZ2BkB5IxL9WUHmGtlsuUzYw2B-5NPc-ZqGVPY,1006
 zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py,sha256=PqcUCXRyJoT7jmR15A5d7N5NJxXkgLKWm8BC3YW6Omw,3789
 zenml/integrations/skypilot_aws/orchestrators/__init__.py,sha256=xxFZ2iM9H9LVKfSB8N-yKGxuB-lJE3Y368lcj7MfhI0,1006
 zenml/integrations/skypilot_aws/orchestrators/skypilot_aws_vm_orchestrator.py,sha256=0uFfRa8lGsc7D_zdrGGMqKoL6eKUwuf2ImtDw_RRg6Q,3094
-zenml/integrations/skypilot_azure/__init__.py,sha256=jAMY88MYwfxcewkTh2ZkXDH-VGyK0icnXMq8NGX8OP0,1692
+zenml/integrations/skypilot_azure/__init__.py,sha256=WckxTBIZMH1UqRIcoUQCdRRNzABXbKqMO-Y-ty6VnSY,1691
 zenml/integrations/skypilot_azure/flavors/__init__.py,sha256=9JB07IZbIZhY6yL9TjkYFTCUUfxajMajsMbbjC1BZ_s,1024
 zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py,sha256=aL8ibfQGo2IcyckaesjAFk6m8kazy5pq6ycArYRD5w0,3858
 zenml/integrations/skypilot_azure/orchestrators/__init__.py,sha256=gW7JOTLdGB542Cv1ydSqF4kVBAo7LUnfkrlj45kCWm8,1017
 zenml/integrations/skypilot_azure/orchestrators/skypilot_azure_vm_orchestrator.py,sha256=T0CfFOLRNJZJD_cOaucjckeONZORe_MYhwKOrVGMW3c,2418
-zenml/integrations/skypilot_gcp/__init__.py,sha256=GSQINq3uoriQi8lWAIvjgSO0vfCJUoCNmeRwPKIB1fo,1667
+zenml/integrations/skypilot_gcp/__init__.py,sha256=MhyhPrjy-gfHasyWEUif3RyDJFNGdh6e2Cxe5J5_RVM,2009
 zenml/integrations/skypilot_gcp/flavors/__init__.py,sha256=cOszr8-BlWpWua9uG-d0Vi3VkUxxq6jl03BD90wZMv4,1006
 zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py,sha256=K2fe63T8zApDnZwuroTle9x_gPW55SUdeJa5JU7ihEo,3946
 zenml/integrations/skypilot_gcp/orchestrators/__init__.py,sha256=Z6ALEu4PixIo_dKa5Vyo039bNFIOT-3O0MlYFIm6CsY,1007
 zenml/integrations/skypilot_gcp/orchestrators/skypilot_gcp_vm_orchestrator.py,sha256=c7YQKMileo27mCSyHgSpJ_JU4wPckkXm9o5MUXmOQAA,2516
-zenml/integrations/skypilot_kubernetes/__init__.py,sha256=FPN7fbIoClkEfblEfpV1e79WQwT0gGbwAf8mKCj0imk,1814
+zenml/integrations/skypilot_kubernetes/__init__.py,sha256=vGX0w4xKV8TOkWsM1kaJqPGiaprRcKiIwf_2QOOyuJc,1750
 zenml/integrations/skypilot_kubernetes/flavors/__init__.py,sha256=KNKGT467zLLqFJCJl5uyQJU1738cGLt2iCiTC_e1hgA,1069
 zenml/integrations/skypilot_kubernetes/flavors/skypilot_orchestrator_kubernetes_vm_flavor.py,sha256=EkhYa7F_Eqk0BC7CAEsgPwzTVB95FXlkXyy2aSLERBE,3997
 zenml/integrations/skypilot_kubernetes/orchestrators/__init__.py,sha256=26Gf5CHZeT39B7UPt-dSyT9pVMz2SxuBihDAmQ9Pwgo,1041
 zenml/integrations/skypilot_kubernetes/orchestrators/skypilot_kubernetes_vm_orchestrator.py,sha256=evrkn8JVX87HAb5tuqaoBQXUH5GdK2Botqp7SqVTRbA,2428
-zenml/integrations/skypilot_lambda/__init__.py,sha256=Nu67nGLvpOZ6OKkebZS45odUjJ8gd2pyCKgeNgXL9J4,1655
+zenml/integrations/skypilot_lambda/__init__.py,sha256=iH5KRgn8TS5Tw8MdDSQFF4oNR-S8bi2G932ivCS_oKM,1654
 zenml/integrations/skypilot_lambda/flavors/__init__.py,sha256=2Chuv2ViYTppU4GltvSU2DJ0C6E6IYgPhBor2YkOL5g,1033
 zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py,sha256=INsSw8yP48v9TvTp_lRNV1TwXvK-fPhkLAlmUb6971o,4063
 zenml/integrations/skypilot_lambda/orchestrators/__init__.py,sha256=3S2SmBfCp-ID78dRXh1QoI1uSxpG0M63Ul8lhDOC3AU,1021
@@ -1170,6 +1171,7 @@ zenml/zen_stores/migrations/versions/0.80.2_release.py,sha256=fsM4kc-d82s0xXqUiD
 zenml/zen_stores/migrations/versions/0.81.0_release.py,sha256=iiNGlHCVt2IMoQTaN9YIJyHMX3XmGtQVHMxZmyjyD8A,462
 zenml/zen_stores/migrations/versions/0.82.0_release.py,sha256=xBZ8ah1X99l_vUZ8bHC7Pa8caiU8kUhs-SZnGEVAIYY,450
 zenml/zen_stores/migrations/versions/0.82.1_release.py,sha256=OBsyh5W2R-ZBie1_Gw789zSwOXhtgXJiu-_aphomo-E,450
+zenml/zen_stores/migrations/versions/0.83.0_release.py,sha256=Y3Pe9I_LJvUgTtehyDC6yLa9mG2QlD0A2ZdXIYF_y8M,450
 zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py,sha256=hXLzvQcylNrbCVD6vha52PFkSPNC2klW9kA0vuQX_cE,1091
 zenml/zen_stores/migrations/versions/03742aa7fdd7_add_secrets.py,sha256=gewKqu1AnzvNTjVvK1eaAwP0hVneWDUyDRSLTvRCdpg,1587
 zenml/zen_stores/migrations/versions/0392807467dc_add_build_duration.py,sha256=YlkDBlfBBv45FsrMO11YcdRn4Maqmlg77t8gWJO4DfA,982
@@ -1325,8 +1327,8 @@ zenml/zen_stores/secrets_stores/sql_secrets_store.py,sha256=LPFW757WCJLP1S8vrvjs
 zenml/zen_stores/sql_zen_store.py,sha256=TxqVUsE-2NlJ1EasyMfIOtLCPMCtbd19Jgye9b4PlVE,465963
 zenml/zen_stores/template_utils.py,sha256=GbJ7LgGVYHSCKPEA8RNTxPoVTWqpC77F_lGzjJ4O1Fw,9220
 zenml/zen_stores/zen_store_interface.py,sha256=_ap55L3_mrHgegsLkMRSmmNXVasYC53LwjcEeuS1YT4,92411
-zenml_nightly-0.82.1.dev20250528.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
-zenml_nightly-0.82.1.dev20250528.dist-info/METADATA,sha256=_UKf8EAqztNQG7y7gIlv5H0iDz0d6UEwayy2MaPJXpI,24317
-zenml_nightly-0.82.1.dev20250528.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-zenml_nightly-0.82.1.dev20250528.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
-zenml_nightly-0.82.1.dev20250528.dist-info/RECORD,,
+zenml_nightly-0.83.0.dev20250530.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
+zenml_nightly-0.83.0.dev20250530.dist-info/METADATA,sha256=GcqahpuJfnFHW_b2HfKEKfc6Z_6sWD_Dt0Q8Dbjdn2s,24317
+zenml_nightly-0.83.0.dev20250530.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+zenml_nightly-0.83.0.dev20250530.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
+zenml_nightly-0.83.0.dev20250530.dist-info/RECORD,,