zenml-nightly 0.82.1.dev20250527__py3-none-any.whl → 0.83.0.dev20250529__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.82.1.dev20250527
+0.83.0.dev20250529

zenml/cli/base.py

@@ -610,6 +610,11 @@ def info(
 
     store_cfg = gc.store_configuration
 
+    try:
+        active_project = client.active_project
+    except RuntimeError:
+        active_project = None
+
     user_info = {
         "zenml_local_version": zenml_version,
         "zenml_server_version": store_info.version,
@@ -622,7 +627,7 @@ def info(
         "python_version": environment.python_version(),
         "environment": get_environment(),
         "system_info": environment.get_system_info(),
-        "active_project": client.active_project.name,
+        "active_project": active_project.name if active_project else None,
         "active_stack": client.active_stack_model.name,
         "active_user": client.active_user.name,
         "telemetry_status": "enabled" if gc.analytics_opt_in else "disabled",

zenml/cli/model.py

@@ -68,12 +68,6 @@ def _model_version_to_print(
         "stage": model_version.stage,
         "run_metadata": model_version.run_metadata,
         "tags": [t.name for t in model_version.tags],
-        "data_artifacts_count": len(model_version.data_artifact_ids),
-        "model_artifacts_count": len(model_version.model_artifact_ids),
-        "deployment_artifacts_count": len(
-            model_version.deployment_artifact_ids
-        ),
-        "pipeline_runs_count": len(model_version.pipeline_run_ids),
         "updated": model_version.updated.date(),
     }
 
@@ -601,21 +595,6 @@ def _print_artifacts_links_generic(
         else "model artifacts"
     )
 
-    if (
-        (only_data_artifacts and not model_version.data_artifact_ids)
-        or (
-            only_deployment_artifacts
-            and not model_version.deployment_artifact_ids
-        )
-        or (only_model_artifacts and not model_version.model_artifact_ids)
-    ):
-        cli_utils.declare(f"No {type_} linked to the model version found.")
-        return
-
-    cli_utils.title(
-        f"{type_} linked to the model version `{model_version.name}[{model_version.number}]`:"
-    )
-
     links = Client().list_model_version_artifact_links(
         model_version_id=model_version.id,
         only_data_artifacts=only_data_artifacts,
@@ -624,6 +603,13 @@ def _print_artifacts_links_generic(
         **kwargs,
     )
 
+    if not links:
+        cli_utils.declare(f"No {type_} linked to the model version found.")
+        return
+
+    cli_utils.title(
+        f"{type_} linked to the model version `{model_version.name}[{model_version.number}]`:"
+    )
     cli_utils.print_pydantic_models(
         links,
         columns=["artifact_version", "created"],
@@ -732,29 +718,23 @@ def list_model_version_pipeline_runs(
         model_name: The ID or name of the model containing version.
         model_version: The name, number or ID of the model version. If not
             provided, the latest version is used.
-        **kwargs: Keyword arguments to filter models.
+        **kwargs: Keyword arguments to filter runs.
     """
     model_version_response_model = Client().get_model_version(
         model_name_or_id=model_name,
         model_version_name_or_number_or_id=model_version,
     )
 
-    if not model_version_response_model.pipeline_run_ids:
-        cli_utils.declare("No pipeline runs attached to model version found.")
-        return
-    cli_utils.title(
-        f"Pipeline runs linked to the model version `{model_version_response_model.name}[{model_version_response_model.number}]`:"
-    )
-
-    links = Client().list_model_version_pipeline_run_links(
+    runs = Client().list_model_version_pipeline_run_links(
         model_version_id=model_version_response_model.id,
         **kwargs,
     )
 
-    cli_utils.print_pydantic_models(
-        links,
-        columns=[
-            "pipeline_run",
-            "created",
-        ],
+    if not runs:
+        cli_utils.declare("No pipeline runs attached to model version found.")
+        return
+
+    cli_utils.title(
+        f"Pipeline runs linked to the model version `{model_version_response_model.name}[{model_version_response_model.number}]`:"
     )
+    cli_utils.print_pydantic_models(runs)

zenml/cli/server.py

@@ -268,9 +268,14 @@ def status() -> None:
         # Write about the active entities
         scope = "repository" if client.uses_local_configuration else "global"
         cli_utils.declare(f"  The active user is: '{client.active_user.name}'")
-        cli_utils.declare(
-            f"  The active project is: '{client.active_project.name}'"
-        )
+
+        try:
+            cli_utils.declare(
+                f"  The active project is: '{client.active_project.name}' ({scope})"
+            )
+        except RuntimeError:
+            cli_utils.declare("  No active project set.")
+
         cli_utils.declare(
             f"  The active stack is: '{client.active_stack_model.name}' ({scope})"
         )

zenml/client.py

@@ -2573,7 +2573,7 @@ class Client(metaclass=ClientMetaClass):
                 self.list_run_templates,
                 pipeline_id=pipeline.id,
                 stack_id=stack.id if stack else None,
-                project=project or pipeline.project.id,
+                project=project or pipeline.project_id,
             )
 
             for template in templates:
@@ -3855,6 +3855,7 @@ class Client(metaclass=ClientMetaClass):
         allow_name_prefix_match: bool = True,
         project: Optional[Union[str, UUID]] = None,
         hydrate: bool = True,
+        include_full_metadata: bool = False,
     ) -> PipelineRunResponse:
         """Gets a pipeline run by name, ID, or prefix.
 
@@ -3864,6 +3865,8 @@ class Client(metaclass=ClientMetaClass):
             project: The project name/ID to filter by.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
+            include_full_metadata: If True, include metadata of all steps in
+                the response.
 
         Returns:
             The pipeline run.
@@ -3875,6 +3878,7 @@ class Client(metaclass=ClientMetaClass):
             allow_name_prefix_match=allow_name_prefix_match,
             project=project,
             hydrate=hydrate,
+            include_full_metadata=include_full_metadata,
         )
 
     def list_pipeline_runs(
@@ -3913,6 +3917,7 @@ class Client(metaclass=ClientMetaClass):
         stack: Optional[Union[UUID, str]] = None,
         stack_component: Optional[Union[UUID, str]] = None,
         hydrate: bool = False,
+        include_full_metadata: bool = False,
     ) -> Page[PipelineRunResponse]:
         """List all pipeline runs.
 
@@ -3953,6 +3958,8 @@ class Client(metaclass=ClientMetaClass):
             stack_component: Filter by stack component name/ID.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
+            include_full_metadata: If True, include metadata of all steps in
+                the response.
 
         Returns:
             A page with Pipeline Runs fitting the filter description
@@ -3995,6 +4002,7 @@ class Client(metaclass=ClientMetaClass):
         return self.zen_store.list_runs(
             runs_filter_model=runs_filter_model,
             hydrate=hydrate,
+            include_full_metadata=include_full_metadata,
         )
 
     def delete_pipeline_run(
@@ -6368,7 +6376,7 @@ class Client(metaclass=ClientMetaClass):
             model_version=ModelVersionRequest(
                 name=name,
                 description=description,
-                project=model.project.id,
+                project=model.project_id,
                 model=model.id,
                 tags=tags,
             )
@@ -6623,7 +6631,7 @@ class Client(metaclass=ClientMetaClass):
         if not is_valid_uuid(model_name_or_id):
             model = self.get_model(model_name_or_id, project=project)
             model_name_or_id = model.id
-            project = project or model.project.id
+            project = project or model.project_id
         if not is_valid_uuid(version_name_or_id):
             version_name_or_id = self.get_model_version(
                 model_name_or_id, version_name_or_id, project=project
@@ -7032,6 +7040,7 @@ class Client(metaclass=ClientMetaClass):
         allow_name_prefix_match: bool = True,
         project: Optional[Union[str, UUID]] = None,
         hydrate: bool = True,
+        **kwargs: Any,
     ) -> AnyResponse:
         """Fetches an entity using the id, name, or partial id/name.
 
@@ -7044,6 +7053,8 @@ class Client(metaclass=ClientMetaClass):
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
             project: The project name/ID to filter by.
+            **kwargs: Additional keyword arguments to pass to the get and list
+                methods.
 
         Returns:
             The entity with the given name, id or partial id.
@@ -7058,13 +7069,14 @@ class Client(metaclass=ClientMetaClass):
 
         # First interpret as full UUID
         if is_valid_uuid(name_id_or_prefix):
-            return get_method(name_id_or_prefix, hydrate=hydrate)
+            return get_method(name_id_or_prefix, hydrate=hydrate, **kwargs)
 
         # If not a UUID, try to find by name
         assert not isinstance(name_id_or_prefix, UUID)
         list_kwargs: Dict[str, Any] = dict(
             name=f"equals:{name_id_or_prefix}",
             hydrate=hydrate,
+            **kwargs,
         )
         scope = ""
         if project:
@@ -7182,6 +7194,7 @@ class Client(metaclass=ClientMetaClass):
         allow_name_prefix_match: bool,
         project: Optional[Union[str, UUID]] = None,
         hydrate: bool = True,
+        **kwargs: Any,
     ) -> AnyResponse:
         """Fetches an entity using a partial ID or name.
 
@@ -7193,6 +7206,8 @@ class Client(metaclass=ClientMetaClass):
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
             project: The project name/ID to filter by.
+            **kwargs: Additional keyword arguments to pass to the get and list
+                methods.
 
         Returns:
             The entity with the given partial ID or name.
@@ -7206,6 +7221,7 @@ class Client(metaclass=ClientMetaClass):
             "logical_operator": LogicalOperators.OR,
             "id": f"startswith:{partial_id_or_name}",
             "hydrate": hydrate,
+            **kwargs,
         }
         if allow_name_prefix_match:
             list_method_args["name"] = f"startswith:{partial_id_or_name}"

zenml/client_lazy_loader.py

@@ -14,6 +14,7 @@
 """Lazy loading functionality for Client methods."""
 
 import contextlib
+import functools
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type
 
 from pydantic import BaseModel, Field
@@ -188,6 +189,7 @@ def evaluate_all_lazy_load_args_in_client_methods(
     def _evaluate_args(
         func: Callable[..., Any], is_instance_method: bool
     ) -> Any:
+        @functools.wraps(func)
         def _inner(*args: Any, **kwargs: Any) -> Any:
             args_ = list(args)
             if not is_instance_method:

zenml/config/docker_settings.py

@@ -72,6 +72,9 @@ class DockerBuildConfig(BaseModel):
     dockerignore: Optional[str] = None
 
 
+_docker_settings_warnings_logged = []
+
+
 class DockerSettings(BaseSettings):
     """Settings for building Docker images to run ZenML pipelines.
 
@@ -372,6 +375,9 @@ class DockerSettings(BaseSettings):
         if isinstance(
             self.replicate_local_python_environment,
             (str, list, PythonEnvironmentExportMethod),
+        ) and (
+            "replicate_local_python_environment"
+            not in _docker_settings_warnings_logged
         ):
             logger.warning(
                 "Specifying a command (`%s`) for "
@@ -383,7 +389,9 @@ class DockerSettings(BaseSettings):
                 "file, use `DockerSettings.pyproject_path` and "
                 "`DockerSettings.pyproject_export_command` instead."
             )
-
+            _docker_settings_warnings_logged.append(
+                "replicate_local_python_environment"
+            )
         return self
 
     @model_validator(mode="before")
@@ -400,7 +408,11 @@ class DockerSettings(BaseSettings):
         Returns:
             The validated settings values.
         """
-        if "python_package_installer" not in data:
+        if (
+            "python_package_installer" not in data
+            and "python_package_installer"
+            not in _docker_settings_warnings_logged
+        ):
             logger.warning(
                 "In a future release, the default Python package installer "
                 "used by ZenML to build container images for your "
@@ -409,6 +421,7 @@ class DockerSettings(BaseSettings):
                 "`python_package_installer=PythonPackageInstaller.PIP` "
                 "in your DockerSettings."
             )
+            _docker_settings_warnings_logged.append("python_package_installer")
         return data
 
     model_config = ConfigDict(

zenml/enums.py

@@ -408,6 +408,8 @@ class OnboardingStep(StrEnum):
     DEVICE_VERIFIED = "device_verified"
     PROJECT_CREATED = "project_created"
     PIPELINE_RUN = "pipeline_run"
+    SECOND_PIPELINE_RUN = "second_pipeline_run"
+    THIRD_PIPELINE_RUN = "third_pipeline_run"
     STARTER_SETUP_COMPLETED = "starter_setup_completed"
     STACK_WITH_REMOTE_ORCHESTRATOR_CREATED = (
         "stack_with_remote_orchestrator_created"
@@ -422,6 +424,7 @@ class OnboardingStep(StrEnum):
         "pipeline_run_with_remote_artifact_store"
     )
     PRODUCTION_SETUP_COMPLETED = "production_setup_completed"
+    PRO_ONBOARDING_COMPLETED = "pro_onboarding_completed"
 
 
 class StackDeploymentProvider(StrEnum):

zenml/event_hub/event_hub.py

@@ -126,7 +126,7 @@ class InternalEventHub(BaseEventHub):
         triggers: List[TriggerResponse] = depaginate(
             self.zen_store.list_triggers,
             trigger_filter_model=TriggerFilter(
-                project=event_source.project.id,
+                project=event_source.project_id,
                 event_source_id=event_source.id,
                 is_active=True,
             ),

zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py

@@ -326,7 +326,7 @@ class BitbucketWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
             config: Event source configuration instantiated from the response.
         """
         assert isinstance(config, BitbucketWebhookEventSourceConfiguration)
-        assert event_source.user is not None, (
+        assert event_source.user_id is not None, (
             "User is not set for event source"
         )
 

zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py

@@ -352,7 +352,7 @@ class GithubWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
             config: Event source configuration instantiated from the response.
         """
         assert isinstance(config, GithubWebhookEventSourceConfiguration)
-        assert event_source.user is not None, (
+        assert event_source.user_id is not None, (
             "User is not set for event source"
         )
 

zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py

@@ -67,6 +67,8 @@ class KubernetesOrchestratorSettings(BaseSettings):
         ttl_seconds_after_finished: The amount of seconds to keep finished jobs
             before deleting them. This only applies to jobs created when
             scheduling a pipeline.
+        prevent_orchestrator_pod_caching: If `True`, the orchestrator pod will
+            not try to compute cached steps before starting the step pods.
     """
 
     synchronous: bool = True
@@ -85,6 +87,7 @@ class KubernetesOrchestratorSettings(BaseSettings):
     successful_jobs_history_limit: Optional[NonNegativeInt] = None
     failed_jobs_history_limit: Optional[NonNegativeInt] = None
     ttl_seconds_after_finished: Optional[NonNegativeInt] = None
+    prevent_orchestrator_pod_caching: bool = False
 
 
 class KubernetesOrchestratorConfig(

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py

@@ -74,23 +74,37 @@ def main() -> None:
     orchestrator_pod_name = socket.gethostname()
 
     client = Client()
+    active_stack = client.active_stack
+    orchestrator = active_stack.orchestrator
+    assert isinstance(orchestrator, KubernetesOrchestrator)
 
-    deployment_config = client.get_deployment(args.deployment_id)
+    deployment = client.get_deployment(args.deployment_id)
+    pipeline_settings = cast(
+        KubernetesOrchestratorSettings,
+        orchestrator.get_settings(deployment),
+    )
 
-    pipeline_dag = {
-        step_name: step.spec.upstream_steps
-        for step_name, step in deployment_config.step_configurations.items()
-    }
     step_command = StepEntrypointConfiguration.get_entrypoint_command()
 
-    active_stack = client.active_stack
+    if args.run_id and not pipeline_settings.prevent_orchestrator_pod_caching:
+        from zenml.orchestrators import cache_utils
+
+        run_required = (
+            cache_utils.create_cached_step_runs_and_prune_deployment(
+                deployment=deployment,
+                pipeline_run=client.get_pipeline_run(args.run_id),
+                stack=active_stack,
+            )
+        )
+
+        if not run_required:
+            return
+
     mount_local_stores = active_stack.orchestrator.config.is_local
 
     # Get a Kubernetes client from the active Kubernetes orchestrator, but
     # override the `incluster` setting to `True` since we are running inside
     # the Kubernetes cluster.
-    orchestrator = active_stack.orchestrator
-    assert isinstance(orchestrator, KubernetesOrchestrator)
     kube_client = orchestrator.get_kube_client(incluster=True)
     core_api = k8s_client.CoreV1Api(kube_client)
 
@@ -121,7 +135,7 @@ def main() -> None:
         Raises:
             Exception: If the pod fails to start.
         """
-        step_config = deployment_config.step_configurations[step_name].config
+        step_config = deployment.step_configurations[step_name].config
         settings = step_config.settings.get("orchestrator.kubernetes", None)
         settings = KubernetesOrchestratorSettings.model_validate(
             settings.model_dump() if settings else {}
@@ -147,10 +161,10 @@ def main() -> None:
         )
 
         image = KubernetesOrchestrator.get_image(
-            deployment=deployment_config, step_name=step_name
+            deployment=deployment, step_name=step_name
         )
         step_args = StepEntrypointConfiguration.get_entrypoint_arguments(
-            step_name=step_name, deployment_id=deployment_config.id
+            step_name=step_name, deployment_id=deployment.id
         )
 
         # We set some default minimum memory resource requests for the step pod
@@ -165,9 +179,7 @@ def main() -> None:
 
         if orchestrator.config.pass_zenml_token_as_secret:
             env.pop("ZENML_STORE_API_TOKEN", None)
-            secret_name = orchestrator.get_token_secret_name(
-                deployment_config.id
-            )
+            secret_name = orchestrator.get_token_secret_name(deployment.id)
             pod_settings.env.append(
                 {
                     "name": "ZENML_STORE_API_TOKEN",
@@ -184,7 +196,7 @@ def main() -> None:
         pod_manifest = build_pod_manifest(
             pod_name=pod_name,
             run_name=args.run_name,
-            pipeline_name=deployment_config.pipeline_configuration.name,
+            pipeline_name=deployment.pipeline_configuration.name,
             image_name=image,
             command=step_command,
             args=step_args,
@@ -251,8 +263,8 @@ def main() -> None:
 
             pipeline_runs = client.list_pipeline_runs(
                 hydrate=True,
-                project=deployment_config.project.id,
-                deployment_id=deployment_config.id,
+                project=deployment.project_id,
+                deployment_id=deployment.id,
                 **list_args,
             )
             if not len(pipeline_runs):
@@ -298,27 +310,26 @@ def main() -> None:
     parallel_node_startup_waiting_period = (
         orchestrator.config.parallel_step_startup_waiting_period or 0.0
     )
-    settings = cast(
-        KubernetesOrchestratorSettings,
-        orchestrator.get_settings(deployment_config),
-    )
+
+    pipeline_dag = {
+        step_name: step.spec.upstream_steps
+        for step_name, step in deployment.step_configurations.items()
+    }
     try:
         ThreadedDagRunner(
             dag=pipeline_dag,
             run_fn=run_step_on_kubernetes,
             finalize_fn=finalize_run,
             parallel_node_startup_waiting_period=parallel_node_startup_waiting_period,
-            max_parallelism=settings.max_parallelism,
+            max_parallelism=pipeline_settings.max_parallelism,
         ).run()
         logger.info("Orchestration pod completed.")
     finally:
         if (
             orchestrator.config.pass_zenml_token_as_secret
-            and deployment_config.schedule is None
+            and deployment.schedule is None
         ):
-            secret_name = orchestrator.get_token_secret_name(
-                deployment_config.id
-            )
+            secret_name = orchestrator.get_token_secret_name(deployment.id)
             try:
                 kube_utils.delete_secret(
                     core_api=core_api,

zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py

@@ -13,7 +13,7 @@
 #  permissions and limitations under the License.
 """Skypilot orchestrator base config and settings."""
 
-from typing import Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import Field
 
@@ -67,6 +67,14 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
         disk_size: the size of the OS disk in GiB.
         disk_tier: the disk performance tier to use. If None, defaults to
             ``'medium'``.
+        ports: Ports to expose. Could be an integer, a range, or a list of
+            integers and ranges. All ports will be exposed to the public internet.
+        labels: Labels to apply to instances as key-value pairs. These are
+            mapped to cloud-specific implementations (instance tags in AWS,
+            instance labels in GCP, etc.)
+        any_of: List of candidate resources to try in order of preference based on
+            cost (determined by the optimizer).
+        ordered: List of candidate resources to try in the specified order.
 
         cluster_name: name of the cluster to create/reuse.  If None,
             auto-generate a name.
@@ -88,6 +96,19 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
         stream_logs: if True, show the logs in the terminal.
         docker_run_args: Optional arguments to pass to the `docker run` command
             running inside the VM.
+        workdir: Working directory to sync to the VM. Synced to ~/sky_workdir.
+        task_name: Task name used for display purposes.
+        file_mounts: File and storage mounts configuration for remote cluster.
+        envs: Environment variables for the task.
+        task_settings: Dictionary of arbitrary settings to pass to sky.Task().
+            This allows passing future parameters added by SkyPilot without
+            requiring updates to ZenML.
+        resources_settings: Dictionary of arbitrary settings to pass to
+            sky.Resources(). This allows passing future parameters added
+            by SkyPilot without requiring updates to ZenML.
+        launch_settings: Dictionary of arbitrary settings to pass to
+            sky.launch(). This allows passing future parameters added
+            by SkyPilot without requiring updates to ZenML.
     """
 
     # Resources
@@ -103,14 +124,18 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
     )
     accelerator_args: Optional[Dict[str, str]] = None
     use_spot: Optional[bool] = None
-    job_recovery: Optional[str] = None
+    job_recovery: Union[None, str, Dict[str, Any]] = Field(
+        default=None, union_mode="left_to_right"
+    )
     region: Optional[str] = None
     zone: Optional[str] = None
     image_id: Union[Dict[str, str], str, None] = Field(
         default=None, union_mode="left_to_right"
     )
     disk_size: Optional[int] = None
-    disk_tier: Optional[Literal["high", "medium", "low"]] = None
+    disk_tier: Optional[Literal["high", "medium", "low", "ultra", "best"]] = (
+        None
+    )
 
     # Run settings
     cluster_name: Optional[str] = None
@@ -118,9 +143,25 @@ class SkypilotBaseOrchestratorSettings(BaseSettings):
     idle_minutes_to_autostop: Optional[int] = 30
     down: bool = True
     stream_logs: bool = True
-
     docker_run_args: List[str] = []
 
+    # Additional SkyPilot features
+    ports: Union[None, int, str, List[Union[int, str]]] = Field(
+        default=None, union_mode="left_to_right"
+    )
+    labels: Optional[Dict[str, str]] = None
+    any_of: Optional[List[Dict[str, Any]]] = None
+    ordered: Optional[List[Dict[str, Any]]] = None
+    workdir: Optional[str] = None
+    task_name: Optional[str] = None
+    file_mounts: Optional[Dict[str, Any]] = None
+    envs: Optional[Dict[str, str]] = None
+
+    # Future-proofing settings dictionaries
+    task_settings: Dict[str, Any] = {}
+    resources_settings: Dict[str, Any] = {}
+    launch_settings: Dict[str, Any] = {}
+
 
 class SkypilotBaseOrchestratorConfig(
     BaseOrchestratorConfig, SkypilotBaseOrchestratorSettings