zenml-nightly 0.83.1.dev20250624__py3-none-any.whl → 0.83.1.dev20250626__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.83.1.dev20250624
+0.83.1.dev20250626

zenml/cli/base.py

@@ -49,6 +49,7 @@ from zenml.integrations.registry import integration_registry
 from zenml.io import fileio
 from zenml.logger import get_logger
 from zenml.utils.io_utils import copy_dir, get_global_config_directory
+from zenml.utils.package_utils import get_package_information
 from zenml.utils.server_utils import get_local_server
 from zenml.utils.yaml_utils import write_yaml
 
@@ -640,7 +641,7 @@ def info(
     }
 
     if all:
-        user_info["packages"] = cli_utils.get_package_information()
+        user_info["packages"] = get_package_information()
     if packages:
         if user_info.get("packages"):
             if isinstance(user_info["packages"], dict):
@@ -650,7 +651,7 @@ def info(
                     if p in packages
                 }
         else:
-            user_info["query_packages"] = cli_utils.get_package_information(
+            user_info["query_packages"] = get_package_information(
                 list(packages)
             )
     if file:

zenml/cli/login.py

@@ -229,6 +229,7 @@ def connect_to_pro_server(
     api_key: Optional[str] = None,
     refresh: bool = False,
     pro_api_url: Optional[str] = None,
+    verify_ssl: Union[str, bool] = True,
 ) -> None:
     """Connect the client to a ZenML Pro server.
 
@@ -238,6 +239,8 @@ def connect_to_pro_server(
         api_key: The API key to use to authenticate with the ZenML Pro server.
         refresh: Whether to force a new login flow with the ZenML Pro server.
         pro_api_url: The URL for the ZenML Pro API.
+        verify_ssl: Whether to verify the server's TLS certificate. If a string
+            is passed, it is interpreted as the path to a CA bundle file.
 
     Raises:
         ValueError: If incorrect parameters are provided.
@@ -280,7 +283,12 @@ def connect_to_pro_server(
         # server to connect to.
         if api_key:
             if server_url:
-                connect_to_server(server_url, api_key=api_key, pro_server=True)
+                connect_to_server(
+                    server_url,
+                    api_key=api_key,
+                    pro_server=True,
+                    verify_ssl=verify_ssl,
+                )
                 return
             else:
                 raise ValueError(
@@ -299,6 +307,7 @@ def connect_to_pro_server(
         try:
             token = web_login(
                 pro_api_url=pro_api_url,
+                verify_ssl=verify_ssl,
             )
         except AuthorizationException as e:
             cli_utils.error(f"Authorization error: {e}")
@@ -418,7 +427,9 @@ def connect_to_pro_server(
         f"Connecting to ZenML Pro server: {server.name} [{str(server.id)}] "
     )
 
-    connect_to_server(server.url, api_key=api_key, pro_server=True)
+    connect_to_server(
+        server.url, api_key=api_key, pro_server=True, verify_ssl=verify_ssl
+    )
 
     # Update the stored server info with more accurate data taken from the
     # ZenML Pro workspace object.
@@ -555,7 +566,7 @@ def _fail_if_authentication_environment_variables_set() -> None:
         not return until the server exits or is stopped with CTRL+C
 
       * `--docker`: start the local ZenML server as a Docker container instead
-        of a local process
+        of a local background process.
 
       * `--port`: use a custom TCP port value for the local ZenML server
 
@@ -775,6 +786,9 @@ def login(
             pro_server=server,
             refresh=True,
             pro_api_url=pro_api_url,
+            verify_ssl=ssl_ca_cert
+            if ssl_ca_cert is not None
+            else not no_verify_ssl,
         )
         return
 
@@ -822,6 +836,7 @@ def login(
                     # Prefer the pro API URL extracted from the server info if
                     # available
                     pro_api_url=server_pro_api_url or pro_api_url,
+                    verify_ssl=verify_ssl,
                 )
             else:
                 connect_to_server(
@@ -837,6 +852,7 @@ def login(
                 api_key=api_key_value,
                 refresh=refresh,
                 pro_api_url=pro_api_url,
+                verify_ssl=verify_ssl,
             )
 
     elif current_non_local_server and not refresh:
@@ -861,6 +877,7 @@ def login(
                 # Prefer the pro API URL extracted from the server info if
                 # available
                 pro_api_url=server_pro_api_url or pro_api_url,
+                verify_ssl=verify_ssl,
             )
         else:
             cli_utils.declare(
@@ -890,6 +907,7 @@ def login(
         connect_to_pro_server(
             api_key=api_key_value,
             pro_api_url=pro_api_url,
+            verify_ssl=verify_ssl,
         )
 
 

zenml/cli/service_connectors.py

@@ -899,14 +899,7 @@ def register_service_connector(
 
         # Prepare the rest of the variables to fall through to the
         # non-interactive configuration case
-        parsed_args = connector_model.configuration
-        parsed_args.update(
-            {
-                k: s.get_secret_value()
-                for k, s in connector_model.secrets.items()
-                if s is not None
-            }
-        )
+        parsed_args = connector_model.configuration.plain
         auto_configure = False
         no_verify = False
         expiration_seconds = connector_model.expiration_seconds
@@ -1137,7 +1130,7 @@ def describe_service_connector(
         try:
             connector = client.get_service_connector(
                 name_id_or_prefix=name_id_or_prefix,
-                load_secrets=True,
+                expand_secrets=show_secrets,
             )
         except KeyError as err:
             cli_utils.error(str(err))
@@ -1385,7 +1378,7 @@ def update_service_connector(
         connector = client.get_service_connector(
             name_id_or_prefix,
             allow_name_prefix_match=False,
-            load_secrets=True,
+            expand_secrets=True,
         )
     except KeyError as e:
         cli_utils.error(str(e))
@@ -1445,7 +1438,7 @@ def update_service_connector(
                 default=False,
             )
 
-        existing_config = connector.full_configuration
+        existing_config = connector.configuration.plain
 
         if confirm:
             # Here we reconfigure the connector or update the existing
@@ -1582,7 +1575,7 @@ def update_service_connector(
         # Non-interactive configuration
 
         # Apply the configuration from the command line arguments
-        config_dict = connector.full_configuration
+        config_dict = connector.configuration.plain
         config_dict.update(parsed_args)
 
         if not resource_type and not connector.is_multi_type:

zenml/cli/stack.py

@@ -1783,14 +1783,10 @@ def _get_service_connector_info(
     answers = {}
     for req_field in required_fields:
         if connector_details:
-            if conf_value := connector_details.configuration.get(
+            if conf_value := connector_details.configuration.get_plain(
                 req_field, None
             ):
                 answers[req_field] = conf_value
-            elif secret_value := connector_details.secrets.get(
-                req_field, None
-            ):
-                answers[req_field] = secret_value.get_secret_value()
         if req_field not in answers:
             answers[req_field] = Prompt.ask(
                 f"Please enter value for `{req_field}`:",

zenml/cli/utils.py

@@ -40,7 +40,6 @@ from typing import (
 )
 
 import click
-import pkg_resources
 import yaml
 from pydantic import BaseModel, SecretStr
 from rich import box, table
@@ -80,6 +79,7 @@ from zenml.stack import StackComponent
 from zenml.stack.flavor import Flavor
 from zenml.stack.stack_component import StackComponentConfig
 from zenml.utils import secret_utils
+from zenml.utils.package_utils import requirement_installed
 from zenml.utils.time_utils import expires_in
 from zenml.utils.typing_utils import get_origin, is_union
 
@@ -1052,7 +1052,7 @@ def install_packages(
         # just return without doing anything
         return
 
-    if use_uv and not is_installed_in_python_environment("uv"):
+    if use_uv and not requirement_installed("uv"):
         # If uv is installed globally, don't run as a python module
         command = []
     else:
@@ -1094,7 +1094,7 @@ def uninstall_package(package: str, use_uv: bool = False) -> None:
         package: The package to uninstall.
         use_uv: Whether to use uv for package uninstallation.
     """
-    if use_uv and not is_installed_in_python_environment("uv"):
+    if use_uv and not requirement_installed("uv"):
         # If uv is installed globally, don't run as a python module
         command = []
     else:
@@ -1110,22 +1110,6 @@ def uninstall_package(package: str, use_uv: bool = False) -> None:
     subprocess.check_call(command)
 
 
-def is_installed_in_python_environment(package: str) -> bool:
-    """Check if a package is installed in the current python environment.
-
-    Args:
-        package: The package to check.
-
-    Returns:
-        True if the package is installed, False otherwise.
-    """
-    try:
-        pkg_resources.get_distribution(package)
-        return True
-    except pkg_resources.DistributionNotFound:
-        return False
-
-
 def is_uv_installed() -> bool:
     """Check if uv is installed.
 
@@ -1141,7 +1125,7 @@ def is_pip_installed() -> bool:
     Returns:
         True if pip is installed, False otherwise.
     """
-    return is_installed_in_python_environment("pip")
+    return requirement_installed("pip")
 
 
 def pretty_print_secret(
@@ -1830,7 +1814,7 @@ def print_service_connector_configuration(
 
     console.print(rich_table)
 
-    if len(connector.configuration) == 0 and len(connector.secrets) == 0:
+    if len(connector.configuration) == 0:
         declare("No configuration options are set for this connector.")
 
     else:
@@ -1842,8 +1826,8 @@ def print_service_connector_configuration(
         rich_table.add_column("PROPERTY")
         rich_table.add_column("VALUE", overflow="fold")
 
-        config = connector.configuration.copy()
-        secrets = connector.secrets.copy()
+        config = connector.configuration.non_secrets
+        secrets = connector.configuration.secrets
         for key, value in secrets.items():
             if not show_secrets:
                 config[key] = "[HIDDEN]"
@@ -2499,30 +2483,6 @@ def temporary_active_stack(
             Client().activate_stack(old_stack_id)
 
 
-def get_package_information(
-    package_names: Optional[List[str]] = None,
-) -> Dict[str, str]:
-    """Get a dictionary of installed packages.
-
-    Args:
-        package_names: Specific package names to get the information for.
-
-    Returns:
-        A dictionary of the name:version for the package names passed in or
-            all packages and their respective versions.
-    """
-    import pkg_resources
-
-    if package_names:
-        return {
-            pkg.key: pkg.version
-            for pkg in pkg_resources.working_set
-            if pkg.key in package_names
-        }
-
-    return {pkg.key: pkg.version for pkg in pkg_resources.working_set}
-
-
 def print_user_info(info: Dict[str, Any]) -> None:
     """Print user information to the terminal.
 
@@ -2617,11 +2577,7 @@ def is_jupyter_installed() -> bool:
     Returns:
         bool: True if Jupyter notebook is installed, False otherwise.
     """
-    try:
-        pkg_resources.get_distribution("notebook")
-        return True
-    except pkg_resources.DistributionNotFound:
-        return False
+    return requirement_installed("notebook")
 
 
 def multi_choice_prompt(

zenml/client.py

@@ -5500,17 +5500,18 @@ class Client(metaclass=ClientMetaClass):
         self,
         name_id_or_prefix: Union[str, UUID],
         allow_name_prefix_match: bool = True,
-        load_secrets: bool = False,
         hydrate: bool = True,
+        expand_secrets: bool = False,
     ) -> ServiceConnectorResponse:
         """Fetches a registered service connector.
 
         Args:
             name_id_or_prefix: The id of the service connector to fetch.
             allow_name_prefix_match: If True, allow matching by name prefix.
-            load_secrets: If True, load the secrets for the service connector.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
+            expand_secrets: If True, expand the secrets for the service
+                connector.
 
         Returns:
             The registered service connector.
@@ -5521,25 +5522,9 @@ class Client(metaclass=ClientMetaClass):
             name_id_or_prefix=name_id_or_prefix,
             allow_name_prefix_match=allow_name_prefix_match,
             hydrate=hydrate,
+            expand_secrets=expand_secrets,
         )
 
-        if load_secrets and connector.secret_id:
-            client = Client()
-            try:
-                secret = client.get_secret(
-                    name_id_or_prefix=connector.secret_id,
-                    allow_partial_id_match=False,
-                    allow_partial_name_match=False,
-                )
-            except KeyError as err:
-                logger.error(
-                    "Unable to retrieve secret values associated with "
-                    f"service connector '{connector.name}': {err}"
-                )
-            else:
-                # Add secret values to connector configuration
-                connector.secrets.update(secret.values)
-
         return connector
 
     def list_service_connectors(
@@ -5558,8 +5543,8 @@ class Client(metaclass=ClientMetaClass):
         resource_id: Optional[str] = None,
         user: Optional[Union[UUID, str]] = None,
         labels: Optional[Dict[str, Optional[str]]] = None,
-        secret_id: Optional[Union[str, UUID]] = None,
         hydrate: bool = False,
+        expand_secrets: bool = False,
     ) -> Page[ServiceConnectorResponse]:
         """Lists all registered service connectors.
 
@@ -5580,10 +5565,10 @@ class Client(metaclass=ClientMetaClass):
             user: Filter by user name/ID.
             name: The name of the service connector to filter by.
             labels: The labels of the service connector to filter by.
-            secret_id: Filter by the id of the secret that is referenced by the
-                service connector.
             hydrate: Flag deciding whether to hydrate the output model(s)
                 by including metadata fields in the response.
+            expand_secrets: If True, expand the secrets for the service
+                connectors.
 
         Returns:
             A page of service connectors.
@@ -5603,11 +5588,11 @@ class Client(metaclass=ClientMetaClass):
             created=created,
             updated=updated,
             labels=labels,
-            secret_id=secret_id,
         )
         return self.zen_store.list_service_connectors(
             filter_model=connector_filter_model,
             hydrate=hydrate,
+            expand_secrets=expand_secrets,
         )
 
     def update_service_connector(
@@ -5691,7 +5676,9 @@ class Client(metaclass=ClientMetaClass):
         connector_model = self.get_service_connector(
             name_id_or_prefix,
             allow_name_prefix_match=False,
-            load_secrets=True,
+            # We need the existing secrets only if a new configuration is not
+            # provided.
+            expand_secrets=configuration is None,
         )
 
         connector_instance: Optional[ServiceConnector] = None
@@ -5736,23 +5723,16 @@ class Client(metaclass=ClientMetaClass):
         )
 
         # Validate and configure the resources
-        if configuration is not None:
+        connector_update.validate_and_configure_resources(
+            connector_type=connector,
+            resource_types=resource_types,
+            resource_id=resource_id,
             # The supplied configuration is a drop-in replacement for the
-            # existing configuration and secrets
-            connector_update.validate_and_configure_resources(
-                connector_type=connector,
-                resource_types=resource_types,
-                resource_id=resource_id,
-                configuration=configuration,
-            )
-        else:
-            connector_update.validate_and_configure_resources(
-                connector_type=connector,
-                resource_types=resource_types,
-                resource_id=resource_id,
-                configuration=connector_model.configuration,
-                secrets=connector_model.secrets,
-            )
+            # existing configuration
+            configuration=configuration
+            if configuration is not None
+            else connector_model.configuration,
+        )
 
         # Add the labels
         if labels is not None:
@@ -5912,6 +5892,12 @@ class Client(metaclass=ClientMetaClass):
                 list_resources=list_resources,
             )
         else:
+            # Get the service connector model, with full secrets
+            service_connector = self.get_service_connector(
+                name_id_or_prefix=name_id_or_prefix,
+                allow_name_prefix_match=False,
+                expand_secrets=True,
+            )
             connector_instance = (
                 service_connector_registry.instantiate_connector(
                     model=service_connector
@@ -6034,6 +6020,12 @@ class Client(metaclass=ClientMetaClass):
                 # server-side implementation may not be able to do so
                 connector_client.verify()
         else:
+            # Get the service connector model, with full secrets
+            service_connector = self.get_service_connector(
+                name_id_or_prefix=name_id_or_prefix,
+                allow_name_prefix_match=False,
+                expand_secrets=True,
+            )
             connector_instance = (
                 service_connector_registry.instantiate_connector(
                     model=service_connector

zenml/config/__init__.py

@@ -23,12 +23,23 @@ configuration. This ``GlobalConfiguration`` object handles the serialization and
 deserialization of the configuration options that are stored in the file in
 order to persist the configuration across sessions.
 """
-from zenml.config.docker_settings import DockerSettings
-from zenml.config.resource_settings import ResourceSettings
+from zenml.config.docker_settings import (
+    DockerSettings,
+    PythonPackageInstaller,
+    PythonEnvironmentExportMethod,
+)
+from zenml.config.resource_settings import ResourceSettings, ByteUnit
 from zenml.config.retry_config import StepRetryConfig
+from zenml.config.schedule import Schedule
+from zenml.config.store_config import StoreConfiguration
 
 __all__ = [
     "DockerSettings",
+    "PythonPackageInstaller",
+    "PythonEnvironmentExportMethod",
     "ResourceSettings",
+    "ByteUnit",
     "StepRetryConfig",
+    "Schedule",
+    "StoreConfiguration",
 ]

zenml/constants.py

@@ -173,7 +173,6 @@ ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
 ENV_ZENML_DISABLE_STEP_NAMES_IN_LOGS = "ZENML_DISABLE_STEP_NAMES_IN_LOGS"
 ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
 ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
-ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
 ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION = (
     "ZENML_PIPELINE_API_TOKEN_EXPIRATION"
 )

zenml/exceptions.py

@@ -220,3 +220,19 @@ class CustomFlavorImportError(ImportError):
 
 class MaxConcurrentTasksError(ZenMLBaseException):
     """Raised when the maximum number of concurrent tasks is reached."""
+
+
+class RunMonitoringError(ZenMLBaseException):
+    """Raised when an error occurs while monitoring a pipeline run."""
+
+    def __init__(
+        self,
+        original_exception: BaseException,
+    ) -> None:
+        """Initializes the error.
+
+        Args:
+            original_exception: The original exception that occurred while
+                monitoring the pipeline run.
+        """
+        self.original_exception = original_exception

zenml/integrations/airflow/orchestrators/airflow_orchestrator.py

@@ -38,7 +38,7 @@ from zenml.integrations.airflow.flavors.airflow_orchestrator_flavor import (
 )
 from zenml.io import fileio
 from zenml.logger import get_logger
-from zenml.orchestrators import ContainerizedOrchestrator
+from zenml.orchestrators import ContainerizedOrchestrator, SubmissionResult
 from zenml.orchestrators.utils import get_orchestrator_run_name
 from zenml.stack import StackValidator
 from zenml.utils import io_utils
@@ -191,21 +191,29 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
         if self.config.local:
             stack.check_local_paths()
 
-    def prepare_or_run_pipeline(
+    def submit_pipeline(
         self,
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
         placeholder_run: Optional["PipelineRunResponse"] = None,
-    ) -> Any:
-        """Creates and writes an Airflow DAG zip file.
+    ) -> Optional[SubmissionResult]:
+        """Submits a pipeline to the orchestrator.
+
+        This method should only submit the pipeline and not wait for it to
+        complete. If the orchestrator is configured to wait for the pipeline run
+        to complete, a function that waits for the pipeline run to complete can
+        be passed as part of the submission result.
 
         Args:
-            deployment: The pipeline deployment to prepare or run.
+            deployment: The pipeline deployment to submit.
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
-                environment.
+                environment. These don't need to be set if running locally.
             placeholder_run: An optional placeholder run for the deployment.
+
+        Returns:
+            Optional submission result.
         """
         pipeline_settings = cast(
             AirflowOrchestratorSettings, self.get_settings(deployment)
@@ -277,6 +285,7 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
             dag_generator_values=dag_generator_values,
             output_dir=pipeline_settings.dag_output_dir or self.dags_directory,
         )
+        return None
 
     def _apply_resource_settings(
         self,

zenml/integrations/aws/container_registries/aws_container_registry.py

@@ -81,7 +81,9 @@ class AWSContainerRegistry(BaseContainerRegistry):
         """
         if self.connector:
             try:
-                model = Client().get_service_connector(self.connector)
+                model = Client().get_service_connector(
+                    self.connector, expand_secrets=True
+                )
                 connector = service_connector_registry.instantiate_connector(
                     model=model
                 )