zenml-nightly 0.58.2.dev20240623__py3-none-any.whl → 0.61.0.dev20240712__py3-none-any.whl

zenml/config/docker_settings.py

@@ -16,7 +16,7 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import Field, model_validator
+from pydantic import BaseModel, Field, model_validator
 from pydantic_settings import SettingsConfigDict
 
 from zenml.config.base_settings import BaseSettings
@@ -65,6 +65,23 @@ class PythonPackageInstaller(Enum):
     UV = "uv"
 
 
+class DockerBuildConfig(BaseModel):
+    """Configuration for a Docker build.
+
+    Attributes:
+        build_options: Additional options that will be passed unmodified to the
+            Docker build call when building an image. You can use this to for
+            example specify build args or a target stage. See
+            https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build
+            for a full list of available options.
+        dockerignore: Path to a dockerignore file to use when building the
+            Docker image.
+    """
+
+    build_options: Dict[str, Any] = {}
+    dockerignore: Optional[str] = None
+
+
 class DockerSettings(BaseSettings):
     """Settings for building Docker images to run ZenML pipelines.
 
@@ -85,10 +102,11 @@ class DockerSettings(BaseSettings):
     Depending on the configuration of this object, requirements will be
     installed in the following order (each step optional):
     - The packages installed in your local python environment
-    - The packages specified via the `requirements` attribute
-    - The packages specified via the `required_integrations` and potentially
-      stack requirements
     - The packages specified via the `required_hub_plugins` attribute
+    - The packages required by the stack unless this is disabled by setting
+      `install_stack_requirements=False`.
+    - The packages specified via the `required_integrations`
+    - The packages specified via the `requirements` attribute
 
     Attributes:
         parent_image: Full name of the Docker image that should be
@@ -116,18 +134,17 @@ class DockerSettings(BaseSettings):
         build_context_root: Build context root for the Docker build, only used
             when the `dockerfile` attribute is set. If this is left empty, the
             build context will only contain the Dockerfile.
-        build_options: Additional options that will be passed unmodified to the
-            Docker build call when building an image using the specified
-            `dockerfile`. You can use this to for example specify build
-            args or a target stage. See
-            https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build
-            for a full list of available options.
+        parent_image_build_config: Configuration for the parent image build.
+        build_options: DEPRECATED, use parent_image_build_config.build_options
+            instead.
         skip_build: If set to `True`, the parent image will be used directly to
             run the steps of your pipeline.
         target_repository: Name of the Docker repository to which the
             image should be pushed. This repository will be appended to the
             registry URI of the container registry of your stack and should
-            therefore **not** include any registry.
+            therefore **not** include any registry. If not specified, the
+            default repository name configured in the container registry
+            stack component settings will be used.
         python_package_installer: The package installer to use for python
             packages.
         python_package_installer_args: Arguments to pass to the python package
@@ -158,8 +175,8 @@ class DockerSettings(BaseSettings):
         apt_packages: APT packages to install inside the Docker image.
         environment: Dictionary of environment variables to set inside the
             Docker image.
-        dockerignore: Path to a dockerignore file to use when building the
-            Docker image.
+        build_config: Configuration for the main image build.
+        dockerignore: DEPRECATED, use build_config.dockerignore instead.
         copy_files: DEPRECATED, use the `source_files` attribute instead.
         copy_global_config: DEPRECATED/UNUSED.
         user: If not `None`, will set the user, make it owner of the `/app`
@@ -184,8 +201,9 @@ class DockerSettings(BaseSettings):
     dockerfile: Optional[str] = None
     build_context_root: Optional[str] = None
     build_options: Dict[str, Any] = {}
+    parent_image_build_config: Optional[DockerBuildConfig] = None
     skip_build: bool = False
-    target_repository: str = "zenml"
+    target_repository: Optional[str] = None
     python_package_installer: PythonPackageInstaller = (
         PythonPackageInstaller.PIP
     )
@@ -205,6 +223,7 @@ class DockerSettings(BaseSettings):
     copy_files: bool = True
     copy_global_config: bool = True
     user: Optional[str] = None
+    build_config: Optional[DockerBuildConfig] = None
 
     source_files: SourceFileMode = SourceFileMode.DOWNLOAD_OR_INCLUDE
 

zenml/constants.py

@@ -263,6 +263,7 @@ DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS = 3
 DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT = 60 * 5  # 5 minutes
 DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING = 5  # seconds
 DEFAULT_HTTP_TIMEOUT = 30
+SERVICE_CONNECTOR_VERIFY_REQUEST_TIMEOUT = 120  # seconds
 ZENML_API_KEY_PREFIX = "ZENKEY_"
 DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48  # 48 hours
 DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
@@ -326,7 +327,7 @@ REQUIRES_CUSTOM_RESOURCE_REPORTING = ["pipeline", "pipeline_run"]
 
 # API Endpoint paths:
 ACTIVATE = "/activate"
-ACTIONS = "/action-flavors"
+ACTIONS = "/actions"
 API = "/api"
 API_KEYS = "/api_keys"
 API_KEY_ROTATE = "/rotate"
@@ -337,6 +338,7 @@ ARTIFACT_VISUALIZATIONS = "/artifact_visualizations"
 CODE_REFERENCES = "/code_references"
 CODE_REPOSITORIES = "/code_repositories"
 COMPONENT_TYPES = "/component-types"
+CONFIG = "/config"
 CURRENT_USER = "/current-user"
 DEACTIVATE = "/deactivate"
 DEVICES = "/devices"
@@ -346,6 +348,7 @@ EMAIL_ANALYTICS = "/email-opt-in"
 EVENT_FLAVORS = "/event-flavors"
 EVENT_SOURCES = "/event-sources"
 FLAVORS = "/flavors"
+FULL_STACK = "/full-stack"
 GET_OR_CREATE = "/get-or-create"
 GRAPH = "/graph"
 HEALTH = "/health"
@@ -372,13 +375,14 @@ SERVICE_CONNECTOR_CLIENT = "/client"
 SERVICE_CONNECTOR_RESOURCES = "/resources"
 SERVICE_CONNECTOR_TYPES = "/service_connector_types"
 SERVICE_CONNECTOR_VERIFY = "/verify"
-SERVICE_CONNECTOR_RESOURCES = "/resources"
 MODELS = "/models"
 MODEL_VERSIONS = "/model_versions"
 MODEL_VERSION_ARTIFACTS = "/model_version_artifacts"
 MODEL_VERSION_PIPELINE_RUNS = "/model_version_pipeline_runs"
 SERVICES = "/services"
 SERVICE_CONNECTORS = "/service_connectors"
+STACK = "/stack"
+STACK_DEPLOYMENT = "/stack-deployment"
 STACKS = "/stacks"
 STACK_COMPONENTS = "/components"
 STATISTICS = "/statistics"
@@ -389,6 +393,7 @@ TAGS = "/tags"
 TRIGGERS = "/triggers"
 TRIGGER_EXECUTIONS = "/trigger_executions"
 USERS = "/users"
+URL = "/url"
 VERSION_1 = "/v1"
 VISUALIZE = "/visualize"
 WEBHOOKS = "/webhooks"
@@ -458,6 +463,7 @@ MLSTACKS_SUPPORTED_STACK_COMPONENTS = [
     "step_operator",
 ]
 
+
 # Parameters for internal ZenML Models
 TEXT_FIELD_MAX_LENGTH = 65535
 STR_FIELD_MAX_LENGTH = 255
@@ -480,3 +486,6 @@ FINISHED_ONBOARDING_SURVEY_KEY = "awareness_channels"
 
 # Name validation
 BANNED_NAME_CHARACTERS = "\t\n\r\v\f"
+
+
+STACK_DEPLOYMENT_API_TOKEN_EXPIRATION = 60 * 6  # 6 hours

zenml/container_registries/base_container_registry.py

@@ -41,6 +41,7 @@ class BaseContainerRegistryConfig(AuthenticationConfigMixin):
     """
 
     uri: str
+    default_repository: Optional[str] = None
 
     @field_validator("uri")
     @classmethod

zenml/enums.py

@@ -384,3 +384,10 @@ class PluginSubType(StrEnum):
     WEBHOOK = "webhook"
     # Action Subtypes
     PIPELINE_RUN = "pipeline_run"
+
+
+class StackDeploymentProvider(StrEnum):
+    """All possible stack deployment providers."""
+
+    AWS = "aws"
+    GCP = "gcp"

zenml/event_hub/base_event_hub.py

@@ -121,7 +121,7 @@ class BaseEventHub(ABC):
             trigger=trigger.id, event_metadata=event.model_dump()
         )
 
-        action_config = trigger.get_metadata().action
+        action_config = trigger.action.configuration
 
         trigger_execution = self.zen_store.create_trigger_execution(request)
 
@@ -130,16 +130,16 @@ class BaseEventHub(ABC):
         # is associated with the service account configured for the trigger
         # and has a validity defined by the trigger's authentication window.
         token = JWTToken(
-            user_id=trigger.service_account.id,
+            user_id=trigger.action.service_account.id,
         )
         expires: Optional[datetime] = None
-        if trigger.auth_window:
+        if trigger.action.auth_window:
             expires = datetime.utcnow() + timedelta(
-                minutes=trigger.auth_window
+                minutes=trigger.action.auth_window
             )
         encoded_token = token.encode(expires=expires)
         auth_context = AuthContext(
-            user=trigger.service_account,
+            user=trigger.action.service_account,
             access_token=token,
             encoded_access_token=encoded_token,
         )

zenml/event_hub/event_hub.py

@@ -13,8 +13,9 @@
 #  permissions and limitations under the License.
 """Base class for all the Event Hub."""
 
-from functools import partial
-from typing import TYPE_CHECKING, List
+from typing import List
+
+from pydantic import ValidationError
 
 from zenml import EventSourceResponse
 from zenml.enums import PluginType
@@ -35,9 +36,6 @@ from zenml.zen_server.utils import plugin_flavor_registry
 
 logger = get_logger(__name__)
 
-if TYPE_CHECKING:
-    pass
-
 
 class InternalEventHub(BaseEventHub):
     """Internal in-server event hub implementation.
@@ -126,13 +124,11 @@ class InternalEventHub(BaseEventHub):
         """
         # get all event sources configured for this flavor
         triggers: List[TriggerResponse] = depaginate(
-            partial(
-                self.zen_store.list_triggers,
-                trigger_filter_model=TriggerFilter(
-                    event_source_id=event_source.id, is_active=True
-                ),
-                hydrate=True,
-            )
+            self.zen_store.list_triggers,
+            trigger_filter_model=TriggerFilter(
+                event_source_id=event_source.id, is_active=True
+            ),
+            hydrate=True,
         )
 
         trigger_list: List[TriggerResponse] = []
@@ -158,9 +154,19 @@ class InternalEventHub(BaseEventHub):
 
             assert issubclass(plugin_flavor, BaseEventSourceFlavor)
 
-            # Get the filter class from the plugin flavor class
             event_filter_config_class = plugin_flavor.EVENT_FILTER_CONFIG_CLASS
-            event_filter = event_filter_config_class(**trigger.event_filter)
+            try:
+                event_filter = event_filter_config_class(
+                    **trigger.event_filter if trigger.event_filter else {}
+                )
+            except ValidationError:
+                logger.exception(
+                    f"Could not instantiate event filter config class for "
+                    f"event source {event_source.id}. Skipping trigger "
+                    f"{trigger.id}."
+                )
+                continue
+
             if event_filter.event_matches_filter(event=event):
                 trigger_list.append(trigger)
 

zenml/event_sources/base_event.py

@@ -13,19 +13,8 @@
 #  permissions and limitations under the License.
 """Base implementation for events."""
 
-from typing import (
-    TYPE_CHECKING,
-)
-
 from pydantic import BaseModel
 
-from zenml.logger import get_logger
-
-if TYPE_CHECKING:
-    pass
-
-logger = get_logger(__name__)
-
 # -------------------- Event Models -----------------------------------
 
 

zenml/event_sources/base_event_source.py

@@ -449,6 +449,13 @@ class BaseEventSourceHandler(BasePlugin, ABC):
             event: The event to dispatch.
             event_source: The event source that produced the event.
         """
+        if not event_source.is_active:
+            logger.debug(
+                f"Event source {event_source.id} is not active. Skipping event "
+                f"dispatch."
+            )
+            return
+
         self.event_hub.publish_event(
             event=event,
             event_source=event_source,

zenml/event_sources/webhooks/base_webhook_event_source.py

@@ -17,7 +17,7 @@ import hashlib
 import hmac
 import json
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type
+from typing import Any, ClassVar, Dict, Optional, Type
 
 from zenml.enums import PluginSubType
 from zenml.event_sources.base_event import BaseEvent
@@ -33,9 +33,6 @@ from zenml.models import EventSourceResponse
 
 logger = get_logger(__name__)
 
-if TYPE_CHECKING:
-    pass
-
 
 # -------------------- Event Models -----------------------------------
 

zenml/exceptions.py

@@ -197,6 +197,10 @@ class EntityExistsError(ZenMLBaseException):
     """Raised when trying to register an entity that already exists."""
 
 
+class ActionExistsError(EntityExistsError):
+    """Raised when registering an action with a name that already exists."""
+
+
 class TriggerExistsError(EntityExistsError):
     """Raised when registering a trigger with name that already exists."""
 

zenml/hooks/hook_validators.py

@@ -14,14 +14,13 @@
 """Validation functions for hooks."""
 
 import inspect
-from types import FunctionType
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING
 
 from zenml.config.source import Source
 from zenml.utils import source_utils
 
 if TYPE_CHECKING:
-    HookSpecification = Union[str, Source, FunctionType]
+    from zenml.types import HookSpecification
 
 
 def resolve_and_validate_hook(hook: "HookSpecification") -> Source:

zenml/integrations/aws/__init__.py

@@ -43,6 +43,7 @@ class AWSIntegration(Integration):
         "kubernetes",
         "aws-profile-manager",
     ]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
 
     @staticmethod
     def activate() -> None:

zenml/integrations/azure/__init__.py

@@ -48,6 +48,7 @@ class AzureIntegration(Integration):
         "azure-storage-blob==12.17.0",  # temporary fix for https://github.com/Azure/azure-sdk-for-python/issues/32056
         "kubernetes",
     ]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
 
     @staticmethod
     def activate() -> None:

zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py

@@ -152,9 +152,9 @@ class BitbucketEvent(BaseEvent):
 class BitbucketWebhookEventFilterConfiguration(WebhookEventFilterConfig):
     """Configuration for Bitbucket event filters."""
 
-    repo: Optional[str]
-    branch: Optional[str]
-    event_type: Optional[BitbucketEventType]
+    repo: Optional[str] = None
+    branch: Optional[str] = None
+    event_type: Optional[BitbucketEventType] = None
 
     def event_matches_filter(self, event: BaseEvent) -> bool:
         """Checks the filter against the inbound event.

zenml/integrations/deepchecks/__init__.py

@@ -44,6 +44,7 @@ class DeepchecksIntegration(Integration):
         "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
     ]
     APT_PACKAGES = ["ffmpeg", "libsm6", "libxext6"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas","torchvision","tenacity"]
 
     @staticmethod
     def activate() -> None:

zenml/integrations/discord/__init__.py

@@ -31,6 +31,7 @@ class DiscordIntegration(Integration):
 
     NAME = DISCORD
     REQUIREMENTS = ["discord.py>=2.3.2", "aiohttp>=3.8.1", "asyncio"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["aiohttp","asyncio"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

zenml/integrations/evidently/__init__.py

@@ -57,6 +57,7 @@ class EvidentlyIntegration(Integration):
         "evidently>=0.4.16,<=0.4.22",
         "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
     ]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

zenml/integrations/facets/__init__.py

@@ -22,6 +22,7 @@ class FacetsIntegration(Integration):
 
     NAME = FACETS
     REQUIREMENTS = ["facets-overview>=1.0.0", "pandas"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
 
     @staticmethod
     def activate() -> None:

zenml/integrations/feast/__init__.py

@@ -34,6 +34,7 @@ class FeastIntegration(Integration):
     # click is added to keep the feast click version
     # in sync with ZenML's click
     REQUIREMENTS = ["feast<=0.37.1", "click>=8.0.1,<8.1.4"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

zenml/integrations/gcp/__init__.py

@@ -43,15 +43,17 @@ class GcpIntegration(Integration):
 
     NAME = GCP
     REQUIREMENTS = [
-        "kfp>=1.8.22",
+        "kfp>=2.6.0",
         "gcsfs",
         "google-cloud-secret-manager",
         "google-cloud-container>=2.21.0",
+        "google-cloud-artifact-registry>=1.11.3",
         "google-cloud-storage>=2.9.0",
         "google-cloud-aiplatform>=1.34.0",  # includes shapely pin fix
         "google-cloud-build>=3.11.0",
         "kubernetes",
     ]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes","kfp"]
 
     @staticmethod
     def activate() -> None:

zenml/integrations/gcp/google_credentials_mixin.py

@@ -85,7 +85,7 @@ class GoogleCredentialsMixin(StackComponent):
                     "trying to use the linked connector, but got "
                     f"{type(credentials)}."
                 )
-            return credentials, connector.config.project_id
+            return credentials, connector.config.gcp_project_id
 
         if self.config.service_account_path:
             credentials, project_id = load_credentials_from_file(