zenml-nightly 0.66.0.dev20240928__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl

zenml/zen_server/deploy/terraform/providers/gcp_provider.py

@@ -1,59 +0,0 @@
-#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Zen Server GCP Terraform deployer implementation."""
-
-from typing import ClassVar, Type
-
-from zenml.enums import ServerProviderType
-from zenml.logger import get_logger
-from zenml.zen_server.deploy.terraform.providers.terraform_provider import (
-    TerraformServerProvider,
-)
-from zenml.zen_server.deploy.terraform.terraform_zen_server import (
-    TerraformServerDeploymentConfig,
-)
-
-logger = get_logger(__name__)
-
-
-class GCPServerDeploymentConfig(TerraformServerDeploymentConfig):
-    """GCP server deployment configuration.
-
-    Attributes:
-        project_id: The project in GCP to deploy the server to.
-        region: The GCP region to deploy to.
-        cloudsql_name: The name of the CloudSQL instance to create
-        db_name: Name of CloudSQL database to create.
-        db_instance_tier: Instance class of CloudSQL database to create.
-        db_disk_size: Allocated storage of CloudSQL database to create.
-    """
-
-    project_id: str
-    region: str = "europe-west3"
-    cloudsql_name: str = "zenmlserver"
-    db_name: str = "zenmlserver"
-    db_instance_tier: str = "db-n1-standard-1"
-    db_disk_size: int = 10
-
-
-class GCPServerProvider(TerraformServerProvider):
-    """GCP ZenML server provider."""
-
-    TYPE: ClassVar[ServerProviderType] = ServerProviderType.GCP
-    CONFIG_TYPE: ClassVar[Type[TerraformServerDeploymentConfig]] = (
-        GCPServerDeploymentConfig
-    )
-
-
-GCPServerProvider.register_as_provider()

zenml/zen_server/deploy/terraform/providers/terraform_provider.py

@@ -1,331 +0,0 @@
-#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at:
-#
-#       https://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-#  or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-"""Zen Server terraform deployer implementation."""
-
-import os
-from typing import ClassVar, List, Optional, Tuple, Type, cast
-from uuid import uuid4
-
-from zenml.config.global_config import GlobalConfiguration
-from zenml.logger import get_logger
-from zenml.services import BaseService, ServiceConfig
-from zenml.services.service_endpoint import (
-    ServiceEndpointConfig,
-    ServiceEndpointProtocol,
-)
-from zenml.services.service_monitor import (
-    HTTPEndpointHealthMonitorConfig,
-    ServiceEndpointHealthMonitorConfig,
-)
-from zenml.zen_server.deploy.base_provider import BaseServerProvider
-from zenml.zen_server.deploy.deployment import (
-    ServerDeploymentConfig,
-    ServerDeploymentStatus,
-)
-from zenml.zen_server.deploy.docker.docker_zen_server import (
-    ZEN_SERVER_HEALTHCHECK_URL_PATH,
-)
-from zenml.zen_server.deploy.terraform.terraform_zen_server import (
-    TERRAFORM_VALUES_FILE_PATH,
-    TERRAFORM_ZENML_SERVER_CONFIG_PATH,
-    TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT,
-    TERRAFORM_ZENML_SERVER_RECIPE_SUBPATH,
-    TerraformServerDeploymentConfig,
-    TerraformZenServer,
-    TerraformZenServerConfig,
-)
-
-logger = get_logger(__name__)
-
-
-class TerraformServerProvider(BaseServerProvider):
-    """Terraform ZenML server provider."""
-
-    CONFIG_TYPE: ClassVar[Type[ServerDeploymentConfig]] = (
-        TerraformServerDeploymentConfig
-    )
-
-    @staticmethod
-    def _get_server_recipe_root_path() -> str:
-        """Get the server recipe root path.
-
-        The Terraform recipe files for all terraform server providers are
-        located in a folder relative to the `zenml.zen_server.deploy.terraform`
-        Python module.
-
-        Returns:
-            The server recipe root path.
-        """
-        import zenml.zen_server.deploy.terraform as terraform_module
-
-        root_path = os.path.join(
-            os.path.dirname(terraform_module.__file__),
-            TERRAFORM_ZENML_SERVER_RECIPE_SUBPATH,
-        )
-        return root_path
-
-    @classmethod
-    def _get_service_configuration(
-        cls,
-        server_config: ServerDeploymentConfig,
-    ) -> Tuple[
-        ServiceConfig,
-        ServiceEndpointConfig,
-        ServiceEndpointHealthMonitorConfig,
-    ]:
-        """Construct the service configuration from a server deployment configuration.
-
-        Args:
-            server_config: server deployment configuration.
-
-        Returns:
-            The service configuration.
-        """
-        assert isinstance(server_config, TerraformServerDeploymentConfig)
-
-        return (
-            TerraformZenServerConfig(
-                name=server_config.name,
-                root_runtime_path=TERRAFORM_ZENML_SERVER_CONFIG_PATH,
-                singleton=True,
-                directory_path=os.path.join(
-                    cls._get_server_recipe_root_path(),
-                    server_config.provider,
-                ),
-                log_level=server_config.log_level,
-                variables_file_path=TERRAFORM_VALUES_FILE_PATH,
-                server=server_config,
-            ),
-            ServiceEndpointConfig(
-                protocol=ServiceEndpointProtocol.HTTP,
-                allocate_port=False,
-            ),
-            HTTPEndpointHealthMonitorConfig(
-                healthcheck_uri_path=ZEN_SERVER_HEALTHCHECK_URL_PATH,
-                use_head_request=True,
-            ),
-        )
-
-    def _create_service(
-        self,
-        config: ServerDeploymentConfig,
-        timeout: Optional[int] = None,
-    ) -> BaseService:
-        """Create, start and return the terraform ZenML server deployment service.
-
-        Args:
-            config: The server deployment configuration.
-            timeout: The timeout in seconds to wait until the service is
-                running.
-
-        Returns:
-            The service instance.
-
-        Raises:
-            RuntimeError: If a terraform service is already running.
-        """
-        assert isinstance(config, TerraformServerDeploymentConfig)
-
-        if timeout is None:
-            timeout = TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT
-
-        existing_service = TerraformZenServer.get_service()
-        if existing_service:
-            raise RuntimeError(
-                f"A terraform ZenML server with name '{existing_service.config.name}' "
-                f"is already running. Please stop it first before starting a "
-                f"new one."
-            )
-
-        (
-            service_config,
-            endpoint_cfg,
-            monitor_cfg,
-        ) = self._get_service_configuration(config)
-
-        service = TerraformZenServer(uuid=uuid4(), config=service_config)
-
-        service.start(timeout=timeout)
-        return service
-
-    def _update_service(
-        self,
-        service: BaseService,
-        config: ServerDeploymentConfig,
-        timeout: Optional[int] = None,
-    ) -> BaseService:
-        """Update the terraform ZenML server deployment service.
-
-        Args:
-            service: The service instance.
-            config: The new server deployment configuration.
-            timeout: The timeout in seconds to wait until the updated service is
-                running.
-
-        Returns:
-            The updated service instance.
-        """
-        if timeout is None:
-            timeout = TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT
-
-        (
-            new_config,
-            endpoint_cfg,
-            monitor_cfg,
-        ) = self._get_service_configuration(config)
-
-        assert isinstance(new_config, TerraformZenServerConfig)
-        assert isinstance(service, TerraformZenServer)
-
-        # preserve the server ID across updates
-        service.config = new_config
-        service.start(timeout=timeout)
-
-        return service
-
-    def _start_service(
-        self,
-        service: BaseService,
-        timeout: Optional[int] = None,
-    ) -> BaseService:
-        """Start the terraform ZenML server deployment service.
-
-        Args:
-            service: The service instance.
-            timeout: The timeout in seconds to wait until the service is
-                running.
-
-        Returns:
-            The updated service instance.
-        """
-        if timeout is None:
-            timeout = TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT
-
-        service.start(timeout=timeout)
-        return service
-
-    def _stop_service(
-        self,
-        service: BaseService,
-        timeout: Optional[int] = None,
-    ) -> BaseService:
-        """Stop the terraform ZenML server deployment service.
-
-        Args:
-            service: The service instance.
-            timeout: The timeout in seconds to wait until the service is
-                stopped.
-
-        Returns:
-            The updated service instance.
-        """
-        if timeout is None:
-            timeout = TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT
-
-        service.stop(timeout=timeout)
-        return service
-
-    def _delete_service(
-        self,
-        service: BaseService,
-        timeout: Optional[int] = None,
-    ) -> None:
-        """Remove the terraform ZenML server deployment service.
-
-        Args:
-            service: The service instance.
-            timeout: The timeout in seconds to wait until the service is
-                removed.
-        """
-        assert isinstance(service, TerraformZenServer)
-
-        if timeout is None:
-            timeout = TERRAFORM_ZENML_SERVER_DEFAULT_TIMEOUT
-
-        service.stop(timeout)
-
-    def _get_service(self, server_name: str) -> BaseService:
-        """Get the terraform ZenML server deployment service.
-
-        Args:
-            server_name: The server deployment name.
-
-        Returns:
-            The service instance.
-
-        Raises:
-            KeyError: If the server deployment is not found.
-        """
-        service = TerraformZenServer.get_service()
-        if service is None:
-            raise KeyError("The terraform ZenML server is not deployed.")
-
-        if service.config.server.name != server_name:
-            raise KeyError(
-                "The terraform ZenML server is deployed but with a different name."
-            )
-        return service
-
-    def _list_services(self) -> List[BaseService]:
-        """Get all service instances for all deployed ZenML servers.
-
-        Returns:
-            A list of service instances.
-        """
-        service = TerraformZenServer.get_service()
-        if service:
-            return [service]
-        return []
-
-    def _get_deployment_config(
-        self, service: BaseService
-    ) -> ServerDeploymentConfig:
-        """Recreate the server deployment configuration from a service instance.
-
-        Args:
-            service: The service instance.
-
-        Returns:
-            The server deployment configuration.
-        """
-        server = cast(TerraformZenServer, service)
-        return server.config.server
-
-    def _get_deployment_status(
-        self, service: BaseService
-    ) -> ServerDeploymentStatus:
-        """Get the status of a server deployment from its service.
-
-        Args:
-            service: The server deployment service.
-
-        Returns:
-            The status of the server deployment.
-        """
-        gc = GlobalConfiguration()
-        url: Optional[str] = None
-        service = cast(TerraformZenServer, service)
-        ca_crt = None
-        if service.is_running:
-            url = service.get_server_url()
-            ca_crt = service.get_certificate()
-        connected = url is not None and gc.store_configuration.url == url
-
-        return ServerDeploymentStatus(
-            url=url,
-            status=service.status.state,
-            status_message=service.status.last_error,
-            connected=connected,
-            ca_crt=ca_crt,
-        )

zenml/zen_server/deploy/terraform/recipes/aws/.gitignore

@@ -1,8 +0,0 @@
-*.exe
-*.hcl
-*.info
-*.backup
-*.tfstate
-*.pem
-kubeconfig_*
-.terraform

zenml/zen_server/deploy/terraform/recipes/aws/helm.tf

@@ -1,20 +0,0 @@
-# check if the host OS is Linux or Windows
-data "external" "os" {
-  working_dir = path.module
-  program     = ["printf", "{\"os\": \"Linux\"}"]
-}
-locals {
-  os                  = data.external.os.result.os
-  kubectl_config_path = local.os == "Windows" ? "%USERPROFILE%\\.kube\\config" : "~/.kube/config"
-}
-
-# A default (non-aliased) provider configuration for "helm"
-provider "helm" {
-  kubernetes {
-    config_path = var.kubectl_config_path == "" ? local.kubectl_config_path : var.kubectl_config_path
-  }
-}
-
-provider "kubernetes" {
-  config_path = var.kubectl_config_path == "" ? local.kubectl_config_path : var.kubectl_config_path
-}

zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf

@@ -1,30 +0,0 @@
-# set up the nginx ingress controller
-resource "kubernetes_namespace" "nginx-ns" {
-  count = var.create_ingress_controller ? 1 : 0
-  metadata {
-    name = "${var.name}-ingress"
-  }
-}
-
-resource "helm_release" "nginx-controller" {
-  name       = "zenml"
-  count      = var.create_ingress_controller ? 1 : 0
-  repository = "https://kubernetes.github.io/ingress-nginx"
-  chart      = "ingress-nginx"
-  # dependency on nginx-ns
-  namespace = var.create_ingress_controller ? kubernetes_namespace.nginx-ns[0].metadata[0].name : ""
-  depends_on = [
-    resource.kubernetes_namespace.nginx-ns
-  ]
-}
-
-data "kubernetes_service" "ingress-controller" {
-  count = var.create_ingress_controller ? 1 : 0
-  metadata {
-    name      = "zenml-ingress-nginx-controller"
-    namespace = var.create_ingress_controller ? kubernetes_namespace.nginx-ns[0].metadata[0].name : ""
-  }
-  depends_on = [
-    resource.helm_release.nginx-controller
-  ]
-}

zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf

@@ -1,7 +0,0 @@
-output "zenml_server_url" {
-  value = var.create_ingress_controller ? "https://${data.kubernetes_service.ingress-controller[0].status.0.load_balancer.0.ingress.0.hostname}" : "https://zenml.${var.ingress_controller_ip}.nip.io"
-}
-output "ca_crt" {
-  value     = base64decode(data.kubernetes_secret.certificates.binary_data["ca.crt"])
-  sensitive = true
-}

zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd

@@ -1,2 +0,0 @@
-@echo off
-echo {"os": "Windows"}

zenml/zen_server/deploy/terraform/recipes/aws/sql.tf

@@ -1,62 +0,0 @@
-# random string for the RDS instance name
-resource "random_string" "rds_suffix" {
-  count  = var.deploy_db ? 1 : 0
-  length = 4
-  upper  = false
-  special = false
-}
-
-module "metadata_store" {
-  source  = "terraform-aws-modules/rds/aws"
-  version = "5.9.0"
-  count   = var.deploy_db ? 1 : 0
-
-  identifier = "${var.name}${var.rds_name}${random_string.rds_suffix[0].result}"
-
-  engine            = var.db_type
-  engine_version    = var.db_version
-  instance_class    = var.db_instance_class
-  allocated_storage = var.db_allocated_storage
-
-  db_name                = var.db_name
-  username               = var.database_username
-  password               = var.database_password
-  create_random_password = var.database_password == "" ? true : false
-  port                   = "3306"
-
-  # configure access
-  publicly_accessible = true
-
-  # DB subnet group
-  create_db_subnet_group = true
-  subnet_ids             = module.vpc[0].public_subnets
-  skip_final_snapshot    = true
-
-  iam_database_authentication_enabled = false
-
-  # we've added MySQL ingress rules to 
-  # this sg so we're using it here
-  vpc_security_group_ids = [module.vpc[0].default_security_group_id]
-
-  # DB parameter group
-  family = "mysql5.7"
-
-  # DB option group
-  major_engine_version = "5.7"
-
-  tags = {
-    Owner       = "user"
-    Environment = "zenml-env"
-  }
-
-  parameters = [
-    {
-      name  = "character_set_client"
-      value = "utf8mb4"
-    },
-    {
-      name  = "character_set_server"
-      value = "utf8mb4"
-    }
-  ]
-}

zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf

@@ -1,44 +0,0 @@
-# defining the providers for the recipe module
-terraform {
-  required_providers {
-    aws = {
-      source = "hashicorp/aws"
-    }
-
-    random = {
-      source  = "hashicorp/random"
-      version = "3.1.0"
-    }
-
-    local = {
-      source  = "hashicorp/local"
-      version = "2.1.0"
-    }
-
-    null = {
-      source  = "hashicorp/null"
-      version = "3.1.0"
-    }
-
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = ">= 2.11.0"
-    }
-
-    kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = "1.14.0"
-    }
-
-    htpasswd = {
-      source  = "loafoe/htpasswd"
-      version = "1.0.3"
-    }
-  }
-
-  required_version = ">= 0.14.8"
-}
-
-provider "aws" {
-  region = var.region
-}