domino-py-iisas 1.0.6__tar.gz → 1.0.8__tar.gz

{domino_py_iisas-1.0.6/src/domino_py_iisas.egg-info → domino_py_iisas-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domino-py-iisas
-Version: 1.0.6
+Version: 1.0.8
 Summary: Fork of the original Python package for Domino.
 Author-email: Stefan Dlugolinsky <stefan.dlugolinsky@savba.sk>, Luiz Tauffer <luiz@taufferconsulting.com>, Vinicius Vaz <vinicius@taufferconsulting.com>
 License: 

domino_py_iisas-1.0.8/src/domino/VERSION

@@ -0,0 +1 @@
+1.0.8

{domino_py_iisas-1.0.6 → domino_py_iisas-1.0.8}/src/domino/cli/cli.py

@@ -48,6 +48,14 @@ def get_cluster_name_from_env():
     return cluster_name
 
 
+def get_cluster_http_port_from_env():
+    return os.environ.get("DOMINO_KIND_CLUSTER_HTTP_PORT", 80)
+
+
+def get_cluster_https_port_from_env():
+    return int(os.environ.get("DOMINO_KIND_CLUSTER_HTTPS_PORT", 443))
+
+
 def get_github_workflows_ssh_private_key_from_env():
     return os.environ.get("DOMINO_GITHUB_WORKFLOWS_SSH_PRIVATE_KEY", "")
 
@@ -83,6 +91,18 @@ def get_github_token_pieces_from_config_or_env():
     default=get_cluster_name_from_env,
     help='Define the name for the local k8s cluster.'
 )
+@click.option(
+    '--http-port',
+    prompt='Local cluster HTTP port',
+    default=get_cluster_http_port_from_env,
+    help='Define the HTTP port for the local k8s cluster.'
+)
+@click.option(
+    '--https-port',
+    prompt='Local cluster HTTPS port',
+    default=get_cluster_https_port_from_env,
+    help='Define the HTTPS port for the local k8s cluster.'
+)
 @click.option(
     '--workflows-repository',
     prompt='Workflows repository',
@@ -145,6 +165,8 @@ def get_github_token_pieces_from_config_or_env():
 )
 def cli_prepare_platform(
     cluster_name,
+    http_port,
+    https_port,
     workflows_repository,
     github_workflows_ssh_private_key,
     github_default_pieces_repository_token,
@@ -160,6 +182,8 @@ def cli_prepare_platform(
     """Prepare local folder for running a Domino platform."""
     platform.prepare_platform(
         cluster_name=cluster_name,
+        http_port=int(http_port),
+        https_port=int(https_port),
         workflows_repository=workflows_repository,
         github_workflows_ssh_private_key=github_workflows_ssh_private_key,
         github_default_pieces_repository_token=github_default_pieces_repository_token,

{domino_py_iisas-1.0.6 → domino_py_iisas-1.0.8}/src/domino/cli/utils/config-domino-local.toml

@@ -1,15 +1,20 @@
 # Configuration file for running Domino platform locally
-
-[airflow]
-AIRFLOW_API_SERVER_PORT_HOST = 8080
+[airflow_db]
+AIRFLOW_DB_IMAGE = "ghcr.io/cloudnative-pg/postgresql"
+AIRFLOW_DB_IMAGE_TAG = "13"
+AIRFLOW_DB_HOST = "airflow-postgresql"
+AIRFLOW_DB_PORT = 5432
+AIRFLOW_DB_USER = "airflow"
+AIRFLOW_DB_PASSWORD = "airflow"
+AIRFLOW_DB_NAME = "postgres"
 
 [path]
 DOMINO_LOCAL_RUNNING_PATH = ""
 
 [kind]
 DOMINO_KIND_CLUSTER_NAME = "domino-cluster"
-HTTP_PORT_HOST = 80
-HTTPS_PORT_HOST = 443
+DOMINO_KIND_CLUSTER_HTTP_PORT = 80
+DOMINO_KIND_CLUSTER_HTTPS_PORT = 443
 
 [github]
 DOMINO_GITHUB_WORKFLOWS_REPOSITORY = ""

{domino_py_iisas-1.0.6 → domino_py_iisas-1.0.8}/src/domino/cli/utils/platform.py

@@ -9,6 +9,7 @@ import requests
 import time
 from concurrent.futures import ThreadPoolExecutor
 import base64
+import sys
 from pathlib import Path
 from rich.console import Console
 from yaml.resolver import BaseResolver
@@ -58,6 +59,8 @@ def create_ssh_pair_key() -> None:
 
 def prepare_platform(
     cluster_name: str,
+    http_port: int,
+    https_port: int,
     workflows_repository: str,
     github_workflows_ssh_private_key: str,
     github_default_pieces_repository_token: str,
@@ -78,6 +81,10 @@ def prepare_platform(
     config_dict["path"]["DOMINO_LOCAL_RUNNING_PATH"] = running_path
     config_dict["kind"]["DOMINO_KIND_CLUSTER_NAME"] = cluster_name
     config_dict['kind']['DOMINO_DEPLOY_MODE'] = deploy_mode
+    config_dict["kind"]["DOMINO_KIND_CLUSTER_HTTP_PORT"] = http_port
+    config_dict["kind"]["DOMINO_KIND_CLUSTER_HTTPS_PORT"] = https_port
+
+    config_dict["domino_frontend"]["API_URL"] = "http://localhost{}/api".format(f":{http_port}")
 
     if deploy_mode == 'local-k8s-dev':
         config_dict['dev']['DOMINO_AIRFLOW_IMAGE'] = local_airflow_image
@@ -171,19 +178,13 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
                 extraPortMappings=[
                     dict(
                         containerPort=80,
-                        hostPort=platform_config['kind'].get('HTTP_PORT_HOST', 80),
+                        hostPort=platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTP_PORT', 80),
                         listenAddress="0.0.0.0",
                         protocol="TCP"
                     ),
                     dict(
                         containerPort=443,
-                        hostPort=platform_config['kind'].get('HTTPS_PORT_HOST', 443),
-                        listenAddress="0.0.0.0",
-                        protocol="TCP"
-                    ),
-                    dict(
-                        containerPort=8080,
-                        hostPort=platform_config['airflow'].get('AIRFLOW_API_SERVER_PORT_HOST', 8080),
+                        hostPort=platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTPS_PORT', 443),
                         listenAddress="0.0.0.0",
                         protocol="TCP"
                     )
@@ -263,7 +264,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
 
     if local_domino_rest_image:
         console.print(f"Loading local REST image {local_domino_rest_image} to Kind cluster...")
-        subprocess.run(["kind", "load", "docker-image", local_domino_rest_image , "--name", cluster_name, "--nodes", f"{cluster_name}-worker"])
+        subprocess.run(["kind", "load", "docker-image", local_domino_rest_image, "--name", cluster_name, "--nodes", f"{cluster_name}-worker"])
         domino_rest_image = f'docker.io/library/{local_domino_rest_image}'
     elif platform_config['kind']["DOMINO_DEPLOY_MODE"] == 'local-k8s-dev':
         domino_rest_image = "ghcr.io/iisas/domino-rest:latest-dev"
@@ -301,7 +302,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
             "image": domino_frontend_image,
             "apiEnv": "dev" if platform_config['kind']["DOMINO_DEPLOY_MODE"] in ['local-k8s-dev', 'local-k8s'] else 'prod',
             "deployMode": platform_config['kind']["DOMINO_DEPLOY_MODE"],
-            "apiUrl": platform_config['domino_frontend'].get('API_URL', 'http://localhost:{}/api'.format(platform_config['kind'].get('HTTP_PORT_HOST', 80))),
+            "apiUrl": platform_config['domino_frontend'].get('API_URL', 'http://localhost:{}/api'.format(platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTP_PORT', 80))),
             "baseName": platform_config['domino_frontend'].get('BASE_NAME', '/')
         },
         "rest": {
@@ -309,14 +310,6 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
             "image": domino_rest_image,
             "workflowsRepository": platform_config['github']['DOMINO_GITHUB_WORKFLOWS_REPOSITORY'],
             "createDefaultUser": platform_config['domino_db'].get('DOMINO_CREATE_DEFAULT_USER', True)
-        },
-        "database": {
-            "enabled": db_enabled,
-            "image": "postgres:13",
-            "name": "postgres",
-            "user": "postgres",
-            "password": "postgres",
-            "port": "5432",
         }
     }
 
@@ -326,7 +319,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
         domino_values_override_config['database'] = {
             **domino_values_override_config['database'],
             "host": platform_config['domino_db']["DOMINO_DB_HOST"],
-            "name":  platform_config['domino_db']["DOMINO_DB_NAME"],
+            "name": platform_config['domino_db']["DOMINO_DB_NAME"],
             "user": platform_config['domino_db']["DOMINO_DB_USER"],
             "password": platform_config['domino_db']["DOMINO_DB_PASSWORD"],
             "port": str(platform_config['domino_db'].get("DOMINO_DB_PORT", 5432))
@@ -430,6 +423,12 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
                 "sidecar.istio.io/inject": "false"
             },
         },
+        "postgresql": {
+            "enabled": False,
+        },
+        "data": {
+            "metadataSecretName": 'airflow-metadata-secret'
+        },
         **workers,
         **scheduler,
     }
@@ -442,6 +441,116 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
     # Install Airflow Helm Chart
     if install_airflow:
         console.print('Installing Apache Airflow...')
+
+        # External database for the Airflow metastore (due to [changes](https://github.com/bitnami/containers/issues/83267) related to bitnami images since [v1.12.0](https://airflow.apache.org/docs/helm-chart/1.12.0/release_notes.html#the-helm-chart-is-now-using-a-newer-version-of-bitnami-postgresql-dependency-34817))
+        if not airflow_values_override_config['postgresql']['enabled']:
+            console.print('Installing external database for the Apache Airflow metastore...')
+
+            # create secret for the Airflow metastore
+            airflow_db_host = platform_config['airflow_db'].get('AIRFLOW_DB_HOST', 'airflow-postgres')
+            airflow_db_port = platform_config['airflow_db'].get('AIRFLOW_DB_PORT', 5432)
+            airflow_db_user = platform_config['airflow_db'].get('AIRFLOW_DB_USER', 'airflow')
+            airflow_db_password = platform_config['airflow_db'].get('AIRFLOW_DB_PASSWORD', 'airflow')
+            airflow_db_name = platform_config['airflow_db'].get('AIRFLOW_DB_NAME', 'postgres')
+            result = subprocess.run(["kubectl", "create", "secret", "generic",
+                                     f"{airflow_values_override_config['data']['metadataSecretName']}",
+                                     f"--from-literal=connection=postgresql://{airflow_db_user}:{airflow_db_password}@{airflow_db_host}:{airflow_db_port}/{airflow_db_name}"])
+            if result.returncode != 0:
+                error_message = result.stderr.strip() if result.stderr else result.stdout.strip()
+
+            # deploy Airflow metastore database
+            airflow_db_image = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE',
+                                                                 'ghcr.io/cloudnative-pg/postgresql')
+            airflow_db_image_tag = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE_TAG', '13')
+            cnpg_operator_add_repo_command = [
+                "helm", "repo", "add", "cnpg", "https://cloudnative-pg.github.io/charts"
+            ]
+            console.print("Adding CloudNativePG repository...")
+            subprocess.run(cnpg_operator_add_repo_command)
+            helm_update_command = ["helm", "repo", "update", "cnpg"]
+            console.print("Updating helm repositories...")
+            subprocess.run(helm_update_command)
+            cnpg_operator_install_command = [
+                "helm", "install", "cnpg",
+                "--namespace", "cnpg-system",
+                "--create-namespace",
+                "--set", "config.clusterWide=false",
+                "cnpg/cloudnative-pg",
+                "--hide-notes"
+            ]
+            console.print("Installing CloudNativePG operator...")
+            subprocess.run(cnpg_operator_install_command)
+
+            airflow_db_manifest = [
+                {
+                    "apiVersion": "postgresql.cnpg.io/v1",
+                    "kind": "Cluster",
+                    "metadata": {
+                        "name": "airflow-postgres",
+                        "namespace": "default"
+                    },
+                    "spec": {
+                        "instances": 1,
+                        "imageName": f"{airflow_db_image}:{airflow_db_image_tag}",
+                        "storage": {
+                            "size": "1Gi"
+                        },
+                        "bootstrap": {
+                            "initdb": {
+                                "database": f"{airflow_db_name}",
+                                "owner": f"{airflow_db_user}",
+                                "secret": {
+                                    "name": f"{airflow_values_override_config['data']['metadataSecretName']}"
+                                }
+                            }
+                        }
+                    }
+                },
+                {
+                    "apiVersion": "v1",
+                    "kind": "Service",
+                    "metadata": {
+                        "name": "airflow-postgres-service",
+                        "namespace": "default"
+                    },
+                    "spec": {
+                        "type": "ClusterIP",
+                        "selector": {
+                            "app": "airflow-postgres"
+                        },
+                        "ports": [
+                            {
+                                "protocol": "TCP",
+                                "port": 5432,
+                                "targetPort": 5432
+                            }
+                        ]
+                    }
+                }
+            ]
+
+            with NamedTemporaryFile(suffix='.yaml', mode="w", delete_on_close=False) as fp:
+                # Dump as separate YAML documents
+                yaml_output = "\n---\n".join(
+                    yaml.safe_dump(doc, sort_keys=False) for doc in airflow_db_manifest
+                )
+                fp.write(yaml_output)
+                fp.close()
+                commands = [
+                    "kubectl", "apply", "-f", str(fp.name), "--validate=false"
+                ]
+                subprocess.run(commands, stdout=subprocess.DEVNULL)
+                commands = [
+                    "kubectl", "wait", "--namespace", "default", "--for", "condition=Ready", "cluster/airflow-postgres", "--timeout=60s"
+                ]
+                result = subprocess.run(commands)
+                if result.returncode != 0:
+                    error_message = result.stderr.strip() if result.stderr else result.stdout.strip() if result.stdout else 'no details given'
+                    raise Exception(f"An error occurred while installing database for Apache Airflow metastore: {error_message}")
+                
+            console.print("database for the Apache Airflow metastore installed successfully!", style=f"bold {COLOR_PALETTE.get('success')}")
+            console.print("")
+
         # Create temporary file with airflow values
         with NamedTemporaryFile(suffix='.yaml', mode="w") as fp:
             yaml.dump(airflow_values_override_config, fp)
@@ -464,7 +573,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
             commands = [
                 "helm", "install",
                 "-f", str(fp.name),
-                "domino-iisas",
+                "domino",
                 helm_domino_path
             ]
             subprocess.run(commands)
@@ -486,7 +595,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
                 commands = [
                     "helm", "install",
                     "-f", str(fp.name),
-                    "domino-iisas",
+                    "domino",
                     f"{tmp_dir}/domino",
                 ]
                 subprocess.run(commands)
@@ -653,9 +762,9 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
 
     console.print("")
     console.print("K8s resources created successfully!", style=f"bold {COLOR_PALETTE.get('success')}")
-    console.print("You can now access the Domino frontend at: http://localhost:{}/".format(platform_config['kind'].get('HTTP_PORT_HOST')))
-    console.print("Domino's REST API: http://localhost:{}/api/".format(platform_config['kind'].get('HTTP_PORT_HOST')))
-    console.print("Domino's REST API Swagger: http://localhost:{}/api/docs".format(platform_config['kind'].get('HTTP_PORT_HOST')))
+    console.print("You can now access the Domino frontend at: http://localhost:{}/".format(platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTP_PORT')))
+    console.print("Domino's REST API: http://localhost:{}/api/".format(platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTP_PORT')))
+    console.print("Domino's REST API Swagger: http://localhost:{}/api/docs".format(platform_config['kind'].get('DOMINO_KIND_CLUSTER_HTTP_PORT')))
     console.print("")
 
 

{domino_py_iisas-1.0.6 → domino_py_iisas-1.0.8/src/domino_py_iisas.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domino-py-iisas
-Version: 1.0.6
+Version: 1.0.8
 Summary: Fork of the original Python package for Domino.
 Author-email: Stefan Dlugolinsky <stefan.dlugolinsky@savba.sk>, Luiz Tauffer <luiz@taufferconsulting.com>, Vinicius Vaz <vinicius@taufferconsulting.com>
 License: 

domino_py_iisas-1.0.6/src/domino/VERSION

@@ -1 +0,0 @@
-1.0.6