domino-py-iisas 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

domino/VERSION

@@ -1 +1 @@
-1.0.9
+1.0.11

domino/cli/utils/platform.py

@@ -9,7 +9,7 @@ import requests
 import time
 from concurrent.futures import ThreadPoolExecutor
 import base64
-import sys
+import secrets
 from pathlib import Path
 from rich.console import Console
 from yaml.resolver import BaseResolver
@@ -429,6 +429,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
         "data": {
             "metadataSecretName": 'airflow-metadata-secret'
         },
+        "webserverSecretKey": secrets.token_hex(16),
         **workers,
         **scheduler,
     }
@@ -444,112 +445,90 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
 
         # External database for the Airflow metastore (due to [changes](https://github.com/bitnami/containers/issues/83267) related to bitnami images since [v1.12.0](https://airflow.apache.org/docs/helm-chart/1.12.0/release_notes.html#the-helm-chart-is-now-using-a-newer-version-of-bitnami-postgresql-dependency-34817))
         if not airflow_values_override_config['postgresql']['enabled']:
-            console.print('Installing external database for the Apache Airflow metastore...')
+            console.print('Using external database for the Apache Airflow metastore.')
 
             # create secret for the Airflow metastore
-            airflow_db_host = platform_config['airflow_db'].get('AIRFLOW_DB_HOST', 'airflow-postgres')
+            airflow_db_host = platform_config['airflow_db'].get('AIRFLOW_DB_HOST', 'airflow-postgres-rw')
             airflow_db_port = platform_config['airflow_db'].get('AIRFLOW_DB_PORT', 5432)
             airflow_db_user = platform_config['airflow_db'].get('AIRFLOW_DB_USER', 'airflow')
             airflow_db_password = platform_config['airflow_db'].get('AIRFLOW_DB_PASSWORD', 'airflow')
             airflow_db_name = platform_config['airflow_db'].get('AIRFLOW_DB_NAME', 'postgres')
-            result = subprocess.run(["kubectl", "create", "secret", "generic",
-                                     f"{airflow_values_override_config['data']['metadataSecretName']}",
-                                     f"--from-literal=connection=postgresql://{airflow_db_user}:{airflow_db_password}@{airflow_db_host}:{airflow_db_port}/{airflow_db_name}"])
-            if result.returncode != 0:
-                error_message = result.stderr.strip() if result.stderr else result.stdout.strip()
-
-            # deploy Airflow metastore database
-            airflow_db_image = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE',
-                                                                 'ghcr.io/cloudnative-pg/postgresql')
-            airflow_db_image_tag = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE_TAG', '13')
-            cnpg_operator_add_repo_command = [
-                "helm", "repo", "add", "cnpg", "https://cloudnative-pg.github.io/charts"
-            ]
-            console.print("Adding CloudNativePG repository...")
-            subprocess.run(cnpg_operator_add_repo_command)
-            helm_update_command = ["helm", "repo", "update", "cnpg"]
-            console.print("Updating helm repositories...")
-            subprocess.run(helm_update_command)
-            cnpg_operator_install_command = [
-                "helm", "install", "cnpg",
-                "--namespace", "cnpg-system",
-                "--create-namespace",
-                "--set", "config.clusterWide=false",
-                "cnpg/cloudnative-pg",
-                "--hide-notes"
-            ]
-            console.print("Installing CloudNativePG operator...")
-            subprocess.run(cnpg_operator_install_command)
-
-            airflow_db_manifest = [
-                {
-                    "apiVersion": "postgresql.cnpg.io/v1",
-                    "kind": "Cluster",
-                    "metadata": {
-                        "name": "airflow-postgres",
-                        "namespace": "default"
-                    },
-                    "spec": {
-                        "instances": 1,
-                        "imageName": f"{airflow_db_image}:{airflow_db_image_tag}",
-                        "storage": {
-                            "size": "1Gi"
+
+            if not platform_config['airflow_db'].get('AIRFLOW_DB_HOST', False):
+                console.print('Installing external database for the Apache Airflow metastore...')
+                result = subprocess.run(
+                    [
+                        "kubectl",
+                        "create",
+                        "secret",
+                        "generic",
+                        f"{airflow_values_override_config['data']['metadataSecretName']}",
+                        f"--from-literal=connection=postgresql://{airflow_db_user}:{airflow_db_password}@{airflow_db_host}:{airflow_db_port}/{airflow_db_name}",
+                        f"--from-literal=username={airflow_db_user}",
+                        f"--from-literal=password={airflow_db_password}",
+                    ]
+                )
+                if result.returncode != 0:
+                    error_message = result.stderr.strip() if result.stderr else result.stdout.strip()
+
+                console.print("Adding CloudNativePG repository...")
+                subprocess.run(["helm", "repo", "add", "cnpg", "https://cloudnative-pg.github.io/charts"])
+
+                console.print("Updating helm repositories...")
+                subprocess.run(["helm", "repo", "update", "cnpg"])
+
+                console.print("Installing CloudNativePG operator...")
+                subprocess.run(["helm", "install", "cnpg", "--namespace", "default", "--set", "config.clusterWide=false", "cnpg/cloudnative-pg", "--hide-notes"])
+
+                console.print("Waiting fot the CloudNativePG operator to be ready...")
+                subprocess.run(["kubectl", "wait", "--namespace", "default", "--for", "condition=Available", "deployment/cnpg-cloudnative-pg", "--timeout=60s"])
+
+                # deploy Airflow metastore database
+                airflow_db_image = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE', 'ghcr.io/cloudnative-pg/postgresql')
+                airflow_db_image_tag = platform_config['airflow_db'].get('AIRFLOW_DB_IMAGE_TAG', '13')
+                airflow_db_manifest = [
+                    {
+                        "apiVersion": "postgresql.cnpg.io/v1",
+                        "kind": "Cluster",
+                        "metadata": {
+                            "name": "airflow-postgres",
+                            "namespace": "default"
                         },
-                        "bootstrap": {
-                            "initdb": {
-                                "database": f"{airflow_db_name}",
-                                "owner": f"{airflow_db_user}",
-                                "secret": {
-                                    "name": f"{airflow_values_override_config['data']['metadataSecretName']}"
+                        "spec": {
+                            "instances": 1,
+                            "imageName": f"{airflow_db_image}:{airflow_db_image_tag}",
+                            "storage": {
+                                "size": "1Gi"
+                            },
+                            "bootstrap": {
+                                "initdb": {
+                                    "database": f"{airflow_db_name}",
+                                    "owner": f"{airflow_db_user}",
+                                    "secret": {
+                                        "name": f"{airflow_values_override_config['data']['metadataSecretName']}"
+                                    },
+                                    "encoding": "UTF-8"
                                 }
                             }
                         }
                     }
-                },
-                {
-                    "apiVersion": "v1",
-                    "kind": "Service",
-                    "metadata": {
-                        "name": "airflow-postgres-service",
-                        "namespace": "default"
-                    },
-                    "spec": {
-                        "type": "ClusterIP",
-                        "selector": {
-                            "app": "airflow-postgres"
-                        },
-                        "ports": [
-                            {
-                                "protocol": "TCP",
-                                "port": 5432,
-                                "targetPort": 5432
-                            }
-                        ]
-                    }
-                }
-            ]
-
-            with NamedTemporaryFile(suffix='.yaml', mode="w", delete_on_close=False) as fp:
-                # Dump as separate YAML documents
-                yaml_output = "\n---\n".join(
-                    yaml.safe_dump(doc, sort_keys=False) for doc in airflow_db_manifest
-                )
-                fp.write(yaml_output)
-                fp.close()
-                commands = [
-                    "kubectl", "apply", "-f", str(fp.name), "--validate=false"
-                ]
-                subprocess.run(commands, stdout=subprocess.DEVNULL)
-                commands = [
-                    "kubectl", "wait", "--namespace", "default", "--for", "condition=Ready", "cluster/airflow-postgres", "--timeout=60s"
                 ]
-                result = subprocess.run(commands)
-                if result.returncode != 0:
-                    error_message = result.stderr.strip() if result.stderr else result.stdout.strip() if result.stdout else 'no details given'
-                    raise Exception(f"An error occurred while installing database for Apache Airflow metastore: {error_message}")
-                
-            console.print("database for the Apache Airflow metastore installed successfully!", style=f"bold {COLOR_PALETTE.get('success')}")
-            console.print("")
+
+                with NamedTemporaryFile(suffix='.yaml', mode="w", delete_on_close=False) as fp:
+                    # Dump as separate YAML documents
+                    yaml_output = "\n---\n".join(
+                        yaml.safe_dump(doc, sort_keys=False) for doc in airflow_db_manifest
+                    )
+                    fp.write(yaml_output)
+                    fp.close()
+                    subprocess.run(["kubectl", "apply", "-f", str(fp.name)], stdout=subprocess.DEVNULL)
+                    result = subprocess.run(["kubectl", "wait", "--namespace", "default", "--for", "condition=Ready", "cluster/airflow-postgres", "--timeout=60s"])
+                    if result.returncode != 0:
+                        error_message = result.stderr.strip() if result.stderr else result.stdout.strip() if result.stdout else 'no details given'
+                        raise Exception(f"An error occurred while installing database for Apache Airflow metastore: {error_message}")
+
+                console.print("database for the Apache Airflow metastore installed successfully!", style=f"bold {COLOR_PALETTE.get('success')}")
+                console.print("")
 
         # Create temporary file with airflow values
         with NamedTemporaryFile(suffix='.yaml', mode="w") as fp:

{domino_py_iisas-1.0.9.dist-info → domino_py_iisas-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domino-py-iisas
-Version: 1.0.9
+Version: 1.0.11
 Summary: Fork of the original Python package for Domino.
 Author-email: Stefan Dlugolinsky <stefan.dlugolinsky@savba.sk>, Luiz Tauffer <luiz@taufferconsulting.com>, Vinicius Vaz <vinicius@taufferconsulting.com>
 License: 

{domino_py_iisas-1.0.9.dist-info → domino_py_iisas-1.0.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-domino/VERSION,sha256=LfK9cBHoc1Hi4-KW3bfyUgJwp2Fg-VstZLM3FnKU8ow,6
+domino/VERSION,sha256=tp2sshlQpVXDKBLvZuwhU-NA7V8OSwIrMCMAVoiQe2s,6
 domino/__init__.py,sha256=I3dmIBqKnlYjo-VFCDdA_7YUqsNT-i9QtQqXnV9pW4U,233
 domino/base_piece.py,sha256=LZE08XqU63Zq_W7G9DuFk8ZEjKNhgSE7pumXBctW9BE,17610
 domino/logger.py,sha256=08Km3_NmgYH3HrxJjZJWGSmOv7dE5E5xvo9_ZnKut_o,889
@@ -12,7 +12,7 @@ domino/cli/utils/config-domino-local.toml,sha256=9Mu2GBHeYNLxt9o3be_7Hzl-li2USNC
 domino/cli/utils/constants.py,sha256=8CbuNpXqCFCFjtcRt1X3_2i0udPerdUbIs4Z35AFRhc,252
 domino/cli/utils/docker-compose.yaml,sha256=r2M5-Unp5MXvkCYLVy9Z4qXVN1onYlncTqlHHJaKJQc,12667
 domino/cli/utils/pieces_repository.py,sha256=5lkUZtY3PH5pSwenNDi2kGfxWLL4v2uWLNYI62Sd16Q,22760
-domino/cli/utils/platform.py,sha256=zmXtLT1L75BKl-wlCHCqRXa4_qDs90wrrKyl25MoPAg,48691
+domino/cli/utils/platform.py,sha256=OV_YHfmfznC8JJuywyO6Ni5fyl5abQjEtyjhsP_cMCs,48330
 domino/cli/utils/templates.py,sha256=b7YhxqblrWeFhxmYiFh_iqRcJxIUZgPiaWhOHeb0shY,1982
 domino/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 domino/client/domino_backend_client.py,sha256=GuE4nZYt7Ke8vrnSa7o2VRAUYvy19RdyYwJD6WpoAik,2861
@@ -63,9 +63,9 @@ domino/utils/__init__.py,sha256=7CET-zz6y1EBuBJQLgF3rXNhKOSj32Z3X_3mKUAirII,231
 domino/utils/metadata_default.py,sha256=4tbmWoVuoRMxd2c2vkwKYLkTTDf1I0OooXB_P5DGWXM,455
 domino/utils/piece_generator.py,sha256=Yww5PwH-EDLqNrnWAinCgpG5-uQSVi_3UbMw4HZCE-E,514
 domino/utils/workflow_shared_storage.py,sha256=USt_Q6nRGrtbcgrWHjtEvBDMSGSDd3BwUrZVe0RItH0,272
-domino_py_iisas-1.0.9.dist-info/licenses/LICENSE,sha256=ZOzivsSTOmVUr23IVL-IMv2nDRhWBp4hVxwYTNe4fBc,11353
-domino_py_iisas-1.0.9.dist-info/METADATA,sha256=gDWgcCR5ZyevlwBJmf03A8lhTnsmZ2qiqfQlR_pRUwk,17095
-domino_py_iisas-1.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-domino_py_iisas-1.0.9.dist-info/entry_points.txt,sha256=EBX10akoWncqaYPjad7nTNFpge2bbooLSZJGPT-Ivzk,46
-domino_py_iisas-1.0.9.dist-info/top_level.txt,sha256=Mo0jr96Ke1GnB5Qa_U9nSu_7yRSWsu5dvPJk8RFiwRw,7
-domino_py_iisas-1.0.9.dist-info/RECORD,,
+domino_py_iisas-1.0.11.dist-info/licenses/LICENSE,sha256=ZOzivsSTOmVUr23IVL-IMv2nDRhWBp4hVxwYTNe4fBc,11353
+domino_py_iisas-1.0.11.dist-info/METADATA,sha256=XQR3hkWFu2gtuwwAH-bjv_SM3I9SiSpBP9mqQX-ovbw,17096
+domino_py_iisas-1.0.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+domino_py_iisas-1.0.11.dist-info/entry_points.txt,sha256=EBX10akoWncqaYPjad7nTNFpge2bbooLSZJGPT-Ivzk,46
+domino_py_iisas-1.0.11.dist-info/top_level.txt,sha256=Mo0jr96Ke1GnB5Qa_U9nSu_7yRSWsu5dvPJk8RFiwRw,7
+domino_py_iisas-1.0.11.dist-info/RECORD,,