qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/terraform_client.py

@@ -1,35 +1,39 @@
 import json
 import logging
+import re
 import shutil
+import tempfile
 from collections import defaultdict
 from collections.abc import (
+    Generator,
     Iterable,
     Mapping,
 )
+from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import (
     datetime,
     timedelta,
 )
+from subprocess import CalledProcessError
 from threading import Lock
 from typing import (
+    IO,
     Any,
-    Optional,
     cast,
 )
 
 from botocore.errorfactory import ClientError
-from python_terraform import (
-    IsFlagged,
-    Terraform,
-    TerraformCommandError,
-)
+from packaging import version as pkg_version
 from sretoolbox.utils import (
     retry,
     threaded,
 )
 
 import reconcile.utils.lean_terraform_client as lean_tf
+from reconcile.typed_queries.app_interface_custom_messages import (
+    get_app_interface_custom_message,
+)
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.aws_helper import get_region_from_availability_zone
 from reconcile.utils.external_resource_spec import (
@@ -37,8 +41,15 @@ from reconcile.utils.external_resource_spec import (
     ExternalResourceSpecInventory,
 )
 
-ALLOWED_TF_SHOW_FORMAT_VERSION = "0.1"
+ALLOWED_TF_SHOW_FORMAT_VERSION = "1.2"
 DATE_FORMAT = "%Y-%m-%d"
+PROVIDER_LOG_REGEX = (
+    r""".*\s(?:\[INFO]|\[WARN]|\[ERROR])\s.+\s(?:\[WARN]|\[ERROR])\s.*"""
+)
+PROVIDER_LOG_IGNORE_REGEX = (
+    r""".*(?:ObjectLockConfigurationNotFoundError|WaitForState).*"""
+)
+TERRAFORM_LOG_LEVEL = "TRACE"  # can change to INFO after tf 0.15
 
 
 @dataclass
@@ -47,6 +58,20 @@ class AccountUser:
     user: str
 
 
+@dataclass(frozen=True, eq=True)
+class TerraformSpec:
+    name: str
+    working_dir: str
+
+
+class TerraformCommandError(CalledProcessError):
+    pass
+
+
+class RdsUpgradeValidationError(Exception):
+    pass
+
+
 class DeletionApprovalExpirationValueError(Exception):
     pass
 
@@ -60,7 +85,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         accounts: Iterable[Mapping[str, Any]],
         working_dirs: Mapping[str, str],
         thread_pool_size: int,
-        aws_api: Optional[AWSApi] = None,
+        aws_api: AWSApi | None = None,
         init_users=False,
     ):
         self.integration = integration
@@ -72,27 +97,32 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         self.thread_pool_size = thread_pool_size
         self._aws_api = aws_api
         self._log_lock = Lock()
-        self.should_apply = False
+        self.apply_count = 0
 
+        self.specs: list[TerraformSpec] = []
         self.init_specs()
+        self.outputs: dict = {}
         self.init_outputs()
 
         self.OUTPUT_TYPE_SECRETS = "Secrets"
         self.OUTPUT_TYPE_PASSWORDS = "enc-passwords"
         self.OUTPUT_TYPE_CONSOLEURLS = "console-urls"
 
+        self.users: dict = {}
         if init_users:
             self.init_existing_users()
 
     def init_existing_users(self):
-        all_users = {}
-        for account, output in self.outputs.items():
-            users = []
-            user_passwords = self.format_output(output, self.OUTPUT_TYPE_PASSWORDS)
-            for user_name in user_passwords:
-                users.append(user_name)
-            all_users[account] = users
-        self.users = all_users
+        self.users = {
+            account: list(self.format_output(output, self.OUTPUT_TYPE_PASSWORDS).keys())
+            for account, output in self.outputs.items()
+        }
+
+    def increment_apply_count(self):
+        self.apply_count += 1
+
+    def should_apply(self) -> bool:
+        return self.apply_count > 0
 
     def get_new_users(self):
         new_users = []
@@ -103,37 +133,54 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             for user_name, enc_password in user_passwords.items():
                 if AccountUser(account, user_name) not in self.created_users:
                     continue
-                new_users.append(
-                    (account, console_urls[account], user_name, enc_password)
-                )
+                new_users.append((
+                    account,
+                    console_urls[account],
+                    user_name,
+                    enc_password,
+                ))
         return new_users
 
     def init_specs(self):
-        wd_specs = [{"name": name, "wd": wd} for name, wd in self.working_dirs.items()]
-        results = threaded.run(self.terraform_init, wd_specs, self.thread_pool_size)
-        self.specs = [{"name": name, "tf": tf} for name, tf in results]
+        self.specs = [
+            TerraformSpec(name=name, working_dir=wd)
+            for name, wd in self.working_dirs.items()
+        ]
+        for spec in self.specs:
+            self.terraform_init(spec)
+
+    @contextmanager
+    def _terraform_log_file(
+        self, working_dir: str
+    ) -> Generator[tuple[IO, dict[str, str]], None, None]:
+        with tempfile.NamedTemporaryFile(dir=working_dir) as f:
+            env = {
+                "TF_LOG": TERRAFORM_LOG_LEVEL,
+                "TF_LOG_PATH": f.name,
+            }
+            yield f, env
 
     @retry(exceptions=TerraformCommandError)
-    def terraform_init(self, init_spec):
-        name = init_spec["name"]
-        wd = init_spec["wd"]
-        tf = Terraform(working_dir=wd)
-        return_code, stdout, stderr = tf.init()
-        error = self.check_output(name, "init", return_code, stdout, stderr)
+    def terraform_init(self, spec: TerraformSpec):
+        with self._terraform_log_file(spec.working_dir) as (f, env):
+            return_code, stdout, stderr = lean_tf.init(spec.working_dir, env=env)
+            log = f.read().decode("utf-8")
+        error = self.check_output(spec.name, "init", return_code, stdout, stderr, log)
         if error:
-            raise TerraformCommandError(return_code, "init", out=stdout, err=stderr)
-        return name, tf
+            raise TerraformCommandError(
+                return_code, "init", output=stdout, stderr=stderr
+            )
 
     def init_outputs(self):
         results = threaded.run(self.terraform_output, self.specs, self.thread_pool_size)
         self.outputs = dict(results)
 
     @retry(exceptions=TerraformCommandError)
-    def terraform_output(self, spec):
-        name = spec["name"]
-        tf = spec["tf"]
-        return_code, stdout, stderr = tf.output_cmd(json=IsFlagged)
-        error = self.check_output(name, "output", return_code, stdout, stderr)
+    def terraform_output(self, spec: TerraformSpec):
+        with self._terraform_log_file(spec.working_dir) as (f, env):
+            return_code, stdout, stderr = lean_tf.output(spec.working_dir, env=env)
+            log = f.read().decode("utf-8")
+        error = self.check_output(spec.name, "output", return_code, stdout, stderr, log)
         no_output_error = (
             "The module root could not be found. There is nothing to output."
         )
@@ -142,9 +189,9 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                 stdout = "{}"
             else:
                 raise TerraformCommandError(
-                    return_code, "output", out=stdout, err=stderr
+                    return_code, "output", output=stdout, stderr=stderr
                 )
-        return name, json.loads(stdout)
+        return spec.name, json.loads(stdout)
 
     # terraform plan
     def plan(self, enable_deletion):
@@ -166,18 +213,31 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             self.created_users.extend(created_users)
         return disabled_deletions_detected, errors
 
-    @retry()
+    def safe_plan(self, enable_deletion: bool) -> None:
+        """Raises exception if errors are detected at plan step"""
+        disable_deletions_detected, errors = self.plan(enable_deletion)
+
+        if errors:
+            raise RuntimeError("Terraform plan has errors")
+        if disable_deletions_detected:
+            raise RuntimeError("Terraform plan has disabled deletions detected")
+
+    @retry(no_retry_exceptions=RdsUpgradeValidationError)
     def terraform_plan(
-        self, plan_spec: dict, enable_deletion: bool
+        self, spec: TerraformSpec, enable_deletion: bool
     ) -> tuple[bool, list[AccountUser], bool]:
-        name = plan_spec["name"]
-        tf = plan_spec["tf"]
-        return_code, stdout, stderr = tf.plan(
-            detailed_exitcode=False, parallelism=self.parallelism, out=name
-        )
-        error = self.check_output(name, "plan", return_code, stdout, stderr)
+        with self._terraform_log_file(spec.working_dir) as (f, env):
+            return_code, stdout, stderr = lean_tf.plan(
+                spec.working_dir,
+                spec.name,
+                env=env,
+            )
+            log = f.read().decode("utf-8")
+        error = self.check_output(spec.name, "plan", return_code, stdout, stderr, log)
+        if error:
+            return False, [], error
         disabled_deletion_detected, created_users = self.log_plan_diff(
-            name, tf, enable_deletion
+            spec, enable_deletion
         )
         return disabled_deletion_detected, created_users, error
 
@@ -216,11 +276,11 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
 
     def log_plan_diff(
         self,
-        name: str,
-        tf: Terraform,
+        spec: TerraformSpec,
         enable_deletion: bool,
     ) -> tuple[bool, list]:
         disabled_deletion_detected = False
+        name = spec.name
         account_enable_deletion = self.accounts[name].get("enableDeletion") or False
         # deletions are allowed
         # if enableDeletion is true for an account
@@ -228,7 +288,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         deletions_allowed = enable_deletion or account_enable_deletion
         created_users: list[AccountUser] = []
 
-        output = self.terraform_show(name, tf.working_dir)
+        output = lean_tf.show_json(spec.working_dir, name)
         format_version = output.get("format_version")
         if format_version != ALLOWED_TF_SHOW_FORMAT_VERSION:
             raise NotImplementedError("terraform show untested format version")
@@ -245,7 +305,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             after = output_change.get("after")
             if before != after:
                 logging.info(["update", name, "output", output_name])
-                self.should_apply = True
+                self.increment_apply_count()
 
         # A way to detect deleted outputs is by comparing
         # the prior state with the output changes.
@@ -258,7 +318,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         deleted_outputs = [po for po in prior_outputs if po not in output_changes]
         for output_name in deleted_outputs:
             logging.info(["delete", name, "output", output_name])
-            self.should_apply = True
+            self.increment_apply_count()
 
         resource_changes = output.get("resource_changes")
         if resource_changes is None:
@@ -267,6 +327,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         always_enabled_deletions = {
             "random_id",
             "aws_lb_target_group_attachment",
+            "aws_iam_user_policy",
             "cloudflare_record",  # This is because a zone can contain up to one thousand records and it's not practical to require adding each record to deletionApprovals
         }
 
@@ -274,14 +335,30 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         for resource_change in resource_changes:
             resource_type = resource_change["type"]
             resource_name = resource_change["name"]
+            resource_address = resource_change["address"]
+            resource_previous_address = resource_change.get("previous_address")
             resource_change = resource_change["change"]
             actions = resource_change["actions"]
             for action in actions:
+                if resource_previous_address:
+                    # the resource is being moved/renamed in the TF state
+                    with self._log_lock:
+                        logging.info([
+                            "move/rename",
+                            name,
+                            resource_previous_address,
+                            resource_address,
+                        ])
+
                 if action == "no-op":
                     logging.debug([action, name, resource_type, resource_name])
-                    continue
+                    if resource_previous_address:
+                        # apply resource renaming with no-op
+                        self.increment_apply_count()
+                    else:
+                        continue
                 if action == "update" and resource_type == "aws_db_instance":
-                    self._validate_db_upgrade(name, resource_name, resource_change)
+                    self.validate_db_upgrade(name, resource_name, resource_change)
                     # Ignore RDS modifications that are going to occur during the next
                     # maintenance window. This can be up to 7 days away and will cause
                     # unnecessary Terraform state updates until they complete.
@@ -294,16 +371,14 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                         )
                         continue
                 with self._log_lock:
-                    logging.info(
-                        [
-                            action,
-                            name,
-                            resource_type,
-                            resource_name,
-                            self._resource_diff_changed_fields(action, resource_change),
-                        ]
-                    )
-                    self.should_apply = True
+                    logging.info([
+                        action,
+                        name,
+                        resource_type,
+                        resource_name,
+                        self._resource_diff_changed_fields(action, resource_change),
+                    ])
+                    self.increment_apply_count()
                 if action == "create":
                     if resource_type == "aws_iam_user_login_profile":
                         created_users.append(AccountUser(name, resource_name))
@@ -315,11 +390,13 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                         name, resource_type, resource_name
                     ):
                         disabled_deletion_detected = True
-                        logging.error(
-                            "'delete' action is not enabled. "
-                            + "Please run the integration manually "
-                            + "with the '--enable-deletion' flag."
+                        instructions = (
+                            get_app_interface_custom_message(
+                                "disabled-deletion-instructions"
+                            )
+                            or ""
                         )
+                        logging.error(f"'delete' action is not enabled. {instructions}")
                     if resource_type == "aws_db_instance":
                         deletion_protected = resource_change["before"].get(
                             "deletion_protection"
@@ -351,7 +428,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                     f"[{account_name}] expiration not does not match "
                     f"date format {DATE_FORMAT}. details: "
                     f"type: {da['type']}, name: {da['name']}"
-                )
+                ) from None
             if (
                 resource_type == da["type"]
                 and resource_name == da["name"]
@@ -361,28 +438,20 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
 
         return False
 
-    @staticmethod
-    def terraform_show(name, working_dir):
-        return lean_tf.show_json(working_dir, name)
-
     # terraform apply
     def apply(self):
-        errors = False
-
-        results = threaded.run(self.terraform_apply, self.specs, self.thread_pool_size)
-
-        for error in results:
-            if error:
-                errors = True
-        return errors
-
-    def terraform_apply(self, apply_spec):
-        name = apply_spec["name"]
-        tf = apply_spec["tf"]
-        # adding var=None to allow applying the saved plan
-        # https://github.com/beelit94/python-terraform/issues/67
-        return_code, stdout, stderr = tf.apply(dir_or_plan=name, var=None)
-        error = self.check_output(name, "apply", return_code, stdout, stderr)
+        errors = threaded.run(self.terraform_apply, self.specs, self.thread_pool_size)
+        return any(errors)
+
+    def terraform_apply(self, spec: TerraformSpec):
+        with self._terraform_log_file(spec.working_dir) as (f, env):
+            return_code, stdout, stderr = lean_tf.apply(
+                spec.working_dir,
+                spec.name,
+                env=env,
+            )
+            log = f.read().decode("utf-8")
+        error = self.check_output(spec.name, "apply", return_code, stdout, stderr, log)
         return error
 
     def get_terraform_output_secrets(self) -> dict[str, dict[str, dict[str, str]]]:
@@ -411,9 +480,9 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                 replica_source_name = f'{replica_src}-{tf_resource.get("provider")}'
                 # Creating a dict that is convenient to use inside the
                 # loop processing the formatted_output
-                replicas_info[spec.provisioner_name][
-                    spec.output_prefix
-                ] = replica_source_name
+                replicas_info[spec.provisioner_name][spec.output_prefix] = (
+                    replica_source_name
+                )
 
         return replicas_info
 
@@ -423,12 +492,8 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         if output is None:
             return data
 
-        enc_pass_pfx = "{}_{}".format(
-            self.integration_prefix, self.OUTPUT_TYPE_PASSWORDS
-        )
-        console_urls_pfx = "{}_{}".format(
-            self.integration_prefix, self.OUTPUT_TYPE_CONSOLEURLS
-        )
+        enc_pass_pfx = f"{self.integration_prefix}_{self.OUTPUT_TYPE_PASSWORDS}"
+        console_urls_pfx = f"{self.integration_prefix}_{self.OUTPUT_TYPE_CONSOLEURLS}"
         for k, v in output.items():
             # the integration creates outputs of the form
             # 0.11: output_secret_name[secret_key] = secret_value
@@ -474,11 +539,11 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                 data[resource_name] = {}
             data[resource_name][field_key] = field_value
 
-        if len(data) == 1 and type in (
+        if len(data) == 1 and type in {
             self.OUTPUT_TYPE_PASSWORDS,
             self.OUTPUT_TYPE_CONSOLEURLS,
-        ):
-            return data[list(data.keys())[0]]
+        }:
+            return data[next(iter(data.keys()))]
         return data
 
     def populate_terraform_output_secrets(
@@ -541,7 +606,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                         secret_copy["db.password"] = replica_src_password
 
             # clean metadata
-            for key in secret.keys():
+            for key in secret:
                 if integration_prefix in key:
                     secret_copy.pop(key)
 
@@ -552,22 +617,29 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         name: str,
         cmd: str,
         return_code: int,
-        stdout: list[str],
-        stderr: list[str],
+        stdout: str,
+        stderr: str,
+        log: str,
     ) -> bool:
-        error_occured = False
+        error_occured = return_code != 0
         line_format = "[{} - {}] {}"
-        stdout, stderr = self.split_to_lines(stdout, stderr)
+        stdout, stderr, log = self.split_to_lines(stdout, stderr, log)
+        provider_log_re = re.compile(PROVIDER_LOG_REGEX)
+        provider_log_ignore_re = re.compile(PROVIDER_LOG_IGNORE_REGEX)
         with self._log_lock:
             for line in stdout:
                 logging.debug(line_format.format(name, cmd, line))
-            if return_code == 0:
+            if error_occured:
                 for line in stderr:
-                    logging.warning(line_format.format(name, cmd, line))
+                    logging.error(line_format.format(name, cmd, line))
             else:
                 for line in stderr:
-                    logging.error(line_format.format(name, cmd, line))
-                error_occured = True
+                    logging.warning(line_format.format(name, cmd, line))
+            for line in log:
+                if provider_log_re.match(line) and not provider_log_ignore_re.match(
+                    line
+                ):
+                    logging.warning(line_format.format(name, cmd, line))
         return error_occured
 
     @staticmethod
@@ -683,7 +755,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             return not set(changed_resource_arguments) - set(changed_values)
         return False
 
-    def _validate_db_upgrade(
+    def validate_db_upgrade(
         self, account_name: str, resource_name: str, resource_change: Mapping[str, Any]
     ):
         """
@@ -711,23 +783,110 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             valid_upgrade_target = self._aws_api.get_db_valid_upgrade_target(
                 account_name, engine, before_version, region_name=region_name
             )
-            found = False
-            for t in valid_upgrade_target:
-                if t["EngineVersion"] == after_version:
-                    found = True
-                    if t["IsMajorVersionUpgrade"] and not after.get(
-                        "allow_major_version_upgrade", False
-                    ):
-                        raise ValueError(
-                            "allow_major_version_upgrade is not enabled for upgrading RDS instance: "
-                            f"{resource_name} to a new major version."
-                        )
-
-            if not found:
-                raise ValueError(
+            target = next(
+                (
+                    t
+                    for t in valid_upgrade_target
+                    if t["EngineVersion"] == after_version
+                ),
+                None,
+            )
+            if target is None:
+                raise RdsUpgradeValidationError(
                     f"Cannot upgrade RDS instance: {resource_name} "
                     f"from {before_version} to {after_version}"
                 )
+            allow_major_version_upgrade = after.get(
+                "allow_major_version_upgrade",
+                False,
+            )
+            if target["IsMajorVersionUpgrade"] and not allow_major_version_upgrade:
+                raise RdsUpgradeValidationError(
+                    "allow_major_version_upgrade is not enabled for upgrading RDS instance: "
+                    f"{resource_name} to a new major version."
+                )
+
+            blue_green_update = after.get("blue_green_update", [])
+            if blue_green_update and blue_green_update[0]["enabled"]:
+                changed_fields = self._resource_diff_changed_fields(
+                    "update", resource_change
+                )
+                replica = before["replicas"] != [] or before["replicate_source_db"]
+                self.validate_blue_green_update_requirements(
+                    account_name,
+                    resource_name,
+                    engine,
+                    before_version,
+                    replica,
+                    before["parameter_group_name"],
+                    region_name,
+                    changed_fields=changed_fields,
+                )
+
+    def validate_blue_green_update_requirements(
+        self,
+        account_name: str,
+        resource_name: str,
+        engine: str,
+        version: str,
+        replica: bool,
+        parameter_group: str,
+        region_name: str,
+        changed_fields: set[str],
+    ) -> None:
+        min_supported_versions = {
+            "mysql": [pkg_version.parse("5.7"), pkg_version.parse("8.0.15")],
+            "postgres": [
+                pkg_version.parse("11.21"),
+                pkg_version.parse("12.16"),
+                pkg_version.parse("13.12"),
+                pkg_version.parse("14.9"),
+                pkg_version.parse("15.4"),
+                pkg_version.parse("16.1"),
+            ],
+        }
+
+        def is_supported(engine, version):
+            parsed_version = pkg_version.parse(version)
+            if engine == "mysql":
+                return any(
+                    parsed_version >= min_version
+                    for min_version in min_supported_versions["mysql"]
+                )
+            elif engine == "postgres":
+                return any(
+                    parsed_version >= min_version
+                    for min_version in min_supported_versions["postgres"]
+                )
+            return False
+
+        if not is_supported(engine, version):
+            raise RdsUpgradeValidationError(
+                f"Cannot upgrade RDS instance: {resource_name}. "
+                f"Engine version {version} is not supported for blue/green updates."
+            )
+
+        if replica:
+            raise RdsUpgradeValidationError(
+                f"Cannot upgrade RDS instance: {resource_name}. "
+                "Blue/green updates are not supported for instances with read replicas."
+            )
+
+        if engine == "postgres" and self._aws_api is not None:
+            pg_details = self._aws_api.describe_db_parameter_group(
+                account_name, parameter_group, region_name
+            )
+            if pg_details.get("rds.logical_replication") != "1":
+                raise RdsUpgradeValidationError(
+                    f"Cannot upgrade RDS instance: {resource_name}. "
+                    f"Blue/green updates require logical replication to be enabled in the Parameter group {parameter_group}."
+                )
+
+        if "storage_type" in changed_fields or "allocated_storage" in changed_fields:
+            raise RdsUpgradeValidationError(
+                f"Cannot upgrade RDS instance: {resource_name}. "
+                f"Blue/green updates are not supported when 'storage_type' or 'allocated_storage' has changed."
+            )
 
 
 class TerraformPlanFailed(Exception):