qontract-reconcile 0.10.1rc1173__py3-none-any.whl → 0.10.1rc1175__py3-none-any.whl

{qontract_reconcile-0.10.1rc1173.dist-info → qontract_reconcile-0.10.1rc1175.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1173
+Version: 0.10.1rc1175
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1173.dist-info → qontract_reconcile-0.10.1rc1175.dist-info}/RECORD

@@ -192,7 +192,7 @@ reconcile/endpoints_discovery/integration.py,sha256=znfnlm8bZesfcNbQnaR2aaVM-DTB
 reconcile/endpoints_discovery/merge_request.py,sha256=_yLb4tnvoZMCko8rta2C_CvOInJa9pa3HzSmHNtjgGU,2978
 reconcile/endpoints_discovery/merge_request_manager.py,sha256=wUMsumxv8RnWaRattax4HfoRlhtVzmgro3GiJJ1C4Vc,6392
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/external_resources/aws.py,sha256=309Zui7rE8XFJA1ZBLupl55Vp8Y5KKgXdsKQWqKbg8I,7069
+reconcile/external_resources/aws.py,sha256=EN2Vz6IRp6TNWX5vwGAGESrD09_8sTFzKgZjdDR6cmg,7072
 reconcile/external_resources/factories.py,sha256=KrJDh52_8PeCEVjwfeVr1jwAJDdhMXRQ_XcBETfnKY4,4988
 reconcile/external_resources/integration.py,sha256=gBVO5dE8JyZ3xYcYik-MTIp_18oU7_hpYc_oztyfElQ,6753
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
@@ -200,7 +200,7 @@ reconcile/external_resources/manager.py,sha256=fXUm09w-9FRWYfJAwtV4_td1UHNLzdoGh
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=nMbyonGZEJDD1lYzpQY2eR9TNwvxYC4ZCcpi6wrExcM,1037
 reconcile/external_resources/model.py,sha256=H3elpiqehg_jACy28fGV5_77n8gKclVO77-7cfbaMNA,9178
-reconcile/external_resources/reconciler.py,sha256=Lhzg0O9Sw65KrCae-J14g9PV82xdp29O4jS_T5YNgEY,9661
+reconcile/external_resources/reconciler.py,sha256=K9QvbQCIOCuOHnPIxQE_P_jFtrkF3dGo8d_cCCh08Ys,8973
 reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
 reconcile/external_resources/state.py,sha256=z086bnIUTOkzFmQvS9rSAhFsM3Aw_9PLKHBACJ-0tQc,9690
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -838,11 +838,11 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=oZPib4HPq0aZ2Zui1QGJGk6qQdfpeihujGDBnSdKyGE,17627
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=ndRUc8mjkubajPkZKaoE2IFVT2e4YXYRVtUaeLt3zzE,140496
+tools/qontract_cli.py,sha256=YMOYkmP9WZFMX8wPxoJZ5ddstK3YyXMMx6ReXnCiH0w,140707
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/cli_commands/erv2.py,sha256=fByRn6D_SBi5YptjAqR49yFedsjyAHBQBaIbsdO1QKQ,16412
+tools/cli_commands/erv2.py,sha256=469qdhyaf7thpPQ4hJSurvmxBqYDJsoI8H4AigQIF7U,20737
 tools/cli_commands/gpg_encrypt.py,sha256=x02JOMn834z89YSNvr5B-oJky7rR1C0begCkPh45eHk,4958
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
 tools/cli_commands/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -880,8 +880,8 @@ tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6d
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1173.dist-info/METADATA,sha256=UAjgR0lD9syc08q6s2PuH2Q3_cOi0HN8W2Q7fqJcLh8,2213
-qontract_reconcile-0.10.1rc1173.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-qontract_reconcile-0.10.1rc1173.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1173.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1173.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1175.dist-info/METADATA,sha256=W7cOYjf2juRnK6QmkGQqjd_MG3Sft1hMo5imMLED1qo,2213
+qontract_reconcile-0.10.1rc1175.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+qontract_reconcile-0.10.1rc1175.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1175.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1175.dist-info/RECORD,,

reconcile/external_resources/aws.py

@@ -9,7 +9,6 @@ from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
 )
 from reconcile.utils.external_resources import ResourceValueResolver
-from reconcile.utils.helpers import generate_random_password
 from reconcile.utils.secret_reader import SecretReaderBase
 
 
@@ -59,10 +58,6 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
             pg_data = rvr._get_values(data["parameter_group"])
             data["parameter_group"] = pg_data
 
-        if data.get("transit_encryption_enabled", False):
-            data["auth_token"] = (
-                spec.get_secret_field("db.auth_token") or generate_random_password()
-            )
         return data
 
     def validate(self, resource: ExternalResource) -> None:
@@ -71,6 +66,11 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
         if data.get("parameter_group"):
             if not data["parameter_group"].get("name"):
                 data["parameter_group"]["name"] = f"{data['replication_group_id']}-pg"
+            else:
+                # prefix the parameter_group name with the replication_group_id
+                data["parameter_group"]["name"] = (
+                    f"{data['replication_group_id']}-{data['parameter_group']['name']}"
+                )
 
             if (
                 data.get("parameter_group_name")

reconcile/external_resources/reconciler.py

@@ -21,6 +21,7 @@ from kubernetes.client import (
 from pydantic import BaseModel
 
 from reconcile.external_resources.model import (
+    Action,
     Reconciliation,
 )
 from reconcile.external_resources.state import ReconcileStatus
@@ -88,6 +89,75 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
         }
 
     def job_spec(self) -> V1JobSpec:
+        job_container = V1Container(
+            name="job",
+            image=self.reconciliation.module_configuration.image_version,
+            image_pull_policy="Always",
+            env=[
+                V1EnvVar(
+                    name="DRY_RUN",
+                    value=str(self.is_dry_run),
+                ),
+                V1EnvVar(
+                    name="ACTION",
+                    value=self.reconciliation.action.value,
+                ),
+            ],
+            volume_mounts=[
+                V1VolumeMount(
+                    name="credentials",
+                    mount_path="/credentials",
+                    sub_path="credentials",
+                ),
+                V1VolumeMount(
+                    name="workdir",
+                    mount_path="/work",
+                ),
+                self.scripts_volume_mount("/inputs"),
+            ],
+        )
+        outputs_secret_container = V1Container(
+            name="outputs",
+            image=self.reconciliation.module_configuration.outputs_secret_image_version,
+            image_pull_policy="Always",
+            env=[
+                V1EnvVar(
+                    name="NAMESPACE",
+                    value_from=V1EnvVarSource(
+                        field_ref=V1ObjectFieldSelector(field_path="metadata.namespace")
+                    ),
+                ),
+                V1EnvVar(
+                    name="ACTION",
+                    value=self.reconciliation.action,
+                ),
+                V1EnvVar(
+                    name="DRY_RUN",
+                    value=str(self.is_dry_run),
+                ),
+            ],
+            volume_mounts=[
+                V1VolumeMount(
+                    name="credentials",
+                    mount_path="/.aws/credentials",
+                    sub_path="credentials",
+                ),
+                V1VolumeMount(
+                    name="workdir",
+                    mount_path="/work",
+                ),
+                self.scripts_volume_mount("/inputs"),
+            ],
+        )
+
+        # For delete actions, we don't need to run the outputs_secrets container
+        if self.reconciliation.action == Action.APPLY:
+            init_containers = [job_container]
+            containers = [outputs_secret_container]
+        else:
+            init_containers = []
+            containers = [job_container]
+
         return V1JobSpec(
             backoff_limit=0,
             active_deadline_seconds=self.reconciliation.module_configuration.reconcile_timeout_minutes
@@ -98,72 +168,8 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
                     annotations=self.annotations(), labels=self.labels()
                 ),
                 spec=V1PodSpec(
-                    init_containers=[
-                        V1Container(
-                            name="job",
-                            image=self.reconciliation.module_configuration.image_version,
-                            image_pull_policy="Always",
-                            env=[
-                                V1EnvVar(
-                                    name="DRY_RUN",
-                                    value=str(self.is_dry_run),
-                                ),
-                                V1EnvVar(
-                                    name="ACTION",
-                                    value=self.reconciliation.action.value,
-                                ),
-                            ],
-                            volume_mounts=[
-                                V1VolumeMount(
-                                    name="credentials",
-                                    mount_path="/credentials",
-                                    sub_path="credentials",
-                                ),
-                                V1VolumeMount(
-                                    name="workdir",
-                                    mount_path="/work",
-                                ),
-                                self.scripts_volume_mount("/inputs"),
-                            ],
-                        )
-                    ],
-                    containers=[
-                        V1Container(
-                            name="outputs",
-                            image=self.reconciliation.module_configuration.outputs_secret_image_version,
-                            image_pull_policy="Always",
-                            env=[
-                                V1EnvVar(
-                                    name="NAMESPACE",
-                                    value_from=V1EnvVarSource(
-                                        field_ref=V1ObjectFieldSelector(
-                                            field_path="metadata.namespace"
-                                        )
-                                    ),
-                                ),
-                                V1EnvVar(
-                                    name="ACTION",
-                                    value=self.reconciliation.action,
-                                ),
-                                V1EnvVar(
-                                    name="DRY_RUN",
-                                    value=str(self.is_dry_run),
-                                ),
-                            ],
-                            volume_mounts=[
-                                V1VolumeMount(
-                                    name="credentials",
-                                    mount_path="/.aws/credentials",
-                                    sub_path="credentials",
-                                ),
-                                V1VolumeMount(
-                                    name="workdir",
-                                    mount_path="/work",
-                                ),
-                                self.scripts_volume_mount("/inputs"),
-                            ],
-                        )
-                    ],
+                    init_containers=init_containers,
+                    containers=containers,
                     image_pull_secrets=[V1LocalObjectReference(name="quay.io")],
                     volumes=[
                         V1Volume(

tools/cli_commands/erv2.py

@@ -4,12 +4,12 @@ import json
 import os
 import sys
 from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from difflib import get_close_matches
 from enum import Enum
 from pathlib import Path
 from subprocess import CalledProcessError, run
-from typing import Protocol
+from typing import Any, Protocol
 
 from pydantic import BaseModel
 from rich import print as rich_print
@@ -178,6 +178,7 @@ class Erv2Cli:
 
             # run cdktf synth
             with task(self.progress_spinner, "-- Running CDKTF synth"):
+                run(["docker", "pull", self.image], check=True, capture_output=True)
                 run(
                     [
                         "docker",
@@ -245,8 +246,14 @@ class TfAction(Enum):
     DESTROY = "delete"
 
 
+class Change(BaseModel):
+    before: Any | None = None
+    after: Any | None = None
+
+
 class TfResource(BaseModel):
     address: str
+    change: Change | None = None
 
     @property
     def id(self) -> str:
@@ -269,13 +276,19 @@ class TfResourceList(BaseModel):
     def __iter__(self) -> Iterator[TfResource]:  # type: ignore
         return iter(self.resources)
 
-    def _get_resource_by_address(self, address: str) -> TfResource | None:
+    def get_resource_by_address(self, address: str) -> TfResource | None:
         for resource in self.resources:
             if resource.address == address:
                 return resource
         return None
 
-    def _get_resources_by_type(self, type: str) -> list[TfResource]:
+    def get_resource_by_type(self, type: str) -> TfResource:
+        results = self.get_resources_by_type(type)
+        if len(results) > 1:
+            raise ValueError(f"More than one resource found for type '{type}'!")
+        return results[0]
+
+    def get_resources_by_type(self, type: str) -> list[TfResource]:
         results = [resource for resource in self.resources if resource.type == type]
         if not results:
             raise KeyError(f"Resource type '{type}' not found!")
@@ -287,13 +300,13 @@ class TfResourceList(BaseModel):
         self holds the source resources (terraform-resources).
         The tf_resource is the destination resource (ERv2).
         """
-        if resource := self._get_resource_by_address(tf_resource.address):
+        if resource := self.get_resource_by_address(tf_resource.address):
             # exact match by AWS address
             return [resource]
 
         # a resource with the same ID does not exist
         # let's try to find the resource by the AWS type
-        results = self._get_resources_by_type(tf_resource.type)
+        results = self.get_resources_by_type(tf_resource.type)
         if len(results) == 1:
             # there is just one resource with the same type
             # this must be the searched resource.
@@ -354,6 +367,25 @@ class TerraformCli:
             if not self._dry_run:
                 self._tf_run(self._path, ["state", "push", str(self.state_file)])
 
+    def _tf_import(self, address: str, value: str) -> None:
+        """Import a resource.
+
+        !!! Attention !!!
+
+        Because terraform import doesn't use the local state file and always imports the resource to the remote state,
+        we need to push and pull the state file again.
+        """
+        # push local changes
+        self._tf_state_push()
+        with task(
+            self.progress_spinner,
+            f"-- Importing resource {address} {'[b red](DRY-RUN)' if self._dry_run else ''}",
+        ):
+            if not self._dry_run:
+                self._tf_run(self._path, ["import", address, value])
+        # and pull the state file again
+        self._tf_state_pull()
+
     def upload_state(self) -> None:
         self._tf_state_push()
 
@@ -362,7 +394,10 @@ class TerraformCli:
         plan = json.loads(self._tf_run(self._path, ["show", "-json", "plan.out"]))
         return TfResourceList(
             resources=[
-                TfResource(address=r["address"])
+                TfResource(
+                    address=r["address"],
+                    change=Change(**r["change"]),
+                )
                 for r in plan["resource_changes"]
                 if action.value.lower() in r["change"]["actions"]
             ]
@@ -372,24 +407,112 @@ class TerraformCli:
         self, source_state_file: Path, source: TfResource, destination: TfResource
     ) -> None:
         """Move the resource from source state file to destination state file."""
-        if self.progress_spinner:
-            self.progress_spinner.log(
-                f"-- Moving {destination} {'[b red](DRY-RUN)' if self._dry_run else ''}"
-            )
+        with task(
+            self.progress_spinner,
+            f"-- Moving {destination} {'[b red](DRY-RUN)' if self._dry_run else ''}",
+        ):
+            if not self._dry_run:
+                self._tf_run(
+                    self._path,
+                    [
+                        "state",
+                        "mv",
+                        "-lock=false",
+                        f"-state={source_state_file!s}",
+                        f"-state-out={self.state_file!s}",
+                        f"{source.address}",
+                        f"{destination.address}",
+                    ],
+                )
+
+    def commit(self, source: TerraformCli) -> None:
+        """Commit the changes."""
         if not self._dry_run:
-            self._tf_run(
-                self._path,
-                [
-                    "state",
-                    "mv",
-                    "-lock=false",
-                    f"-state={source_state_file!s}",
-                    f"-state-out={self.state_file!s}",
-                    f"{source.address}",
-                    f"{destination.address}",
-                ],
+            if self.progress_spinner:
+                self.progress_spinner.stop()
+            if not Confirm.ask(
+                "\nEverything ok? Would you like to upload the modified terraform states",
+                default=False,
+            ):
+                return
+
+            if self.progress_spinner:
+                self.progress_spinner.start()
+
+            # finally push the terraform states
+            self.upload_state()
+            source.upload_state()
+
+    def _elasticache_import_password(
+        self, destination: TfResource, password: str
+    ) -> None:
+        """Import the elasticache auth_token random_password."""
+        self._tf_import(address=destination.address, value=password)
+
+        if (
+            not destination.change
+            or not destination.change.after
+            or not destination.change.after.get("override_special")
+        ):
+            # nothing to change, nothing to do
+            return
+
+        state_data = json.loads(self.state_file.read_text())
+        state_data["serial"] += 1
+        if self._dry_run:
+            # in dry-run mode, tf_import is a no-op, therefore, the password is not in the state yet.
+            return
+        state_password_obj = next(
+            r for r in state_data["resources"] if r["name"] == destination.id
+        )
+
+        # Set the "override_special" to disable the password reset
+        state_password_obj["instances"][0]["attributes"]["override_special"] = (
+            destination.change.after["override_special"]
+        )
+        # Write the state,
+        self.state_file.write_text(json.dumps(state_data, indent=2))
+
+    def migrate_elasticache_resources(self, source: TerraformCli) -> None:
+        source_resources = source.resource_changes(TfAction.DESTROY)
+        destination_resources = self.resource_changes(TfAction.CREATE)
+        if not source_resources or not destination_resources:
+            raise ValueError("No resource changes found!")
+
+        source_ec = source_resources.get_resource_by_type(
+            "aws_elasticache_replication_group"
+        )
+        if not source_ec.change or not source_ec.change.before:
+            raise ValueError(
+                "Something went wrong with the source elasticache instance!"
             )
 
+        current_auth_token = source_ec.change.before.get("auth_token")
+        if current_auth_token:
+            with suppress(KeyError):
+                self._elasticache_import_password(
+                    destination_resources.get_resource_by_type("random_password"),
+                    current_auth_token,
+                )
+
+        # migrate resources
+        for destination_resource in destination_resources:
+            if current_auth_token and destination_resource.type == "random_password":
+                # random password handled above
+                continue
+
+            possible_source_resouces = source_resources[destination_resource]
+            if not possible_source_resouces or len(possible_source_resouces) > 1:
+                raise ValueError(
+                    f"Either source resource for {destination_resource} not found or more than one resource found!"
+                )
+            self.move_resource(
+                source_state_file=source.state_file,
+                source=possible_source_resouces[0],
+                destination=destination_resource,
+            )
+        self.commit(source)
+
     def migrate_resources(self, source: TerraformCli) -> None:
         """Migrate the resources from source."""
         # if not self.initialized or not source.initialized:
@@ -456,18 +579,4 @@ class TerraformCli:
                 destination=destination_resource,
             )
 
-        if not self._dry_run:
-            if self.progress_spinner:
-                self.progress_spinner.stop()
-            if not Confirm.ask(
-                "\nEverything ok? Would you like to upload the modified terraform states",
-                default=False,
-            ):
-                return
-
-            if self.progress_spinner:
-                self.progress_spinner.start()
-
-            # finally push the terraform states
-            self.upload_state()
-            source.upload_state()
+        self.commit(source)

tools/qontract_cli.py

@@ -4304,7 +4304,11 @@ def migrate(ctx, dry_run: bool, skip_build: bool) -> None:
             progress,
             "Migrating the resources from terraform-resources to ERv2",
         ):
-            erv2_tf_cli.migrate_resources(source=tfr_tf_cli)
+            if ctx.obj["provider"] == "elasticache":
+                # Elasticache migration is a bit different
+                erv2_tf_cli.migrate_elasticache_resources(source=tfr_tf_cli)
+            else:
+                erv2_tf_cli.migrate_resources(source=tfr_tf_cli)
 
     rich_print(f"[b red]Please remove the temporary directory ({tempdir}) manually!")