qontract-reconcile 0.10.2.dev403__py3-none-any.whl → 0.10.2.dev404__py3-none-any.whl

{qontract_reconcile-0.10.2.dev403.dist-info → qontract_reconcile-0.10.2.dev404.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev403
+Version: 0.10.2.dev404
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev403.dist-info → qontract_reconcile-0.10.2.dev404.dist-info}/RECORD

@@ -1,7 +1,7 @@
 reconcile/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/acs_policies.py,sha256=1LKnRRPa6h3QewsDhv0epINi9RJtQmT88mPgcU2LhDM,8667
 reconcile/acs_rbac.py,sha256=15vNfNzdG_DeXaJ-f5m8DSaJh__LUK766_xAECqyTsg,22657
-reconcile/aws_ami_share.py,sha256=M_gT7y3cSAyT_Pm90PBCNDSmbZtqREqe2jNETh0i9Qs,3808
+reconcile/aws_ami_share.py,sha256=JqevGzjLKlktRDeL01ukBko5ope5Xhl_0NvdjfGdENA,3503
 reconcile/aws_ecr_image_pull_secrets.py,sha256=r9Dy01w4kNPWh3LO2tSGH_x4rQg3B2FJc5sVjPJxZdI,2622
 reconcile/aws_iam_keys.py,sha256=YHp-K8K8Dqm7vVKzry0RyhM6Egmpp7eCWMNdKk0vbME,4118
 reconcile/aws_iam_password_reset.py,sha256=5oajSspJVAvpGd445hKsxtEGYb75dM4l1_PJTzrfHk0,3253
@@ -88,7 +88,7 @@ reconcile/quay_mirror.py,sha256=pA1_OujRduwQ6dYljoWXU_VJgAwlv7DzThk26ymKmGs,1432
 reconcile/quay_mirror_org.py,sha256=ltPbHuWUI8Wnl8gV4aeYmvoYFA1uXLWqlXqEPpw7Hi0,11065
 reconcile/quay_permissions.py,sha256=BF539lRxjpgwm88WzazklzgaCF_ipRALwbO2AdpqUqE,4388
 reconcile/quay_repos.py,sha256=fBleLzMtfDmTidpzbrTt8kGCy-Bk3J06EO4hhyghGnQ,7570
-reconcile/queries.py,sha256=_XCgyGyZib2YM90W3qCgqVPoOkuiwjFjNTTAhKM2tLo,56477
+reconcile/queries.py,sha256=L0NbIr6-M12P7xqU2s_2-tgi5BOC5QEo6Otu33WG5tI,56598
 reconcile/query_validator.py,sha256=csOSkKxcf6ZlpchJu4ck2jLYKUN6y1l-UmSQUFHgssY,1618
 reconcile/requests_sender.py,sha256=g-tlrudvIqhneQPDMrfYF0Xsq7BSW2QcBPirl7hFM6I,4058
 reconcile/resource_scraper.py,sha256=TcMhXga7konX9x97NhpoijnDGWA-ZjdpiiXjm5qCmPk,2249
@@ -584,7 +584,7 @@ reconcile/unleash_feature_toggles/integration.py,sha256=Etp6D-UPkGy16IGHMsWBJscg
 reconcile/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/aggregated_list.py,sha256=IxQoMpKV2HHuZOxRmVIew2WQ7SutuLJ_VxlewieK3nY,4036
 reconcile/utils/amtool.py,sha256=Ng5VVNCiPYEK67eDjIwfuuTLs5JsfltCwt6w5UfXbcY,2289
-reconcile/utils/aws_api.py,sha256=fEf9Gtv0fz6NchYxJogjMQnMRfTLykmGSpYm7V52Ub4,61187
+reconcile/utils/aws_api.py,sha256=M2EQfDg2Aa8gKDh5aZDL8-Zd0vBXD7FQSySv-ivmYE4,62002
 reconcile/utils/aws_helper.py,sha256=9u4m94bnc3chFIDDEXwYXRXj5b--D6CTRy6g234jhNY,2972
 reconcile/utils/batches.py,sha256=TtEm64a8lWhFuNbUVpFEmXVdU2Q0sTBrP_I0Cjbgh7g,320
 reconcile/utils/binary.py,sha256=lSIevhilMeoGMePPHD7A-pxe45LVpBT0LksecYbM-EA,2477
@@ -800,7 +800,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=uQv2QJAmUXP1g2GPIH30WTlvL9soY6m9lefpZEVDM5w,3965
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=KcYVfa3UmJHVP_ocgrKe8NkrO5IDB9aWEDydSokPcRk,975
-qontract_reconcile-0.10.2.dev403.dist-info/METADATA,sha256=J9syq10cvCdSVu54yWmY5g4xeigFcWBapp8Qlc61bsk,24946
-qontract_reconcile-0.10.2.dev403.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev403.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev403.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev404.dist-info/METADATA,sha256=8yOHo8P888-GRCR1xE6xEJZyxETTQeIpsDS3cSlKIlc,24946
+qontract_reconcile-0.10.2.dev404.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev404.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev404.dist-info/RECORD,,

reconcile/aws_ami_share.py

@@ -1,17 +1,19 @@
 import logging
+import re
 from collections.abc import (
-    Callable,
     Iterable,
     Mapping,
 )
 from typing import Any
 
 from reconcile import queries
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.defer import defer
 
 QONTRACT_INTEGRATION = "aws-ami-share"
-MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
+
+MANAGED_TAG = {"managed_by_integration": QONTRACT_INTEGRATION}
 
 
 def filter_accounts(accounts: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -37,65 +39,70 @@ def get_region(
     return region
 
 
-@defer
-def run(dry_run: bool, defer: Callable | None = None) -> None:
+def share_ami(
+    dry_run: bool,
+    src_account: Mapping[str, Any],
+    share: Mapping[str, Any],
+    default_tags: dict[str, str],
+    aws_api: AWSApi,
+) -> None:
+    dst_account = share["account"]
+    regex = re.compile(share["regex"])
+    region = get_region(share, src_account, dst_account)
+    src_amis = aws_api.get_amis_details(src_account, src_account, regex, region)
+    dst_amis = aws_api.get_amis_details(dst_account, src_account, regex, region)
+
+    for ami_id, src_ami_tags in src_amis.items():
+        dst_ami_tags = dst_amis.get(ami_id)
+        if dst_ami_tags is None:
+            logging.info([
+                "share_ami",
+                src_account["name"],
+                dst_account["name"],
+                ami_id,
+            ])
+            if not dry_run:
+                aws_api.share_ami(src_account, dst_account["uid"], ami_id, region)
+        dst_account_tags = default_tags | get_aws_account_tags(
+            dst_account.get("organization", None)
+        )
+        desired_tags = src_ami_tags | dst_account_tags | MANAGED_TAG
+        current_tags = dst_ami_tags or {}
+
+        if desired_tags != current_tags:
+            logging.info([
+                "tag_shared_ami",
+                dst_account["name"],
+                ami_id,
+                desired_tags,
+            ])
+            if not dry_run:
+                aws_api.create_tags(dst_account, ami_id, desired_tags)
+
+
+def run(dry_run: bool) -> None:
     accounts = queries.get_aws_accounts(sharing=True)
     sharing_accounts = filter_accounts(accounts)
     settings = queries.get_app_interface_settings()
-    aws_api = AWSApi(1, sharing_accounts, settings=settings, init_users=False)
-    if defer:
-        defer(aws_api.cleanup)
-
-    for src_account in sharing_accounts:
-        sharing = src_account.get("sharing")
-        if not sharing:
-            continue
-        for share in sharing:
-            if share["provider"] != "ami":
-                continue
-            dst_account = share["account"]
-            regex = share["regex"]
-            region = get_region(share, src_account, dst_account)
-            src_amis = aws_api.get_amis_details(src_account, src_account, regex, region)
-            dst_amis = aws_api.get_amis_details(dst_account, src_account, regex, region)
-
-            for src_ami in src_amis:
-                src_ami_id = src_ami["image_id"]
-                found_dst_amis = [d for d in dst_amis if d["image_id"] == src_ami_id]
-                if not found_dst_amis:
-                    logging.info([
-                        "share_ami",
-                        src_account["name"],
-                        dst_account["name"],
-                        src_ami_id,
-                    ])
-                    if not dry_run:
-                        aws_api.share_ami(
-                            src_account, dst_account["uid"], src_ami_id, region
-                        )
-                    # we assume an unshared ami does not have tags
-                    found_dst_amis = [{"image_id": src_ami_id, "tags": []}]
-
-                dst_ami = found_dst_amis[0]
-                dst_ami_id = dst_ami["image_id"]
-                dst_ami_tags = dst_ami["tags"]
-                if MANAGED_TAG not in dst_ami_tags:
-                    logging.info([
-                        "tag_shared_ami",
-                        dst_account["name"],
-                        dst_ami_id,
-                        MANAGED_TAG,
-                    ])
-                    if not dry_run:
-                        aws_api.create_tag(dst_account, dst_ami_id, MANAGED_TAG)
-                src_ami_tags = src_ami["tags"]
-                for src_tag in src_ami_tags:
-                    if src_tag not in dst_ami_tags:
-                        logging.info([
-                            "tag_shared_ami",
-                            dst_account["name"],
-                            dst_ami_id,
-                            src_tag,
-                        ])
-                        if not dry_run:
-                            aws_api.create_tag(dst_account, dst_ami_id, src_tag)
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = {}
+
+    with AWSApi(
+        1,
+        sharing_accounts,
+        settings=settings,
+        init_users=False,
+    ) as aws_api:
+        for src_account in sharing_accounts:
+            for share in src_account.get("sharing") or []:
+                if share["provider"] == "ami":
+                    share_ami(
+                        dry_run=dry_run,
+                        src_account=src_account,
+                        share=share,
+                        default_tags=default_tags,
+                        aws_api=aws_api,
+                    )

reconcile/queries.py

@@ -505,6 +505,12 @@ AWS_ACCOUNTS_QUERY = """
         name
         uid
         supportedDeploymentRegions
+        organization {
+          payerAccount {
+            organizationAccountTags
+          }
+          tags
+        }
       }
       ... on AWSAccountSharingOptionAMI_v1 {
         regex

reconcile/utils/aws_api.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import operator
 import os
-import re
 from functools import lru_cache
 from threading import Lock
 from typing import (
@@ -25,6 +24,7 @@ import reconcile.utils.lean_terraform_client as terraform
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 
 if TYPE_CHECKING:
+    import re
     from collections.abc import (
         Iterable,
         Iterator,
@@ -1074,28 +1074,40 @@ class AWSApi:
         return [rt["RouteTableId"] for rt in vpc_route_tables]
 
     @staticmethod
-    def _filter_amis(
-        images: Iterable[ImageTypeDef], regex: str
-    ) -> list[dict[str, Any]]:
-        results = []
-        pattern = re.compile(regex)
-        for i in images:
-            if not re.search(pattern, i["Name"]):
-                continue
-            if i["State"] != "available":
-                continue
-            item = {"image_id": i["ImageId"], "tags": i.get("Tags", [])}
-            results.append(item)
+    def normalize_tags(tags: Iterable[TagTypeDef]) -> dict[str, str]:
+        return {tag["Key"]: tag["Value"] for tag in tags}
 
-        return results
+    @staticmethod
+    def _filter_amis(
+        images: Iterable[ImageTypeDef],
+        regex: re.Pattern,
+    ) -> dict[str, dict[str, str]]:
+        return {
+            image["ImageId"]: AWSApi.normalize_tags(image.get("Tags", []))
+            for image in images
+            if regex.search(image["Name"]) and image["State"] == "available"
+        }
 
     def get_amis_details(
         self,
         account: Mapping[str, Any],
         owner_account: Mapping[str, Any],
-        regex: str,
+        regex: re.Pattern,
         region: str | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, dict[str, str]]:
+        """
+        Get AMI details for an account, find AMI name matches regex and state is available.
+        Return ImageId and normalized tags.
+
+        Args:
+            account: AWS account
+            owner_account: AMI owner AWS account uid
+            regex: regex to filter AMI name
+            region: AWS account region
+
+        Returns:
+            dict[str, dict[str, str]]: Key is AMI ImageId, value is AMI normalized tags.
+        """
         ec2 = self._account_ec2_client(account["name"], region_name=region)
         images = self.get_account_amis(ec2, owner=owner_account["uid"])
         return self._filter_amis(images, regex)
@@ -1175,12 +1187,31 @@ class AWSApi:
         client = self._account_cloudwatch_client(account_name, region_name=region_name)
         client.delete_log_group(logGroupName=group_name)
 
-    def create_tag(
-        self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
+    def create_tags(
+        self,
+        account: Mapping[str, Any],
+        resource_id: str,
+        tags: Mapping[str, str],
     ) -> None:
+        """
+        Create tags on EC2 resources (AMI)
+
+        Args:
+            account: AWS account
+            resource_id: AWS resource id
+            tags: tags to update
+
+        Returns:
+            None
+        """
         ec2 = self._account_ec2_client(account["name"])
-        tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
-        ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
+        formatted_tags: list[TagTypeDef] = [
+            {"Key": k, "Value": v} for k, v in tags.items()
+        ]
+        ec2.create_tags(
+            Resources=[resource_id],
+            Tags=formatted_tags,
+        )
 
     def get_alb_network_interface_ips(
         self, account: awsh.Account, service_name: str