qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/aws_api.py

@@ -1,9 +1,12 @@
 import logging
+import os
 import re
 import time
 from collections.abc import (
     Iterable,
+    Iterator,
     Mapping,
+    Sequence,
 )
 from datetime import datetime
 from functools import lru_cache
@@ -12,12 +15,11 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Literal,
-    Optional,
-    Union,
 )
 
 import botocore
 from boto3 import Session
+from botocore.config import Config
 from pydantic import BaseModel
 from sretoolbox.utils import threaded
 
@@ -39,10 +41,12 @@ if TYPE_CHECKING:
         TagTypeDef,
         TransitGatewayTypeDef,
         TransitGatewayVpcAttachmentTypeDef,
+        VpcEndpointTypeDef,
         VpcTypeDef,
     )
     from mypy_boto3_iam import IAMClient
     from mypy_boto3_iam.type_defs import AccessKeyMetadataTypeDef
+    from mypy_boto3_organizations import OrganizationsClient
     from mypy_boto3_rds import RDSClient
     from mypy_boto3_rds.type_defs import (
         DBInstanceMessageTypeDef,
@@ -54,40 +58,19 @@ if TYPE_CHECKING:
         ResourceRecordSetTypeDef,
         ResourceRecordTypeDef,
     )
+    from mypy_boto3_s3 import S3Client
 else:
-    EC2Client = (
-        EC2ServiceResource
-    ) = (
-        RouteTableTypeDef
-    ) = (
-        SubnetTypeDef
-    ) = (
+    EC2Client = EC2ServiceResource = RouteTableTypeDef = SubnetTypeDef = (
         TransitGatewayTypeDef
-    ) = (
-        TransitGatewayVpcAttachmentTypeDef
-    ) = (
-        VpcTypeDef
-    ) = (
-        IAMClient
-    ) = (
+    ) = TransitGatewayVpcAttachmentTypeDef = VpcTypeDef = IAMClient = (
         AccessKeyMetadataTypeDef
-    ) = (
-        ImageTypeDef
-    ) = (
-        TagTypeDef
-    ) = (
-        LaunchPermissionModificationsTypeDef
-    ) = (
+    ) = ImageTypeDef = TagTypeDef = LaunchPermissionModificationsTypeDef = (
         FilterTypeDef
-    ) = (
-        Route53Client
-    ) = (
-        ResourceRecordSetTypeDef
-    ) = (
-        ResourceRecordTypeDef
-    ) = (
+    ) = Route53Client = ResourceRecordSetTypeDef = ResourceRecordTypeDef = (
         HostedZoneTypeDef
-    ) = RDSClient = DBInstanceMessageTypeDef = UpgradeTargetTypeDef = object
+    ) = RDSClient = DBInstanceMessageTypeDef = UpgradeTargetTypeDef = (
+        OrganizationsClient
+    ) = S3Client = object
 
 
 class InvalidResourceTypeError(Exception):
@@ -98,7 +81,7 @@ class MissingARNError(Exception):
     pass
 
 
-KeyStatus = Union[Literal["Active"], Literal["Inactive"]]
+KeyStatus = Literal["Active"] | Literal["Inactive"]
 
 GOVCLOUD_PARTITION = "aws-us-gov"
 
@@ -153,6 +136,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
         self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
         self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
+        self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
 
     def init_sessions_and_resources(self, accounts: Iterable[awsh.Account]):
         results = threaded.run(
@@ -168,6 +152,13 @@ class AWSApi:  # pylint: disable=too-many-public-methods
             access_key = secret["aws_access_key_id"]
             secret_key = secret["aws_secret_access_key"]
             region_name = account["resourcesDefaultRegion"]
+            self.use_fips = False
+
+            # ensure that govcloud accounts use FIPs endpoints
+            if "partition" in account and account["partition"] == GOVCLOUD_PARTITION:
+                logging.debug(f"FIPS endpoint enabled for AWS account: {account_name}")
+                self.use_fips = True
+
             session = Session(
                 aws_access_key_id=access_key,
                 aws_secret_access_key=secret_key,
@@ -198,45 +189,67 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         self,
         session: Session,
         service_name,
-        region_name: Optional[str] = None,
+        region_name: str | None = None,
     ):
         region = region_name if region_name else session.region_name
-        client = session.client(service_name, region_name=region)
+        client = session.client(
+            service_name,
+            region_name=region,
+            config=Config(use_fips_endpoint=self.use_fips),
+        )
         self._session_clients.append(client)
         return client
 
     @staticmethod
     # pylint: disable=method-hidden
     def _get_session_resource(
-        session: Session, service_name, region_name: Optional[str] = None
+        session: Session, service_name, region_name: str | None = None
     ):
         region = region_name if region_name else session.region_name
         return session.resource(service_name, region_name=region)
 
     def _account_ec2_client(
-        self, account_name: str, region_name: Optional[str] = None
+        self, account_name: str, region_name: str | None = None
     ) -> EC2Client:
         session = self.get_session(account_name)
         return self.get_session_client(session, "ec2", region_name)
 
     def _account_ec2_resource(
-        self, account_name: str, region_name: Optional[str] = None
+        self, account_name: str, region_name: str | None = None
     ) -> EC2ServiceResource:
         session = self.get_session(account_name)
         return self._get_session_resource(session, "ec2", region_name)
 
     def _account_route53_client(
-        self, account_name: str, region_name: Optional[str] = None
+        self, account_name: str, region_name: str | None = None
     ) -> Route53Client:
         session = self.get_session(account_name)
         return self.get_session_client(session, "route53", region_name)
 
     def _account_rds_client(
-        self, account_name: str, region_name: Optional[str] = None
+        self, account_name: str, region_name: str | None = None
     ) -> RDSClient:
         session = self.get_session(account_name)
         return self.get_session_client(session, "rds", region_name)
 
+    def _account_cloudwatch_client(
+        self, account_name: str, region_name: str | None = None
+    ):
+        session = self.get_session(account_name)
+        return self.get_session_client(session, "logs", region_name)
+
+    def _account_organizations_client(
+        self, account_name: str, region_name: str | None = None
+    ) -> OrganizationsClient:
+        session = self.get_session(account_name)
+        return self.get_session_client(session, "organizations", region_name)
+
+    def _account_s3_client(
+        self, account_name: str, region_name: str | None = None
+    ) -> S3Client:
+        session = self.get_session(account_name)
+        return self.get_session_client(session, "s3", region_name)
+
     def init_users(self):
         self.users = {}
         for account, s in self.sessions.items():
@@ -478,7 +491,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
     @staticmethod
     def resource_has_special_name(account, type, resource):
-        skip_msg = "[{}] skipping {} ".format(account, type) + "({} related) {}"
+        skip_msg = f"[{account}] skipping {type} " + "({} related) {}"
 
         ignore_names = {
             "production": ["prod"],
@@ -495,7 +508,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         return False
 
     def resource_has_special_tags(self, account, type, resource, tags):
-        skip_msg = "[{}] skipping {} ".format(account, type) + "({}={}) {}"
+        skip_msg = f"[{account}] skipping {type} " + "({}={}) {}"
 
         ignore_tags = {
             "ENV": ["prod", "stage", "staging"],
@@ -601,8 +614,8 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         if managed_by_integration_tag[0] == "terraform_resources":
             return "service_account"
 
-        huh = "unrecognized managed_by_integration tag: {}".format(
-            managed_by_integration_tag[0]
+        huh = (
+            f"unrecognized managed_by_integration tag: {managed_by_integration_tag[0]}"
         )
         raise InvalidResourceTypeError(huh)
 
@@ -725,7 +738,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
     def get_user_key_status(self, iam: IAMClient, user: str, key: str) -> KeyStatus:
         key_list = self._get_user_key_list(iam, user)
-        return [k["Status"] for k in key_list if k["AccessKeyId"] == key][0]
+        return next(k["Status"] for k in key_list if k["AccessKeyId"] == key)
 
     def get_support_cases(self):
         all_support_cases = {}
@@ -796,17 +809,20 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         self.auth_tokens = auth_tokens
 
     @staticmethod
-    def _get_account_assume_data(account: awsh.Account) -> tuple[str, str, str]:
+    def _get_account_assume_data(
+        account: awsh.Account,
+    ) -> tuple[str, str | None, str]:
         """
         returns mandatory data to be able to assume a role with this account:
         (account_name, assume_role, assume_region)
+        assume_role may be None for ROSA (CCS) clusters where we own the account
         """
-        required_keys = ["name", "assume_role", "assume_region"]
-        ok = all(elem in account.keys() for elem in required_keys)
+        required_keys = ["name", "assume_region"]
+        ok = all(elem in account for elem in required_keys)
         if not ok:
             account_name = account.get("name")
-            raise KeyError("[{}] account is missing required keys".format(account_name))
-        return (account["name"], account["assume_role"], account["assume_region"])
+            raise KeyError(f"[{account_name}] account is missing required keys")
+        return (account["name"], account.get("assume_role"), account["assume_region"])
 
     @staticmethod
     # pylint: disable=method-hidden
@@ -843,9 +859,17 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         return assumed_session
 
     def _get_assumed_role_client(
-        self, account_name: str, assume_role: str, assume_region: str, client_type="ec2"
+        self,
+        account_name: str,
+        assume_role: str | None,
+        assume_region: str,
+        client_type="ec2",
     ) -> EC2Client:
         session = self.get_session(account_name)
+        if not assume_role:
+            return self.get_session_client(
+                session, client_type, region_name=assume_region
+            )
         sts = self.get_session_client(session, "sts")
         assumed_session = self._get_assume_role_session(
             sts, account_name, assume_role, assume_region
@@ -867,7 +891,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
     # filters a list of aws resources according to tags
     @staticmethod
     def filter_on_tags(
-        items: Iterable[Any], tags: Optional[Mapping[str, str]] = None
+        items: Iterable[Any], tags: Mapping[str, str] | None = None
     ) -> list[Any]:
         if tags is None:
             tags = {}
@@ -892,12 +916,15 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
         return subnets.get("Subnets", [])
 
-    def get_cluster_vpc_details(self, account, route_tables=False, subnets=False):
+    def get_cluster_vpc_details(
+        self, account, route_tables=False, subnets=False, hcp_vpc_endpoint_sg=False
+    ):
         """
         Returns a cluster VPC details:
             - VPC ID
             - Route table IDs (optional)
             - Subnets list including Subnet ID and Subnet Availability zone
+            - VPC Endpoint default security group of the private API router (optional)
         :param account: a dictionary containing the following keys:
                         - name - name of the AWS account
                         - assume_role - role to assume to get access
@@ -917,6 +944,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
         route_table_ids = None
         subnets_id_az = None
+        api_security_group_id = None
         if vpc_id:
             if route_tables:
                 vpc_route_tables = self.get_vpc_route_tables(vpc_id, assumed_ec2)
@@ -927,15 +955,55 @@ class AWSApi:  # pylint: disable=too-many-public-methods
                     {"id": s["SubnetId"], "az": s["AvailabilityZone"]}
                     for s in vpc_subnets
                 ]
+            if hcp_vpc_endpoint_sg:
+                api_security_group_id = self._get_api_security_group_id(
+                    assumed_ec2, vpc_id
+                )
 
-        return vpc_id, route_table_ids, subnets_id_az
+        return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
 
-    def get_cluster_nat_gateways_egress_ips(self, account):
+    def _get_api_security_group_id(self, assumed_ec2, vpc_id):
+        endpoints = AWSApi._get_vpc_endpoints(
+            [
+                {"Name": "vpc-id", "Values": [vpc_id]},
+                {
+                    "Name": "tag:AWSEndpointService",
+                    "Values": ["private-router"],
+                },
+            ],
+            assumed_ec2,
+        )
+        if not endpoints:
+            return None
+        if len(endpoints) > 1:
+            raise ValueError(
+                f"exactly one VPC endpoint for private API router in VPC {vpc_id} expected but {len(endpoints)} found"
+            )
+        endpoint = endpoints[0]
+        vpc_endpoint_id = endpoint["VpcEndpointId"]
+        # https://github.com/openshift/hypershift/blob/c855f68e84e78924ccc9c2132b75dc7e30c4e1d8/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go#L4243
+        # https://github.com/openshift/hypershift/blob/2569f3353ef5ac0858eace9ee77310c3cc38b8e0/control-plane-operator/controllers/awsprivatelink/awsprivatelink_controller.go#L787
+        security_groups = [
+            sg
+            for sg in endpoint["Groups"]
+            if sg["GroupName"].endswith("-default-sg")
+            or sg["GroupName"].endswith("-vpce-private-router")
+        ]
+        if len(security_groups) != 1:
+            raise ValueError(
+                f"exactly one VPC endpoint default security group for private API router {vpc_endpoint_id} "
+                f"in VPC {vpc_id} expected but {len(security_groups)} found"
+            )
+        return security_groups[0]["GroupId"]
+
+    def get_cluster_nat_gateways_egress_ips(self, account: dict[str, Any], vpc_id: str):
         assumed_role_data = self._get_account_assume_data(account)
         assumed_ec2 = self._get_assumed_role_client(*assumed_role_data)
         nat_gateways = assumed_ec2.describe_nat_gateways()
         egress_ips = set()
-        for nat in nat_gateways.get("NatGateways"):
+        for nat in nat_gateways.get("NatGateways") or []:
+            if nat["VpcId"] != vpc_id:
+                continue
             for address in nat["NatGatewayAddresses"]:
                 egress_ips.add(address["PublicIp"])
 
@@ -994,7 +1062,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         account: Mapping[str, Any],
         owner_account: Mapping[str, Any],
         regex: str,
-        region: Optional[str] = None,
+        region: str | None = None,
     ) -> list[dict[str, Any]]:
         ec2 = self._account_ec2_client(account["name"], region_name=region)
         images = self.get_account_amis(ec2, owner=owner_account["uid"])
@@ -1005,7 +1073,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         account: Mapping[str, Any],
         share_account_uid: str,
         image_id: str,
-        region: Optional[str] = None,
+        region: str | None = None,
     ):
         ec2 = self._account_ec2_resource(account["name"], region)
         image = ec2.Image(image_id)
@@ -1014,6 +1082,67 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         }
         image.modify_attribute(LaunchPermission=launch_permission)
 
+    @staticmethod
+    def _normalize_log_group_arn(arn: str) -> str:
+        # DescribeLogGroups response arn has additional :* at the end
+        return arn.rstrip(":*")
+
+    def create_cloudwatch_tag(
+        self,
+        account_name: str,
+        arn: str,
+        new_tag: dict[str, str],
+        region_name: str | None = None,
+    ) -> None:
+        client = self._account_cloudwatch_client(account_name, region_name=region_name)
+        client.tag_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+            tags=new_tag,
+        )
+
+    def get_cloudwatch_log_groups(
+        self,
+        account_name: str,
+        region_name: str | None = None,
+    ) -> Iterator[dict]:
+        client = self._account_cloudwatch_client(account_name, region_name=region_name)
+        paginator = client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            yield from page["logGroups"]
+
+    def get_cloudwatch_log_group_tags(
+        self,
+        account_name: str,
+        arn: str,
+        region_name: str | None = None,
+    ) -> dict[str, str]:
+        client = self._account_cloudwatch_client(account_name, region_name=region_name)
+        tags = client.list_tags_for_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+        )
+        return tags.get("tags", {})
+
+    def set_cloudwatch_log_retention(
+        self,
+        account_name: str,
+        group_name: str,
+        retention_days: int,
+        region_name: str | None = None,
+    ):
+        client = self._account_cloudwatch_client(account_name, region_name=region_name)
+        client.put_retention_policy(
+            logGroupName=group_name, retentionInDays=retention_days
+        )
+
+    def delete_cloudwatch_log_group(
+        self,
+        account_name: str,
+        group_name: str,
+        region_name: str | None = None,
+    ):
+        client = self._account_cloudwatch_client(account_name, region_name=region_name)
+        client.delete_log_group(logGroupName=group_name)
+
     def create_tag(
         self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
     ):
@@ -1053,7 +1182,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
     @staticmethod
     # pylint: disable=method-hidden
-    def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> Optional[str]:
+    def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> str | None:
         vpc_security_groups = ec2.describe_security_groups(
             Filters=[
                 {"Name": "vpc-id", "Values": [vpc_id]},
@@ -1074,7 +1203,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
     def get_tgw_default_route_table_id(
         self, ec2: EC2Client, tgw_id: str, tags: Mapping[str, str]
-    ) -> Optional[str]:
+    ) -> str | None:
         tgws = self.get_transit_gateways(ec2)
         tgws = self.filter_on_tags(tgws, tags)
         # we know the party TGW exists, so we can be
@@ -1245,6 +1374,14 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
         return results
 
+    @staticmethod
+    # pylint: disable=method-hidden
+    def _get_vpc_endpoints(
+        filters: Sequence[FilterTypeDef], ec2: EC2Client
+    ) -> list["VpcEndpointTypeDef"]:
+        atts = ec2.describe_vpc_endpoints(Filters=filters)
+        return atts.get("VpcEndpoints", [])
+
     @staticmethod
     def _get_hosted_zone_id(zone: HostedZoneTypeDef) -> str:
         # 'Id': '/hostedzone/THISISTHEZONEID'
@@ -1257,7 +1394,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         if not zones:
             return []
         zone_id = self._get_hosted_zone_id(zones[0])
-        return route53.list_resource_record_sets(HostedZoneId=zone_id)[
+        return route53.list_resource_record_sets(HostedZoneId=zone_id)[  # type: ignore[return-value]
             "ResourceRecordSets"
         ]
 
@@ -1425,7 +1562,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
 
     def get_image_id(
         self, account_name: str, region_name: str, tags: Iterable[AmiTag]
-    ) -> Optional[str]:
+    ) -> str | None:
         """
         Get AMI ID matching the specified criteria.
 
@@ -1454,7 +1591,7 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         self,
         account_name: str,
         db_instance_name: str,
-        region_name: Optional[str] = None,
+        region_name: str | None = None,
     ) -> DBInstanceMessageTypeDef:
         """
         Describe a single RDS instance.
@@ -1471,12 +1608,20 @@ class AWSApi:  # pylint: disable=too-many-public-methods
         rds = self._account_rds_client(account_name, **optional_kwargs)
         return rds.describe_db_instances(DBInstanceIdentifier=db_instance_name)
 
+    def describe_rds_recommendations(
+        self,
+        account_name: str,
+        region_name: str | None = None,
+    ):
+        rds = self._account_rds_client(account_name, region_name)
+        return rds.describe_db_recommendations()
+
     def get_db_valid_upgrade_target(
         self,
         account_name: str,
         engine: str,
         engine_version: str,
-        region_name: Optional[str] = None,
+        region_name: str | None = None,
     ) -> list[UpgradeTargetTypeDef]:
         """
         Get a list version of the database engine that a DB instance can be upgraded to.
@@ -1493,6 +1638,74 @@ class AWSApi:  # pylint: disable=too-many-public-methods
             optional_kwargs["region_name"] = region_name
 
         rds = self._account_rds_client(account_name, **optional_kwargs)
-        return rds.describe_db_engine_versions(
-            Engine=engine, EngineVersion=engine_version
-        )["DBEngineVersions"][0]["ValidUpgradeTarget"]
+        response = rds.describe_db_engine_versions(
+            Engine=engine,
+            EngineVersion=engine_version,
+            IncludeAll=True,
+        )
+
+        if versions := response["DBEngineVersions"]:
+            return versions[0]["ValidUpgradeTarget"]
+        return []
+
+    def describe_db_parameter_group(
+        self,
+        account_name: str,
+        db_parameter_group_name: str,
+        region_name: str | None = None,
+    ) -> dict[str, str]:
+        optional_kwargs = {}
+
+        if region_name:
+            optional_kwargs["region_name"] = region_name
+
+        rds = self._account_rds_client(account_name, **optional_kwargs)
+        paginator = rds.get_paginator("describe_db_parameters")
+        parameters = {}
+        for page in paginator.paginate(DBParameterGroupName=db_parameter_group_name):
+            for param in page.get("Parameters", []):
+                parameters[param["ParameterName"]] = param.get("ParameterValue", "")
+        return parameters
+
+    def get_organization_billing_account(self, account_name: str) -> str:
+        org = self._account_organizations_client(account_name)
+        return org.describe_organization()["Organization"]["MasterAccountId"]
+
+    def get_s3_object_content(
+        self,
+        account_name: str,
+        bucket_name: str,
+        path: str,
+        region_name: str | None = None,
+    ) -> str:
+        s3 = self._account_s3_client(account_name, region_name=region_name)
+        return (
+            s3.get_object(Bucket=bucket_name, Key=path)["Body"].read().decode("utf-8")
+        )
+
+    def list_s3_objects(
+        self,
+        account_name: str,
+        bucket_name: str,
+        path: str,
+        region_name: str | None = None,
+    ) -> list[str]:
+        s3 = self._account_s3_client(account_name, region_name=region_name)
+        objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=path, Delimiter="/")[
+            "Contents"
+        ]
+        return [
+            obj["Key"]
+            for obj in sorted(
+                objects, key=lambda obj: obj["LastModified"], reverse=True
+            )
+        ]
+
+
+def aws_config_file_path() -> str | None:
+    config_file_path = os.path.expanduser(
+        os.environ.get("AWS_CONFIG_FILE", "~/.aws/config")
+    )
+    if not os.path.isfile(config_file_path):
+        return None
+    return config_file_path

reconcile/utils/aws_api_typed/account.py

@@ -0,0 +1,23 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mypy_boto3_account import AccountClient
+else:
+    AccountClient = object
+
+
+class AWSApiAccount:
+    def __init__(self, client: AccountClient) -> None:
+        self.client = client
+
+    def set_security_contact(
+        self, name: str, title: str, email: str, phone_number: str
+    ) -> None:
+        """Set the security contact for the account."""
+        self.client.put_alternate_contact(
+            AlternateContactType="SECURITY",
+            EmailAddress=email,
+            Name=name,
+            Title=title,
+            PhoneNumber=phone_number,
+        )