qontract-reconcile 0.10.2.dev334__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/acs/policies.py

@@ -11,7 +11,7 @@ class Scope(BaseModel):
     """
 
     cluster: str
-    namespace: str | None
+    namespace: str | None = None
 
 
 class PolicyCondition(BaseModel):
@@ -23,7 +23,7 @@ class PolicyCondition(BaseModel):
     """
 
     field_name: str
-    negate: bool | None
+    negate: bool | None = None
     values: list[str]
 
 

reconcile/utils/aggregated_list.py

@@ -1,8 +1,9 @@
-import json
 import logging
 from collections.abc import Callable, KeysView
 from typing import Any, TypedDict
 
+from reconcile.utils.json import json_dumps
+
 Action = Callable[[Any, list[Any]], bool]
 Cond = Callable[[Any], bool]
 
@@ -89,11 +90,11 @@ class AggregatedList:
         return list(self._dict.values())
 
     def to_json(self) -> str:
-        return json.dumps(self.dump(), indent=4)
+        return json_dumps(self.dump(), indent=4)
 
     @staticmethod
     def hash_params(params: Any) -> int:
-        return hash(json.dumps(params, sort_keys=True))
+        return hash(json_dumps(params))
 
 
 class AggregatedDiffRunner:

reconcile/utils/aws_api.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import operator
 import os
-import re
 from functools import lru_cache
 from threading import Lock
 from typing import (
@@ -25,6 +24,7 @@ import reconcile.utils.lean_terraform_client as terraform
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 
 if TYPE_CHECKING:
+    import re
     from collections.abc import (
         Iterable,
         Iterator,
@@ -1074,28 +1074,40 @@ class AWSApi:
         return [rt["RouteTableId"] for rt in vpc_route_tables]
 
     @staticmethod
-    def _filter_amis(
-        images: Iterable[ImageTypeDef], regex: str
-    ) -> list[dict[str, Any]]:
-        results = []
-        pattern = re.compile(regex)
-        for i in images:
-            if not re.search(pattern, i["Name"]):
-                continue
-            if i["State"] != "available":
-                continue
-            item = {"image_id": i["ImageId"], "tags": i.get("Tags", [])}
-            results.append(item)
+    def normalize_tags(tags: Iterable[TagTypeDef]) -> dict[str, str]:
+        return {tag["Key"]: tag["Value"] for tag in tags}
 
-        return results
+    @staticmethod
+    def _filter_amis(
+        images: Iterable[ImageTypeDef],
+        regex: re.Pattern,
+    ) -> dict[str, dict[str, str]]:
+        return {
+            image["ImageId"]: AWSApi.normalize_tags(image.get("Tags", []))
+            for image in images
+            if regex.search(image["Name"]) and image["State"] == "available"
+        }
 
     def get_amis_details(
         self,
         account: Mapping[str, Any],
         owner_account: Mapping[str, Any],
-        regex: str,
+        regex: re.Pattern,
         region: str | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, dict[str, str]]:
+        """
+        Get AMI details for an account, find AMI name matches regex and state is available.
+        Return ImageId and normalized tags.
+
+        Args:
+            account: AWS account
+            owner_account: AMI owner AWS account uid
+            regex: regex to filter AMI name
+            region: AWS account region
+
+        Returns:
+            dict[str, dict[str, str]]: Key is AMI ImageId, value is AMI normalized tags.
+        """
         ec2 = self._account_ec2_client(account["name"], region_name=region)
         images = self.get_account_amis(ec2, owner=owner_account["uid"])
         return self._filter_amis(images, regex)
@@ -1175,12 +1187,31 @@ class AWSApi:
         client = self._account_cloudwatch_client(account_name, region_name=region_name)
         client.delete_log_group(logGroupName=group_name)
 
-    def create_tag(
-        self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
+    def create_tags(
+        self,
+        account: Mapping[str, Any],
+        resource_id: str,
+        tags: Mapping[str, str],
     ) -> None:
+        """
+        Create tags on EC2 resources (AMI)
+
+        Args:
+            account: AWS account
+            resource_id: AWS resource id
+            tags: tags to update
+
+        Returns:
+            None
+        """
         ec2 = self._account_ec2_client(account["name"])
-        tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
-        ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
+        formatted_tags: list[TagTypeDef] = [
+            {"Key": k, "Value": v} for k, v in tags.items()
+        ]
+        ec2.create_tags(
+            Resources=[resource_id],
+            Tags=formatted_tags,
+        )
 
     def get_alb_network_interface_ips(
         self, account: awsh.Account, service_name: str

reconcile/utils/aws_api_typed/api.py

@@ -6,9 +6,11 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from boto3 import Session
+from botocore.config import Config
 from pydantic import BaseModel
 
 import reconcile.utils.aws_api_typed.account
+import reconcile.utils.aws_api_typed.cloudformation
 import reconcile.utils.aws_api_typed.dynamodb
 import reconcile.utils.aws_api_typed.iam
 import reconcile.utils.aws_api_typed.organization
@@ -17,8 +19,10 @@ import reconcile.utils.aws_api_typed.service_quotas
 import reconcile.utils.aws_api_typed.sts
 import reconcile.utils.aws_api_typed.support
 from reconcile.utils.aws_api_typed.account import AWSApiAccount
+from reconcile.utils.aws_api_typed.cloudformation import AWSApiCloudFormation
 from reconcile.utils.aws_api_typed.dynamodb import AWSApiDynamoDB
 from reconcile.utils.aws_api_typed.iam import AWSApiIam
+from reconcile.utils.aws_api_typed.logs import AWSApiLogs
 from reconcile.utils.aws_api_typed.organization import AWSApiOrganizations
 from reconcile.utils.aws_api_typed.s3 import AWSApiS3
 from reconcile.utils.aws_api_typed.service_quotas import AWSApiServiceQuotas
@@ -31,7 +35,9 @@ if TYPE_CHECKING:
 SubApi = TypeVar(
     "SubApi",
     AWSApiAccount,
+    AWSApiCloudFormation,
     AWSApiDynamoDB,
+    AWSApiLogs,
     AWSApiIam,
     AWSApiOrganizations,
     AWSApiS3,
@@ -40,6 +46,13 @@ SubApi = TypeVar(
     AWSApiSupport,
 )
 
+DEFAULT_CONFIG = Config(
+    retries={
+        "mode": "standard",
+        "total_max_attempts": 10,
+    },
+)
+
 
 class AWSCredentials(ABC):
     @abstractmethod
@@ -174,28 +187,34 @@ class AWSApi:
         """Return a new or cached sub api client."""
         match api_cls:
             case reconcile.utils.aws_api_typed.account.AWSApiAccount:
-                client = self.session.client("account")
+                client = self.session.client("account", config=DEFAULT_CONFIG)
+                api = api_cls(client)
+            case reconcile.utils.aws_api_typed.cloudformation.AWSApiCloudFormation:
+                client = self.session.client("cloudformation", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.dynamodb.AWSApiDynamoDB:
-                client = self.session.client("dynamodb")
+                client = self.session.client("dynamodb", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.iam.AWSApiIam:
-                client = self.session.client("iam")
+                client = self.session.client("iam", config=DEFAULT_CONFIG)
+                api = api_cls(client)
+            case reconcile.utils.aws_api_typed.logs.AWSApiLogs:
+                client = self.session.client("logs", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.organization.AWSApiOrganizations:
-                client = self.session.client("organizations")
+                client = self.session.client("organizations", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.s3.AWSApiS3:
-                client = self.session.client("s3")
+                client = self.session.client("s3", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.service_quotas.AWSApiServiceQuotas:
-                client = self.session.client("service-quotas")
+                client = self.session.client("service-quotas", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.sts.AWSApiSts:
-                client = self.session.client("sts")
+                client = self.session.client("sts", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.support.AWSApiSupport:
-                client = self.session.client("support")
+                client = self.session.client("support", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case _:
                 raise ValueError(f"Unknown API class: {api_cls}")
@@ -205,9 +224,14 @@ class AWSApi:
 
     @cached_property
     def account(self) -> AWSApiAccount:
-        """Return an AWS Acount Api client"""
+        """Return an AWS Account Api client"""
         return self._init_sub_api(AWSApiAccount)
 
+    @cached_property
+    def cloudformation(self) -> AWSApiCloudFormation:
+        """Return an AWS CloudFormation Api client"""
+        return self._init_sub_api(AWSApiCloudFormation)
+
     @cached_property
     def dynamodb(self) -> AWSApiDynamoDB:
         """Return an AWS DynamoDB Api client"""
@@ -218,6 +242,11 @@ class AWSApi:
         """Return an AWS IAM Api client."""
         return self._init_sub_api(AWSApiIam)
 
+    @cached_property
+    def logs(self) -> AWSApiLogs:
+        """Return an AWS Logs Api client."""
+        return self._init_sub_api(AWSApiLogs)
+
     @cached_property
     def organizations(self) -> AWSApiOrganizations:
         """Return an AWS Organizations Api client."""

reconcile/utils/aws_api_typed/cloudformation.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from reconcile.utils.json import json_dumps
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudformation import CloudFormationClient
+    from mypy_boto3_cloudformation.type_defs import (
+        ParameterTypeDef,
+        StackTypeDef,
+        TagTypeDef,
+    )
+
+
+class AWSApiCloudFormation:
+    def __init__(self, client: CloudFormationClient) -> None:
+        self.client = client
+
+    def create_stack(
+        self,
+        stack_name: str,
+        change_set_name: str,
+        template_body: str,
+        parameters: dict[str, str] | None = None,
+        tags: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Create a CloudFormation stack using a change set with import existing resources.
+        This method creates a change set of type "CREATE" with the provided template and parameters,
+        waits for the change set to be created, executes it, and then waits for the stack creation to complete.
+        It returns the StackId of the created stack.
+
+        Args:
+            stack_name (str): The name of the stack to create.
+            change_set_name (str): The name of the change set to create.
+            template_body (str): The CloudFormation template body as a string.
+            parameters (dict[str, str] | None): A dictionary of parameter key-value pairs for
+                the stack. Defaults to None.
+            tags (dict[str, str] | None): A dictionary of tag key-value pairs to
+                associate with the stack. Defaults to None.
+
+        Returns:
+            str: The StackId of the created stack.
+        """
+        response = self.client.create_change_set(
+            StackName=stack_name,
+            ChangeSetName=change_set_name,
+            TemplateBody=template_body,
+            ChangeSetType="CREATE",
+            Parameters=self._build_parameters(parameters or {}),
+            Tags=self._build_tags(tags or {}),
+            ImportExistingResources=True,
+        )
+        change_set_arn = response["Id"]
+        self.client.get_waiter("change_set_create_complete").wait(
+            ChangeSetName=change_set_arn
+        )
+        self.client.execute_change_set(ChangeSetName=change_set_arn)
+        self.client.get_waiter("stack_create_complete").wait(StackName=stack_name)
+        return response["StackId"]
+
+    def update_stack(
+        self,
+        stack_name: str,
+        template_body: str,
+        parameters: dict[str, str] | None = None,
+        tags: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Update a CloudFormation stack with the provided template and parameters.
+        This method updates the specified stack, waits for the update to complete,
+        and returns the StackId of the updated stack.
+
+        Args:
+            stack_name (str): The name of the stack to update.
+            template_body (str): The CloudFormation template body as a string.
+            parameters (dict[str, str] | None): A dictionary of parameter key-value pairs for
+                the stack. Defaults to None.
+            tags (dict[str, str] | None): A dictionary of tag key-value pairs to
+                associate with the stack. Defaults to None.
+
+        Returns:
+            str: The StackId of the updated stack.
+        """
+        response = self.client.update_stack(
+            StackName=stack_name,
+            TemplateBody=template_body,
+            Parameters=self._build_parameters(parameters or {}),
+            Tags=self._build_tags(tags or {}),
+        )
+        self.client.get_waiter("stack_update_complete").wait(StackName=stack_name)
+        return response["StackId"]
+
+    def get_stack(self, stack_name: str) -> StackTypeDef | None:
+        """
+        Retrieve information about a CloudFormation stack by its name.
+        If the stack exists, it returns the stack details as a dictionary.
+        If the stack does not exist, it returns None.
+
+        Args:
+            stack_name (str): The name of the stack to retrieve.
+
+        Returns:
+            StackTypeDef | None: The stack details if found, otherwise None.
+        """
+        try:
+            response = self.client.describe_stacks(StackName=stack_name)
+            return response["Stacks"][0]
+        except self.client.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "ValidationError":
+                return None
+            raise
+
+    def get_template_body(self, stack_name: str) -> str:
+        """
+        Retrieve the CloudFormation template body for a specified stack.
+
+        Args:
+            stack_name (str): The name of the stack whose template is to be retrieved.
+
+        Returns:
+            str: The CloudFormation template body as a string.
+        """
+        response = self.client.get_template(StackName=stack_name)
+        # TemplateBody is str when using yaml
+        if isinstance(response["TemplateBody"], str):
+            return response["TemplateBody"]
+        return json_dumps(response["TemplateBody"])
+
+    @staticmethod
+    def _build_parameters(parameters: dict[str, str]) -> list[ParameterTypeDef]:
+        return [
+            {
+                "ParameterKey": key,
+                "ParameterValue": value,
+            }
+            for key, value in sorted(parameters.items())
+        ]
+
+    @staticmethod
+    def _build_tags(tags: dict[str, str]) -> list[TagTypeDef]:
+        return [
+            {
+                "Key": key,
+                "Value": value,
+            }
+            for key, value in sorted(tags.items())
+        ]

reconcile/utils/aws_api_typed/logs.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from mypy_boto3_logs import CloudWatchLogsClient
+    from mypy_boto3_logs.type_defs import LogGroupTypeDef
+
+
+class AWSApiLogs:
+    def __init__(self, client: CloudWatchLogsClient) -> None:
+        self.client = client
+
+    def get_log_groups(self) -> Iterator[LogGroupTypeDef]:
+        paginator = self.client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            yield from page["logGroups"]
+
+    def delete_log_group(self, log_group_name: str) -> None:
+        self.client.delete_log_group(logGroupName=log_group_name)
+
+    def put_retention_policy(
+        self,
+        log_group_name: str,
+        retention_in_days: int,
+    ) -> None:
+        self.client.put_retention_policy(
+            logGroupName=log_group_name,
+            retentionInDays=retention_in_days,
+        )
+
+    def get_tags(self, arn: str) -> dict[str, str]:
+        tags = self.client.list_tags_for_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+        )
+        return tags.get("tags") or {}
+
+    def set_tags(
+        self,
+        arn: str,
+        tags: dict[str, str],
+    ) -> None:
+        self.client.tag_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+            tags=tags,
+        )
+
+    def delete_tags(
+        self,
+        arn: str,
+        tag_keys: Iterable[str],
+    ) -> None:
+        self.client.untag_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+            tagKeys=list(tag_keys),
+        )
+
+    @staticmethod
+    def _normalize_log_group_arn(arn: str) -> str:
+        """
+        Normalize a log group ARN by removing any trailing ":*".
+
+        DescribeLogGroups response arn has additional :* at the end.
+
+        Args:
+            arn: The ARN of the log group.
+
+        Returns:
+            The normalized ARN without the trailing ":*".
+        """
+        return arn.rstrip(":*")

reconcile/utils/aws_api_typed/organization.py

@@ -52,9 +52,11 @@ class AwsOrganizationOU(BaseModel):
 class AWSAccountStatus(BaseModel):
     id: str = Field(..., alias="Id")
     name: str = Field(..., alias="AccountName")
-    uid: str | None = Field(alias="AccountId")
+    uid: str | None = Field(None, alias="AccountId")
     state: str = Field(..., alias="State")
-    failure_reason: CreateAccountFailureReasonType | None = Field(alias="FailureReason")
+    failure_reason: CreateAccountFailureReasonType | None = Field(
+        None, alias="FailureReason"
+    )
 
 
 class AWSAccount(BaseModel):

reconcile/utils/binary.py

@@ -38,10 +38,7 @@ def binary_version(
     def deco_binary_version(f: Callable) -> Callable:
         @wraps(f)
         def f_binary_version(*args: Any, **kwargs: Any) -> None:
-            regex = re.compile(search_regex)
-
-            cmd = [binary]
-            cmd.extend(version_args)
+            cmd = [binary, *version_args]
             try:
                 result = subprocess.run(cmd, capture_output=True, check=True)
             except subprocess.CalledProcessError as e:
@@ -50,15 +47,13 @@ def binary_version(
                 )
                 raise Exception(msg) from e
 
-            found = False
-            match = None
-            for line in result.stdout.splitlines():
-                match = regex.search(line.decode("utf-8"))
-                if match is not None:
-                    found = True
-                    break
+            match = re.search(
+                search_regex,
+                result.stdout.decode("utf-8"),
+                re.MULTILINE,
+            )
 
-            if not found or not match:
+            if match is None:
                 raise Exception(
                     f"Could not find version for binary '{binary}' via regex "
                     f"for binary version check: "

reconcile/utils/datetime_util.py

@@ -0,0 +1,67 @@
+from datetime import UTC, datetime
+
+ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
+ISO8601_MICRO = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+def utc_now() -> datetime:
+    """
+    Get the current UTC datetime.
+
+    Returns:
+        A datetime object representing the current time in UTC.
+    """
+    return datetime.now(tz=UTC)
+
+
+def ensure_utc(dt: datetime) -> datetime:
+    """
+    Ensure the provided datetime is in UTC timezone.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A datetime object in UTC timezone.
+    """
+    if dt.tzinfo is None:
+        return dt.replace(tzinfo=UTC)
+    return dt.astimezone(UTC)
+
+
+def to_utc_seconds_iso_format(dt: datetime) -> str:
+    """
+    Convert a datetime object to ISO 8601 format YYYY-MM-DDTHH:MM:SSZ.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A string representing the datetime in ISO 8601 format.
+    """
+    return ensure_utc(dt).strftime(ISO8601)
+
+
+def to_utc_microseconds_iso_format(dt: datetime) -> str:
+    """
+    Convert a datetime object to ISO 8601 format with microseconds YYYY-MM-DDTHH:MM:SS.mmmmmmZ.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A string representing the datetime in ISO 8601 format with microseconds.
+    """
+    return ensure_utc(dt).strftime(ISO8601_MICRO)
+
+
+def from_utc_iso_format(dt_str: str) -> datetime:
+    """
+    Parse a datetime string in ISO 8601 format to a datetime object.
+
+    Args:
+        dt_str: A string representing the datetime in ISO 8601 format.
+    Returns:
+        A datetime object in UTC timezone.
+    """
+    return ensure_utc(datetime.fromisoformat(dt_str))

reconcile/utils/deadmanssnitch_api.py

@@ -26,7 +26,7 @@ class Snitch(BaseModel):
     interval: str
     alert_type: str
     alert_email: list[str]
-    vault_data: str | None
+    vault_data: str | None = None
 
     def needs_vault_update(self) -> bool:
         return self.vault_data is not None and self.check_in_url != self.vault_data

reconcile/utils/differ.py

@@ -7,7 +7,6 @@ from collections.abc import (
 from dataclasses import dataclass
 from typing import (
     Any,
-    Generic,
     TypeVar,
 )
 
@@ -18,13 +17,13 @@ Key = TypeVar("Key")
 
 
 @dataclass(frozen=True, eq=True)
-class DiffPair(Generic[Current, Desired]):
+class DiffPair[Current, Desired]:
     current: Current
     desired: Desired
 
 
 @dataclass(frozen=True, eq=True)
-class DiffResult(Generic[Current, Desired, Key]):
+class DiffResult[Current, Desired, Key]:
     add: dict[Key, Desired]
     delete: dict[Key, Current]
     change: dict[Key, DiffPair[Current, Desired]]