qontract-reconcile 0.10.2.dev349__py3-none-any.whl → 0.10.2.dev414__py3-none-any.whl

reconcile/terraform_vpc_resources/integration.py

@@ -20,6 +20,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.aws_vpc_requests import get_aws_vpc_requests
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
@@ -161,13 +162,21 @@ class TerraformVpcResources(QontractReconcileIntegration[TerraformVpcResourcesPa
             logging.debug("No VPC requests found, nothing to do.")
             sys.exit(ExitCodes.SUCCESS)
 
-        accounts_untyped: list[dict] = [acc.dict(by_alias=True) for acc in accounts]
+        accounts_untyped: list[dict] = [
+            acc.model_dump(by_alias=True) for acc in accounts
+        ]
+        try:
+            default_tags = get_settings().default_tags
+        except ValueError:
+            # no external resources settings found
+            default_tags = None
         with TerrascriptClient(
             integration=QONTRACT_INTEGRATION,
             integration_prefix=QONTRACT_TF_PREFIX,
             thread_pool_size=thread_pool_size,
             accounts=accounts_untyped,
             secret_reader=secret_reader,
+            default_tags=default_tags,
         ) as ts_client:
             ts_client.populate_vpc_requests(data, AWS_PROVIDER_VERSION)
 

reconcile/typed_queries/aws_account_tags.py

@@ -0,0 +1,41 @@
+import json
+from collections.abc import Mapping
+
+from reconcile.gql_definitions.fragments.aws_organization import (
+    AWSOrganization,
+)
+
+
+def get_aws_account_tags(
+    organization: AWSOrganization | Mapping | None,
+) -> dict[str, str]:
+    """
+    Get AWS account tags by merging payer account tags
+
+    Args:
+        organization: AWSOrganization | Mapping | None - The organization object from which to extract tags.
+
+    Returns:
+        dict[str, str]: A dictionary containing the merged tags from the payer account and the account itself.
+    """
+    if organization is None:
+        return {}
+
+    match organization:
+        case AWSOrganization():
+            payer_account_tags = (
+                organization.payer_account.organization_account_tags or {}
+            )
+            account_tags = organization.tags or {}
+        case Mapping():
+            payer_account_tags = organization.get("payerAccount", {}).get(
+                "organizationAccountTags", {}
+            )
+            if isinstance(payer_account_tags, str):
+                payer_account_tags = json.loads(payer_account_tags)
+
+            account_tags = organization.get("tags", {})
+            if isinstance(account_tags, str):
+                account_tags = json.loads(account_tags)
+
+    return payer_account_tags | account_tags

reconcile/typed_queries/cost_report/app_names.py

@@ -6,7 +6,7 @@ from reconcile.utils.gql import GqlApi
 
 class App(BaseModel):
     name: str
-    parent_app_name: str | None
+    parent_app_name: str | None = None
 
 
 def get_app_names(

reconcile/typed_queries/cost_report/cost_namespaces.py

@@ -13,7 +13,7 @@ class CostNamespace(BaseModel, frozen=True):
     labels: CostNamespaceLabels
     app_name: str
     cluster_name: str
-    cluster_external_id: str | None
+    cluster_external_id: str | None = None
 
 
 def get_cost_namespaces(
@@ -32,7 +32,7 @@ def get_cost_namespaces(
     return [
         CostNamespace(
             name=namespace.name,
-            labels=CostNamespaceLabels.parse_obj(namespace.labels or {}),
+            labels=CostNamespaceLabels.model_validate(namespace.labels or {}),
             app_name=namespace.app.name,
             cluster_name=namespace.cluster.name,
             cluster_external_id=namespace.cluster.spec.external_id

reconcile/typed_queries/saas_files.py

@@ -1,5 +1,4 @@
 import hashlib
-import json
 from collections.abc import Callable
 from threading import Lock
 from typing import Any
@@ -7,7 +6,6 @@ from typing import Any
 from jsonpath_ng.exceptions import JsonPathParserError
 from pydantic import (
     BaseModel,
-    Extra,
     Field,
     Json,
 )
@@ -48,10 +46,16 @@ from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     ParameterError,
 )
+from reconcile.utils.json import json_dumps
 from reconcile.utils.jsonpath import parse_jsonpath
 
 
-class SaasResourceTemplateTarget(ConfiguredBaseModel):
+class SaasResourceTemplateTarget(
+    ConfiguredBaseModel,
+    validate_by_alias=True,
+    # ignore `namespaceSelector` and 'provider' fields from the GQL schema
+    extra="ignore",
+):
     path: str | None = Field(..., alias="path")
     name: str | None = Field(..., alias="name")
     # the namespace must be required to fulfill the saas file schema (utils.saasherder.interface.SaasFile)
@@ -79,12 +83,8 @@ class SaasResourceTemplateTarget(ConfiguredBaseModel):
             digest_size=20,
         ).hexdigest()
 
-    class Config:
-        # ignore `namespaceSelector` and 'provider' fields from the GQL schema
-        extra = Extra.ignore
-
 
-class SaasResourceTemplate(ConfiguredBaseModel):
+class SaasResourceTemplate(ConfiguredBaseModel, validate_by_alias=True):
     name: str = Field(..., alias="name")
     url: str = Field(..., alias="url")
     path: str = Field(..., alias="path")
@@ -97,7 +97,7 @@ class SaasResourceTemplate(ConfiguredBaseModel):
     targets: list[SaasResourceTemplateTarget] = Field(..., alias="targets")
 
 
-class SaasFile(ConfiguredBaseModel):
+class SaasFile(ConfiguredBaseModel, validate_by_alias=True):
     path: str = Field(..., alias="path")
     name: str = Field(..., alias="name")
     labels: Json | None = Field(..., alias="labels")
@@ -221,7 +221,7 @@ class SaasFileList:
             with self._namespaces_as_dict_lock:
                 self._namespaces_as_dict_cache = {
                     "namespace": [
-                        ns.dict(by_alias=True, exclude_none=True)
+                        ns.model_dump(by_alias=True, exclude_none=True)
                         for ns in self.namespaces
                     ]
                 }
@@ -283,7 +283,7 @@ class SaasFileList:
             if app_name and saas_file.app.name != app_name:
                 continue
 
-            sf = saas_file.copy(deep=True)
+            sf = saas_file.model_copy(deep=True)
             if env_name:
                 for rt in sf.resource_templates[:]:
                     for target in rt.targets[:]:
@@ -302,7 +302,7 @@ def convert_parameters_to_json_string(root: dict[str, Any]) -> dict[str, Any]:
     """Find all parameter occurrences and convert them to a json string."""
     for key, value in root.items():
         if key in {"parameters", "labels"}:
-            root[key] = json.dumps(value) if value is not None else None
+            root[key] = json_dumps(value) if value is not None else None
         elif isinstance(value, dict):
             root[key] = convert_parameters_to_json_string(value)
         elif isinstance(value, list):
@@ -314,7 +314,7 @@ def convert_parameters_to_json_string(root: dict[str, Any]) -> dict[str, Any]:
 
 
 def export_model(model: BaseModel) -> dict[str, Any]:
-    return convert_parameters_to_json_string(model.dict(by_alias=True))
+    return convert_parameters_to_json_string(model.model_dump(by_alias=True))
 
 
 def get_saas_files(

reconcile/typed_queries/status_board.py

@@ -32,7 +32,7 @@ def get_selected_app_names(
             prefix = f"{namespace.app.parent_app.name}-"
         name = f"{prefix}{namespace.app.name}"
         selected_app_names.add(name)
-        app = namespace.app.dict(by_alias=True)
+        app = namespace.app.model_dump(by_alias=True)
         app["name"] = name
         apps["apps"].append(app)
 
@@ -40,7 +40,7 @@ def get_selected_app_names(
             name = f"{namespace.app.name}-{child.name}"
             if name not in selected_app_names:
                 selected_app_names.add(f"{namespace.app.name}-{child.name}")
-                child_dict = child.dict(by_alias=True)
+                child_dict = child.model_dump(by_alias=True)
                 child_dict["name"] = name
                 apps["apps"].append(child_dict)
 

reconcile/unleash_feature_toggles/integration.py

@@ -31,7 +31,7 @@ QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
 
 
 class UnleashTogglesIntegrationParams(PydanticRunParams):
-    instance: str | None
+    instance: str | None = None
 
 
 def feature_toggle_equal(c: FeatureToggle, d: FeatureToggleUnleashV1) -> bool:
@@ -68,7 +68,9 @@ class UnleashTogglesIntegration(
         if not query_func:
             query_func = gql.get_api().query
         return {
-            "toggles": [ft.dict() for ft in self.get_unleash_instances(query_func)],
+            "toggles": [
+                ft.model_dump() for ft in self.get_unleash_instances(query_func)
+            ],
         }
 
     def get_unleash_instances(

reconcile/utils/acs/base.py

@@ -6,7 +6,7 @@ from typing import (
 )
 
 import requests
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from reconcile.gql_definitions.acs.acs_instances import AcsInstanceV1
 from reconcile.gql_definitions.acs.acs_instances import query as acs_instances_query
@@ -19,8 +19,11 @@ class AcsBaseApi(BaseModel):
     timeout: int = 30
     session: requests.Session = requests.Session()
 
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(
+        validate_by_name=True,
+        validate_by_alias=True,
+        arbitrary_types_allowed=True,
+    )
 
     def __enter__(self) -> Self:
         return self

reconcile/utils/acs/policies.py

@@ -11,7 +11,7 @@ class Scope(BaseModel):
     """
 
     cluster: str
-    namespace: str | None
+    namespace: str | None = None
 
 
 class PolicyCondition(BaseModel):
@@ -23,7 +23,7 @@ class PolicyCondition(BaseModel):
     """
 
     field_name: str
-    negate: bool | None
+    negate: bool | None = None
     values: list[str]
 
 

reconcile/utils/aggregated_list.py

@@ -1,8 +1,9 @@
-import json
 import logging
 from collections.abc import Callable, KeysView
 from typing import Any, TypedDict
 
+from reconcile.utils.json import json_dumps
+
 Action = Callable[[Any, list[Any]], bool]
 Cond = Callable[[Any], bool]
 
@@ -89,11 +90,11 @@ class AggregatedList:
         return list(self._dict.values())
 
     def to_json(self) -> str:
-        return json.dumps(self.dump(), indent=4)
+        return json_dumps(self.dump(), indent=4)
 
     @staticmethod
     def hash_params(params: Any) -> int:
-        return hash(json.dumps(params, sort_keys=True))
+        return hash(json_dumps(params))
 
 
 class AggregatedDiffRunner:

reconcile/utils/aws_api.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import operator
 import os
-import re
 from functools import lru_cache
 from threading import Lock
 from typing import (
@@ -25,6 +24,7 @@ import reconcile.utils.lean_terraform_client as terraform
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 
 if TYPE_CHECKING:
+    import re
     from collections.abc import (
         Iterable,
         Iterator,
@@ -583,40 +583,6 @@ class AWSApi:
 
         return error, service_account_recycle_complete
 
-    def disable_keys(
-        self,
-        dry_run: bool,
-        keys_to_disable: Mapping,
-    ) -> bool:
-        error = False
-        users_keys = self.get_users_keys()
-        for account, s in self.sessions.items():
-            iam = self.get_session_client(s, "iam")
-            keys = keys_to_disable.get(account, [])
-            for key in keys:
-                user_and_user_keys = [
-                    (user, user_keys)
-                    for user, user_keys in users_keys[account].items()
-                    if key in user_keys
-                ]
-                if not user_and_user_keys:
-                    continue
-                if len(user_and_user_keys) > 1:
-                    raise RuntimeError(
-                        f"key {key} returned multiple users: {user_and_user_keys}"
-                    )
-                user = user_and_user_keys[0][0]
-                key_status = self.get_user_key_status(iam, user, key)
-                if key_status == "Active":
-                    logging.info(["disable_key", account, user, key])
-                    if not dry_run:
-                        iam.update_access_key(
-                            UserName=user, AccessKeyId=key, Status="Inactive"
-                        )
-                else:
-                    logging.info(["key_already_disabled", account, user, key])
-        return error
-
     def get_users_keys(self) -> dict:
         users_keys = {}
         for account, s in self.sessions.items():
@@ -1108,28 +1074,40 @@ class AWSApi:
         return [rt["RouteTableId"] for rt in vpc_route_tables]
 
     @staticmethod
-    def _filter_amis(
-        images: Iterable[ImageTypeDef], regex: str
-    ) -> list[dict[str, Any]]:
-        results = []
-        pattern = re.compile(regex)
-        for i in images:
-            if not re.search(pattern, i["Name"]):
-                continue
-            if i["State"] != "available":
-                continue
-            item = {"image_id": i["ImageId"], "tags": i.get("Tags", [])}
-            results.append(item)
+    def normalize_tags(tags: Iterable[TagTypeDef]) -> dict[str, str]:
+        return {tag["Key"]: tag["Value"] for tag in tags}
 
-        return results
+    @staticmethod
+    def _filter_amis(
+        images: Iterable[ImageTypeDef],
+        regex: re.Pattern,
+    ) -> dict[str, dict[str, str]]:
+        return {
+            image["ImageId"]: AWSApi.normalize_tags(image.get("Tags", []))
+            for image in images
+            if regex.search(image["Name"]) and image["State"] == "available"
+        }
 
     def get_amis_details(
         self,
         account: Mapping[str, Any],
         owner_account: Mapping[str, Any],
-        regex: str,
+        regex: re.Pattern,
         region: str | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, dict[str, str]]:
+        """
+        Get AMI details for an account, find AMI name matches regex and state is available.
+        Return ImageId and normalized tags.
+
+        Args:
+            account: AWS account
+            owner_account: AMI owner AWS account uid
+            regex: regex to filter AMI name
+            region: AWS account region
+
+        Returns:
+            dict[str, dict[str, str]]: Key is AMI ImageId, value is AMI normalized tags.
+        """
         ec2 = self._account_ec2_client(account["name"], region_name=region)
         images = self.get_account_amis(ec2, owner=owner_account["uid"])
         return self._filter_amis(images, regex)
@@ -1209,12 +1187,31 @@ class AWSApi:
         client = self._account_cloudwatch_client(account_name, region_name=region_name)
         client.delete_log_group(logGroupName=group_name)
 
-    def create_tag(
-        self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
+    def create_tags(
+        self,
+        account: Mapping[str, Any],
+        resource_id: str,
+        tags: Mapping[str, str],
     ) -> None:
+        """
+        Create tags on EC2 resources (AMI)
+
+        Args:
+            account: AWS account
+            resource_id: AWS resource id
+            tags: tags to update
+
+        Returns:
+            None
+        """
         ec2 = self._account_ec2_client(account["name"])
-        tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
-        ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
+        formatted_tags: list[TagTypeDef] = [
+            {"Key": k, "Value": v} for k, v in tags.items()
+        ]
+        ec2.create_tags(
+            Resources=[resource_id],
+            Tags=formatted_tags,
+        )
 
     def get_alb_network_interface_ips(
         self, account: awsh.Account, service_name: str

reconcile/utils/aws_api_typed/api.py

@@ -6,9 +6,11 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from boto3 import Session
+from botocore.config import Config
 from pydantic import BaseModel
 
 import reconcile.utils.aws_api_typed.account
+import reconcile.utils.aws_api_typed.cloudformation
 import reconcile.utils.aws_api_typed.dynamodb
 import reconcile.utils.aws_api_typed.iam
 import reconcile.utils.aws_api_typed.organization
@@ -17,8 +19,10 @@ import reconcile.utils.aws_api_typed.service_quotas
 import reconcile.utils.aws_api_typed.sts
 import reconcile.utils.aws_api_typed.support
 from reconcile.utils.aws_api_typed.account import AWSApiAccount
+from reconcile.utils.aws_api_typed.cloudformation import AWSApiCloudFormation
 from reconcile.utils.aws_api_typed.dynamodb import AWSApiDynamoDB
 from reconcile.utils.aws_api_typed.iam import AWSApiIam
+from reconcile.utils.aws_api_typed.logs import AWSApiLogs
 from reconcile.utils.aws_api_typed.organization import AWSApiOrganizations
 from reconcile.utils.aws_api_typed.s3 import AWSApiS3
 from reconcile.utils.aws_api_typed.service_quotas import AWSApiServiceQuotas
@@ -31,7 +35,9 @@ if TYPE_CHECKING:
 SubApi = TypeVar(
     "SubApi",
     AWSApiAccount,
+    AWSApiCloudFormation,
     AWSApiDynamoDB,
+    AWSApiLogs,
     AWSApiIam,
     AWSApiOrganizations,
     AWSApiS3,
@@ -40,6 +46,13 @@ SubApi = TypeVar(
     AWSApiSupport,
 )
 
+DEFAULT_CONFIG = Config(
+    retries={
+        "mode": "standard",
+        "total_max_attempts": 10,
+    },
+)
+
 
 class AWSCredentials(ABC):
     @abstractmethod
@@ -174,28 +187,34 @@ class AWSApi:
         """Return a new or cached sub api client."""
         match api_cls:
             case reconcile.utils.aws_api_typed.account.AWSApiAccount:
-                client = self.session.client("account")
+                client = self.session.client("account", config=DEFAULT_CONFIG)
+                api = api_cls(client)
+            case reconcile.utils.aws_api_typed.cloudformation.AWSApiCloudFormation:
+                client = self.session.client("cloudformation", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.dynamodb.AWSApiDynamoDB:
-                client = self.session.client("dynamodb")
+                client = self.session.client("dynamodb", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.iam.AWSApiIam:
-                client = self.session.client("iam")
+                client = self.session.client("iam", config=DEFAULT_CONFIG)
+                api = api_cls(client)
+            case reconcile.utils.aws_api_typed.logs.AWSApiLogs:
+                client = self.session.client("logs", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.organization.AWSApiOrganizations:
-                client = self.session.client("organizations")
+                client = self.session.client("organizations", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.s3.AWSApiS3:
-                client = self.session.client("s3")
+                client = self.session.client("s3", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.service_quotas.AWSApiServiceQuotas:
-                client = self.session.client("service-quotas")
+                client = self.session.client("service-quotas", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.sts.AWSApiSts:
-                client = self.session.client("sts")
+                client = self.session.client("sts", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case reconcile.utils.aws_api_typed.support.AWSApiSupport:
-                client = self.session.client("support")
+                client = self.session.client("support", config=DEFAULT_CONFIG)
                 api = api_cls(client)
             case _:
                 raise ValueError(f"Unknown API class: {api_cls}")
@@ -205,9 +224,14 @@ class AWSApi:
 
     @cached_property
     def account(self) -> AWSApiAccount:
-        """Return an AWS Acount Api client"""
+        """Return an AWS Account Api client"""
         return self._init_sub_api(AWSApiAccount)
 
+    @cached_property
+    def cloudformation(self) -> AWSApiCloudFormation:
+        """Return an AWS CloudFormation Api client"""
+        return self._init_sub_api(AWSApiCloudFormation)
+
     @cached_property
     def dynamodb(self) -> AWSApiDynamoDB:
         """Return an AWS DynamoDB Api client"""
@@ -218,6 +242,11 @@ class AWSApi:
         """Return an AWS IAM Api client."""
         return self._init_sub_api(AWSApiIam)
 
+    @cached_property
+    def logs(self) -> AWSApiLogs:
+        """Return an AWS Logs Api client."""
+        return self._init_sub_api(AWSApiLogs)
+
     @cached_property
     def organizations(self) -> AWSApiOrganizations:
         """Return an AWS Organizations Api client."""