qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev430__py3-none-any.whl

reconcile/statuspage/status.py

@@ -2,18 +2,15 @@ from abc import (
     ABC,
     abstractmethod,
 )
-from datetime import (
-    UTC,
-    datetime,
-)
+from datetime import datetime
 
-from dateutil.parser import isoparse
 from pydantic import BaseModel
 
 from reconcile.gql_definitions.statuspage.statuspages import (
     ManualStatusProviderV1,
     StatusProviderV1,
 )
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 
 # This module defines the interface for status providers for components on status
 # pages. A status provider is responsible for determining the status of a component.
@@ -70,7 +67,7 @@ class ManualStatusProvider(StatusProvider, BaseModel):
             raise ValueError(
                 "manual component status time window is invalid: end before start"
             )
-        now = datetime.now(UTC)
+        now = utc_now()
         if self.start and now < self.start:
             return False
         return not (self.end and self.end < now)
@@ -84,8 +81,8 @@ def build_status_provider_config(
     provider specific implementation that provides the status resolution logic.
     """
     if isinstance(cfg, ManualStatusProviderV1):
-        start = isoparse(cfg.manual.q_from) if cfg.manual.q_from else None
-        end = isoparse(cfg.manual.until) if cfg.manual.until else None
+        start = from_utc_iso_format(cfg.manual.q_from) if cfg.manual.q_from else None
+        end = from_utc_iso_format(cfg.manual.until) if cfg.manual.until else None
         return ManualStatusProvider(
             component_status=cfg.manual.component_status,
             start=start,

reconcile/templating/lib/rendering.py

@@ -18,7 +18,7 @@ from reconcile.utils.secret_reader import SecretReaderBase
 
 class TemplateData(BaseModel):
     variables: dict[str, Any]
-    current: dict[str, Any] | None
+    current: dict[str, Any] | None = None
     current_with_explicit_start: bool | None = False
 
 
@@ -26,7 +26,7 @@ class TemplatePatch(Protocol):
     path: str
     identifier: str | None
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class Template(Protocol):
@@ -36,7 +36,7 @@ class Template(Protocol):
     template: str
     overwrite: bool | None
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
     @property
     def patch(self) -> TemplatePatch | None:

reconcile/templating/renderer.py

@@ -239,8 +239,8 @@ def unpack_dynamic_variables(
 
 class TemplateRendererIntegrationParams(PydanticRunParams):
     clone_repo: bool = False
-    app_interface_root_path: str | None
-    template_collection_name: str | None
+    app_interface_root_path: str | None = None
+    template_collection_name: str | None = None
 
 
 def join_path(base: str, sub: str) -> str:

reconcile/terraform_aws_route53.py

@@ -9,6 +9,7 @@ from typing import Any
 
 from reconcile import queries
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import dnsutils
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
@@ -227,13 +228,18 @@ def run(
             f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
         )
         return
-
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         "",
         thread_pool_size,
         participating_accounts,
         settings=settings,
+        default_tags=default_tags,
     )
 
     desired_state = build_desired_state(zones, all_accounts, settings)

reconcile/terraform_cloudflare_dns.py

@@ -155,7 +155,7 @@ class TerraformCloudflareDNSIntegration(
 
         accts_per_zone = []
         for zone in query_zones.zones or []:
-            acct = zone.account.dict(by_alias=True)
+            acct = zone.account.model_dump(by_alias=True)
             acct["name"] = f"{zone.account.name}-{zone.identifier}"
             accts_per_zone.append(acct)
 
@@ -369,11 +369,11 @@ def cloudflare_dns_zone_to_external_resource(
             provision_provider=DEFAULT_PROVISIONER_PROVIDER,
             provisioner={"name": f"{zone.account.name}-{zone.identifier}"},
             namespace=DEFAULT_NAMESPACE,
-            resource=zone.dict(by_alias=True, exclude=DEFAULT_EXCLUDE_KEY),
+            resource=zone.model_dump(by_alias=True, exclude=DEFAULT_EXCLUDE_KEY),
         )
         external_resource_spec.resource["provider"] = DEFAULT_PROVIDER
         external_resource_spec.resource["records"] = [
-            record.dict(by_alias=True) for record in zone.records or []
+            record.model_dump(by_alias=True) for record in zone.records or []
         ]
         external_resource_specs.append(external_resource_spec)
     return external_resource_specs

reconcile/terraform_cloudflare_resources.py

@@ -168,7 +168,7 @@ def _build_oc_resources(
         internal=internal,
     )
 
-    namespace_mapping = [ns.dict() for ns in cloudflare_namespaces]
+    namespace_mapping = [ns.model_dump() for ns in cloudflare_namespaces]
 
     state_specs = init_specs_to_fetch(
         ri, oc_map, namespaces=namespace_mapping, override_managed_types=["Secret"]
@@ -338,7 +338,7 @@ def run(
     )
 
     if not cloudflare_namespaces:
-        logging.info("No cloudflare namespaces were detected, nothing to do.")
+        logging.debug("No cloudflare namespaces were detected, nothing to do.")
         sys.exit(ExitCodes.SUCCESS)
 
     # Build Cloudflare clients
@@ -351,7 +351,7 @@ def run(
         spec
         for namespace in query_resources.namespaces
         for spec in get_external_resource_specs(
-            namespace.dict(by_alias=True), PROVIDER_CLOUDFLARE
+            namespace.model_dump(by_alias=True), PROVIDER_CLOUDFLARE
         )
         if not selected_account or spec.provisioner_name == selected_account
     ]
@@ -383,7 +383,7 @@ def run(
         QONTRACT_INTEGRATION_VERSION,
         QONTRACT_TF_PREFIX,
         [
-            acct.dict(by_alias=True)  # convert CloudflareAccountV1 to dict
+            acct.model_dump(by_alias=True)  # convert CloudflareAccountV1 to dict
             for acct in query_accounts.accounts or []
             if acct.name in cf_clients.dump()  # use only if it is a registered client
         ],
@@ -442,4 +442,4 @@ def _get_cloudflare_desired_state() -> tuple[
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
     desired_state = _get_cloudflare_desired_state()
 
-    return {state.__repr_name__(): state.dict() for state in desired_state}
+    return {str(state): state.model_dump() for state in desired_state}

reconcile/terraform_cloudflare_users.py

@@ -88,7 +88,7 @@ class TerraformCloudflareUsers(
         if not settings.settings:
             raise RuntimeError("App interface setting not defined")
 
-        early_exit_desired_state = cloudflare_roles.dict()
+        early_exit_desired_state = cloudflare_roles.model_dump()
         early_exit_desired_state.update({
             CLOUDFLARE_EMAIL_DOMAIN_ALLOW_LIST_KEY: settings.settings
         })
@@ -149,7 +149,8 @@ class TerraformCloudflareUsers(
         }
 
         accounts = [
-            acct.dict(by_alias=True) for _, acct in account_names_to_account.items()
+            acct.model_dump(by_alias=True)
+            for _, acct in account_names_to_account.items()
         ]
 
         self._run_terraform(

reconcile/terraform_init/integration.py

@@ -1,6 +1,5 @@
 import logging
 from collections.abc import Callable
-from datetime import UTC, datetime
 from typing import Any
 
 import jinja2
@@ -12,12 +11,16 @@ from reconcile.gql_definitions.terraform_init.aws_accounts import (
 from reconcile.terraform_init.merge_request import Renderer, create_parser
 from reconcile.terraform_init.merge_request_manager import MergeRequestManager, MrData
 from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
 from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.gql import GqlApi
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
@@ -31,10 +34,16 @@ QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
 
 
 class TerraformInitIntegrationParams(PydanticRunParams):
-    account_name: str | None
+    account_name: str | None = None
     # To avoid the accidental deletion of the resource file, explicitly set the
     # qontract.cli option in the integration extraArgs!
     state_tmpl_resource: str = "/terraform-init/terraform-state.yml"
+    cloudformation_template_resource: str = (
+        "/terraform-init/terraform-state-s3-bucket.yaml"
+    )
+    cloudformation_import_template_resource: str = (
+        "/terraform-init/terraform-state-s3-bucket-import.yaml"
+    )
     template_collection_root_path: str = "data/templating/collections/terraform-init"
 
 
@@ -55,25 +64,35 @@ class TerraformInitIntegration(
             query_func = gql.get_api().query
         return {
             "accounts": [
-                account.dict() for account in self.get_aws_accounts(query_func)
+                account.model_dump() for account in self.get_aws_accounts(query_func)
             ],
         }
 
     def get_aws_accounts(
         self, query_func: Callable, account_name: str | None = None
     ) -> list[AWSAccountV1]:
-        """Return all AWS accounts with terraform username but no terraform state set."""
+        """Return all AWS accounts with terraform username."""
         return [
             account
             for account in aws_accounts_query(query_func).accounts or []
             if integration_is_enabled(self.name, account)
             and (not account_name or account.name == account_name)
             and account.terraform_username
-            and not account.terraform_state
         ]
 
+    @staticmethod
+    def get_default_tags(gql_api: GqlApi) -> dict[str, str]:
+        try:
+            return get_settings(gql_api.query).default_tags
+        except ValueError:
+            # no settings found
+            return {}
+
+    @staticmethod
     def render_state_collection(
-        self, template: str, bucket_name: str, account: AWSAccountV1
+        template: str,
+        bucket_name: str,
+        account: AWSAccountV1,
     ) -> str:
         return jinja2.Template(
             template,
@@ -85,24 +104,114 @@ class TerraformInitIntegration(
             "account_name": account.name,
             "bucket_name": bucket_name,
             "region": account.resources_default_region,
-            "timestamp": int(datetime.now(tz=UTC).timestamp()),
+            "timestamp": int(utc_now().timestamp()),
         })
 
     def reconcile_account(
         self,
-        account_aws_api: AWSApi,
+        aws_api: AWSApi,
+        merge_request_manager: MergeRequestManager,
+        dry_run: bool,
+        account: AWSAccountV1,
+        state_template: str,
+        cloudformation_template: str,
+        cloudformation_import_template: str,
+        default_tags: dict[str, str],
+    ) -> None:
+        """
+        Reconcile the terraform state for a given account.
+
+        Create S3 bucket via CloudFormation and Merge Request to update template file on init.
+        Import existing bucket if it exists but not managed by CloudFormation.
+        Update CloudFormation stack if tags or template body differ.
+
+        CloudFormation stack name and bucket name is `terraform-<account_name>`.
+        `cloudformation_import_template` should be minimal template with identifier fields only.
+        `cloudformation_template` should be the full template.
+        Use import template to import then update stack to match full template.
+        This will ensure imported resources match CloudFormation template.
+        And all desired changes in full template are applied.
+        Both templates must have BucketName in Parameters.
+
+        Args:
+            aws_api: AWSApi: AWS API client for the target account.
+            merge_request_manager: MergeRequestManager: Manager to handle merge requests.
+            dry_run: bool: If True, do not make any changes.
+            account: AWSAccountV1: The AWS account to reconcile.
+            state_template: str: Jinja2 template for the Terraform state configuration.
+            cloudformation_template: str: CloudFormation template to create the S3 bucket.
+            cloudformation_import_template: str: CloudFormation template to import existing S3 bucket.
+            default_tags: dict[str, str]: Default tags to apply to the CloudFormation stack.
+
+        Returns:
+            None
+        """
+        bucket_name = (
+            account.terraform_state.bucket
+            if account.terraform_state
+            else f"terraform-{account.name}"
+        )
+
+        tags = default_tags | get_aws_account_tags(account.organization)
+
+        if account.terraform_state is None:
+            return self._provision_terraform_state(
+                aws_api=aws_api,
+                merge_request_manager=merge_request_manager,
+                dry_run=dry_run,
+                account=account,
+                bucket_name=bucket_name,
+                cloudformation_template=cloudformation_template,
+                state_template=state_template,
+                tags=tags,
+            )
+
+        stack = aws_api.cloudformation.get_stack(stack_name=bucket_name)
+
+        if stack is None:
+            return self._import_cloudformation_stack(
+                aws_api=aws_api,
+                dry_run=dry_run,
+                bucket_name=bucket_name,
+                cloudformation_import_template=cloudformation_import_template,
+                cloudformation_template=cloudformation_template,
+                tags=tags,
+            )
+
+        return self._reconcile_cloudformation_stack(
+            aws_api=aws_api,
+            dry_run=dry_run,
+            bucket_name=bucket_name,
+            cloudformation_template=cloudformation_template,
+            tags=tags,
+            current_tags={tag["Key"]: tag["Value"] for tag in stack.get("Tags", [])},
+        )
+
+    def _provision_terraform_state(
+        self,
+        aws_api: AWSApi,
         merge_request_manager: MergeRequestManager,
         dry_run: bool,
-        state_collection: str,
-        bucket_name: str,
         account: AWSAccountV1,
+        bucket_name: str,
+        cloudformation_template: str,
+        state_template: str,
+        tags: dict[str, str],
     ) -> None:
         logging.info("Creating bucket '%s' for account '%s'", bucket_name, account.name)
         if not dry_run:
-            # the creation of the bucket is idempotent
-            account_aws_api.s3.create_bucket(
-                name=bucket_name, region=account.resources_default_region
+            aws_api.cloudformation.create_stack(
+                stack_name=bucket_name,
+                change_set_name=f"create-{bucket_name}",
+                template_body=cloudformation_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
             )
+        state_collection = self.render_state_collection(
+            template=state_template,
+            bucket_name=bucket_name,
+            account=account,
+        )
         merge_request_manager.create_merge_request(
             data=MrData(
                 account=account.name,
@@ -111,6 +220,55 @@ class TerraformInitIntegration(
             )
         )
 
+    @staticmethod
+    def _import_cloudformation_stack(
+        aws_api: AWSApi,
+        dry_run: bool,
+        bucket_name: str,
+        cloudformation_import_template: str,
+        cloudformation_template: str,
+        tags: dict[str, str],
+    ) -> None:
+        logging.info("Importing existing bucket %s", bucket_name)
+        if not dry_run:
+            aws_api.cloudformation.create_stack(
+                stack_name=bucket_name,
+                change_set_name=f"import-{bucket_name}",
+                template_body=cloudformation_import_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
+            )
+            logging.info("Syncing stack %s after import", bucket_name)
+            aws_api.cloudformation.update_stack(
+                stack_name=bucket_name,
+                template_body=cloudformation_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
+            )
+
+    @staticmethod
+    def _reconcile_cloudformation_stack(
+        aws_api: AWSApi,
+        dry_run: bool,
+        bucket_name: str,
+        cloudformation_template: str,
+        tags: dict[str, str],
+        current_tags: dict[str, str],
+    ) -> None:
+        if (
+            current_tags != tags
+            or aws_api.cloudformation.get_template_body(stack_name=bucket_name)
+            != cloudformation_template
+        ):
+            logging.info("Updating stack %s", bucket_name)
+            if not dry_run:
+                aws_api.cloudformation.update_stack(
+                    stack_name=bucket_name,
+                    template_body=cloudformation_template,
+                    parameters={"BucketName": bucket_name},
+                    tags=tags,
+                )
+
     @defer
     def run(self, dry_run: bool, defer: Callable | None = None) -> None:
         """Run the integration."""
@@ -144,6 +302,14 @@ class TerraformInitIntegration(
         state_template = gql_api.get_resource(path=self.params.state_tmpl_resource)[
             "content"
         ]
+        cloudformation_template = gql_api.get_resource(
+            path=self.params.cloudformation_template_resource
+        )["content"]
+        cloudformation_import_template = gql_api.get_resource(
+            path=self.params.cloudformation_import_template_resource
+        )["content"]
+        default_tags = self.get_default_tags(gql_api)
+
         for account in accounts:
             secret = self.secret_reader.read_all_secret(account.automation_token)
             with AWSApi(
@@ -153,15 +319,13 @@ class TerraformInitIntegration(
                     region=account.resources_default_region,
                 )
             ) as account_aws_api:
-                bucket_name = f"terraform-{account.name}"
-                state_collection = self.render_state_collection(
-                    state_template, bucket_name, account
-                )
                 self.reconcile_account(
-                    account_aws_api,
-                    merge_request_manager,
-                    dry_run,
-                    state_collection,
-                    bucket_name,
-                    account,
+                    aws_api=account_aws_api,
+                    merge_request_manager=merge_request_manager,
+                    dry_run=dry_run,
+                    account=account,
+                    state_template=state_template,
+                    cloudformation_template=cloudformation_template,
+                    cloudformation_import_template=cloudformation_import_template,
+                    default_tags=default_tags,
                 )

reconcile/terraform_repo.py

@@ -41,10 +41,10 @@ class RepoOutput(BaseModel):
     project_path: str
     delete: bool
     aws_creds: VaultSecret
-    variables: TerraformRepoVariablesV1 | None
-    bucket: str | None
-    region: str | None
-    bucket_path: str | None
+    variables: TerraformRepoVariablesV1 | None = None
+    bucket: str | None = None
+    region: str | None = None
+    bucket_path: str | None = None
     require_fips: bool
     tf_version: str
 
@@ -62,8 +62,8 @@ class OutputFile(BaseModel):
 class TerraformRepoIntegrationParams(PydanticRunParams):
     output_file: str | None
     validate_git: bool
-    gitlab_project_id: str | None
-    gitlab_merge_request_id: int | None
+    gitlab_project_id: str | None = None
+    gitlab_merge_request_id: int | None = None
 
 
 class TerraformRepoIntegration(
@@ -168,7 +168,7 @@ class TerraformRepoIntegration(
                     self.params.output_file, "w", encoding="locale"
                 ) as output_file:
                     yaml.safe_dump(
-                        data=output.dict(),
+                        data=output.model_dump(),
                         stream=output_file,
                         explicit_start=True,
                     )
@@ -177,7 +177,7 @@ class TerraformRepoIntegration(
                     f"Unable to write to '{self.params.output_file}'"
                 ) from None
         else:
-            print(yaml.safe_dump(data=output.dict(), explicit_start=True))
+            print(yaml.safe_dump(data=output.model_dump(), explicit_start=True))
 
         return output
 
@@ -206,7 +206,7 @@ class TerraformRepoIntegration(
         keys = state.ls()
         for key in keys:
             if value := state.get(key.lstrip("/"), None):
-                repo = TerraformRepoV1.parse_obj(value)
+                repo = TerraformRepoV1.model_validate(value)
                 repo_list.append(repo)
 
         return repo_list
@@ -283,7 +283,7 @@ class TerraformRepoIntegration(
             for add_key, add_val in diff_result.add.items():
                 # state.add already performs a json.dumps(key) so we export the
                 # pydantic model as a dict to avoid a double json dump with extra quotes
-                state.add(add_key, add_val.dict(by_alias=True), force=True)
+                state.add(add_key, add_val.model_dump(by_alias=True), force=True)
             for delete_key in diff_result.delete:
                 state.rm(delete_key)
             for change_key, change_val in diff_result.change.items():
@@ -291,7 +291,9 @@ class TerraformRepoIntegration(
                     state.rm(change_key)
                 else:
                     state.add(
-                        change_key, change_val.desired.dict(by_alias=True), force=True
+                        change_key,
+                        change_val.desired.model_dump(by_alias=True),
+                        force=True,
                     )
         except KeyError:
             pass
@@ -394,5 +396,7 @@ class TerraformRepoIntegration(
     def early_exit_desired_state(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
         gqlapi = gql.get_api()
         return {
-            "repos": [repo.dict() for repo in self.get_repos(query_func=gqlapi.query)]
+            "repos": [
+                repo.model_dump() for repo in self.get_repos(query_func=gqlapi.query)
+            ]
         }

reconcile/terraform_resources.py

@@ -137,7 +137,7 @@ def fetch_current_state(
         use_jump_host=use_jump_host,
         thread_pool_size=thread_pool_size,
     )
-    namespaces_dicts = [ns.dict(by_alias=True) for ns in namespaces]
+    namespaces_dicts = [ns.model_dump(by_alias=True) for ns in namespaces]
     state_specs = ob.init_specs_to_fetch(
         ri, oc_map, namespaces=namespaces_dicts, override_managed_types=["Secret"]
     )
@@ -273,7 +273,7 @@ def setup(
         )
     else:
         ocm_map = None
-    tf_namespaces_dicts = [ns.dict(by_alias=True) for ns in tf_namespaces]
+    tf_namespaces_dicts = [ns.model_dump(by_alias=True) for ns in tf_namespaces]
 
     provider_exclusions = settings.get("terraformResourcesProviderExclusions") or []
     ts.init_populate_specs(
@@ -295,16 +295,16 @@ def filter_tf_namespaces(
 ) -> list[NamespaceV1]:
     tf_namespaces = []
     for namespace_info in namespaces:
-        if ob.is_namespace_deleted(namespace_info.dict(by_alias=True)):
+        if ob.is_namespace_deleted(namespace_info.model_dump(by_alias=True)):
             continue
-        if not managed_external_resources(namespace_info.dict(by_alias=True)):
+        if not managed_external_resources(namespace_info.model_dump(by_alias=True)):
             continue
 
         if not account_names:
             tf_namespaces.append(namespace_info)
             continue
 
-        specs = get_external_resource_specs(namespace_info.dict(by_alias=True))
+        specs = get_external_resource_specs(namespace_info.model_dump(by_alias=True))
         if not specs:
             tf_namespaces.append(namespace_info)
             continue
@@ -567,7 +567,7 @@ def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
     }
     for ns_info in get_tf_namespaces():
         for spec in get_external_resource_specs(
-            ns_info.dict(by_alias=True), provision_provider=PROVIDER_AWS
+            ns_info.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
         ):
             resource_paths = [
                 spec.resource.get("defaults"),