qontract-reconcile 0.10.1rc735__py3-none-any.whl → 0.10.1rc737__py3-none-any.whl

{qontract_reconcile-0.10.1rc735.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc735
+Version: 0.10.1rc737
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc735.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=XBUVgDXU9vIUWu4F3qkNN39FlMH7XN8kaGjzfShPWGM,97162
+reconcile/cli.py,sha256=Fra8s6kxIuCoLF-moSto81gDFoJ6Q-WrsaRcVxR-KRI,98281
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -331,8 +331,8 @@ reconcile/gql_definitions/status_board/status_board.py,sha256=vHEzncabujkqbjJ-ib
 reconcile/gql_definitions/statuspage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/statuspage/statuspages.py,sha256=gxDb42H93nwtBg7oFRb6Gk9pbAZpsWk_y4Y0s3_g3nE,3520
 reconcile/gql_definitions/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/templating/template_collection.py,sha256=QYqEsy8BBdX9GK4SuOdSe_hqOHK4IHNz9eJ5P2f4TRQ,3007
-reconcile/gql_definitions/templating/templates.py,sha256=ujPPFZm9BbkRrxkcR7ZyReDYfFem4uxONYoPuzgiBTc,2735
+reconcile/gql_definitions/templating/template_collection.py,sha256=lS0vzEKV2ZrzOqOEriqpy0yBgKjb2Ftrzgx6PIH46_4,3310
+reconcile/gql_definitions/templating/templates.py,sha256=ejAvQ13zfNMQTz3FWtRUic6dSvio3aAgBKEqt600hbk,2821
 reconcile/gql_definitions/terraform_cloudflare_dns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_cloudflare_dns/app_interface_cloudflare_dns_settings.py,sha256=eyGX9HcTF6MZbOYZ6Kl6Mg3k6nJTUtwqs9gDxBP_8Dk,1920
 reconcile/gql_definitions/terraform_cloudflare_dns/terraform_cloudflare_zones.py,sha256=uVZYu5EUcvdAQYBK5YKD0mjoMKDb5inSuCJrrOD5KpE,5704
@@ -342,6 +342,8 @@ reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_re
 reconcile/gql_definitions/terraform_cloudflare_users/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_cloudflare_users/app_interface_setting_cloudflare_and_vault.py,sha256=KFey-0ItgpGPeIwViGKqb55HAFJoKdi3eCNSIzf6Rc8,1960
 reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.py,sha256=1KiTikTKSSRYmISN8tY29rJ_fVtgBnT7sK85IJjN420,4027
+reconcile/gql_definitions/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/terraform_init/aws_accounts.py,sha256=OJ0hDbRachRaDkL-OGT6-byr9cKdBiQDnNCpwUe3oJ8,2674
 reconcile/gql_definitions/terraform_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_repo/terraform_repo.py,sha256=pefTRhb0ZcWm_j6dYz6m6qNqZFteqgrQ25SgUqRAVaI,3173
 reconcile/gql_definitions/terraform_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -419,12 +421,16 @@ reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9
 reconcile/templates/rosa-classic-cluster-creation.sh.j2,sha256=0UHfYtXRVJqP07VJQx456cRI6EbZNBgamtP_8nb4WPY,2353
 reconcile/templates/rosa-hcp-cluster-creation.sh.j2,sha256=O7Bf3WQIJhsZoEqaYA0wRktUO4yXXCb4BQkuvvp-C80,2385
 reconcile/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/renderer.py,sha256=JUUzXGXzhcq-bhj81se7lLWgVfrUIEpO5vh6JgppbrI,10178
-reconcile/templating/validator.py,sha256=QGH2VSk7sVBuojhk9quRAbj_XBykHN-KZ53DbEneUJs,4391
+reconcile/templating/renderer.py,sha256=2FWbnefT2siozQpXXkuvVKUo6cePMqLY4BMYpqXg6xM,10652
+reconcile/templating/validator.py,sha256=pvDEc6veznEZzjypkoRJUGMMFLWosU-zd7i3j7JeNjE,4670
 reconcile/templating/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/lib/merge_request_manager.py,sha256=El3ufdVjHP4EW-LztX7zPiI9syJBJ6ETGRmiM9K4Nqw,5112
-reconcile/templating/lib/model.py,sha256=TYiH2xL63awd30U-fkZDLEzuxkLdUEgzJ913DEzpFoM,265
+reconcile/templating/lib/merge_request_manager.py,sha256=JUkfF3smaQ8onzKF5F7UpmA7MWaQpftANy6dDo1FCug,5464
+reconcile/templating/lib/model.py,sha256=fb6FYYLQjmoh2DjVKO7TEWCuDPf1Q34xmOx0M9Z07ek,324
 reconcile/templating/lib/rendering.py,sha256=_BVQ2gqip8K1AgLYfaTWh8NKJFTW6VjUZ6rBI_GH30E,5061
+reconcile/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/terraform_init/integration.py,sha256=xcFKTc_or3xB3kE_I3OECNkkgbwALIwwdiktnF-MyWI,6114
+reconcile/terraform_init/merge_request.py,sha256=3CYtgSd7Q9zjKg4wsDz437EPCRfGeZZ8fZ0Y-ChKXJY,1475
+reconcile/terraform_init/merge_request_manager.py,sha256=fMcT6hbdEF3nFATJpvr8BedvQHq_MzFkgVJSloBNwOQ,3101
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/conftest.py,sha256=rQousYrxUz-EwAIbsYO6bIwR1B4CrOz9y_zaUVo2lfI,4466
 reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
@@ -664,9 +670,10 @@ reconcile/utils/acs/notifiers.py,sha256=LfWw9LGq7hA90A69n8Ie9f-NozvCGdYwXEXRLIc1
 reconcile/utils/acs/policies.py,sha256=_jAz6cv8KRYtDsXjGoJgNbD8_9PUa5LSwwVlpK4A_cQ,5505
 reconcile/utils/acs/rbac.py,sha256=ugsLM9Pb7FbUbdq85E3VzXGMaB9ZovXob7tdWCxwqZ8,8808
 reconcile/utils/aws_api_typed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/utils/aws_api_typed/api.py,sha256=gqfZISSQLp6tHEAwEsroLWwyU4ZdbwHi9p0rNBQyLuI,7901
+reconcile/utils/aws_api_typed/api.py,sha256=rGUh7-gc8AcjUFuLgQxKyPy1KGY7ffNe6zT1EDdurOM,8283
 reconcile/utils/aws_api_typed/iam.py,sha256=ka46H2-SzTCgy6EJYapKTzyZK9vR1bkfD0wF8bDdy1Q,2201
 reconcile/utils/aws_api_typed/organization.py,sha256=oXftcLVuSs9qej6efdssl38FvjeZaQC5R2Wj3NzxX4U,5529
+reconcile/utils/aws_api_typed/s3.py,sha256=J2uOTtEFgMyKT22pa4DbFnV7zfg575m2DeidQaeselM,1034
 reconcile/utils/aws_api_typed/service_quotas.py,sha256=OU1D8LCmMw1IT87nt45LqXhguzcWwC8AaBdDTI7tz98,3018
 reconcile/utils/aws_api_typed/sts.py,sha256=5Sauncj9Fif3YDLkJYkBZrtOX0v0bGAqOmY0A5Bh9yA,1237
 reconcile/utils/aws_api_typed/support.py,sha256=PH3UW96Ne4_8I1J-_Vqj-DsK73gYGeVdOH13eD1783c,2447
@@ -770,8 +777,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=w2l4BHB09k1d-BGJ1jBUNCqDv7zkqYrMHojQXg-21kQ,4155
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc735.dist-info/METADATA,sha256=AQww9xL2v9VdTWvywruYZoPnQRbH2TO1wlIpHlpfGqE,2382
-qontract_reconcile-0.10.1rc735.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc735.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc735.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc735.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc737.dist-info/METADATA,sha256=cSDUzKY1nBbpIuGIWfYeQyeIlIvmyU8slP8iVQOiR7Q,2382
+qontract_reconcile-0.10.1rc737.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc737.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
+qontract_reconcile-0.10.1rc737.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc737.dist-info/RECORD,,

reconcile/cli.py

@@ -1032,6 +1032,41 @@ def aws_account_manager(
     )
 
 
+@integration.command(short_help="Initialize AWS accounts for Terraform usage.")
+@account_name
+@click.option(
+    "--state-tmpl-resource",
+    help="Resource name of the state template-collection template in the app-interface.",
+    required=True,
+    default="/terraform-init/terraform-state.yml",
+)
+@click.option(
+    "--template-collection-root-path",
+    help="File path to the root directory to store new state template-collections.",
+    required=True,
+    default="data/templating/collections/terraform-init",
+)
+@click.pass_context
+def terraform_init(
+    ctx, account_name, state_tmpl_resource, template_collection_root_path
+):
+    from reconcile.terraform_init.integration import (
+        TerraformInitIntegration,
+        TerraformInitIntegrationParams,
+    )
+
+    run_class_integration(
+        integration=TerraformInitIntegration(
+            TerraformInitIntegrationParams(
+                account_name=account_name,
+                state_tmpl_resource=state_tmpl_resource,
+                template_collection_root_path=template_collection_root_path,
+            )
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage Jenkins roles association via REST API.")
 @click.pass_context
 def jenkins_roles(ctx):

reconcile/gql_definitions/templating/template_collection.py

@@ -22,7 +22,9 @@ DEFINITION = """
 query TemplateCollection_v1 {
   template_collection_v1 {
     name
+    additionalMrLabels
     description
+    enableAutoApproval
     variables {
       static
       dynamic {
@@ -32,6 +34,7 @@ query TemplateCollection_v1 {
     }
     templates {
       name
+      autoApproved
       condition
       targetPath
       patch {
@@ -68,6 +71,7 @@ class TemplatePatchV1(ConfiguredBaseModel):
 
 class TemplateV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    auto_approved: Optional[bool] = Field(..., alias="autoApproved")
     condition: Optional[str] = Field(..., alias="condition")
     target_path: str = Field(..., alias="targetPath")
     patch: Optional[TemplatePatchV1] = Field(..., alias="patch")
@@ -76,7 +80,9 @@ class TemplateV1(ConfiguredBaseModel):
 
 class TemplateCollectionV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    additional_mr_labels: Optional[list[str]] = Field(..., alias="additionalMrLabels")
     description: str = Field(..., alias="description")
+    enable_auto_approval: Optional[bool] = Field(..., alias="enableAutoApproval")
     variables: Optional[TemplateCollectionVariablesV1] = Field(..., alias="variables")
     templates: list[TemplateV1] = Field(..., alias="templates")
 

reconcile/gql_definitions/templating/templates.py

@@ -22,6 +22,7 @@ DEFINITION = """
 query Templatev1 {
   template_v1 {
     name
+    autoApproved
     condition
     patch {
       path
@@ -64,6 +65,7 @@ class TemplateTestV1(ConfiguredBaseModel):
 
 class TemplateV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    auto_approved: Optional[bool] = Field(..., alias="autoApproved")
     condition: Optional[str] = Field(..., alias="condition")
     patch: Optional[TemplatePatchV1] = Field(..., alias="patch")
     target_path: str = Field(..., alias="targetPath")

reconcile/gql_definitions/terraform_init/aws_accounts.py

@@ -0,0 +1,93 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query TerraformInitAWSAccounts {
+  accounts: awsaccounts_v1 {
+    name
+    terraformUsername
+    terraformState {
+      region
+    }
+    resourcesDefaultRegion
+    automationToken {
+      ...VaultSecret
+    }
+    disable {
+      integrations
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class TerraformStateAWSV1(ConfiguredBaseModel):
+    region: str = Field(..., alias="region")
+
+
+class DisableClusterAutomationsV1(ConfiguredBaseModel):
+    integrations: Optional[list[str]] = Field(..., alias="integrations")
+
+
+class AWSAccountV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    terraform_username: Optional[str] = Field(..., alias="terraformUsername")
+    terraform_state: Optional[TerraformStateAWSV1] = Field(..., alias="terraformState")
+    resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
+    automation_token: VaultSecret = Field(..., alias="automationToken")
+    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
+
+
+class TerraformInitAWSAccountsQueryData(ConfiguredBaseModel):
+    accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
+
+
+def query(query_func: Callable, **kwargs: Any) -> TerraformInitAWSAccountsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        TerraformInitAWSAccountsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return TerraformInitAWSAccountsQueryData(**raw_data)

reconcile/templating/lib/merge_request_manager.py

@@ -13,6 +13,7 @@ from reconcile.utils.merge_request_manager.parser import (
     Parser,
 )
 from reconcile.utils.mr import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
 from reconcile.utils.vcs import VCS
 
 DATA_SEPARATOR = (
@@ -120,16 +121,23 @@ class TemplateRenderingMR(MergeRequestBase):
                 )
 
 
+class MrData(BaseModel):
+    data: list[TemplateOutput]
+    auto_approved: bool
+
+
 class MergeRequestManager(MergeRequestManagerBase[TemplateInfo]):
     def __init__(self, vcs: VCS, parser: Parser):
         super().__init__(vcs, parser, TR_LABEL)
 
-    def create_merge_request(self, output: list[TemplateOutput]) -> None:
+    def create_merge_request(self, data: MrData) -> None:
         if not self._housekeeping_ran:
             self.housekeeping()
 
-        collections = {o.input.collection for o in output if o.input}
-        collection_hashes = {o.input.collection_hash for o in output if o.input}
+        output = data.data
+        collections = {o.input.collection for o in output}
+        collection_hashes = {o.input.collection_hash for o in output}
+        additional_labels = {label for o in output for label in o.input.labels}
         # From the way the code is written, we can assert that there is only one collection and one template hash
         assert len(collections) == 1
         assert len(collection_hashes) == 1
@@ -158,6 +166,12 @@ class MergeRequestManager(MergeRequestManagerBase[TemplateInfo]):
         logging.info("Opening MR for %s with hash (%s)", collection, collection_hash)
         mr_labels = [TR_LABEL]
 
+        if data.auto_approved:
+            mr_labels.append(AUTO_MERGE)
+
+        if additional_labels:
+            mr_labels.extend(additional_labels)
+
         self._vcs.open_app_interface_merge_request(
             mr=TemplateRenderingMR(
                 title=title,

reconcile/templating/lib/model.py

@@ -1,15 +1,16 @@
-from typing import Optional
-
 from pydantic import BaseModel
 
 
 class TemplateInput(BaseModel):
     collection: str
     collection_hash: str
+    enable_auto_approval: bool = False
+    labels: list[str] = []
 
 
 class TemplateOutput(BaseModel):
-    input: Optional[TemplateInput]
+    input: TemplateInput
     is_new: bool = False
     path: str
     content: str
+    auto_approved: bool = False

reconcile/templating/renderer.py

@@ -17,6 +17,7 @@ from reconcile.gql_definitions.templating.template_collection import (
 )
 from reconcile.templating.lib.merge_request_manager import (
     MergeRequestManager,
+    MrData,
     create_parser,
 )
 from reconcile.templating.lib.model import TemplateInput, TemplateOutput
@@ -59,6 +60,19 @@ class FilePersistence(ABC):
     def read(self, path: str) -> Optional[str]:
         pass
 
+    @staticmethod
+    def _read_local_file(path: str) -> Optional[str]:
+        try:
+            with open(
+                path,
+                "r",
+                encoding="utf-8",
+            ) as f:
+                return f.read()
+        except FileNotFoundError:
+            logging.debug(f"File not found: {path}, need to create it")
+        return None
+
 
 class LocalFilePersistence(FilePersistence):
     """
@@ -80,16 +94,7 @@ class LocalFilePersistence(FilePersistence):
                 f.write(output.content)
 
     def read(self, path: str) -> Optional[str]:
-        try:
-            with open(
-                f"{join_path(self.app_interface_data_path, path)}",
-                "r",
-                encoding="utf-8",
-            ) as f:
-                return f.read()
-        except FileNotFoundError:
-            logging.debug(f"File not found: {path}, need to create it")
-        return None
+        return self._read_local_file(join_path(self.app_interface_data_path, path))
 
 
 class PersistenceTransaction(FilePersistence):
@@ -127,6 +132,8 @@ class ClonedRepoGitlabPersistence(FilePersistence):
     """
     This class is used to persist the rendered templates in a cloned gitlab repo
     Reads are from the local filesystem, writes are done via utils.VCS abstraction
+
+    Only one MR is created per run. Auto-approval MRs are prefered.
     """
 
     def __init__(self, local_path: str, vcs: VCS, mr_manager: MergeRequestManager):
@@ -136,19 +143,19 @@ class ClonedRepoGitlabPersistence(FilePersistence):
 
     def write(self, outputs: list[TemplateOutput]) -> None:
         self.mr_manager.housekeeping()
-        self.mr_manager.create_merge_request(outputs)
+
+        if any([o.input.enable_auto_approval for o in outputs]):
+            auto_approved = [o for o in outputs if o.auto_approved]
+            if auto_approved:
+                self.mr_manager.create_merge_request(
+                    MrData(data=auto_approved, auto_approved=True)
+                )
+                return
+
+        self.mr_manager.create_merge_request(MrData(data=outputs, auto_approved=False))
 
     def read(self, path: str) -> Optional[str]:
-        try:
-            with open(
-                f"{join_path(self.local_path, path)}",
-                "r",
-                encoding="utf-8",
-            ) as f:
-                return f.read()
-        except FileNotFoundError:
-            logging.debug(f"File not found: {path}, need to create it")
-        return None
+        return self._read_local_file(join_path(self.local_path, path))
 
 
 def unpack_static_variables(
@@ -187,6 +194,7 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         variables: dict,
         persistence: FilePersistence,
         ruaml_instance: yaml.YAML,
+        template_input: TemplateInput,
     ) -> Optional[TemplateOutput]:
         r = create_renderer(
             template,
@@ -217,6 +225,8 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                     path=target_path,
                     content=output,
                     is_new=current_str is None,
+                    auto_approved=template.auto_approved or False,
+                    input=template_input,
                 )
         return None
 
@@ -244,18 +254,17 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
             ).hexdigest()
 
             with PersistenceTransaction(persistence, dry_run) as p:
+                input = TemplateInput(
+                    collection=c.name,
+                    collection_hash=template_hash,
+                    enable_auto_approval=c.enable_auto_approval or False,
+                    labels=c.additional_mr_labels or [],
+                )
                 for template in c.templates:
                     output = self.process_template(
-                        template,
-                        variables,
-                        p,
-                        ruamel_instance,
+                        template, variables, p, ruamel_instance, input
                     )
                     if output:
-                        output.input = TemplateInput(
-                            collection=c.name,
-                            collection_hash=template_hash,
-                        )
                         outputs.append(output)
 
                 if not dry_run:

reconcile/templating/validator.py

@@ -127,7 +127,11 @@ class TemplateValidatorIntegration(QontractReconcileIntegration):
         if diffs:
             for diff in diffs:
                 logging.error(f"template: {diff.template}, test: {diff.test}")
-                logging.debug(diff.diff)
+                # This log should never be added except for local debugging.
+                # Credentials could be leaked, i.e. creating an MR with a diff,
+                # using a template, that uses the vault function.
+                # Use template-validator CLI instead.
+                # logging.debug(diff.diff)
             raise ValueError("Template validation failed")
 
     @property

reconcile/terraform_init/integration.py

@@ -0,0 +1,165 @@
+import logging
+from collections.abc import Callable
+from datetime import datetime, timezone
+from typing import Any
+
+import jinja2
+
+from reconcile.gql_definitions.terraform_init.aws_accounts import AWSAccountV1
+from reconcile.gql_definitions.terraform_init.aws_accounts import (
+    query as aws_accounts_query,
+)
+from reconcile.terraform_init.merge_request import Renderer, create_parser
+from reconcile.terraform_init.merge_request_manager import MergeRequestManager, MrData
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils import gql
+from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
+from reconcile.utils.defer import defer
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.unleash import get_feature_toggle_state
+from reconcile.utils.vcs import VCS
+
+QONTRACT_INTEGRATION = "terraform-init"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+
+
+class TerraformInitIntegrationParams(PydanticRunParams):
+    account_name: str | None
+    state_tmpl_resource: str = "/terraform-init/terraform-state.yml"
+    template_collection_root_path: str = "data/templating/collections/terraform-init"
+
+
+class TerraformInitIntegration(
+    QontractReconcileIntegration[TerraformInitIntegrationParams]
+):
+    """Initialize AWS accounts for Terraform usage."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(
+        self, query_func: Callable | None = None
+    ) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        if not query_func:
+            query_func = gql.get_api().query
+        return {
+            "accounts": [
+                account.dict() for account in self.get_aws_accounts(query_func)
+            ],
+        }
+
+    def get_aws_accounts(
+        self, query_func: Callable, account_name: str | None = None
+    ) -> list[AWSAccountV1]:
+        """Return all AWS accounts with terraform username but no terraform state set."""
+        return [
+            account
+            for account in aws_accounts_query(query_func).accounts or []
+            if integration_is_enabled(self.name, account)
+            and (not account_name or account.name == account_name)
+            and account.terraform_username
+            and not account.terraform_state
+        ]
+
+    def render_state_collection(
+        self, template: str, bucket_name: str, account: AWSAccountV1
+    ) -> str:
+        return jinja2.Template(
+            template,
+            undefined=jinja2.StrictUndefined,
+            trim_blocks=True,
+            lstrip_blocks=True,
+            keep_trailing_newline=True,
+        ).render({
+            "account_name": account.name,
+            "bucket_name": bucket_name,
+            "region": account.resources_default_region,
+            "timestamp": int(datetime.now(tz=timezone.utc).timestamp()),
+        })
+
+    def reconcile_account(
+        self,
+        account_aws_api: AWSApi,
+        merge_request_manager: MergeRequestManager,
+        dry_run: bool,
+        state_collection: str,
+        bucket_name: str,
+        account: AWSAccountV1,
+    ) -> None:
+        logging.info("Creating bucket '%s' for account '%s'", bucket_name, account.name)
+        if not dry_run:
+            # the creation of the bucket is idempotent
+            account_aws_api.s3.create_bucket(
+                name=bucket_name, region=account.resources_default_region
+            )
+        merge_request_manager.create_merge_request(
+            data=MrData(
+                account=account.name,
+                content=state_collection,
+                path=f"{self.params.template_collection_root_path}/{account.name}.yml",
+            )
+        )
+
+    @defer
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
+        """Run the integration."""
+        gql_api = gql.get_api()
+        accounts = self.get_aws_accounts(
+            gql_api.query, account_name=self.params.account_name
+        )
+        if not accounts:
+            # nothing to do
+            return
+
+        vcs = VCS(
+            secret_reader=self.secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=False,
+            allow_opening_mrs=True,
+        )
+        if defer:
+            defer(vcs.cleanup)
+        merge_request_manager = MergeRequestManager(
+            vcs=vcs,
+            renderer=Renderer(),
+            parser=create_parser(),
+            auto_merge_enabled=get_feature_toggle_state(
+                integration_name=f"{self.name}-allow-auto-merge-mrs", default=False
+            ),
+        )
+        state_template = gql_api.get_resource(path=self.params.state_tmpl_resource)[
+            "content"
+        ]
+        for account in accounts:
+            secret = self.secret_reader.read_all_secret(account.automation_token)
+            with AWSApi(
+                AWSStaticCredentials(
+                    access_key_id=secret["aws_access_key_id"],
+                    secret_access_key=secret["aws_secret_access_key"],
+                    region=account.resources_default_region,
+                )
+            ) as account_aws_api:
+                bucket_name = f"terraform-{account.name}"
+                state_collection = self.render_state_collection(
+                    state_template, bucket_name, account
+                )
+                self.reconcile_account(
+                    account_aws_api,
+                    merge_request_manager,
+                    dry_run,
+                    state_collection,
+                    bucket_name,
+                    account,
+                )

reconcile/terraform_init/merge_request.py

@@ -0,0 +1,57 @@
+import re
+import string
+
+from pydantic import BaseModel
+
+from reconcile.utils.merge_request_manager.parser import Parser
+
+PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
+VERSION = "1.0.0"
+LABEL = "terraform-init"
+
+VERSION_REF = "tf_init_version"
+ACCOUNT_REF = "account"
+COMPILED_REGEXES = {
+    i: re.compile(rf".*{i}: (.*)$", re.MULTILINE) for i in [VERSION_REF, ACCOUNT_REF]
+}
+
+DESC = string.Template(
+    f"""
+This MR is triggered by app-interface's [terraform-init](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/terraform_init).
+
+Please **do not remove** the **{LABEL}** label from this MR!
+
+Parts of this description are used by integration to manage the MR.
+
+{PROMOTION_DATA_SEPARATOR}
+
+* {VERSION_REF}: {VERSION}
+* {ACCOUNT_REF}: $account
+"""
+)
+
+
+class Info(BaseModel):
+    account: str
+
+
+def create_parser() -> Parser:
+    """Create a parser for MRs created by terraform-init."""
+
+    return Parser[Info](
+        klass=Info,
+        compiled_regexes=COMPILED_REGEXES,
+        version_ref=VERSION_REF,
+        expected_version=VERSION,
+        data_separator=PROMOTION_DATA_SEPARATOR,
+    )
+
+
+class Renderer:
+    """This class is only concerned with rendering text for MRs."""
+
+    def render_description(self, account: str) -> str:
+        return DESC.safe_substitute(account=account)
+
+    def render_title(self, account: str) -> str:
+        return f"[auto] Terraform State settings for AWS account {account}"

reconcile/terraform_init/merge_request_manager.py

@@ -0,0 +1,102 @@
+import logging
+
+from gitlab.exceptions import GitlabGetError
+from pydantic import BaseModel
+
+from reconcile.terraform_init.merge_request import (
+    LABEL,
+    Info,
+    Renderer,
+)
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.merge_request_manager.parser import Parser
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+
+class TerraformInitMR(MergeRequestBase):
+    name = "TerraformInit"
+
+    def __init__(
+        self, title: str, description: str, path: str, content: str, labels: list[str]
+    ):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self._path = path
+        self._content = content
+        self.labels = labels
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        gitlab_cli.create_file(
+            branch_name=self.branch,
+            file_path=self._path,
+            commit_message="add terraform state template collection",
+            content=self._content,
+        )
+
+
+class MrData(BaseModel):
+    account: str
+    content: str
+    path: str
+
+
+class MergeRequestManager(MergeRequestManagerBase[Info]):
+    """Manager for the merge requests.
+
+    This class is responsible for housekeeping (closing old/bad MRs) and
+    opening new MRs.
+    """
+
+    def __init__(
+        self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
+    ):
+        super().__init__(vcs, parser, LABEL)
+        self._renderer = renderer
+        self._auto_merge_enabled = auto_merge_enabled
+
+    def create_merge_request(self, data: MrData) -> None:
+        """Open a new MR, if not already present, for an AWS account and close any outdated before."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        if self._merge_request_already_exists({"account": data.account}):
+            logging.info("MR already exists for %s", data.account)
+            return None
+
+        try:
+            self._vcs.get_file_content_from_app_interface_master(file_path=data.path)
+            # the file exists, nothing to do
+            return None
+        except GitlabGetError as e:
+            if e.response_code != 404:
+                raise
+
+        description = self._renderer.render_description(account=data.account)
+        title = self._renderer.render_title(account=data.account)
+        logging.info("Open MR for %s", data.account)
+        mr_labels = [LABEL]
+        if self._auto_merge_enabled:
+            mr_labels.append(AUTO_MERGE)
+        self._vcs.open_app_interface_merge_request(
+            mr=TerraformInitMR(
+                path=data.path,
+                title=title,
+                description=description,
+                content=data.content,
+                labels=mr_labels,
+            )
+        )

reconcile/utils/aws_api_typed/api.py

@@ -11,11 +11,13 @@ from pydantic import BaseModel
 
 import reconcile.utils.aws_api_typed.iam
 import reconcile.utils.aws_api_typed.organization
+import reconcile.utils.aws_api_typed.s3
 import reconcile.utils.aws_api_typed.service_quotas
 import reconcile.utils.aws_api_typed.sts
 import reconcile.utils.aws_api_typed.support
 from reconcile.utils.aws_api_typed.iam import AWSApiIam
 from reconcile.utils.aws_api_typed.organization import AWSApiOrganizations
+from reconcile.utils.aws_api_typed.s3 import AWSApiS3
 from reconcile.utils.aws_api_typed.service_quotas import AWSApiServiceQuotas
 from reconcile.utils.aws_api_typed.sts import AWSApiSts
 from reconcile.utils.aws_api_typed.support import AWSApiSupport
@@ -161,6 +163,9 @@ class AWSApi:
             case reconcile.utils.aws_api_typed.organization.AWSApiOrganizations:
                 client = self.session.client("organizations")
                 api = api_cls(client)
+            case reconcile.utils.aws_api_typed.s3.AWSApiS3:
+                client = self.session.client("s3")
+                api = api_cls(client)
             case reconcile.utils.aws_api_typed.service_quotas.AWSApiServiceQuotas:
                 client = self.session.client("service-quotas")
                 api = api_cls(client)
@@ -186,6 +191,11 @@ class AWSApi:
         """Return an AWS Organizations Api client."""
         return self._init_sub_api(AWSApiOrganizations)
 
+    @cached_property
+    def s3(self) -> AWSApiS3:
+        """Return an AWS S3 Api client."""
+        return self._init_sub_api(AWSApiS3)
+
     @cached_property
     def service_quotas(self) -> AWSApiServiceQuotas:
         """Return an AWS Service Quotas Api client."""

reconcile/utils/aws_api_typed/s3.py

@@ -0,0 +1,26 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client
+    from mypy_boto3_s3.literals import BucketLocationConstraintType
+else:
+    S3Client = BucketLocationConstraintType = object
+
+
+class AWSApiS3:
+    def __init__(self, client: S3Client) -> None:
+        self.client = client
+
+    def create_bucket(self, name: str, region: str) -> str:
+        """Create an S3 bucket without any ACLs therefore the creator will be the owner and returns the ARN."""
+        bucket_kwargs = {}
+        if region != "us-east-1":
+            # you can't specify the location if it's us-east-1 :(
+            # see valid values "LocationConstraint" here: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucketConfiguration.html
+            bucket_kwargs = {
+                "CreateBucketConfiguration": {
+                    "LocationConstraint": region,
+                },
+            }
+        self.client.create_bucket(Bucket=name, **bucket_kwargs)  # type: ignore
+        return f"arn:aws:s3:::{name}"