qontract-reconcile 0.10.1rc41__py3-none-any.whl → 0.10.1rc43__py3-none-any.whl

{qontract_reconcile-0.10.1rc41.dist-info → qontract_reconcile-0.10.1rc43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc41
+Version: 0.10.1rc43
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc41.dist-info → qontract_reconcile-0.10.1rc43.dist-info}/RECORD

@@ -7,7 +7,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=i6bSWnlH9fh14P14PjVhFLwNl-q3fD733_rXKM_O51c,2992
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=figtZRuWUvdpdSnkhAqeGvO5dI02TT6J3heyeFhlwqM,5016
-reconcile/cli.py,sha256=bPn61Jm35bH2FZfD2fXj_ITmbK6SdHQZjL-3ixjpBIY,70379
+reconcile/cli.py,sha256=b9m9QgNakXJOS-b9MQgHEESFF5-qc4J1sE4iUuMMVfA,71194
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=0xg_d8dwd36Y8GY1mE-LLO1LQpPEMM77bzAfc_KdgzU,4870
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=Ca75-OQiu5HeA8Q6zQpEYuhyCSjeuWe99K4y9ipTORM,4032
@@ -111,6 +111,7 @@ reconcile/terraform_aws_route53.py,sha256=06VIlIb95BzVkxV_1TPiaY9sQO-TkvQXL4V_qz
 reconcile/terraform_cloudflare_dns.py,sha256=auU4bzeLwd4S8D8oqpqJbrCUoEdELXrgi7vHOedjYFk,13332
 reconcile/terraform_cloudflare_resources.py,sha256=BQg12mHm1iaxf086FFPZutPbWKUMaddqu-nREPR8ptA,14887
 reconcile/terraform_cloudflare_users.py,sha256=Bv0f9lOO_wTM7st8iltb8FR8gu4KpKu3qavMzAYcoMc,13965
+reconcile/terraform_repo.py,sha256=9Gs5Xbt6qNR_Q_78evgvYWlRkyDQK_4v-_7mS6GQw0k,11112
 reconcile/terraform_resources.py,sha256=gQ-LT0TGwf9OR4RF5EWDmNHUnKWnbhrIMtyIdUgP4D4,16782
 reconcile/terraform_tgw_attachments.py,sha256=ootT8zPxcm3-VHy9OiG0zBP0X7wzrvTCh53eYbxJvfI,13725
 reconcile/terraform_users.py,sha256=AzDvEQCdLpsXoS3nLbIQRraQvJHa8JmL40lZFv8YXMk,9321
@@ -119,8 +120,8 @@ reconcile/unleash_watcher.py,sha256=xNLUFpIr66XESEyXUkmHTTmHghVWHiMtnS_k0OC7gd8,
 reconcile/vault_replication.py,sha256=xobxnsOfUcwvdQ-RZ7JH_sZCDh8rpEY7MJ36nkvfFqE,17262
 reconcile/vpc_peerings_validator.py,sha256=10igLYTQpBMGXO9mTO7sJBzgr4jXQ2hf1OH5r5DKugE,3586
 reconcile/aus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aus/advanced_upgrade_service.py,sha256=X3_mjNsocBEwgyGHs1O2hG6lTL0BOlM2DX_cvsmXtZ8,10953
-reconcile/aus/base.py,sha256=z76_aJrxJZHu8XZmVeGHlVKF3SjMwE8Ayrr5NL1VRqc,26221
+reconcile/aus/advanced_upgrade_service.py,sha256=R0V-APkx12JOQNySLSM1HvJDJSzKp3QyBO0hBBoVzlM,11316
+reconcile/aus/base.py,sha256=qnO8kmnesMho6a-EgFdrVejjPtwS_jkdlSPHaXYB1Rs,26259
 reconcile/aus/models.py,sha256=Qj4hmJr6J3fsH1acOaudqvGnXIuk5pXakckOf3L_qHA,4536
 reconcile/aus/ocm_addons_upgrade_scheduler_org.py,sha256=b9qGFWwY8aSEzXeTtw89dqQeK_dJGBEfopOOkdjO8V8,6026
 reconcile/aus/ocm_upgrade_scheduler.py,sha256=Wh2ZbODeOF7_hEbXfsFUGnAJ9CLSy76lNKEmir-GHuM,3447
@@ -244,6 +245,8 @@ reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_re
 reconcile/gql_definitions/terraform_cloudflare_users/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_cloudflare_users/app_interface_setting_cloudflare_and_vault.py,sha256=OHfIzX9qAePtRwARYNERuvafwVv0Zy0YUbT83Frt3eA,1984
 reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.py,sha256=CopEDfqnz6M-rW4kwkbFK_5FvAj7t8NzzffGZUhCTuo,4059
+reconcile/gql_definitions/terraform_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/terraform_repo/terraform_repo.py,sha256=lr9a9pCbhrrwrZCodCzmzQmFTS8gqfry1sGtQ-4v7Z4,2423
 reconcile/gql_definitions/terraform_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py,sha256=_v6Grk8TtqYcIquwgfiE_Ex24QIxKRPUZaoYn3HsAoc,39688
 reconcile/gql_definitions/terraform_tgw_attachments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -288,7 +291,7 @@ reconcile/templates/aws_access_key_email.j2,sha256=2MUr1ERmyISzKgHqsWYLd-1Wbl-pe
 reconcile/templates/email.yml.j2,sha256=OZgczNRgXPj2gVYTgwQyHAQrMGu7xp-e4W1rX19GcrU,690
 reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9igCUQhCnFDYh6xw-zcIbU,570
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/test/conftest.py,sha256=_pvENzQ-5vCvWi40PK70iVVDRrkKUfbC9bTQsjXSj0A,3046
+reconcile/test/conftest.py,sha256=dBWwQMkcdONERlnBJg4J6Td5Fexpbvme5y-UuoI-c9M,3185
 reconcile/test/fixtures.py,sha256=VhvLXH0AWXEyu3FgPp7bcSTPmDPfMEa2v-_9cd8dCmw,572
 reconcile/test/test_aggregated_list.py,sha256=iiWitQuNYC58aimWaiBoE4NROHjr1NCgQ91MnHEG_Ro,6412
 reconcile/test/test_amtool.py,sha256=vxRhGieeydMBOb9UI2ziMHjJa8puMeGNsUhGhy-yMnk,1032
@@ -346,6 +349,7 @@ reconcile/test/test_sql_query.py,sha256=l0QyIflcErIrAwSP8kOIub0jO6oi0Ncuns5IJtnu
 reconcile/test/test_terraform_cloudflare_dns.py,sha256=aQTXX8Vr4h9aWvJZTnpZEhMGYoBpT2d45ZxU_ECIQ6o,3425
 reconcile/test/test_terraform_cloudflare_resources.py,sha256=cWNE2UIhz19rLSWdpJG8xRwuEEYoIZWEkDZY7e2QN_g,3426
 reconcile/test/test_terraform_cloudflare_users.py,sha256=8iAFjz-zbUW4xLS10Lk1XvYSk4B_W__YT9rgrBuigcQ,27482
+reconcile/test/test_terraform_repo.py,sha256=fCr14via4GHmuzAuGIr53PZwNQ2hq4Ys1Iv8Pgro398,6039
 reconcile/test/test_terraform_resources.py,sha256=dEpJwaTzE_FzkRjCozDtGzE4egBrb-VrwSoWr2Benv4,7955
 reconcile/test/test_terraform_tgw_attachments.py,sha256=ddf04h_uKYroJOWKOFGZxuJNL-1PSjW5EyddQB3CLSw,33744
 reconcile/test/test_terraform_users.py,sha256=Yt4iN5FMtn7cfVlVqBJ1MMH94Z0DGchyByhpfNUJFxM,1570
@@ -537,13 +541,13 @@ reconcile/utils/mr/user_maintenance.py,sha256=_4VwAMJsBxD7maM7AZsMl_GjYRgQtZb_rl
 reconcile/utils/ocm/__init__.py,sha256=5Pcf5cyftDWT5XRi1EzvNklOVxGplJi-v12HN3TDarc,57
 reconcile/utils/ocm/base.py,sha256=8pCZB_V6pZhc-qZQFTuIr1kM5nxrzz9kT86-DW3rgq0,168
 reconcile/utils/ocm/cluster_groups.py,sha256=TBb3mIzw68BHBcCzacmAN8IsNPo8FfOnTMphuctwuU0,2679
-reconcile/utils/ocm/clusters.py,sha256=0QuvFUQIRGXlPBvVdQrUeKbVj67eL2KcATwOpoOp12c,7407
+reconcile/utils/ocm/clusters.py,sha256=4Ddc0Ah-s5gmNUK0K_4e2zaHyip50fyCCSFuAirPO2s,7766
 reconcile/utils/ocm/labels.py,sha256=5Edjk9F3o4AnhzYc-YUiwDkG_pnEPcXrIXT9g1uE3x0,4883
 reconcile/utils/ocm/ocm.py,sha256=pp-T6cT0RQf91oqrWFm5QSd_LyH8n5gJmPaetDtMioI,64455
 reconcile/utils/ocm/search_filters.py,sha256=zExZpYBh7_tucG-xKoPHUxz1b_6l9qwbEMpMihQg7nA,15043
 reconcile/utils/ocm/service_log.py,sha256=6_RNLG6KlnXNmJW6xBrlqGdyyz-fuaSeFmiJFOLN51g,2535
 reconcile/utils/ocm/sre_capability_labels.py,sha256=HY5v3ndMu1hwbLipX24l1R-dvm0SFKFettnV7detVBI,1530
-reconcile/utils/ocm/subscriptions.py,sha256=kuHster01yLy0_s5DWFLXh9zse3NIpbGTuKW78q_b90,2322
+reconcile/utils/ocm/subscriptions.py,sha256=PO5Rp440NEmZxIVVvhQpPzDRcKL1jYoQq5j2N9AkM1c,2344
 reconcile/utils/runtime/__init__.py,sha256=l9o8APZxyED5Q6ylGoLIESksQF4f3O8cdke3IdMFOTQ,108
 reconcile/utils/runtime/desired_state_diff.py,sha256=AQhJmq3CP2YOWP-KpmVtYKnhZ46sxERfbk_R6PHO-zc,8272
 reconcile/utils/runtime/environment.py,sha256=cJgCMRBeschdeKJuk_N6BhDWaOCZbo-41i2a9L9DpBE,1328
@@ -570,7 +574,7 @@ tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/app_interface_reporter.py,sha256=M_5C7ySACJG6Hp0UcY-w9ZY0sTVE7oQ48qoijp10jag,21933
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=PXN5wxl6OX8sxddPaakDF3X79nFLvpm-lz0mWLVelw0,2806
-tools/qontract_cli.py,sha256=GLnFJS2FQs1yX_bjAVR4PNgEDlCPcNqDv1i4pEOiqVU,90439
+tools/qontract_cli.py,sha256=pGBwz3TjvfSDCYlKSISTzCG5ab1ljxJQIfTh3lkljOY,90671
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/cli_commands/gpg_encrypt.py,sha256=JryinrDdvztN931enUY3FuDeLVnfs6y58mnK7itNK6Y,4940
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
@@ -578,8 +582,8 @@ tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y
 tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_qontract_cli.py,sha256=awwTHEc2DWlykuqGIYM0WOBoSL0KRnOraCLk3C7izis,1401
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc41.dist-info/METADATA,sha256=FcNoC48l0wdeWYF2QclApVirXpYew_Ew1bJtzZD_n38,2288
-qontract_reconcile-0.10.1rc41.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-qontract_reconcile-0.10.1rc41.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
-qontract_reconcile-0.10.1rc41.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc41.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc43.dist-info/METADATA,sha256=wdqh9jrOR8RDo-xKPaCMIoa3MzNi2zTn_Jliv0jWgsQ,2288
+qontract_reconcile-0.10.1rc43.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+qontract_reconcile-0.10.1rc43.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
+qontract_reconcile-0.10.1rc43.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc43.dist-info/RECORD,,

reconcile/aus/advanced_upgrade_service.py

@@ -75,7 +75,11 @@ class AdvancedUpgradeServiceIntegration(OCMClusterUpgradeSchedulerOrgIntegration
         self, ocm_env: OCMEnvironment, org_ids: Optional[set[str]]
     ) -> dict[str, OrganizationUpgradeSpec]:
         ocm_api = init_ocm_base_client(ocm_env, self.secret_reader)
-        clusters_by_org = _discover_clusters(ocm_api=ocm_api, org_ids=org_ids)
+        clusters_by_org = discover_clusters(
+            ocm_api=ocm_api,
+            org_ids=org_ids,
+            ignore_sts_clusters=self.params.ignore_sts_clusters,
+        )
         labels_by_org = _get_org_labels(ocm_api=ocm_api, org_ids=org_ids)
 
         return _build_org_upgrade_specs_for_ocm_env(
@@ -96,8 +100,10 @@ class AdvancedUpgradeServiceIntegration(OCMClusterUpgradeSchedulerOrgIntegration
             )
 
 
-def _discover_clusters(
-    ocm_api: OCMBaseClient, org_ids: Optional[set[str]] = None
+def discover_clusters(
+    ocm_api: OCMBaseClient,
+    org_ids: Optional[set[str]] = None,
+    ignore_sts_clusters: bool = False,
 ) -> dict[str, list[ClusterDetails]]:
     """
     Discover all clusters that are part of the AUS service.
@@ -111,7 +117,10 @@ def _discover_clusters(
     # group by org and filter if org_id is specified
     clusters_by_org: dict[str, list[ClusterDetails]] = defaultdict(list)
     for c in clusters:
-        if org_ids is None or c.organization_id in org_ids:
+        is_sts_cluster = c.ocm_cluster.aws and c.ocm_cluster.aws.sts_enabled
+        passed_sts_filter = not ignore_sts_clusters or not is_sts_cluster
+        passed_ocm_filters = org_ids is None or c.organization_id in org_ids
+        if passed_ocm_filters and passed_sts_filter:
             clusters_by_org[c.organization_id].append(c)
 
     return clusters_by_org

reconcile/aus/base.py

@@ -59,6 +59,7 @@ class AdvancedUpgradeSchedulerBaseIntegrationParams(PydanticRunParams):
 
     ocm_environment: Optional[str] = None
     ocm_organization_ids: Optional[set[str]] = None
+    ignore_sts_clusters: bool = False
 
 
 class AdvancedUpgradeSchedulerBaseIntegration(

reconcile/cli.py

@@ -1546,6 +1546,26 @@ def ldap_users(ctx, infra_project_id, app_interface_project_id):
     )
 
 
+@integration.command(short_help="Manages raw HCL Terraform from a separate repository.")
+@click.option(
+    "-d",
+    "--output-dir",
+    help="Specify a directory to individually output each repo plan to for the executor",
+)
+@click.pass_context
+def terraform_repo(ctx, output_dir):
+    from reconcile import terraform_repo
+
+    run_class_integration(
+        integration=terraform_repo.TerraformRepoIntegration(
+            terraform_repo.TerraformRepoIntegrationParams(
+                output_dir=output_dir, validate_git=True
+            )
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage AWS Resources using Terraform.")
 @print_to_file
 @vault_output_path
@@ -1996,8 +2016,14 @@ def ocm_addons_upgrade_scheduler_org(ctx):
     required=False,
     envvar="AUS_OCM_ORG_IDS",
 )
+@click.option(
+    "--ignore-sts-clusters",
+    is_flag=True,
+    default=os.environ.get("IGNORE_STS_CLUSTERS", False),
+    help="Ignore STS clusters",
+)
 @click.pass_context
-def aus_upgrade_scheduler_org(ctx, ocm_env, ocm_org_ids):
+def aus_upgrade_scheduler_org(ctx, ocm_env, ocm_org_ids, ignore_sts_clusters):
     from reconcile.aus.advanced_upgrade_service import AdvancedUpgradeServiceIntegration
     from reconcile.aus.base import AdvancedUpgradeSchedulerBaseIntegrationParams
 
@@ -2007,6 +2033,7 @@ def aus_upgrade_scheduler_org(ctx, ocm_env, ocm_org_ids):
             AdvancedUpgradeSchedulerBaseIntegrationParams(
                 ocm_environment=ocm_env,
                 ocm_organization_ids=parsed_ocm_org_ids,
+                ignore_sts_clusters=ignore_sts_clusters,
             )
         ),
         ctx=ctx.obj,

reconcile/gql_definitions/terraform_repo/terraform_repo.py

@@ -0,0 +1,91 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query TerraformRepo {
+  repos: terraform_repo_v1 {
+    account {
+      name
+      uid
+      automationToken {
+        ...VaultSecret
+      }
+    }
+    name
+    repository
+    ref
+    projectPath
+    delete
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union = True
+        extra = Extra.forbid
+
+
+class AWSAccountV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    uid: str = Field(..., alias="uid")
+    automation_token: VaultSecret = Field(..., alias="automationToken")
+
+
+class TerraformRepoV1(ConfiguredBaseModel):
+    account: AWSAccountV1 = Field(..., alias="account")
+    name: str = Field(..., alias="name")
+    repository: str = Field(..., alias="repository")
+    ref: str = Field(..., alias="ref")
+    project_path: str = Field(..., alias="projectPath")
+    delete: Optional[bool] = Field(..., alias="delete")
+
+
+class TerraformRepoQueryData(ConfiguredBaseModel):
+    repos: Optional[list[TerraformRepoV1]] = Field(..., alias="repos")
+
+
+def query(query_func: Callable, **kwargs: Any) -> TerraformRepoQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        TerraformRepoQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return TerraformRepoQueryData(**raw_data)

reconcile/terraform_repo.py

@@ -0,0 +1,302 @@
+import logging
+from collections.abc import Callable
+from typing import (
+    Any,
+    Optional,
+)
+
+import yaml
+from pydantic import (
+    BaseModel,
+    ValidationError,
+)
+
+from reconcile import queries
+from reconcile.gql_definitions.terraform_repo.terraform_repo import (
+    TerraformRepoV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.defer import defer
+from reconcile.utils.differ import (
+    DiffResult,
+    diff_iterables,
+)
+from reconcile.utils.exceptions import ParameterError
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.state import (
+    State,
+    init_state,
+)
+
+
+class RepoSecret(BaseModel):
+    path: str
+    version: Optional[int]
+
+
+class RepoOutput(BaseModel):
+    """
+    Output of the QR terraform-repo integration and input to the executor
+    which removes some information that is unnecessary for the executor to parse
+    """
+
+    dry_run: bool
+    repository: str
+    name: str
+    ref: str
+    project_path: str
+    delete: bool
+    secret: RepoSecret
+
+
+class TerraformRepoIntegrationParams(PydanticRunParams):
+    output_dir: Optional[str]
+    validate_git: bool
+
+
+class TerraformRepoIntegration(
+    QontractReconcileIntegration[TerraformRepoIntegrationParams]
+):
+    def __init__(self, params: TerraformRepoIntegrationParams) -> None:
+        super().__init__(params)
+        self.qontract_integration = "terraform_repo"
+        self.qontract_integration_version = make_semver(0, 1, 0)
+        self.qontract_tf_prefix = "qrtfrepo"
+
+    @property
+    def name(self) -> str:
+        return self.qontract_integration.replace("_", "-")
+
+    @defer
+    def run(
+        self,
+        dry_run: bool,
+        defer: Optional[Callable] = None,
+    ) -> None:
+
+        gqlapi = gql.get_api()
+
+        state = init_state(integration=self.name)
+        if defer:
+            defer(state.cleanup)
+
+        desired = self.get_repos(query_func=gqlapi.query)
+        existing = self.get_existing_state(state)
+
+        repo_diff = self.calculate_diff(
+            existing_state=existing, desired_state=desired, dry_run=dry_run, state=state
+        )
+
+        for repo in repo_diff:
+            # format each repo into the input the executor expects
+            repo_output = RepoOutput(
+                dry_run=dry_run,
+                repository=repo.repository,
+                name=repo.name,
+                ref=repo.ref,
+                project_path=repo.project_path,
+                delete=repo.delete or False,
+                secret=RepoSecret(
+                    path=repo.account.automation_token.path,
+                    version=repo.account.automation_token.version,
+                ),
+            )
+
+            if self.params.output_dir:
+                try:
+                    output_filename = f"{self.params.output_dir}/{repo.name}.yaml"
+                    with open(output_filename, "w") as output_file:
+                        yaml.safe_dump(
+                            data=repo_output.dict(),
+                            stream=output_file,
+                            explicit_start=True,
+                        )
+                except FileNotFoundError:
+                    raise ParameterError(f"Unable to write to '{output_filename}'")
+            else:
+                print(yaml.safe_dump(data=repo_output.dict(), explicit_start=True))
+
+    def get_repos(self, query_func: Callable) -> list[TerraformRepoV1]:
+        """Gets a list of terraform repos defined in App Interface
+
+        :param query_func: function which queries GQL server
+        :type query_func: Callable
+        :return: list of Terraform repos or empty list if none are defined in A-I
+        :rtype: list[TerraformRepoV1]
+        """
+        query_results = query(query_func=query_func).repos
+        if query_results:
+            return query_results
+        return []
+
+    def get_existing_state(self, state: State) -> list[TerraformRepoV1]:
+        """Gets the state of terraform infrastructure currently deployed (stored in S3)
+
+        :param state: S3 state class to retrieve from
+        :type state: State
+        :return: list of terraform repos or empty list if state is unparsable or no repos are deployed
+        :rtype: list[TerraformRepoV1]
+        """
+        repo_list: list[TerraformRepoV1] = []
+        keys = state.ls()
+        for key in keys:
+            if value := state.get(key.lstrip("/"), None):
+                try:
+                    repo = TerraformRepoV1.parse_raw(value)
+                    repo_list.append(repo)
+                except ValidationError as err:
+                    logging.error(
+                        f"{err}\nUnable to parse existing state for repo: '{key}', skipping"
+                    )
+
+        return repo_list
+
+    def check_ref(self, repo_url: str, ref: str) -> None:
+        """Validates that a Git SHA exists
+
+        :param repo_url: full project URL including https/http
+        :type repo_url: str
+        :param ref: git SHA
+        :type ref: str
+        :raises ParameterError: if the Git ref is invalid or project is not reachable
+        """
+        instance = queries.get_gitlab_instance()
+        with GitLabApi(
+            instance,
+            settings=queries.get_secret_reader_settings(),
+            project_url=repo_url,
+        ) as gl:
+            try:
+                gl.get_commit_sha(ref=ref, repo_url=repo_url)
+            except (KeyError, AttributeError):
+                raise ParameterError(
+                    f'Invalid ref: "{ref}" on repo: "{repo_url}". Or the project repo is not reachable'
+                )
+
+    def merge_results(
+        self,
+        diff_result: DiffResult[TerraformRepoV1, TerraformRepoV1, str],
+    ) -> list[TerraformRepoV1]:
+        """Transforms the diff or repos into a list of repos that need to be changed or deleted
+
+        :param diff_result: diff result of existing and desired state
+        :type diff_result: DiffResult[TerraformRepoV1, TerraformRepoV1, str]
+        :return: list of repos that need to be changed or deleted
+        :rtype: list[TerraformRepoV1]
+        """
+        output: list[TerraformRepoV1] = []
+        for add_key, add_val in diff_result.add.items():
+            logging.info(["create_repo", add_val.account.name, add_key])
+            output.append(add_val)
+        for change_key, change_val in diff_result.change.items():
+            if change_val.desired.delete:
+                logging.info(
+                    ["delete_repo", change_val.desired.account.name, change_key]
+                )
+                output.append(change_val.desired)
+            else:
+                logging.info(
+                    ["update_repo", change_val.desired.account.name, change_key]
+                )
+                output.append(change_val.desired)
+        return output
+
+    def update_state(
+        self,
+        diff_result: DiffResult[TerraformRepoV1, TerraformRepoV1, str],
+        state: State,
+    ) -> None:
+        """The state of deployed terraform infrastructure is tracked using AWS S3.
+        Each repo is saved as a JSON dump of a TerraformRepoV1 object meaning that it can
+        be easily compared against the GQL representation in App Interface
+
+        :param diff_result: diff of existing and desired state
+        :type diff_result: DiffResult[TerraformRepoV1, TerraformRepoV1, str]
+        :param state: S3 state class
+        :type state: State
+        """
+        try:
+            for add_key, add_val in diff_result.add.items():
+                # state.add already performs a json.dumps(key) so we export the
+                # pydantic model as a dict to avoid a double json dump with extra quotes
+                state.add(add_key, add_val.dict(by_alias=True), force=True)
+            for delete_key in diff_result.delete.keys():
+                state.rm(delete_key)
+            for change_key, change_val in diff_result.change.items():
+                if change_val.desired.delete:
+                    state.rm(change_key)
+                else:
+                    state.add(
+                        change_key, change_val.desired.dict(by_alias=True), force=True
+                    )
+        except KeyError:
+            pass
+
+    def calculate_diff(
+        self,
+        existing_state: list[TerraformRepoV1],
+        desired_state: list[TerraformRepoV1],
+        dry_run: bool,
+        state: Optional[State],
+    ) -> list[TerraformRepoV1]:
+        """Calculated the difference between existing and desired state
+        to determine what actions the executor will need to take
+
+        :param existing_state: list of Terraform infrastructure that is currently applied
+        :type existing_state: list[TerraformRepoV1]
+        :param desired_state: list of Terraform infrastructure we want
+        :type desired_state: list[TerraformRepoV1]
+        :param dry_run: determines whether State should be updated
+        :type dry_run: bool
+        :param state: AWS S3 state
+        :type state: Optional[State]
+        :raises ParameterError: if there is an invalid operation performed like trying to delete
+        a representation in A-I before setting the delete flag
+        :return: list of Terraform Repos for the executor to act on
+        :rtype: list[TerraformRepoV1]
+        """
+        diff = diff_iterables(existing_state, desired_state, lambda x: x.name)
+
+        # added repos: do standard validation that SHA is valid
+        if self.params.validate_git:
+            for add_repo in diff.add.values():
+                self.check_ref(add_repo.repository, add_repo.ref)
+        # removed repos: ensure that delete = true already
+        for delete_repo in diff.delete.values():
+            if not delete_repo.delete:
+                raise ParameterError(
+                    f'To delete the terraform repo "{delete_repo.name}", you must set delete: true in the repo definition'
+                )
+        # changed repos: prevent non deterministic terraform behavior by disabling updating key parameters
+        # also do SHA verification
+        for changes in diff.change.values():
+            c = changes.current
+            d = changes.desired
+            if (
+                c.account != d.account
+                or c.name != d.name
+                or c.project_path != d.project_path
+                or c.repository != d.repository
+            ):
+                raise ParameterError(
+                    f'Only the `ref` and `delete` parameters for a terraform repo may be updated in merge requests on repo: "{d.name}"'
+                )
+            if self.params.validate_git:
+                self.check_ref(d.repository, d.ref)
+
+        if not dry_run and state:
+            self.update_state(diff, state)
+
+        return self.merge_results(diff)
+
+    def early_exit_desired_state(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        gqlapi = gql.get_api()
+        return {
+            "repos": [repo.dict() for repo in self.get_repos(query_func=gqlapi.query)]
+        }

reconcile/test/conftest.py

@@ -65,8 +65,13 @@ def s3_state_builder() -> Callable[[Mapping], State]:
     """
 
     def builder(data: Mapping) -> State:
-        def get(key: str) -> dict:
-            return data["get"][key]
+        def get(key: str, *args) -> dict:
+            try:
+                return data["get"][key]
+            except KeyError:
+                if args:
+                    return args[0]
+                raise
 
         state = create_autospec(spec=State)
         state.get = get

reconcile/test/test_terraform_repo.py

@@ -0,0 +1,215 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.gql_definitions.terraform_repo.terraform_repo import (
+    AWSAccountV1,
+    TerraformRepoV1,
+)
+from reconcile.terraform_repo import (
+    TerraformRepoIntegration,
+    TerraformRepoIntegrationParams,
+)
+from reconcile.utils.exceptions import ParameterError
+from reconcile.utils.state import State
+
+A_REPO = "https://git-example/tf-repo-example"
+A_REPO_SHA = "a390f5cb20322c90861d6d80e9b70c6a579be1d0"
+B_REPO = "https://git-example/tf-repo-example2"
+B_REPO_SHA = "94edb90815e502b387c25358f5ec602e52d0bfbb"
+AWS_UID = "000000000000"
+AUTOMATION_TOKEN_PATH = "aws-secrets/terraform/foo"
+
+
+@pytest.fixture
+def existing_repo(aws_account) -> TerraformRepoV1:
+    return TerraformRepoV1(
+        name="a_repo",
+        repository=A_REPO,
+        ref=A_REPO_SHA,
+        account=aws_account,
+        projectPath="tf",
+        delete=False,
+    )
+
+
+@pytest.fixture
+def new_repo(aws_account) -> TerraformRepoV1:
+    return TerraformRepoV1(
+        name="b_repo",
+        repository=B_REPO,
+        ref=B_REPO_SHA,
+        account=aws_account,
+        projectPath="tf",
+        delete=False,
+    )
+
+
+@pytest.fixture()
+def automation_token() -> VaultSecret:
+    return VaultSecret(path=AUTOMATION_TOKEN_PATH, version=1, field="all", format=None)
+
+
+@pytest.fixture
+def aws_account(automation_token) -> AWSAccountV1:
+    return AWSAccountV1(
+        name="foo",
+        uid="000000000000",
+        automationToken=automation_token,
+    )
+
+
+@pytest.fixture
+def int_params() -> TerraformRepoIntegrationParams:
+    return TerraformRepoIntegrationParams(print_to_file=None, validate_git=False)
+
+
+@pytest.fixture()
+def a_repo_json() -> str:
+    # terraform repo expects a JSON string not a dict so we have to encode a multi-line JSON string
+    return f"""
+            {{
+                "name": "a_repo",
+                "repository": "{A_REPO}",
+                "ref": "{A_REPO_SHA}",
+                "projectPath": "tf",
+                "delete": false,
+                "account": {{
+                    "name": "foo",
+                    "uid": "{AWS_UID}",
+                    "automationToken": {{
+                        "path": "{AUTOMATION_TOKEN_PATH}",
+                        "field": "all",
+                        "version": 1,
+                        "format": null
+                    }}
+                }}
+            }}
+            """
+
+
+@pytest.fixture()
+def state_mock() -> MagicMock:
+    return MagicMock(spec=State)
+
+
+def test_addition_to_existing_repo(existing_repo, new_repo, int_params, state_mock):
+    existing = [existing_repo]
+    desired = [existing_repo, new_repo]
+
+    integration = TerraformRepoIntegration(params=int_params)
+    diff = integration.calculate_diff(
+        existing_state=existing, desired_state=desired, dry_run=False, state=state_mock
+    )
+
+    assert diff == [new_repo]
+
+    # ensure that the state is saved for the new repo
+    state_mock.add.assert_called_once_with(
+        new_repo.name, new_repo.dict(by_alias=True), force=True
+    )
+
+
+def test_updating_repo_ref(existing_repo, int_params, state_mock):
+    existing = [existing_repo]
+    updated_repo = TerraformRepoV1.copy(existing_repo)
+    updated_repo.ref = B_REPO_SHA
+
+    integration = TerraformRepoIntegration(params=int_params)
+    diff = integration.calculate_diff(
+        existing_state=existing,
+        desired_state=[updated_repo],
+        dry_run=False,
+        state=state_mock,
+    )
+
+    assert diff == [updated_repo]
+
+    state_mock.add.assert_called_once_with(
+        updated_repo.name, updated_repo.dict(by_alias=True), force=True
+    )
+
+
+def test_fail_on_update_invalid_repo_params(existing_repo, int_params):
+    existing = [existing_repo]
+    updated_repo = TerraformRepoV1.copy(existing_repo)
+    updated_repo.name = "c_repo"
+    updated_repo.project_path = "c_repo"
+    updated_repo.repository = B_REPO
+    updated_repo.ref = B_REPO_SHA
+    updated_repo.delete = True
+
+    integration = TerraformRepoIntegration(params=int_params)
+
+    with pytest.raises(ParameterError):
+        integration.calculate_diff(
+            existing_state=existing,
+            desired_state=[updated_repo],
+            dry_run=True,
+            state=None,
+        )
+
+
+def test_delete_repo(existing_repo, int_params, state_mock):
+    existing = [existing_repo]
+    updated_repo = TerraformRepoV1.copy(existing_repo)
+    updated_repo.delete = True
+
+    integration = TerraformRepoIntegration(params=int_params)
+
+    diff = integration.calculate_diff(
+        existing_state=existing,
+        desired_state=[updated_repo],
+        dry_run=False,
+        state=state_mock,
+    )
+
+    assert diff == [updated_repo]
+
+    state_mock.rm.assert_called_once_with(updated_repo.name)
+
+
+def test_delete_repo_without_flag(existing_repo, int_params):
+    existing = [existing_repo]
+
+    integration = TerraformRepoIntegration(params=int_params)
+
+    with pytest.raises(ParameterError):
+        integration.calculate_diff(
+            existing_state=existing, desired_state=[], dry_run=True, state=None
+        )
+
+
+def test_get_repo_state(s3_state_builder, int_params, existing_repo, a_repo_json):
+    state = s3_state_builder(
+        {
+            "ls": [
+                "/a_repo",
+            ],
+            "get": {"a_repo": a_repo_json},
+        }
+    )
+
+    integration = TerraformRepoIntegration(params=int_params)
+
+    existing_state = integration.get_existing_state(state=state)
+    assert existing_state == [existing_repo]
+
+
+def test_update_repo_state(int_params, existing_repo, state_mock):
+    integration = TerraformRepoIntegration(params=int_params)
+
+    existing_state: list = []
+    desired_state = [existing_repo]
+
+    integration.calculate_diff(
+        existing_state=existing_state,
+        desired_state=desired_state,
+        dry_run=False,
+        state=state_mock,
+    )
+
+    state_mock.add.assert_called_once_with(
+        existing_repo.name, existing_repo.dict(by_alias=True), force=True
+    )

reconcile/utils/ocm/clusters.py

@@ -24,6 +24,8 @@ from reconcile.utils.ocm.subscriptions import (
 )
 from reconcile.utils.ocm_base_client import OCMBaseClient
 
+ACTIVE_SUBSCRIPTION_STATES = {"Active", "Reserved"}
+
 
 class OCMClusterState(Enum):
     ERROR = "error"
@@ -39,6 +41,19 @@ class OCMClusterState(Enum):
     WAITING = "waiting"
 
 
+class OCMClusterFlag(BaseModel):
+
+    enabled: bool
+
+
+class OCMClusterAWSSettings(BaseModel):
+    sts: Optional[OCMClusterFlag]
+
+    @property
+    def sts_enabled(self) -> bool:
+        return self.sts is not None and self.sts.enabled
+
+
 class OCMCluster(BaseModel):
 
     kind: str = "Cluster"
@@ -52,8 +67,6 @@ class OCMCluster(BaseModel):
     display_name: str
 
     managed: bool
-
-    openshift_version: str
     state: OCMClusterState
 
     subscription: OCMModelLink
@@ -61,6 +74,8 @@ class OCMCluster(BaseModel):
     cloud_provider: OCMModelLink
     product: OCMModelLink
 
+    aws: Optional[OCMClusterAWSSettings]
+
 
 class ClusterDetails(BaseModel):
 
@@ -188,7 +203,8 @@ def get_cluster_details_for_subscriptions(
     # get subscription details
     subscriptions = get_subscriptions(
         ocm_api=ocm_api,
-        filter=(subscription_filter or Filter()) & build_subscription_filter(),
+        filter=(subscription_filter or Filter())
+        & build_subscription_filter(states=ACTIVE_SUBSCRIPTION_STATES, managed=True),
     )
     if not subscriptions:
         return

reconcile/utils/ocm/subscriptions.py

@@ -77,9 +77,11 @@ def get_subscriptions(
     return subscriptions
 
 
-def build_subscription_filter(state: str = "Active", managed: bool = True) -> Filter:
+def build_subscription_filter(
+    states: Optional[set[str]] = None, managed: bool = True
+) -> Filter:
     """
     Helper function to create a subscription search filer for two very common
     fields: status and managed.
     """
-    return Filter().eq("status", state).eq("managed", str(managed).lower())
+    return Filter().is_in("status", states).eq("managed", str(managed).lower())

tools/qontract_cli.py

@@ -630,8 +630,14 @@ def ocm_fleet_upgrade_policies(
     required=False,
     envvar="AUS_OCM_ORG_IDS",
 )
+@click.option(
+    "--ignore-sts-clusters",
+    is_flag=True,
+    default=os.environ.get("IGNORE_STS_CLUSTERS", False),
+    help="Ignore STS clusters",
+)
 @click.pass_context
-def aus_fleet_upgrade_policies(ctx, ocm_env, ocm_org_ids):
+def aus_fleet_upgrade_policies(ctx, ocm_env, ocm_org_ids, ignore_sts_clusters):
     from reconcile.aus.advanced_upgrade_service import AdvancedUpgradeServiceIntegration
 
     parsed_ocm_org_ids = set(ocm_org_ids.split(",")) if ocm_org_ids else None
@@ -641,6 +647,7 @@ def aus_fleet_upgrade_policies(ctx, ocm_env, ocm_org_ids):
             AdvancedUpgradeSchedulerBaseIntegrationParams(
                 ocm_environment=ocm_env,
                 ocm_organization_ids=parsed_ocm_org_ids,
+                ignore_sts_clusters=ignore_sts_clusters,
             )
         ),
     )