qontract-reconcile 0.10.1rc990__py3-none-any.whl → 0.10.1rc992__py3-none-any.whl

{qontract_reconcile-0.10.1rc990.dist-info → qontract_reconcile-0.10.1rc992.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc990
+Version: 0.10.1rc992
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc990.dist-info → qontract_reconcile-0.10.1rc992.dist-info}/RECORD

@@ -112,7 +112,7 @@ reconcile/terraform_aws_route53.py,sha256=1C1_xd_xjysPLSYKvohS6bfSsjSMP0U3sWVcRN
 reconcile/terraform_cloudflare_dns.py,sha256=-aLEe2QnH5cJPu7HWqs-R9NmQ1NlFbcVUm0v7alVL3I,13431
 reconcile/terraform_cloudflare_resources.py,sha256=41Mj1WkuS75slCDpmhG2GGf1nh3BwfxcdNC73-PNadc,15000
 reconcile/terraform_cloudflare_users.py,sha256=iyTG5sj20Jg4J4qWJ144KVptfIHGOSfH8wQKxu0imq0,13942
-reconcile/terraform_repo.py,sha256=GN_wQytiXqL7SbSKpwm2TLtoLPOkN6atHsS1uynzN28,16269
+reconcile/terraform_repo.py,sha256=E-tVE62H5-B8CStVOi_DwqQ5l3qIYGd-l8MBFSo41mQ,16414
 reconcile/terraform_resources.py,sha256=-sgMMHDtNvnQyNR05-MKebI_pSiyxSWAg8LmeA2_Ntk,19326
 reconcile/terraform_tgw_attachments.py,sha256=EucuF4p3RWKTS4GTPd8oZmR79GpIW_grQl2PAeeNQeI,18665
 reconcile/terraform_users.py,sha256=HqSm3ev3b8dZ9J6F_phDZB-FQsnlsdeKp9RPoY1cU94,10188
@@ -179,10 +179,11 @@ reconcile/cna/assets/asset.py,sha256=KWgA4fuDAEGsJwmR52WwK_YgSJMW-1cV2la3lmNf4iE
 reconcile/cna/assets/asset_factory.py,sha256=7T7X_J6xIsoGETqBRI45_EyIKEdQcnRPt_GAuVuLQcc,785
 reconcile/cna/assets/null.py,sha256=85mVh97atCoC0aLuX47poTZiyOthmziJeBsUw0c924w,1658
 reconcile/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/dynatrace_token_provider/dependencies.py,sha256=uJLvR48kxfqjnBuP60XQx5RbkWPL3__FCgWjqwhKEjo,2160
-reconcile/dynatrace_token_provider/integration.py,sha256=n_t_x-9USDtkT_8koOn7SxxXXxc3lAYbkZENlVm6t5c,14909
+reconcile/dynatrace_token_provider/dependencies.py,sha256=41q05A4C_eS3E8-MR4veeMxtQNsPoGdxmEa3d-OKxq4,2814
+reconcile/dynatrace_token_provider/integration.py,sha256=ffH4BpMNb3AafwengwzurZ-aiztGP1MUlnVU4FnO3IY,21540
 reconcile/dynatrace_token_provider/metrics.py,sha256=xiKkl8fTEBQaXJelGCPNTZhHAWdO1M3pCXNr_Tei63c,1285
-reconcile/dynatrace_token_provider/ocm.py,sha256=IwksRMyGcJnamV88ORlBoyOr7uRENhMaHBoSXaGfwDY,2784
+reconcile/dynatrace_token_provider/model.py,sha256=gkpqo5rRRueBXnIMjp4EEHqBUBuU65TRI8zpdb8GJ0A,241
+reconcile/dynatrace_token_provider/ocm.py,sha256=iHMsgbsLs-dlrB9UXmWNDF7E4UDe49JOsLa9rnowKfo,4282
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
@@ -275,6 +276,7 @@ reconcile/gql_definitions/dashdotdb_slo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5J
 reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py,sha256=zUa-CmpOwiymVmOV6KwDHH5mMl06p000320FcOas6hU,4315
 reconcile/gql_definitions/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py,sha256=5gTuAnR2rnx2k6Rn7FMEAzw6GCZ6F5HZbqkmJ9-3NI4,2244
+reconcile/gql_definitions/dynatrace_token_provider/token_specs.py,sha256=XGsMuB8gowRpqJjkD_KRomx-1OswzyWbF4qjVdhionk,2555
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
 reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
@@ -337,6 +339,7 @@ reconcile/gql_definitions/ocm_labels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeu
 reconcile/gql_definitions/ocm_labels/clusters.py,sha256=K0DaTnOEdCZAziDbmq3Ktrxzzmzy6sdb3-9kIDk6yao,2932
 reconcile/gql_definitions/ocm_labels/organizations.py,sha256=ylNna62pG3XidrLQtMwu0LIOsKh6qyC2QImCbur2tV4,2252
 reconcile/gql_definitions/ocm_oidc_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/ocm_subscription_labels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/openshift_cluster_bots/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/openshift_cluster_bots/clusters.py,sha256=QshhCQeFRu_o0DLpD-4ltT5X_xZHjsCLc5jB3an3UXs,3688
 reconcile/gql_definitions/openshift_groups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -382,7 +385,7 @@ reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.
 reconcile/gql_definitions/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_init/aws_accounts.py,sha256=OJ0hDbRachRaDkL-OGT6-byr9cKdBiQDnNCpwUe3oJ8,2674
 reconcile/gql_definitions/terraform_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/terraform_repo/terraform_repo.py,sha256=_rdq3efy5Q3QFpI-vcs3-wacsXo_1fu1kVix_E83h5Q,3599
+reconcile/gql_definitions/terraform_repo/terraform_repo.py,sha256=nm4CH7Vog4aabdvCKmhVSUvoUb7dxSLx8nwAEJAVqG0,3706
 reconcile/gql_definitions/terraform_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_resources/database_access_manager.py,sha256=yv0_YC-LmhaKD_gyGG3le1w5BtypBjlsO894-Zgdg4U,4813
 reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py,sha256=1PQBvcJjErRaG529kAzMwfw-ShNaM92IQovlwEiYrV8,42491
@@ -551,7 +554,7 @@ reconcile/test/test_terraform_aws_route53.py,sha256=xHggb8K1P76OyCfFcogbkmyKle-N
 reconcile/test/test_terraform_cloudflare_dns.py,sha256=aQTXX8Vr4h9aWvJZTnpZEhMGYoBpT2d45ZxU_ECIQ6o,3425
 reconcile/test/test_terraform_cloudflare_resources.py,sha256=1mdSZS-38mtTSg7teJgDCJU1b_yKbwnrr3N5m8kwV4k,13595
 reconcile/test/test_terraform_cloudflare_users.py,sha256=2VGBtMUhckLPtUnQlHIzpGsCnyVJZPNLFf-ABELkxbQ,27456
-reconcile/test/test_terraform_repo.py,sha256=j9mLfwiK707U2KRxYpvzAbOYywk__pL9SXAH3xbP1t8,12184
+reconcile/test/test_terraform_repo.py,sha256=INfl-VlUtpV87J0neQt4wliptnX7PKvxLPF-ZgweTFA,12960
 reconcile/test/test_terraform_resources.py,sha256=8C97yXIEihaQ3DZrtjxLNt4y4G12IOhD01ydm7JjliY,15359
 reconcile/test/test_terraform_tgw_attachments.py,sha256=SM6QwogMZNLh0BkUyaxzFafuOLp23-hBtYTu_F53C4I,40922
 reconcile/test/test_terraform_users.py,sha256=XOAfGvITCJPI1LTlISmHbA4ONMQMkxYUMTsny7pQCFw,4319
@@ -598,6 +601,7 @@ reconcile/typed_queries/clusters_with_dms.py,sha256=JDSKZXwO3QxT-uA1FaHxP8d4XiYA
 reconcile/typed_queries/clusters_with_peering.py,sha256=lIai7SJJD0bqIJbe7virgrbYRqjLouSL2OpJD0itpAY,330
 reconcile/typed_queries/dynatrace.py,sha256=8vXDXDIDf9_vN_efYwysDr4gLN7SCx4I2bOoNxQhbio,312
 reconcile/typed_queries/dynatrace_environments.py,sha256=VV_7KzKG9wqGDV9wZLbcCJtfuzPhTV1wdg0YwAOaq3A,413
+reconcile/typed_queries/dynatrace_token_provider_token_specs.py,sha256=x41KG6JRDNYw5QGJYtIFNwSeejUUgxrL-agS8qFf6q0,433
 reconcile/typed_queries/external_resources.py,sha256=h1uzZzmtEGzoqSFhDMSAdxauGJoGy0stPuWbA0rkVKE,1503
 reconcile/typed_queries/get_state_aws_account.py,sha256=CSJjVPWsUZ2rkGIt8ehoQt7hokFqrUDgG9HFlg2lVD8,492
 reconcile/typed_queries/github_orgs.py,sha256=UZhoPl8qvA_tcO7CZlN8GuMKckt3ywd47Suu61rgHsc,258
@@ -776,7 +780,7 @@ reconcile/utils/ocm/clusters.py,sha256=Fn4swizm1qq-XiNlIZ9SvahkftWAyNT8hF4kqRBpK
 reconcile/utils/ocm/identity_providers.py,sha256=dKed09N8iWmn39tI_MpwgVe47x23eLsknGbjMUxtwr4,2175
 reconcile/utils/ocm/label_sources.py,sha256=ES_5VP4X6gsRxMFZ95WgbwE_HqqIUo_JRjHjdGYw6Ss,1846
 reconcile/utils/ocm/labels.py,sha256=aCsL5QkRk32hZeJwsSJuCCT9sbojWMn8LL5Zo-aoFb4,5916
-reconcile/utils/ocm/manifests.py,sha256=8kCVwTiaYHyjiKfP2DrkoT9eFxROy_M3rLom_hdsEIU,1193
+reconcile/utils/ocm/manifests.py,sha256=Q6kgOeiAwLbJY_vO_BEW2oePvbLDZcMZk20YpJJGpOA,1195
 reconcile/utils/ocm/ocm.py,sha256=EwhCymt7r8cL8UF2XbwmQ6IiRE016AUuPEiMAtYMepE,36707
 reconcile/utils/ocm/products.py,sha256=XDmTkVv4eWEifloz_f2I8GmdM97tY33PLf2p4d5GMI0,25972
 reconcile/utils/ocm/search_filters.py,sha256=jdj2sMGArcQrZLluzxeypPSbMFX_5zSE3ACvhNpsnOc,14814
@@ -850,8 +854,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc990.dist-info/METADATA,sha256=gPHQgGX8QfVXyoDNJ5Lx-SBWvxs9-idwSy5huEDo0GE,2262
-qontract_reconcile-0.10.1rc990.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc990.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc990.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc990.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc992.dist-info/METADATA,sha256=DRFJMM8ZzS7zsrDv77wTXC9PngnJiNg-SphE3U7qjrQ,2262
+qontract_reconcile-0.10.1rc992.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc992.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc992.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc992.dist-info/RECORD,,

reconcile/dynatrace_token_provider/dependencies.py

@@ -1,5 +1,13 @@
+from collections.abc import Mapping
+
 from reconcile.dynatrace_token_provider.ocm import OCMClient
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+)
 from reconcile.typed_queries.dynatrace_environments import get_dynatrace_environments
+from reconcile.typed_queries.dynatrace_token_provider_token_specs import (
+    get_dynatrace_token_provider_token_specs,
+)
 from reconcile.typed_queries.ocm import get_ocm_environments
 from reconcile.utils.dynatrace.client import DynatraceClient
 from reconcile.utils.ocm_base_client import (
@@ -17,18 +25,25 @@ class Dependencies:
     def __init__(
         self,
         secret_reader: SecretReaderBase,
-        dynatrace_client_by_tenant_id: dict[str, DynatraceClient],
-        ocm_client_by_env_name: dict[str, OCMClient],
+        dynatrace_client_by_tenant_id: Mapping[str, DynatraceClient],
+        ocm_client_by_env_name: Mapping[str, OCMClient],
+        token_spec_by_name: Mapping[str, DynatraceTokenProviderTokenSpecV1],
     ):
         self.secret_reader = secret_reader
-        self.dynatrace_client_by_tenant_id: dict[str, DynatraceClient] = (
+        self.dynatrace_client_by_tenant_id: dict[str, DynatraceClient] = dict(
             dynatrace_client_by_tenant_id
         )
-        self.ocm_client_by_env_name: dict[str, OCMClient] = ocm_client_by_env_name
+        self.ocm_client_by_env_name: dict[str, OCMClient] = dict(ocm_client_by_env_name)
+        self.token_spec_by_name = dict(token_spec_by_name)
 
     def populate(self) -> None:
         self._populate_dynatrace_client_map()
         self._populate_ocm_clients()
+        self._populate_token_specs()
+
+    def _populate_token_specs(self) -> None:
+        token_specs = get_dynatrace_token_provider_token_specs()
+        self.token_spec_by_name = {spec.name: spec for spec in token_specs}
 
     def _populate_dynatrace_client_map(self) -> None:
         dynatrace_environments = get_dynatrace_environments()

reconcile/dynatrace_token_provider/integration.py

@@ -9,28 +9,33 @@ from reconcile.dynatrace_token_provider.metrics import (
     DTPClustersManagedGauge,
     DTPOrganizationErrorRate,
 )
-from reconcile.dynatrace_token_provider.ocm import Cluster, OCMClient
+from reconcile.dynatrace_token_provider.model import DynatraceAPIToken, K8sSecret
+from reconcile.dynatrace_token_provider.ocm import (
+    DTP_LABEL_SEARCH,
+    DTP_TENANT_LABEL,
+    Cluster,
+    OCMClient,
+)
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceAPITokenV1,
+    DynatraceTokenProviderTokenSpecV1,
+)
 from reconcile.utils import (
     metrics,
 )
-from reconcile.utils.dynatrace.client import DynatraceAPITokenCreated, DynatraceClient
+from reconcile.utils.dynatrace.client import DynatraceClient
 from reconcile.utils.ocm.base import (
     OCMClusterServiceLogCreateModel,
     OCMServiceLogSeverity,
 )
 from reconcile.utils.ocm.labels import subscription_label_filter
-from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
 )
 
 QONTRACT_INTEGRATION = "dynatrace-token-provider"
-SYNCSET_ID = "ext-dynatrace-tokens-dtp"
-SECRET_NAME = "dynatrace-token-dtp"
-SECRET_NAMESPACE = "dynatrace"
-DYNATRACE_INGESTION_TOKEN_NAME = "dynatrace-ingestion-token"
-DYNATRACE_OPERATOR_TOKEN_NAME = "dynatrace-operator-token"
+SYNCSET_AND_MANIFEST_ID = "ext-dynatrace-tokens-dtp"
 
 
 class DynatraceTokenProviderIntegrationParams(PydanticRunParams):
@@ -58,6 +63,7 @@ class DynatraceTokenProviderIntegration(
             secret_reader=self.secret_reader,
             dynatrace_client_by_tenant_id={},
             ocm_client_by_env_name={},
+            token_spec_by_name={},
         )
         dependencies.populate()
         self.reconcile(dry_run=dry_run, dependencies=dependencies)
@@ -70,7 +76,7 @@ class DynatraceTokenProviderIntegration(
                 try:
                     clusters = ocm_client.discover_clusters_by_labels(
                         label_filter=subscription_label_filter().like(
-                            "key", dtp_label_key("%")
+                            "key", DTP_LABEL_SEARCH
                         ),
                     )
                 except Exception as e:
@@ -90,7 +96,6 @@ class DynatraceTokenProviderIntegration(
                         for cluster in clusters
                         if cluster.organization_id in self.params.ocm_organization_ids
                     ]
-                dtp_tenant_label_key = f"{dtp_label_key(None)}.tenant"
                 existing_dtp_tokens = {}
 
                 for cluster in clusters:
@@ -105,7 +110,7 @@ class DynatraceTokenProviderIntegration(
                                 _expose_errors_as_service_log(
                                     ocm_client,
                                     cluster_uuid=cluster.external_id,
-                                    error=f"Missing label {dtp_tenant_label_key}",
+                                    error=f"Missing label {DTP_TENANT_LABEL}",
                                 )
                                 continue
                             if (
@@ -122,6 +127,16 @@ class DynatraceTokenProviderIntegration(
                                 tenant_id
                             ]
 
+                            token_spec = dependencies.token_spec_by_name.get(
+                                cluster.token_spec_name
+                            )
+                            if not token_spec:
+                                _expose_errors_as_service_log(
+                                    ocm_client,
+                                    cluster_uuid=cluster.external_id,
+                                    error=f"Token spec {cluster.token_spec_name} does not exist",
+                                )
+                                continue
                             if tenant_id not in existing_dtp_tokens:
                                 existing_dtp_tokens[tenant_id] = (
                                     dt_client.get_token_ids_for_name_prefix(
@@ -129,13 +144,19 @@ class DynatraceTokenProviderIntegration(
                                     )
                                 )
 
+                            """
+                            Note, that we consciously do not parallelize cluster processing
+                            for now. We want to keep stress on OCM at a minimum. The amount
+                            of tagged clusters is currently feasible to be processed sequentially.
+                            """
                             self.process_cluster(
-                                dry_run,
-                                cluster,
-                                dt_client,
-                                ocm_client,
-                                existing_dtp_tokens[tenant_id],
-                                tenant_id,
+                                dry_run=dry_run,
+                                cluster=cluster,
+                                dt_client=dt_client,
+                                ocm_client=ocm_client,
+                                existing_dtp_tokens=existing_dtp_tokens[tenant_id],
+                                tenant_id=tenant_id,
+                                token_spec=token_spec,
                             )
                     except Exception as e:
                         unhandled_exceptions.append(
@@ -153,89 +174,192 @@ class DynatraceTokenProviderIntegration(
         ocm_client: OCMClient,
         existing_dtp_tokens: Iterable[str],
         tenant_id: str,
+        token_spec: DynatraceTokenProviderTokenSpecV1,
     ) -> None:
-        existing_syncset = self.get_syncset(ocm_client, cluster)
+        if cluster.organization_id not in token_spec.ocm_org_ids:
+            logging.info(
+                f"[{token_spec.name=}] Cluster {cluster.external_id} is not part of ocm orgs defined in {token_spec.ocm_org_ids=}"
+            )
+            return
+        existing_data = {}
+        if cluster.is_hcp:
+            existing_data = self.get_manifest(ocm_client=ocm_client, cluster=cluster)
+        else:
+            existing_data = self.get_syncset(ocm_client=ocm_client, cluster=cluster)
         dt_api_url = f"https://{tenant_id}.live.dynatrace.com/api"
-        if not existing_syncset:
+        if not existing_data:
             if not dry_run:
                 try:
-                    (ingestion_token, operator_token) = self.create_dynatrace_tokens(
-                        dt_client, cluster.external_id
-                    )
-                    ocm_client.create_syncset(
-                        cluster.id,
-                        self.construct_syncset(
-                            ingestion_token, operator_token, dt_api_url
-                        ),
+                    k8s_secrets = self.construct_secrets(
+                        token_spec=token_spec,
+                        dt_client=dt_client,
+                        cluster_uuid=cluster.external_id,
                     )
+                    if cluster.is_hcp:
+                        ocm_client.create_manifest(
+                            cluster_id=cluster.id,
+                            manifest_map=self.construct_manifest(
+                                with_id=True,
+                                dt_api_url=dt_api_url,
+                                secrets=k8s_secrets,
+                            ),
+                        )
+                    else:
+                        ocm_client.create_syncset(
+                            cluster_id=cluster.id,
+                            syncset_map=self.construct_syncset(
+                                with_id=True,
+                                dt_api_url=dt_api_url,
+                                secrets=k8s_secrets,
+                            ),
+                        )
                 except Exception as e:
                     _expose_errors_as_service_log(
                         ocm_client,
                         cluster.external_id,
-                        f"DTP can't create Syncset with the tokens {str(e.args)}",
+                        f"DTP can't create {token_spec.name=} {str(e.args)}",
                     )
             logging.info(
-                f"Ingestion and operator tokens created in Dynatrace for cluster {cluster.external_id}."
+                f"{token_spec.name=} created in {dt_api_url} for {cluster.external_id=}."
             )
             logging.info(
-                f"SyncSet {SYNCSET_ID} created in cluster {cluster.external_id}."
+                f"{SYNCSET_AND_MANIFEST_ID} created for {cluster.external_id=}."
             )
         else:
-            tokens = self.get_tokens_from_syncset(existing_syncset)
-            need_patching = False
-            for token_name, token in tokens.items():
-                if token.id not in existing_dtp_tokens:
-                    need_patching = True
-                    logging.info(f"{token_name} missing in Dynatrace.")
-                    if token_name == DYNATRACE_INGESTION_TOKEN_NAME:
-                        if not dry_run:
-                            ingestion_token = self.create_dynatrace_ingestion_token(
-                                dt_client, cluster.external_id
-                            )
-                            token.id = ingestion_token.id
-                            token.token = ingestion_token.token
-                        logging.info(
-                            f"Ingestion token created in Dynatrace for cluster {cluster.external_id}."
-                        )
-                    elif token_name == DYNATRACE_OPERATOR_TOKEN_NAME:
-                        if not dry_run:
-                            operator_token = self.create_dynatrace_operator_token(
-                                dt_client, cluster.external_id
-                            )
-                            token.id = operator_token.id
-                            token.token = operator_token.token
-                        logging.info(
-                            f"Operator token created in Dynatrace for cluster {cluster.external_id}."
-                        )
-                elif token_name == DYNATRACE_INGESTION_TOKEN_NAME:
-                    ingestion_token = token
-                elif token_name == DYNATRACE_OPERATOR_TOKEN_NAME:
-                    operator_token = token
-            if need_patching:
+            current_k8s_secrets: list[K8sSecret] = []
+            if cluster.is_hcp:
+                current_k8s_secrets = self.get_secrets_from_manifest(
+                    manifest=existing_data, token_spec=token_spec
+                )
+            else:
+                current_k8s_secrets = self.get_secrets_from_syncset(
+                    syncset=existing_data, token_spec=token_spec
+                )
+            has_diff, desired_secrets = self.generate_desired(
+                dry_run=dry_run,
+                current_k8s_secrets=current_k8s_secrets,
+                desired_spec=token_spec,
+                existing_dtp_tokens=existing_dtp_tokens,
+                dt_client=dt_client,
+                cluster_uuid=cluster.external_id,
+            )
+            if has_diff:
                 if not dry_run:
-                    patch_syncset_payload = self.construct_base_syncset(
-                        ingestion_token=ingestion_token,
-                        operator_token=operator_token,
-                        dt_api_url=dt_api_url,
-                    )
                     try:
-                        logging.info(f"Patching syncset {SYNCSET_ID}.")
-                        ocm_client.patch_syncset(
-                            cluster_id=cluster.id,
-                            syncset_id=SYNCSET_ID,
-                            syncset_map=patch_syncset_payload,
-                        )
+                        if cluster.is_hcp:
+                            ocm_client.patch_manifest(
+                                cluster_id=cluster.id,
+                                manifest_id=SYNCSET_AND_MANIFEST_ID,
+                                manifest_map=self.construct_manifest(
+                                    dt_api_url=dt_api_url,
+                                    secrets=desired_secrets,
+                                    with_id=False,
+                                ),
+                            )
+                        else:
+                            ocm_client.patch_syncset(
+                                cluster_id=cluster.id,
+                                syncset_id=SYNCSET_AND_MANIFEST_ID,
+                                syncset_map=self.construct_syncset(
+                                    dt_api_url=dt_api_url,
+                                    secrets=desired_secrets,
+                                    with_id=False,
+                                ),
+                            )
                     except Exception as e:
                         _expose_errors_as_service_log(
                             ocm_client,
                             cluster.external_id,
-                            f"DTP can't patch Syncset {SYNCSET_ID} due to {str(e.args)}",
+                            f"DTP can't patch {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} due to {str(e.args)}",
                         )
-                logging.info(f"Syncset {SYNCSET_ID} patched.")
+                logging.info(
+                    f"Patched {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} in {cluster.external_id=}."
+                )
+
+    def generate_desired(
+        self,
+        dry_run: bool,
+        current_k8s_secrets: Iterable[K8sSecret],
+        desired_spec: DynatraceTokenProviderTokenSpecV1,
+        existing_dtp_tokens: Iterable[str],
+        dt_client: DynatraceClient,
+        cluster_uuid: str,
+    ) -> tuple[bool, Iterable[K8sSecret]]:
+        has_diff = False
+        desired: list[K8sSecret] = []
+
+        current_secrets_by_name = {
+            secret.secret_name: secret for secret in current_k8s_secrets
+        }
+
+        for secret in desired_spec.secrets:
+            desired_tokens: list[DynatraceAPIToken] = []
+            current_secret = current_secrets_by_name.get(secret.name)
+            current_tokens_by_name = (
+                {token.name: token for token in current_secret.tokens}
+                if current_secret
+                else {}
+            )
+            for desired_token in secret.tokens:
+                new_token = current_tokens_by_name.get(desired_token.name)
+                if not new_token or new_token.id not in existing_dtp_tokens:
+                    has_diff = True
+                    if not dry_run:
+                        new_token = self.create_dynatrace_token(
+                            dt_client, cluster_uuid, desired_token
+                        )
+                if new_token:
+                    desired_tokens.append(new_token)
+            desired.append(
+                K8sSecret(
+                    secret_name=secret.name,
+                    namespace_name=secret.namespace,
+                    tokens=desired_tokens,
+                )
+            )
+
+        return (has_diff, desired)
+
+    def create_dynatrace_token(
+        self, dt_client: DynatraceClient, cluster_uuid: str, token: DynatraceAPITokenV1
+    ) -> DynatraceAPIToken:
+        token_name = f"dtp-{token.name}-{cluster_uuid}"
+        new_token = dt_client.create_api_token(
+            name=token_name,
+            scopes=token.scopes,
+        )
+        secret_key = token.key_name_in_secret or token.name
+        return DynatraceAPIToken(
+            id=new_token.id,
+            token=new_token.token,
+            name=token_name,
+            secret_key=secret_key,
+        )
+
+    def construct_secrets(
+        self,
+        token_spec: DynatraceTokenProviderTokenSpecV1,
+        dt_client: DynatraceClient,
+        cluster_uuid: str,
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        for secret in token_spec.secrets:
+            new_tokens: list[DynatraceAPIToken] = []
+            for token in secret.tokens:
+                new_token = self.create_dynatrace_token(dt_client, cluster_uuid, token)
+                new_tokens.append(new_token)
+            secrets.append(
+                K8sSecret(
+                    secret_name=secret.name,
+                    namespace_name=secret.namespace,
+                    tokens=new_tokens,
+                )
+            )
+        return secrets
 
     def get_syncset(self, ocm_client: OCMClient, cluster: Cluster) -> dict[str, Any]:
         try:
-            syncset = ocm_client.get_syncset(cluster.id, SYNCSET_ID)
+            syncset = ocm_client.get_syncset(cluster.id, SYNCSET_AND_MANIFEST_ID)
         except Exception as e:
             if "Not Found" in e.args[0]:
                 syncset = None
@@ -243,50 +367,136 @@ class DynatraceTokenProviderIntegration(
                 raise e
         return syncset
 
-    def get_tokens_from_syncset(
-        self, syncset: Mapping[str, Any]
-    ) -> dict[str, DynatraceAPITokenCreated]:
-        tokens: dict[str, Any] = {}
-        for resource in syncset["resources"]:
-            if resource["kind"] == "Secret":
-                operator_token_id = self.base64_decode(resource["data"]["apiTokenId"])
-                operator_token = self.base64_decode(resource["data"]["apiToken"])
-                ingest_token_id = self.base64_decode(
-                    resource["data"]["dataIngestTokenId"]
+    def get_manifest(self, ocm_client: OCMClient, cluster: Cluster) -> dict[str, Any]:
+        try:
+            manifest = ocm_client.get_manifest(cluster.id, SYNCSET_AND_MANIFEST_ID)
+        except Exception as e:
+            if "Not Found" in e.args[0]:
+                manifest = None
+            else:
+                raise e
+        return manifest
+
+    def get_secrets_from_syncset(
+        self, syncset: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        secret_data_by_name = {
+            resource.get("metadata", {}).get("name"): resource.get("data", {})
+            for resource in syncset.get("resources", [])
+            if resource.get("kind") == "Secret"
+        }
+        for secret in token_spec.secrets:
+            secret_data = secret_data_by_name.get(secret.name)
+            if secret_data:
+                tokens = []
+                for token in secret.tokens:
+                    token_id = self.base64_decode(
+                        secret_data.get(f"{token.key_name_in_secret}Id", "")
+                    )
+                    token_value = self.base64_decode(
+                        secret_data.get(token.key_name_in_secret, "")
+                    )
+                    tokens.append(
+                        DynatraceAPIToken(
+                            id=token_id,
+                            token=token_value,
+                            name=token.name,
+                            secret_key=token.key_name_in_secret,
+                        )
+                    )
+                secrets.append(
+                    K8sSecret(
+                        secret_name=secret.name,
+                        namespace_name=secret.namespace,
+                        tokens=tokens,
+                    )
                 )
-                ingest_token = self.base64_decode(resource["data"]["dataIngestToken"])
-        tokens[DYNATRACE_INGESTION_TOKEN_NAME] = DynatraceAPITokenCreated(
-            id=ingest_token_id,
-            token=ingest_token,
-        )
-        tokens[DYNATRACE_OPERATOR_TOKEN_NAME] = DynatraceAPITokenCreated(
-            id=operator_token_id,
-            token=operator_token,
-        )
-        return tokens
+        return secrets
+
+    def get_secrets_from_manifest(
+        self, manifest: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        secret_data_by_name = {
+            resource.get("metadata", {}).get("name"): resource.get("data", {})
+            for resource in manifest.get("workloads", [])
+            if resource.get("kind") == "Secret"
+        }
+        for secret in token_spec.secrets:
+            secret_data = secret_data_by_name.get(secret.name)
+            if secret_data:
+                tokens = []
+                for token in secret.tokens:
+                    token_id = self.base64_decode(
+                        secret_data.get(f"{token.key_name_in_secret}Id", "")
+                    )
+                    token_value = self.base64_decode(
+                        secret_data.get(token.key_name_in_secret, "")
+                    )
+                    tokens.append(
+                        DynatraceAPIToken(
+                            id=token_id,
+                            token=token_value,
+                            name=token.name,
+                            secret_key=token.key_name_in_secret,
+                        )
+                    )
+                secrets.append(
+                    K8sSecret(
+                        secret_name=secret.name,
+                        namespace_name=secret.namespace,
+                        tokens=tokens,
+                    )
+                )
+        return secrets
+
+    def construct_secrets_data(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+    ) -> list[dict[str, Any]]:
+        secrets_data: list[dict[str, Any]] = []
+        for secret in secrets:
+            data: dict[str, str] = {
+                "apiUrl": f"{self.base64_encode_str(dt_api_url)}",
+            }
+            for token in secret.tokens:
+                data[token.secret_key] = f"{self.base64_encode_str(token.token)}"
+                data[f"{token.secret_key}Id"] = f"{self.base64_encode_str(token.id)}"
+            secrets_data.append({
+                "apiVersion": "v1",
+                "kind": "Secret",
+                "metadata": {
+                    "name": secret.secret_name,
+                    "namespace": secret.namespace_name,
+                },
+                "data": data,
+            })
+        return secrets_data
 
     def construct_base_syncset(
         self,
-        ingestion_token: DynatraceAPITokenCreated,
-        operator_token: DynatraceAPITokenCreated,
+        secrets: Iterable[K8sSecret],
         dt_api_url: str,
     ) -> dict[str, Any]:
         return {
             "kind": "SyncSet",
-            "resources": [
-                {
-                    "apiVersion": "v1",
-                    "kind": "Secret",
-                    "metadata": {"name": SECRET_NAME, "namespace": SECRET_NAMESPACE},
-                    "data": {
-                        "apiUrl": f"{self.base64_encode_str(dt_api_url)}",
-                        "dataIngestTokenId": f"{self.base64_encode_str(ingestion_token.id)}",
-                        "dataIngestToken": f"{self.base64_encode_str(ingestion_token.token)}",
-                        "apiTokenId": f"{self.base64_encode_str(operator_token.id)}",
-                        "apiToken": f"{self.base64_encode_str(operator_token.token)}",
-                    },
-                },
-            ],
+            "resources": self.construct_secrets_data(
+                secrets=secrets, dt_api_url=dt_api_url
+            ),
+        }
+
+    def construct_base_manifest(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+    ) -> dict[str, Any]:
+        return {
+            "kind": "Manifest",
+            "workloads": self.construct_secrets_data(
+                secrets=secrets, dt_api_url=dt_api_url
+            ),
         }
 
     def base64_decode(self, encoded: str) -> str:
@@ -300,51 +510,31 @@ class DynatraceTokenProviderIntegration(
 
     def construct_syncset(
         self,
-        ingestion_token: DynatraceAPITokenCreated,
-        operator_token: DynatraceAPITokenCreated,
+        secrets: Iterable[K8sSecret],
         dt_api_url: str,
+        with_id: bool,
     ) -> dict[str, Any]:
         syncset = self.construct_base_syncset(
-            ingestion_token=ingestion_token,
-            operator_token=operator_token,
+            secrets=secrets,
             dt_api_url=dt_api_url,
         )
-        syncset["id"] = SYNCSET_ID
+        if with_id:
+            syncset["id"] = SYNCSET_AND_MANIFEST_ID
         return syncset
 
-    def create_dynatrace_ingestion_token(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> DynatraceAPITokenCreated:
-        return dt_client.create_api_token(
-            name=f"dtp-ingestion-token-{cluster_uuid}",
-            scopes=["metrics.ingest", "logs.ingest", "events.ingest"],
-        )
-
-    def create_dynatrace_operator_token(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> DynatraceAPITokenCreated:
-        return dt_client.create_api_token(
-            name=f"dtp-operator-token-{cluster_uuid}",
-            scopes=[
-                "activeGateTokenManagement.create",
-                "entities.read",
-                "settings.write",
-                "settings.read",
-                "DataExport",
-                "InstallerDownload",
-            ],
+    def construct_manifest(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+        with_id: bool,
+    ) -> dict[str, Any]:
+        manifest = self.construct_base_manifest(
+            secrets=secrets,
+            dt_api_url=dt_api_url,
         )
-
-    def create_dynatrace_tokens(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> tuple[DynatraceAPITokenCreated, DynatraceAPITokenCreated]:
-        ingestion_token = self.create_dynatrace_ingestion_token(dt_client, cluster_uuid)
-        operation_token = self.create_dynatrace_operator_token(dt_client, cluster_uuid)
-        return (ingestion_token, operation_token)
-
-
-def dtp_label_key(config_atom: str | None) -> str:
-    return sre_capability_label_key("dtp", config_atom)
+        if with_id:
+            manifest["id"] = SYNCSET_AND_MANIFEST_ID
+        return manifest
 
 
 def _expose_errors_as_service_log(

reconcile/dynatrace_token_provider/model.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+
+class DynatraceAPIToken(BaseModel):
+    token: str
+    id: str
+    name: str
+    secret_key: str
+
+
+class K8sSecret(BaseModel):
+    namespace_name: str
+    secret_name: str
+    tokens: list[DynatraceAPIToken]

reconcile/dynatrace_token_provider/ocm.py

@@ -14,6 +14,11 @@ from reconcile.utils.ocm.clusters import (
     discover_clusters_by_labels,
 )
 from reconcile.utils.ocm.labels import Filter
+from reconcile.utils.ocm.manifests import (
+    create_manifest,
+    get_manifest,
+    patch_manifest,
+)
 from reconcile.utils.ocm.service_log import create_service_log
 from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
 from reconcile.utils.ocm.syncsets import (
@@ -29,23 +34,39 @@ from reconcile.utils.ocm_base_client import (
 Thin abstractions of reconcile.ocm module to reduce coupling.
 """
 
+DTP_LABEL = sre_capability_label_key("dtp", None)
+DTP_TENANT_LABEL = sre_capability_label_key("dtp", "tenant")
+DTP_LABEL_SEARCH = sre_capability_label_key("dtp", "%")
+
 
 class Cluster(BaseModel):
     id: str
     external_id: str
     organization_id: str
     dt_tenant: str
+    token_spec_name: str
+    is_hcp: bool
 
     @staticmethod
     def from_cluster_details(cluster: ClusterDetails) -> Cluster:
-        dt_tenant = cluster.labels.get_label_value(
-            f"{sre_capability_label_key('dtp', None)}.tenant"
-        )
+        dt_tenant = cluster.labels.get_label_value(DTP_TENANT_LABEL)
+        token_spec_name = cluster.labels.get_label_value(DTP_LABEL)
+        if not token_spec_name:
+            """
+            We want to stay backwards compatible.
+            Earlier version of DTP did not set a value for the label.
+            We fall back to a default token in that case.
+
+            Long-term, we want to remove this behavior.
+            """
+            token_spec_name = "default"
         return Cluster(
             id=cluster.ocm_cluster.id,
             external_id=cluster.ocm_cluster.external_id,
             organization_id=cluster.organization_id,
             dt_tenant=dt_tenant,
+            token_spec_name=token_spec_name,
+            is_hcp=cluster.ocm_cluster.is_rosa_hypershift(),
         )
 
 
@@ -77,6 +98,28 @@ class OCMClient:
             syncset_map=syncset_map,
         )
 
+    def create_manifest(self, cluster_id: str, manifest_map: Mapping) -> None:
+        create_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_map=manifest_map,
+        )
+
+    def get_manifest(self, cluster_id: str, manifest_id: str) -> Any:
+        return get_manifest(
+            ocm_client=self._ocm_client, cluster_id=cluster_id, manifest_id=manifest_id
+        )
+
+    def patch_manifest(
+        self, cluster_id: str, manifest_id: str, manifest_map: Mapping
+    ) -> None:
+        patch_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_id=manifest_id,
+            manifest_map=manifest_map,
+        )
+
     def discover_clusters_by_labels(self, label_filter: Filter) -> list[Cluster]:
         return [
             Cluster.from_cluster_details(cluster)

reconcile/gql_definitions/dynatrace_token_provider/token_specs.py

@@ -0,0 +1,84 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query DynatraceTokenProviderTokenSpecs {
+  token_specs: dynatrace_token_provider_token_spec_v1 {
+    name
+    ocm_org_ids
+    secrets {
+      name
+      namespace
+      tokens {
+        name
+        keyNameInSecret
+        scopes
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class DynatraceAPITokenV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    key_name_in_secret: Optional[str] = Field(..., alias="keyNameInSecret")
+    scopes: list[str] = Field(..., alias="scopes")
+
+
+class DynatraceTokenProviderTokenSecretV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    namespace: str = Field(..., alias="namespace")
+    tokens: list[DynatraceAPITokenV1] = Field(..., alias="tokens")
+
+
+class DynatraceTokenProviderTokenSpecV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    ocm_org_ids: list[str] = Field(..., alias="ocm_org_ids")
+    secrets: list[DynatraceTokenProviderTokenSecretV1] = Field(..., alias="secrets")
+
+
+class DynatraceTokenProviderTokenSpecsQueryData(ConfiguredBaseModel):
+    token_specs: Optional[list[DynatraceTokenProviderTokenSpecV1]] = Field(..., alias="token_specs")
+
+
+def query(query_func: Callable, **kwargs: Any) -> DynatraceTokenProviderTokenSpecsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        DynatraceTokenProviderTokenSpecsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return DynatraceTokenProviderTokenSpecsQueryData(**raw_data)

reconcile/gql_definitions/terraform_repo/terraform_repo.py

@@ -53,6 +53,7 @@ query TerraformRepo {
     delete
     requireFips
     tfVersion
+    forceRerunTimestamp
     variables {
       inputs {
         ...VaultSecret
@@ -105,6 +106,7 @@ class TerraformRepoV1(ConfiguredBaseModel):
     delete: Optional[bool] = Field(..., alias="delete")
     require_fips: Optional[bool] = Field(..., alias="requireFips")
     tf_version: str = Field(..., alias="tfVersion")
+    force_rerun_timestamp: Optional[str] = Field(..., alias="forceRerunTimestamp")
     variables: Optional[TerraformRepoVariablesV1] = Field(..., alias="variables")
 
 

reconcile/terraform_repo.py

@@ -361,6 +361,8 @@ class TerraformRepoIntegration(
                 )
             if self.params.validate_git:
                 self.check_ref(d.repository, d.ref)
+            if c.force_rerun_timestamp != d.force_rerun_timestamp:
+                logging.info("user has forced a re-run of tf-repo execution")
 
         if len(merged) != 0:
             if not dry_run and state:

reconcile/test/test_terraform_repo.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from unittest.mock import MagicMock
 
 import pytest
@@ -46,6 +47,7 @@ def existing_repo(aws_account, tf_variables) -> TerraformRepoV1:
         requireFips=True,
         tfVersion=A_REPO_VERSION,
         variables=tf_variables,
+        forceRerunTimestamp=None,
     )
 
 
@@ -86,6 +88,7 @@ def new_repo(aws_account_no_state) -> TerraformRepoV1:
         requireFips=False,
         tfVersion=B_REPO_VERSION,
         variables=None,
+        forceRerunTimestamp=None,
     )
 
 
@@ -220,6 +223,27 @@ def test_updating_repo_ref(existing_repo, int_params, state_mock):
     )
 
 
+def test_force_rerun(existing_repo, int_params, state_mock):
+    existing = [existing_repo]
+    updated_repo = TerraformRepoV1.copy(existing_repo)
+    updated_repo.force_rerun_timestamp = datetime.now().isoformat()
+
+    integration = TerraformRepoIntegration(params=int_params)
+    diff = integration.calculate_diff(
+        existing_state=existing,
+        desired_state=[updated_repo],
+        dry_run=False,
+        state=state_mock,
+        recreate_state=False,
+    )
+
+    assert diff == [updated_repo]
+
+    state_mock.add.assert_called_once_with(
+        updated_repo.name, updated_repo.dict(by_alias=True), force=True
+    )
+
+
 def test_fail_on_update_invalid_repo_params(existing_repo, int_params):
     existing = [existing_repo]
     updated_repo = TerraformRepoV1.copy(existing_repo)
@@ -291,6 +315,7 @@ def test_get_repo_state(s3_state_builder, int_params, existing_repo, tf_variable
                 "requireFips": True,
                 "tfVersion": A_REPO_VERSION,
                 "variables": tf_variables,
+                "forceRerunTimestamp": None,
                 "account": {
                     "name": "foo",
                     "uid": AWS_UID,

reconcile/typed_queries/dynatrace_token_provider_token_specs.py

@@ -0,0 +1,14 @@
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_dynatrace_token_provider_token_specs(
+    api: GqlApi | None = None,
+) -> list[DynatraceTokenProviderTokenSpecV1]:
+    api = api if api else gql.get_api()
+    data = query(api.query)
+    return list(data.token_specs or [])

reconcile/utils/ocm/manifests.py

@@ -33,7 +33,7 @@ def create_manifest(
 
 
 def patch_manifest(
-    ocm_client: OCMBaseClient, cluster_id: str, syncset_id: str, manifest_map: Mapping
+    ocm_client: OCMBaseClient, cluster_id: str, manifest_id: str, manifest_map: Mapping
 ) -> None:
     manifest = Manifest(cluster_id)
-    ocm_client.patch(api_path=manifest.href + "/" + syncset_id, data=manifest_map)
+    ocm_client.patch(api_path=manifest.href + "/" + manifest_id, data=manifest_map)