qontract-reconcile 0.10.1rc989__py3-none-any.whl → 0.10.1rc991__py3-none-any.whl

{qontract_reconcile-0.10.1rc989.dist-info → qontract_reconcile-0.10.1rc991.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc989
+Version: 0.10.1rc991
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc989.dist-info → qontract_reconcile-0.10.1rc991.dist-info}/RECORD

@@ -179,10 +179,11 @@ reconcile/cna/assets/asset.py,sha256=KWgA4fuDAEGsJwmR52WwK_YgSJMW-1cV2la3lmNf4iE
 reconcile/cna/assets/asset_factory.py,sha256=7T7X_J6xIsoGETqBRI45_EyIKEdQcnRPt_GAuVuLQcc,785
 reconcile/cna/assets/null.py,sha256=85mVh97atCoC0aLuX47poTZiyOthmziJeBsUw0c924w,1658
 reconcile/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/dynatrace_token_provider/dependencies.py,sha256=uJLvR48kxfqjnBuP60XQx5RbkWPL3__FCgWjqwhKEjo,2160
-reconcile/dynatrace_token_provider/integration.py,sha256=n_t_x-9USDtkT_8koOn7SxxXXxc3lAYbkZENlVm6t5c,14909
+reconcile/dynatrace_token_provider/dependencies.py,sha256=41q05A4C_eS3E8-MR4veeMxtQNsPoGdxmEa3d-OKxq4,2814
+reconcile/dynatrace_token_provider/integration.py,sha256=ffH4BpMNb3AafwengwzurZ-aiztGP1MUlnVU4FnO3IY,21540
 reconcile/dynatrace_token_provider/metrics.py,sha256=xiKkl8fTEBQaXJelGCPNTZhHAWdO1M3pCXNr_Tei63c,1285
-reconcile/dynatrace_token_provider/ocm.py,sha256=IwksRMyGcJnamV88ORlBoyOr7uRENhMaHBoSXaGfwDY,2784
+reconcile/dynatrace_token_provider/model.py,sha256=gkpqo5rRRueBXnIMjp4EEHqBUBuU65TRI8zpdb8GJ0A,241
+reconcile/dynatrace_token_provider/ocm.py,sha256=iHMsgbsLs-dlrB9UXmWNDF7E4UDe49JOsLa9rnowKfo,4282
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
@@ -275,6 +276,7 @@ reconcile/gql_definitions/dashdotdb_slo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5J
 reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py,sha256=zUa-CmpOwiymVmOV6KwDHH5mMl06p000320FcOas6hU,4315
 reconcile/gql_definitions/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py,sha256=5gTuAnR2rnx2k6Rn7FMEAzw6GCZ6F5HZbqkmJ9-3NI4,2244
+reconcile/gql_definitions/dynatrace_token_provider/token_specs.py,sha256=XGsMuB8gowRpqJjkD_KRomx-1OswzyWbF4qjVdhionk,2555
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
 reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
@@ -598,6 +600,7 @@ reconcile/typed_queries/clusters_with_dms.py,sha256=JDSKZXwO3QxT-uA1FaHxP8d4XiYA
 reconcile/typed_queries/clusters_with_peering.py,sha256=lIai7SJJD0bqIJbe7virgrbYRqjLouSL2OpJD0itpAY,330
 reconcile/typed_queries/dynatrace.py,sha256=8vXDXDIDf9_vN_efYwysDr4gLN7SCx4I2bOoNxQhbio,312
 reconcile/typed_queries/dynatrace_environments.py,sha256=VV_7KzKG9wqGDV9wZLbcCJtfuzPhTV1wdg0YwAOaq3A,413
+reconcile/typed_queries/dynatrace_token_provider_token_specs.py,sha256=x41KG6JRDNYw5QGJYtIFNwSeejUUgxrL-agS8qFf6q0,433
 reconcile/typed_queries/external_resources.py,sha256=h1uzZzmtEGzoqSFhDMSAdxauGJoGy0stPuWbA0rkVKE,1503
 reconcile/typed_queries/get_state_aws_account.py,sha256=CSJjVPWsUZ2rkGIt8ehoQt7hokFqrUDgG9HFlg2lVD8,492
 reconcile/typed_queries/github_orgs.py,sha256=UZhoPl8qvA_tcO7CZlN8GuMKckt3ywd47Suu61rgHsc,258
@@ -776,7 +779,7 @@ reconcile/utils/ocm/clusters.py,sha256=Fn4swizm1qq-XiNlIZ9SvahkftWAyNT8hF4kqRBpK
 reconcile/utils/ocm/identity_providers.py,sha256=dKed09N8iWmn39tI_MpwgVe47x23eLsknGbjMUxtwr4,2175
 reconcile/utils/ocm/label_sources.py,sha256=ES_5VP4X6gsRxMFZ95WgbwE_HqqIUo_JRjHjdGYw6Ss,1846
 reconcile/utils/ocm/labels.py,sha256=aCsL5QkRk32hZeJwsSJuCCT9sbojWMn8LL5Zo-aoFb4,5916
-reconcile/utils/ocm/manifests.py,sha256=8kCVwTiaYHyjiKfP2DrkoT9eFxROy_M3rLom_hdsEIU,1193
+reconcile/utils/ocm/manifests.py,sha256=Q6kgOeiAwLbJY_vO_BEW2oePvbLDZcMZk20YpJJGpOA,1195
 reconcile/utils/ocm/ocm.py,sha256=EwhCymt7r8cL8UF2XbwmQ6IiRE016AUuPEiMAtYMepE,36707
 reconcile/utils/ocm/products.py,sha256=XDmTkVv4eWEifloz_f2I8GmdM97tY33PLf2p4d5GMI0,25972
 reconcile/utils/ocm/search_filters.py,sha256=jdj2sMGArcQrZLluzxeypPSbMFX_5zSE3ACvhNpsnOc,14814
@@ -799,7 +802,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=C2wrw34OXypshVocAsPrVZsSHptgw4g9u7Haa2wulZQ,9087
 reconcile/utils/saasherder/models.py,sha256=z8ln03zi2a8cu716NcNUDHp8Dv1VcVbhqdWVxCl7x9A,10148
-reconcile/utils/saasherder/saasherder.py,sha256=yi9xG_QElJlJgQlr3ADLHNwaptZVYA3SzYdVxXPXq4s,84489
+reconcile/utils/saasherder/saasherder.py,sha256=sC48LMtbKW2Mhyql2VwTMEYHugCXI9imdz8yeYlq0dk,84729
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
@@ -850,8 +853,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc989.dist-info/METADATA,sha256=t6u-8a7LRyLMJ6e6ODemRnC5wQz7WwdkOi-ntwKUKwQ,2262
-qontract_reconcile-0.10.1rc989.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc989.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc989.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc989.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc991.dist-info/METADATA,sha256=LT6qfb0l7vFc3yL6lrb_VBUmi57gqCi4gBdxOsZ18u0,2262
+qontract_reconcile-0.10.1rc991.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc991.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc991.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc991.dist-info/RECORD,,

reconcile/dynatrace_token_provider/dependencies.py

@@ -1,5 +1,13 @@
+from collections.abc import Mapping
+
 from reconcile.dynatrace_token_provider.ocm import OCMClient
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+)
 from reconcile.typed_queries.dynatrace_environments import get_dynatrace_environments
+from reconcile.typed_queries.dynatrace_token_provider_token_specs import (
+    get_dynatrace_token_provider_token_specs,
+)
 from reconcile.typed_queries.ocm import get_ocm_environments
 from reconcile.utils.dynatrace.client import DynatraceClient
 from reconcile.utils.ocm_base_client import (
@@ -17,18 +25,25 @@ class Dependencies:
     def __init__(
         self,
         secret_reader: SecretReaderBase,
-        dynatrace_client_by_tenant_id: dict[str, DynatraceClient],
-        ocm_client_by_env_name: dict[str, OCMClient],
+        dynatrace_client_by_tenant_id: Mapping[str, DynatraceClient],
+        ocm_client_by_env_name: Mapping[str, OCMClient],
+        token_spec_by_name: Mapping[str, DynatraceTokenProviderTokenSpecV1],
     ):
         self.secret_reader = secret_reader
-        self.dynatrace_client_by_tenant_id: dict[str, DynatraceClient] = (
+        self.dynatrace_client_by_tenant_id: dict[str, DynatraceClient] = dict(
             dynatrace_client_by_tenant_id
         )
-        self.ocm_client_by_env_name: dict[str, OCMClient] = ocm_client_by_env_name
+        self.ocm_client_by_env_name: dict[str, OCMClient] = dict(ocm_client_by_env_name)
+        self.token_spec_by_name = dict(token_spec_by_name)
 
     def populate(self) -> None:
         self._populate_dynatrace_client_map()
         self._populate_ocm_clients()
+        self._populate_token_specs()
+
+    def _populate_token_specs(self) -> None:
+        token_specs = get_dynatrace_token_provider_token_specs()
+        self.token_spec_by_name = {spec.name: spec for spec in token_specs}
 
     def _populate_dynatrace_client_map(self) -> None:
         dynatrace_environments = get_dynatrace_environments()

reconcile/dynatrace_token_provider/integration.py

@@ -9,28 +9,33 @@ from reconcile.dynatrace_token_provider.metrics import (
     DTPClustersManagedGauge,
     DTPOrganizationErrorRate,
 )
-from reconcile.dynatrace_token_provider.ocm import Cluster, OCMClient
+from reconcile.dynatrace_token_provider.model import DynatraceAPIToken, K8sSecret
+from reconcile.dynatrace_token_provider.ocm import (
+    DTP_LABEL_SEARCH,
+    DTP_TENANT_LABEL,
+    Cluster,
+    OCMClient,
+)
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceAPITokenV1,
+    DynatraceTokenProviderTokenSpecV1,
+)
 from reconcile.utils import (
     metrics,
 )
-from reconcile.utils.dynatrace.client import DynatraceAPITokenCreated, DynatraceClient
+from reconcile.utils.dynatrace.client import DynatraceClient
 from reconcile.utils.ocm.base import (
     OCMClusterServiceLogCreateModel,
     OCMServiceLogSeverity,
 )
 from reconcile.utils.ocm.labels import subscription_label_filter
-from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
 )
 
 QONTRACT_INTEGRATION = "dynatrace-token-provider"
-SYNCSET_ID = "ext-dynatrace-tokens-dtp"
-SECRET_NAME = "dynatrace-token-dtp"
-SECRET_NAMESPACE = "dynatrace"
-DYNATRACE_INGESTION_TOKEN_NAME = "dynatrace-ingestion-token"
-DYNATRACE_OPERATOR_TOKEN_NAME = "dynatrace-operator-token"
+SYNCSET_AND_MANIFEST_ID = "ext-dynatrace-tokens-dtp"
 
 
 class DynatraceTokenProviderIntegrationParams(PydanticRunParams):
@@ -58,6 +63,7 @@ class DynatraceTokenProviderIntegration(
             secret_reader=self.secret_reader,
             dynatrace_client_by_tenant_id={},
             ocm_client_by_env_name={},
+            token_spec_by_name={},
         )
         dependencies.populate()
         self.reconcile(dry_run=dry_run, dependencies=dependencies)
@@ -70,7 +76,7 @@ class DynatraceTokenProviderIntegration(
                 try:
                     clusters = ocm_client.discover_clusters_by_labels(
                         label_filter=subscription_label_filter().like(
-                            "key", dtp_label_key("%")
+                            "key", DTP_LABEL_SEARCH
                         ),
                     )
                 except Exception as e:
@@ -90,7 +96,6 @@ class DynatraceTokenProviderIntegration(
                         for cluster in clusters
                         if cluster.organization_id in self.params.ocm_organization_ids
                     ]
-                dtp_tenant_label_key = f"{dtp_label_key(None)}.tenant"
                 existing_dtp_tokens = {}
 
                 for cluster in clusters:
@@ -105,7 +110,7 @@ class DynatraceTokenProviderIntegration(
                                 _expose_errors_as_service_log(
                                     ocm_client,
                                     cluster_uuid=cluster.external_id,
-                                    error=f"Missing label {dtp_tenant_label_key}",
+                                    error=f"Missing label {DTP_TENANT_LABEL}",
                                 )
                                 continue
                             if (
@@ -122,6 +127,16 @@ class DynatraceTokenProviderIntegration(
                                 tenant_id
                             ]
 
+                            token_spec = dependencies.token_spec_by_name.get(
+                                cluster.token_spec_name
+                            )
+                            if not token_spec:
+                                _expose_errors_as_service_log(
+                                    ocm_client,
+                                    cluster_uuid=cluster.external_id,
+                                    error=f"Token spec {cluster.token_spec_name} does not exist",
+                                )
+                                continue
                             if tenant_id not in existing_dtp_tokens:
                                 existing_dtp_tokens[tenant_id] = (
                                     dt_client.get_token_ids_for_name_prefix(
@@ -129,13 +144,19 @@ class DynatraceTokenProviderIntegration(
                                     )
                                 )
 
+                            """
+                            Note, that we consciously do not parallelize cluster processing
+                            for now. We want to keep stress on OCM at a minimum. The amount
+                            of tagged clusters is currently feasible to be processed sequentially.
+                            """
                             self.process_cluster(
-                                dry_run,
-                                cluster,
-                                dt_client,
-                                ocm_client,
-                                existing_dtp_tokens[tenant_id],
-                                tenant_id,
+                                dry_run=dry_run,
+                                cluster=cluster,
+                                dt_client=dt_client,
+                                ocm_client=ocm_client,
+                                existing_dtp_tokens=existing_dtp_tokens[tenant_id],
+                                tenant_id=tenant_id,
+                                token_spec=token_spec,
                             )
                     except Exception as e:
                         unhandled_exceptions.append(
@@ -153,89 +174,192 @@ class DynatraceTokenProviderIntegration(
         ocm_client: OCMClient,
         existing_dtp_tokens: Iterable[str],
         tenant_id: str,
+        token_spec: DynatraceTokenProviderTokenSpecV1,
     ) -> None:
-        existing_syncset = self.get_syncset(ocm_client, cluster)
+        if cluster.organization_id not in token_spec.ocm_org_ids:
+            logging.info(
+                f"[{token_spec.name=}] Cluster {cluster.external_id} is not part of ocm orgs defined in {token_spec.ocm_org_ids=}"
+            )
+            return
+        existing_data = {}
+        if cluster.is_hcp:
+            existing_data = self.get_manifest(ocm_client=ocm_client, cluster=cluster)
+        else:
+            existing_data = self.get_syncset(ocm_client=ocm_client, cluster=cluster)
         dt_api_url = f"https://{tenant_id}.live.dynatrace.com/api"
-        if not existing_syncset:
+        if not existing_data:
             if not dry_run:
                 try:
-                    (ingestion_token, operator_token) = self.create_dynatrace_tokens(
-                        dt_client, cluster.external_id
-                    )
-                    ocm_client.create_syncset(
-                        cluster.id,
-                        self.construct_syncset(
-                            ingestion_token, operator_token, dt_api_url
-                        ),
+                    k8s_secrets = self.construct_secrets(
+                        token_spec=token_spec,
+                        dt_client=dt_client,
+                        cluster_uuid=cluster.external_id,
                     )
+                    if cluster.is_hcp:
+                        ocm_client.create_manifest(
+                            cluster_id=cluster.id,
+                            manifest_map=self.construct_manifest(
+                                with_id=True,
+                                dt_api_url=dt_api_url,
+                                secrets=k8s_secrets,
+                            ),
+                        )
+                    else:
+                        ocm_client.create_syncset(
+                            cluster_id=cluster.id,
+                            syncset_map=self.construct_syncset(
+                                with_id=True,
+                                dt_api_url=dt_api_url,
+                                secrets=k8s_secrets,
+                            ),
+                        )
                 except Exception as e:
                     _expose_errors_as_service_log(
                         ocm_client,
                         cluster.external_id,
-                        f"DTP can't create Syncset with the tokens {str(e.args)}",
+                        f"DTP can't create {token_spec.name=} {str(e.args)}",
                     )
             logging.info(
-                f"Ingestion and operator tokens created in Dynatrace for cluster {cluster.external_id}."
+                f"{token_spec.name=} created in {dt_api_url} for {cluster.external_id=}."
             )
             logging.info(
-                f"SyncSet {SYNCSET_ID} created in cluster {cluster.external_id}."
+                f"{SYNCSET_AND_MANIFEST_ID} created for {cluster.external_id=}."
             )
         else:
-            tokens = self.get_tokens_from_syncset(existing_syncset)
-            need_patching = False
-            for token_name, token in tokens.items():
-                if token.id not in existing_dtp_tokens:
-                    need_patching = True
-                    logging.info(f"{token_name} missing in Dynatrace.")
-                    if token_name == DYNATRACE_INGESTION_TOKEN_NAME:
-                        if not dry_run:
-                            ingestion_token = self.create_dynatrace_ingestion_token(
-                                dt_client, cluster.external_id
-                            )
-                            token.id = ingestion_token.id
-                            token.token = ingestion_token.token
-                        logging.info(
-                            f"Ingestion token created in Dynatrace for cluster {cluster.external_id}."
-                        )
-                    elif token_name == DYNATRACE_OPERATOR_TOKEN_NAME:
-                        if not dry_run:
-                            operator_token = self.create_dynatrace_operator_token(
-                                dt_client, cluster.external_id
-                            )
-                            token.id = operator_token.id
-                            token.token = operator_token.token
-                        logging.info(
-                            f"Operator token created in Dynatrace for cluster {cluster.external_id}."
-                        )
-                elif token_name == DYNATRACE_INGESTION_TOKEN_NAME:
-                    ingestion_token = token
-                elif token_name == DYNATRACE_OPERATOR_TOKEN_NAME:
-                    operator_token = token
-            if need_patching:
+            current_k8s_secrets: list[K8sSecret] = []
+            if cluster.is_hcp:
+                current_k8s_secrets = self.get_secrets_from_manifest(
+                    manifest=existing_data, token_spec=token_spec
+                )
+            else:
+                current_k8s_secrets = self.get_secrets_from_syncset(
+                    syncset=existing_data, token_spec=token_spec
+                )
+            has_diff, desired_secrets = self.generate_desired(
+                dry_run=dry_run,
+                current_k8s_secrets=current_k8s_secrets,
+                desired_spec=token_spec,
+                existing_dtp_tokens=existing_dtp_tokens,
+                dt_client=dt_client,
+                cluster_uuid=cluster.external_id,
+            )
+            if has_diff:
                 if not dry_run:
-                    patch_syncset_payload = self.construct_base_syncset(
-                        ingestion_token=ingestion_token,
-                        operator_token=operator_token,
-                        dt_api_url=dt_api_url,
-                    )
                     try:
-                        logging.info(f"Patching syncset {SYNCSET_ID}.")
-                        ocm_client.patch_syncset(
-                            cluster_id=cluster.id,
-                            syncset_id=SYNCSET_ID,
-                            syncset_map=patch_syncset_payload,
-                        )
+                        if cluster.is_hcp:
+                            ocm_client.patch_manifest(
+                                cluster_id=cluster.id,
+                                manifest_id=SYNCSET_AND_MANIFEST_ID,
+                                manifest_map=self.construct_manifest(
+                                    dt_api_url=dt_api_url,
+                                    secrets=desired_secrets,
+                                    with_id=False,
+                                ),
+                            )
+                        else:
+                            ocm_client.patch_syncset(
+                                cluster_id=cluster.id,
+                                syncset_id=SYNCSET_AND_MANIFEST_ID,
+                                syncset_map=self.construct_syncset(
+                                    dt_api_url=dt_api_url,
+                                    secrets=desired_secrets,
+                                    with_id=False,
+                                ),
+                            )
                     except Exception as e:
                         _expose_errors_as_service_log(
                             ocm_client,
                             cluster.external_id,
-                            f"DTP can't patch Syncset {SYNCSET_ID} due to {str(e.args)}",
+                            f"DTP can't patch {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} due to {str(e.args)}",
                         )
-                logging.info(f"Syncset {SYNCSET_ID} patched.")
+                logging.info(
+                    f"Patched {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} in {cluster.external_id=}."
+                )
+
+    def generate_desired(
+        self,
+        dry_run: bool,
+        current_k8s_secrets: Iterable[K8sSecret],
+        desired_spec: DynatraceTokenProviderTokenSpecV1,
+        existing_dtp_tokens: Iterable[str],
+        dt_client: DynatraceClient,
+        cluster_uuid: str,
+    ) -> tuple[bool, Iterable[K8sSecret]]:
+        has_diff = False
+        desired: list[K8sSecret] = []
+
+        current_secrets_by_name = {
+            secret.secret_name: secret for secret in current_k8s_secrets
+        }
+
+        for secret in desired_spec.secrets:
+            desired_tokens: list[DynatraceAPIToken] = []
+            current_secret = current_secrets_by_name.get(secret.name)
+            current_tokens_by_name = (
+                {token.name: token for token in current_secret.tokens}
+                if current_secret
+                else {}
+            )
+            for desired_token in secret.tokens:
+                new_token = current_tokens_by_name.get(desired_token.name)
+                if not new_token or new_token.id not in existing_dtp_tokens:
+                    has_diff = True
+                    if not dry_run:
+                        new_token = self.create_dynatrace_token(
+                            dt_client, cluster_uuid, desired_token
+                        )
+                if new_token:
+                    desired_tokens.append(new_token)
+            desired.append(
+                K8sSecret(
+                    secret_name=secret.name,
+                    namespace_name=secret.namespace,
+                    tokens=desired_tokens,
+                )
+            )
+
+        return (has_diff, desired)
+
+    def create_dynatrace_token(
+        self, dt_client: DynatraceClient, cluster_uuid: str, token: DynatraceAPITokenV1
+    ) -> DynatraceAPIToken:
+        token_name = f"dtp-{token.name}-{cluster_uuid}"
+        new_token = dt_client.create_api_token(
+            name=token_name,
+            scopes=token.scopes,
+        )
+        secret_key = token.key_name_in_secret or token.name
+        return DynatraceAPIToken(
+            id=new_token.id,
+            token=new_token.token,
+            name=token_name,
+            secret_key=secret_key,
+        )
+
+    def construct_secrets(
+        self,
+        token_spec: DynatraceTokenProviderTokenSpecV1,
+        dt_client: DynatraceClient,
+        cluster_uuid: str,
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        for secret in token_spec.secrets:
+            new_tokens: list[DynatraceAPIToken] = []
+            for token in secret.tokens:
+                new_token = self.create_dynatrace_token(dt_client, cluster_uuid, token)
+                new_tokens.append(new_token)
+            secrets.append(
+                K8sSecret(
+                    secret_name=secret.name,
+                    namespace_name=secret.namespace,
+                    tokens=new_tokens,
+                )
+            )
+        return secrets
 
     def get_syncset(self, ocm_client: OCMClient, cluster: Cluster) -> dict[str, Any]:
         try:
-            syncset = ocm_client.get_syncset(cluster.id, SYNCSET_ID)
+            syncset = ocm_client.get_syncset(cluster.id, SYNCSET_AND_MANIFEST_ID)
         except Exception as e:
             if "Not Found" in e.args[0]:
                 syncset = None
@@ -243,50 +367,136 @@ class DynatraceTokenProviderIntegration(
                 raise e
         return syncset
 
-    def get_tokens_from_syncset(
-        self, syncset: Mapping[str, Any]
-    ) -> dict[str, DynatraceAPITokenCreated]:
-        tokens: dict[str, Any] = {}
-        for resource in syncset["resources"]:
-            if resource["kind"] == "Secret":
-                operator_token_id = self.base64_decode(resource["data"]["apiTokenId"])
-                operator_token = self.base64_decode(resource["data"]["apiToken"])
-                ingest_token_id = self.base64_decode(
-                    resource["data"]["dataIngestTokenId"]
+    def get_manifest(self, ocm_client: OCMClient, cluster: Cluster) -> dict[str, Any]:
+        try:
+            manifest = ocm_client.get_manifest(cluster.id, SYNCSET_AND_MANIFEST_ID)
+        except Exception as e:
+            if "Not Found" in e.args[0]:
+                manifest = None
+            else:
+                raise e
+        return manifest
+
+    def get_secrets_from_syncset(
+        self, syncset: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        secret_data_by_name = {
+            resource.get("metadata", {}).get("name"): resource.get("data", {})
+            for resource in syncset.get("resources", [])
+            if resource.get("kind") == "Secret"
+        }
+        for secret in token_spec.secrets:
+            secret_data = secret_data_by_name.get(secret.name)
+            if secret_data:
+                tokens = []
+                for token in secret.tokens:
+                    token_id = self.base64_decode(
+                        secret_data.get(f"{token.key_name_in_secret}Id", "")
+                    )
+                    token_value = self.base64_decode(
+                        secret_data.get(token.key_name_in_secret, "")
+                    )
+                    tokens.append(
+                        DynatraceAPIToken(
+                            id=token_id,
+                            token=token_value,
+                            name=token.name,
+                            secret_key=token.key_name_in_secret,
+                        )
+                    )
+                secrets.append(
+                    K8sSecret(
+                        secret_name=secret.name,
+                        namespace_name=secret.namespace,
+                        tokens=tokens,
+                    )
                 )
-                ingest_token = self.base64_decode(resource["data"]["dataIngestToken"])
-        tokens[DYNATRACE_INGESTION_TOKEN_NAME] = DynatraceAPITokenCreated(
-            id=ingest_token_id,
-            token=ingest_token,
-        )
-        tokens[DYNATRACE_OPERATOR_TOKEN_NAME] = DynatraceAPITokenCreated(
-            id=operator_token_id,
-            token=operator_token,
-        )
-        return tokens
+        return secrets
+
+    def get_secrets_from_manifest(
+        self, manifest: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
+    ) -> list[K8sSecret]:
+        secrets: list[K8sSecret] = []
+        secret_data_by_name = {
+            resource.get("metadata", {}).get("name"): resource.get("data", {})
+            for resource in manifest.get("workloads", [])
+            if resource.get("kind") == "Secret"
+        }
+        for secret in token_spec.secrets:
+            secret_data = secret_data_by_name.get(secret.name)
+            if secret_data:
+                tokens = []
+                for token in secret.tokens:
+                    token_id = self.base64_decode(
+                        secret_data.get(f"{token.key_name_in_secret}Id", "")
+                    )
+                    token_value = self.base64_decode(
+                        secret_data.get(token.key_name_in_secret, "")
+                    )
+                    tokens.append(
+                        DynatraceAPIToken(
+                            id=token_id,
+                            token=token_value,
+                            name=token.name,
+                            secret_key=token.key_name_in_secret,
+                        )
+                    )
+                secrets.append(
+                    K8sSecret(
+                        secret_name=secret.name,
+                        namespace_name=secret.namespace,
+                        tokens=tokens,
+                    )
+                )
+        return secrets
+
+    def construct_secrets_data(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+    ) -> list[dict[str, Any]]:
+        secrets_data: list[dict[str, Any]] = []
+        for secret in secrets:
+            data: dict[str, str] = {
+                "apiUrl": f"{self.base64_encode_str(dt_api_url)}",
+            }
+            for token in secret.tokens:
+                data[token.secret_key] = f"{self.base64_encode_str(token.token)}"
+                data[f"{token.secret_key}Id"] = f"{self.base64_encode_str(token.id)}"
+            secrets_data.append({
+                "apiVersion": "v1",
+                "kind": "Secret",
+                "metadata": {
+                    "name": secret.secret_name,
+                    "namespace": secret.namespace_name,
+                },
+                "data": data,
+            })
+        return secrets_data
 
     def construct_base_syncset(
         self,
-        ingestion_token: DynatraceAPITokenCreated,
-        operator_token: DynatraceAPITokenCreated,
+        secrets: Iterable[K8sSecret],
         dt_api_url: str,
     ) -> dict[str, Any]:
         return {
             "kind": "SyncSet",
-            "resources": [
-                {
-                    "apiVersion": "v1",
-                    "kind": "Secret",
-                    "metadata": {"name": SECRET_NAME, "namespace": SECRET_NAMESPACE},
-                    "data": {
-                        "apiUrl": f"{self.base64_encode_str(dt_api_url)}",
-                        "dataIngestTokenId": f"{self.base64_encode_str(ingestion_token.id)}",
-                        "dataIngestToken": f"{self.base64_encode_str(ingestion_token.token)}",
-                        "apiTokenId": f"{self.base64_encode_str(operator_token.id)}",
-                        "apiToken": f"{self.base64_encode_str(operator_token.token)}",
-                    },
-                },
-            ],
+            "resources": self.construct_secrets_data(
+                secrets=secrets, dt_api_url=dt_api_url
+            ),
+        }
+
+    def construct_base_manifest(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+    ) -> dict[str, Any]:
+        return {
+            "kind": "Manifest",
+            "workloads": self.construct_secrets_data(
+                secrets=secrets, dt_api_url=dt_api_url
+            ),
         }
 
     def base64_decode(self, encoded: str) -> str:
@@ -300,51 +510,31 @@ class DynatraceTokenProviderIntegration(
 
     def construct_syncset(
         self,
-        ingestion_token: DynatraceAPITokenCreated,
-        operator_token: DynatraceAPITokenCreated,
+        secrets: Iterable[K8sSecret],
         dt_api_url: str,
+        with_id: bool,
     ) -> dict[str, Any]:
         syncset = self.construct_base_syncset(
-            ingestion_token=ingestion_token,
-            operator_token=operator_token,
+            secrets=secrets,
             dt_api_url=dt_api_url,
         )
-        syncset["id"] = SYNCSET_ID
+        if with_id:
+            syncset["id"] = SYNCSET_AND_MANIFEST_ID
         return syncset
 
-    def create_dynatrace_ingestion_token(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> DynatraceAPITokenCreated:
-        return dt_client.create_api_token(
-            name=f"dtp-ingestion-token-{cluster_uuid}",
-            scopes=["metrics.ingest", "logs.ingest", "events.ingest"],
-        )
-
-    def create_dynatrace_operator_token(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> DynatraceAPITokenCreated:
-        return dt_client.create_api_token(
-            name=f"dtp-operator-token-{cluster_uuid}",
-            scopes=[
-                "activeGateTokenManagement.create",
-                "entities.read",
-                "settings.write",
-                "settings.read",
-                "DataExport",
-                "InstallerDownload",
-            ],
+    def construct_manifest(
+        self,
+        secrets: Iterable[K8sSecret],
+        dt_api_url: str,
+        with_id: bool,
+    ) -> dict[str, Any]:
+        manifest = self.construct_base_manifest(
+            secrets=secrets,
+            dt_api_url=dt_api_url,
         )
-
-    def create_dynatrace_tokens(
-        self, dt_client: DynatraceClient, cluster_uuid: str
-    ) -> tuple[DynatraceAPITokenCreated, DynatraceAPITokenCreated]:
-        ingestion_token = self.create_dynatrace_ingestion_token(dt_client, cluster_uuid)
-        operation_token = self.create_dynatrace_operator_token(dt_client, cluster_uuid)
-        return (ingestion_token, operation_token)
-
-
-def dtp_label_key(config_atom: str | None) -> str:
-    return sre_capability_label_key("dtp", config_atom)
+        if with_id:
+            manifest["id"] = SYNCSET_AND_MANIFEST_ID
+        return manifest
 
 
 def _expose_errors_as_service_log(

reconcile/dynatrace_token_provider/model.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+
+class DynatraceAPIToken(BaseModel):
+    token: str
+    id: str
+    name: str
+    secret_key: str
+
+
+class K8sSecret(BaseModel):
+    namespace_name: str
+    secret_name: str
+    tokens: list[DynatraceAPIToken]

reconcile/dynatrace_token_provider/ocm.py

@@ -14,6 +14,11 @@ from reconcile.utils.ocm.clusters import (
     discover_clusters_by_labels,
 )
 from reconcile.utils.ocm.labels import Filter
+from reconcile.utils.ocm.manifests import (
+    create_manifest,
+    get_manifest,
+    patch_manifest,
+)
 from reconcile.utils.ocm.service_log import create_service_log
 from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
 from reconcile.utils.ocm.syncsets import (
@@ -29,23 +34,39 @@ from reconcile.utils.ocm_base_client import (
 Thin abstractions of reconcile.ocm module to reduce coupling.
 """
 
+DTP_LABEL = sre_capability_label_key("dtp", None)
+DTP_TENANT_LABEL = sre_capability_label_key("dtp", "tenant")
+DTP_LABEL_SEARCH = sre_capability_label_key("dtp", "%")
+
 
 class Cluster(BaseModel):
     id: str
     external_id: str
     organization_id: str
     dt_tenant: str
+    token_spec_name: str
+    is_hcp: bool
 
     @staticmethod
     def from_cluster_details(cluster: ClusterDetails) -> Cluster:
-        dt_tenant = cluster.labels.get_label_value(
-            f"{sre_capability_label_key('dtp', None)}.tenant"
-        )
+        dt_tenant = cluster.labels.get_label_value(DTP_TENANT_LABEL)
+        token_spec_name = cluster.labels.get_label_value(DTP_LABEL)
+        if not token_spec_name:
+            """
+            We want to stay backwards compatible.
+            Earlier version of DTP did not set a value for the label.
+            We fall back to a default token in that case.
+
+            Long-term, we want to remove this behavior.
+            """
+            token_spec_name = "default"
         return Cluster(
             id=cluster.ocm_cluster.id,
             external_id=cluster.ocm_cluster.external_id,
             organization_id=cluster.organization_id,
             dt_tenant=dt_tenant,
+            token_spec_name=token_spec_name,
+            is_hcp=cluster.ocm_cluster.is_rosa_hypershift(),
         )
 
 
@@ -77,6 +98,28 @@ class OCMClient:
             syncset_map=syncset_map,
         )
 
+    def create_manifest(self, cluster_id: str, manifest_map: Mapping) -> None:
+        create_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_map=manifest_map,
+        )
+
+    def get_manifest(self, cluster_id: str, manifest_id: str) -> Any:
+        return get_manifest(
+            ocm_client=self._ocm_client, cluster_id=cluster_id, manifest_id=manifest_id
+        )
+
+    def patch_manifest(
+        self, cluster_id: str, manifest_id: str, manifest_map: Mapping
+    ) -> None:
+        patch_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_id=manifest_id,
+            manifest_map=manifest_map,
+        )
+
     def discover_clusters_by_labels(self, label_filter: Filter) -> list[Cluster]:
         return [
             Cluster.from_cluster_details(cluster)

reconcile/gql_definitions/dynatrace_token_provider/token_specs.py

@@ -0,0 +1,84 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query DynatraceTokenProviderTokenSpecs {
+  token_specs: dynatrace_token_provider_token_spec_v1 {
+    name
+    ocm_org_ids
+    secrets {
+      name
+      namespace
+      tokens {
+        name
+        keyNameInSecret
+        scopes
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class DynatraceAPITokenV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    key_name_in_secret: Optional[str] = Field(..., alias="keyNameInSecret")
+    scopes: list[str] = Field(..., alias="scopes")
+
+
+class DynatraceTokenProviderTokenSecretV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    namespace: str = Field(..., alias="namespace")
+    tokens: list[DynatraceAPITokenV1] = Field(..., alias="tokens")
+
+
+class DynatraceTokenProviderTokenSpecV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    ocm_org_ids: list[str] = Field(..., alias="ocm_org_ids")
+    secrets: list[DynatraceTokenProviderTokenSecretV1] = Field(..., alias="secrets")
+
+
+class DynatraceTokenProviderTokenSpecsQueryData(ConfiguredBaseModel):
+    token_specs: Optional[list[DynatraceTokenProviderTokenSpecV1]] = Field(..., alias="token_specs")
+
+
+def query(query_func: Callable, **kwargs: Any) -> DynatraceTokenProviderTokenSpecsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        DynatraceTokenProviderTokenSpecsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return DynatraceTokenProviderTokenSpecsQueryData(**raw_data)

reconcile/typed_queries/dynatrace_token_provider_token_specs.py

@@ -0,0 +1,14 @@
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_dynatrace_token_provider_token_specs(
+    api: GqlApi | None = None,
+) -> list[DynatraceTokenProviderTokenSpecV1]:
+    api = api if api else gql.get_api()
+    data = query(api.query)
+    return list(data.token_specs or [])

reconcile/utils/ocm/manifests.py

@@ -33,7 +33,7 @@ def create_manifest(
 
 
 def patch_manifest(
-    ocm_client: OCMBaseClient, cluster_id: str, syncset_id: str, manifest_map: Mapping
+    ocm_client: OCMBaseClient, cluster_id: str, manifest_id: str, manifest_map: Mapping
 ) -> None:
     manifest = Manifest(cluster_id)
-    ocm_client.patch(api_path=manifest.href + "/" + syncset_id, data=manifest_map)
+    ocm_client.patch(api_path=manifest.href + "/" + manifest_id, data=manifest_map)

reconcile/utils/saasherder/saasherder.py

@@ -969,11 +969,16 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                 else True
             )
             consolidated_parameters = spec.parameters(adjust=False)
-            image = consolidated_parameters.get("image", {})
-            if isinstance(image, dict) and not image.get("tag"):
-                commit_sha = self._get_commit_sha(url, ref, github)
-                image_tag = commit_sha[:hash_length]
-                consolidated_parameters.setdefault("image", {})["tag"] = image_tag
+            commit_sha = self._get_commit_sha(url, ref, github)
+            image_tag = commit_sha[:hash_length]
+            image = consolidated_parameters.setdefault("image", {})
+            if isinstance(image, dict):
+                image.setdefault("tag", image_tag)
+            global_parameters = consolidated_parameters.setdefault("global", {})
+            if isinstance(global_parameters, dict):
+                image = global_parameters.setdefault("image", {})
+                if isinstance(image, dict):
+                    image.setdefault("tag", image_tag)
             resources = helm.template_all(
                 url=url,
                 path=path,