qontract-reconcile 0.10.2.dev192__py3-none-any.whl → 0.10.2.dev194__py3-none-any.whl

{qontract_reconcile-0.10.2.dev192.dist-info → qontract_reconcile-0.10.2.dev194.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev192
+Version: 0.10.2.dev194
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev192.dist-info → qontract_reconcile-0.10.2.dev194.dist-info}/RECORD

@@ -71,7 +71,7 @@ reconcile/openshift_network_policies.py,sha256=p81ShFK1WSEGiWHVURopDpg8YvtA3RE3O
 reconcile/openshift_prometheus_rules.py,sha256=onowXab248zmHH8SbYDTc1W1bl7JiqRFU1xdTkZyLFg,1332
 reconcile/openshift_resourcequotas.py,sha256=yUi56PiOn3inMMfq_x_FEHmaW-reGipzoorjdar372g,2415
 reconcile/openshift_resources.py,sha256=I2nO_C37mG3rfyGrd4cGwN3mVseVGuTAHAyhFzLyqF4,1518
-reconcile/openshift_resources_base.py,sha256=p4jt5R_wmGKWwFQrFz8IH_guz3UoTxnMnW5t-6uLkbw,41483
+reconcile/openshift_resources_base.py,sha256=3HudPdM7EE0HNWUn1eu0O20Ij25fqGisaDBMVvTk1fk,41768
 reconcile/openshift_rolebindings.py,sha256=9mlJ2FjWUoH-rsjtasreA_hV-K5Z_YR00qR_RR60OZM,6555
 reconcile/openshift_routes.py,sha256=fXvuPSjcjVw1X3j2EQvUAdbOepmIFdKk-M3qP8QzPiw,1075
 reconcile/openshift_saas_deploy.py,sha256=T1dvb9zajisaJNjbnR6-AZHU-itscHtr4oCqLj8KCK0,13037
@@ -105,7 +105,7 @@ reconcile/service_dependencies.py,sha256=G2qCuYFc8wQLpRxkdhmibxSAl3nUM3hcan4x50W
 reconcile/signalfx_endpoint_monitoring.py,sha256=Nqgsg1cflSd2nNnm89y_e8c--7xLUqTrKOHkDs-qADE,2868
 reconcile/slack_base.py,sha256=I-msunWxfgu5bSwXYulGbtLjxUB_tRmTCAUCU-3nabI,3484
 reconcile/slack_usergroups.py,sha256=xFkVe67RXSUj8JvpfSFEiRdQzB0TnJJEHW_b5PEwLng,30213
-reconcile/sql_query.py,sha256=OEzEZaqgv-kzG3GR2x9w3uMIfSFXP6EdhlW4u5mc1Dg,25895
+reconcile/sql_query.py,sha256=auZCWe6dytsDp83Imfo4zqkpMCLRXU007IUlPeUE3j4,26376
 reconcile/status.py,sha256=cY4IJFXemhxptRJqR4qaaOWqei9e4jgLXuVSGajMsjg,544
 reconcile/status_board.py,sha256=kJ0bus_wdyX3zsFJuUPrH4n9BNG_jhDbiQ3waOLVRBE,8538
 reconcile/terraform_aws_route53.py,sha256=dQzzT46YhwRA902_H6pi-f7WlX4EaH187wXSdmJAUkQ,9958
@@ -187,9 +187,9 @@ reconcile/cna/assets/asset_factory.py,sha256=7T7X_J6xIsoGETqBRI45_EyIKEdQcnRPt_G
 reconcile/cna/assets/null.py,sha256=85mVh97atCoC0aLuX47poTZiyOthmziJeBsUw0c924w,1658
 reconcile/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/dynatrace_token_provider/dependencies.py,sha256=lvkdwqHMsn_2kgj-tUIJdTUnUNxVoS6z8k4nPkGglnQ,3129
-reconcile/dynatrace_token_provider/integration.py,sha256=1D6gIEwKvkzE9JDOcLb9EAP6JW8OK2OLNuiHAyot_XQ,28329
+reconcile/dynatrace_token_provider/integration.py,sha256=RTGy4A6U4EgE1G4rMdS8gqgw2XIfDcdYd-eF5DL9bo0,27166
 reconcile/dynatrace_token_provider/metrics.py,sha256=oP-6NTZENFdvWiS0krnmX6tq3xyOzQ8e6vS0CZWYUuw,1496
-reconcile/dynatrace_token_provider/model.py,sha256=L6THhpPnSIeJ5n61IHhDT_JTiSEr_uWmgJAw83RUC_w,477
+reconcile/dynatrace_token_provider/model.py,sha256=VU2tZT_NAdoCovGFVj5ZoEKhWfMsC1PPPB8Iu9WMSAw,641
 reconcile/dynatrace_token_provider/ocm.py,sha256=EPknDhLXkySs8Nv8jrrl12oRoe2bRFWx_CMiHpPQhmM,3734
 reconcile/dynatrace_token_provider/validate.py,sha256=40_9QmHoB3-KBc0k_0D4QO00PpNNPS-gU9Z6cIcWga8,1920
 reconcile/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -809,7 +809,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev192.dist-info/METADATA,sha256=k0G4P25wbsCOkHg4TS0eD_mHgGuihKi2U1BvpwEh8HA,24555
-qontract_reconcile-0.10.2.dev192.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev192.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev192.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev194.dist-info/METADATA,sha256=-RyDt7W0DypwBtTgRaMQj0BLugj2YxX2eC_XlfpkDLk,24555
+qontract_reconcile-0.10.2.dev194.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev194.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev194.dist-info/RECORD,,

reconcile/dynatrace_token_provider/integration.py

@@ -2,7 +2,7 @@ import base64
 import hashlib
 import logging
 from collections import Counter, defaultdict
-from collections.abc import Iterable, Mapping, MutableMapping
+from collections.abc import Iterable, Mapping
 from datetime import timedelta
 from threading import Lock
 from typing import Any
@@ -57,6 +57,9 @@ SYNCSET_AND_MANIFEST_ID = "ext-dynatrace-tokens-dtp"
 DTP_LABEL_SEARCH = sre_capability_label_key("dtp", "%")
 DTP_TENANT_V2_LABEL = sre_capability_label_key("dtp.v2", "tenant")
 DTP_SPEC_V2_LABEL = sre_capability_label_key("dtp.v2", "token-spec")
+DTP_V3_PREFIX = sre_capability_label_key("dtp", "v3")
+DTP_V3_SPEC_SUFFIX = "token-spec"
+DTP_V3_TENANT_SUFFIX = "tenant"
 
 
 class ReconcileErrorSummary(Exception):
@@ -110,47 +113,106 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                     cnt,
                 )
 
-    def _parse_ocm_data_to_cluster(self, ocm_cluster: OCMCluster) -> Cluster | None:
-        dt_tenant = ocm_cluster.labels.get(DTP_TENANT_V2_LABEL)
-        token_spec_name = ocm_cluster.labels.get(DTP_SPEC_V2_LABEL)
-        if not dt_tenant or not token_spec_name:
-            logging.warning(
-                f"[Missing DTP labels] {ocm_cluster.id=} {ocm_cluster.subscription_id=} {dt_tenant=} {token_spec_name=}"
+    def _parse_ocm_data_to_cluster(
+        self, ocm_cluster: OCMCluster, dependencies: Dependencies
+    ) -> Cluster | None:
+        bindings: dict[str, TokenSpecTenantBinding] = {}
+        for label in ocm_cluster.labels:
+            if not label.startswith(DTP_V3_PREFIX):
+                continue
+            if not (
+                label.endswith(DTP_V3_TENANT_SUFFIX)
+                or label.endswith(DTP_V3_SPEC_SUFFIX)
+            ):
+                logging.warning(
+                    f"[Bad DTPv3 label key] {label=} {ocm_cluster.id=} {ocm_cluster.subscription_id=}"
+                )
+                continue
+            common_prefix = label.rsplit(".", 1)[0]
+            if not (
+                tenant := ocm_cluster.labels.get(
+                    f"{common_prefix}.{DTP_V3_TENANT_SUFFIX}"
+                )
+            ):
+                logging.warning(
+                    f"[Missing {DTP_V3_TENANT_SUFFIX} for common label prefix {common_prefix=}] {ocm_cluster.id=} {ocm_cluster.subscription_id=}"
+                )
+                continue
+            if not (
+                spec_name := ocm_cluster.labels.get(
+                    f"{common_prefix}.{DTP_V3_SPEC_SUFFIX}"
+                )
+            ):
+                logging.warning(
+                    f"[Missing {DTP_V3_SPEC_SUFFIX} for common label prefix {common_prefix=}] {ocm_cluster.id=} {ocm_cluster.subscription_id=}"
+                )
+                continue
+            if not (spec := dependencies.token_spec_by_name.get(spec_name)):
+                logging.warning(
+                    f"[Missing spec '{spec_name}'] {ocm_cluster.id=} {ocm_cluster.subscription_id=}"
+                )
+                continue
+            bindings[common_prefix] = TokenSpecTenantBinding(
+                spec=spec,
+                tenant_id=tenant,
             )
-            return None
+
+        if not bindings:
+            # Stay backwards compatible with v2 for now
+            dt_tenant = ocm_cluster.labels.get(DTP_TENANT_V2_LABEL)
+            token_spec_name = ocm_cluster.labels.get(DTP_SPEC_V2_LABEL)
+            token_spec = dependencies.token_spec_by_name.get(token_spec_name or "")
+            if not dt_tenant or not token_spec:
+                logging.warning(
+                    f"[Missing DTP labels] {ocm_cluster.id=} {ocm_cluster.subscription_id=} {dt_tenant=} {token_spec_name=}"
+                )
+                return None
+            bindings["v2"] = TokenSpecTenantBinding(
+                spec=token_spec,
+                tenant_id=dt_tenant,
+            )
+
+        bindings_list = list(bindings.values())
+
+        for binding in bindings_list:
+            if binding.tenant_id not in dependencies.dynatrace_client_by_tenant_id:
+                logging.warning(
+                    f"[{ocm_cluster.id=}] Dynatrace {binding.tenant_id=} does not exist"
+                )
+                return None
+
         return Cluster(
             id=ocm_cluster.id,
             external_id=ocm_cluster.external_id,
             organization_id=ocm_cluster.organization_id,
             is_hcp=ocm_cluster.is_hcp,
-            dt_token_bindings=[
-                TokenSpecTenantBinding(
-                    spec_name=token_spec_name,
-                    tenant_id=dt_tenant,
-                )
-            ],
+            dt_token_bindings=bindings_list,
         )
 
     def _filter_clusters(
         self,
         clusters: Iterable[Cluster],
-        token_spec_by_name: Mapping[str, DynatraceTokenProviderTokenSpecV1],
     ) -> list[Cluster]:
         filtered_clusters = []
         for cluster in clusters:
+            # Check if any token binding is valid for this cluster
+            has_valid_binding = False
             for token_binding in cluster.dt_token_bindings:
-                token_spec = token_spec_by_name.get(token_binding.spec_name)
-                if not token_spec:
-                    logging.debug(
-                        f"[{cluster.id=}] Skipping cluster. {token_binding.spec_name=} does not exist."
-                    )
-                    continue
+                token_spec = token_binding.spec
                 if cluster.organization_id in token_spec.ocm_org_ids:
-                    filtered_clusters.append(cluster)
+                    has_valid_binding = True
+                    break
                 else:
                     logging.debug(
-                        f"[{cluster.id=}] Skipping cluster for {token_spec.name=}. {cluster.organization_id=} is not defined in {token_spec.ocm_org_ids=}."
+                        f"[{cluster.id=}] Skipping token binding for {token_spec.name=}. {cluster.organization_id=} is not defined in {token_spec.ocm_org_ids=}."
                     )
+
+            if has_valid_binding:
+                filtered_clusters.append(cluster)
+            else:
+                logging.debug(
+                    f"[{cluster.id=}] Skipping cluster as it has no valid token bindings."
+                )
         return filtered_clusters
 
     def reconcile(self, dry_run: bool, dependencies: Dependencies) -> None:
@@ -176,13 +238,13 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                     for ocm_cluster in ocm_clusters
                     if (
                         cluster := self._parse_ocm_data_to_cluster(
-                            ocm_cluster=ocm_cluster
+                            ocm_cluster=ocm_cluster,
+                            dependencies=dependencies,
                         )
                     )
                 ]
                 filtered_clusters = self._filter_clusters(
                     clusters=clusters,
-                    token_spec_by_name=dependencies.token_spec_by_name,
                 )
 
                 existing_dtp_tokens: dict[str, dict[str, str]] = {}
@@ -195,76 +257,21 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                     len(clusters),
                 )
                 for cluster in filtered_clusters:
-                    for token_binding in cluster.dt_token_bindings:
+                    with DTPOrganizationErrorRate(
+                        integration=self.name,
+                        ocm_env=ocm_env_name,
+                        org_id=cluster.organization_id,
+                    ):
                         try:
-                            with DTPOrganizationErrorRate(
-                                integration=self.name,
-                                ocm_env=ocm_env_name,
-                                org_id=cluster.organization_id,
-                            ):
-                                tenant_id = token_binding.tenant_id
-                                if not tenant_id:
-                                    _expose_errors_as_service_log(
-                                        ocm_client,
-                                        cluster_uuid=cluster.external_id,
-                                        error=f"Missing label {DTP_TENANT_V2_LABEL}",
-                                    )
-                                    logging.warning(
-                                        f"[{cluster.id=}] Missing value for label {DTP_TENANT_V2_LABEL}"
-                                    )
-                                    continue
-                                if (
-                                    tenant_id
-                                    not in dependencies.dynatrace_client_by_tenant_id
-                                ):
-                                    _expose_errors_as_service_log(
-                                        ocm_client,
-                                        cluster_uuid=cluster.external_id,
-                                        error=f"Dynatrace tenant {tenant_id} does not exist",
-                                    )
-                                    logging.warning(
-                                        f"[{cluster.id=}] Dynatrace {tenant_id=} does not exist"
-                                    )
-                                    continue
-                                dt_client = dependencies.dynatrace_client_by_tenant_id[
-                                    tenant_id
-                                ]
-
-                                token_spec = dependencies.token_spec_by_name.get(
-                                    token_binding.spec_name
-                                )
-                                if not token_spec:
-                                    _expose_errors_as_service_log(
-                                        ocm_client,
-                                        cluster_uuid=cluster.external_id,
-                                        error=f"Token spec {token_binding.spec_name} does not exist",
-                                    )
-                                    logging.warning(
-                                        f"[{cluster.id=}] Token spec '{token_binding.spec_name}' does not exist"
-                                    )
-                                    continue
-                                if tenant_id not in existing_dtp_tokens:
-                                    existing_dtp_tokens[tenant_id] = (
-                                        dt_client.get_token_ids_map_for_name_prefix(
-                                            prefix="dtp"
-                                        )
-                                    )
-
-                                """
-                                Note, that we consciously do not parallelize cluster processing
-                                for now. We want to keep stress on OCM at a minimum. The amount
-                                of tagged clusters is currently feasible to be processed sequentially.
-                                """
-                                self.process_cluster(
-                                    dry_run=dry_run,
-                                    cluster=cluster,
-                                    dt_client=dt_client,
-                                    ocm_client=ocm_client,
-                                    existing_dtp_tokens=existing_dtp_tokens[tenant_id],
-                                    tenant_id=tenant_id,
-                                    token_spec=token_spec,
-                                    ocm_env_name=ocm_env_name,
-                                )
+                            self.process_cluster(
+                                dry_run=dry_run,
+                                cluster=cluster,
+                                ocm_client=ocm_client,
+                                existing_dtp_tokens=existing_dtp_tokens,
+                                ocm_env_name=ocm_env_name,
+                                dependencies=dependencies,
+                            )
+
                         except Exception as e:
                             unhandled_exceptions.append(
                                 f"{ocm_env_name}/{cluster.organization_id}/{cluster.id}: {e}"
@@ -278,34 +285,57 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
         self,
         dry_run: bool,
         cluster: Cluster,
-        dt_client: DynatraceClient,
         ocm_client: OCMClient,
-        existing_dtp_tokens: MutableMapping[str, str],
-        tenant_id: str,
-        token_spec: DynatraceTokenProviderTokenSpecV1,
+        existing_dtp_tokens: dict[str, dict[str, str]],
         ocm_env_name: str,
+        dependencies: Dependencies,
     ) -> None:
-        existing_data = {}
+        current_secrets: list[K8sSecret] = []
         if cluster.is_hcp:
-            existing_data = self.get_manifest(ocm_client=ocm_client, cluster=cluster)
+            data = self.get_manifest(ocm_client=ocm_client, cluster=cluster)
+            for binding in cluster.dt_token_bindings:
+                current_secrets.extend(
+                    self.get_secrets_from_manifest(
+                        manifest=data, token_spec=binding.spec
+                    )
+                )
         else:
-            existing_data = self.get_syncset(ocm_client=ocm_client, cluster=cluster)
-        dt_api_url = f"https://{tenant_id}.live.dynatrace.com/api"
-        if not existing_data:
+            data = self.get_syncset(ocm_client=ocm_client, cluster=cluster)
+            for binding in cluster.dt_token_bindings:
+                current_secrets.extend(
+                    self.get_secrets_from_syncset(syncset=data, token_spec=binding.spec)
+                )
+
+        desired_secrets: list[K8sSecret] = []
+        has_diff = False
+        for binding in cluster.dt_token_bindings:
+            dt_client = dependencies.dynatrace_client_by_tenant_id[binding.tenant_id]
+            if binding.tenant_id not in existing_dtp_tokens:
+                existing_dtp_tokens[binding.tenant_id] = (
+                    dt_client.get_token_ids_map_for_name_prefix(prefix="dtp")
+                )
+            cur_diff, cur_desired_secrets = self.generate_desired(
+                dry_run=dry_run,
+                current_k8s_secrets=current_secrets,
+                desired_spec=binding.spec,
+                existing_dtp_tokens=existing_dtp_tokens[binding.tenant_id],
+                dt_client=dt_client,
+                cluster_uuid=cluster.external_id,
+                dt_tenant_id=binding.tenant_id,
+                ocm_env_name=ocm_env_name,
+            )
+            desired_secrets.extend(cur_desired_secrets)
+            has_diff |= cur_diff
+
+        if not current_secrets:
             if not dry_run:
                 try:
-                    k8s_secrets = self.construct_secrets(
-                        token_spec=token_spec,
-                        dt_client=dt_client,
-                        cluster_uuid=cluster.external_id,
-                    )
                     if cluster.is_hcp:
                         ocm_client.create_manifest(
                             cluster_id=cluster.id,
                             manifest_map=self.construct_manifest(
                                 with_id=True,
-                                dt_api_url=dt_api_url,
-                                secrets=k8s_secrets,
+                                secrets=desired_secrets,
                             ),
                         )
                     else:
@@ -313,72 +343,47 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                             cluster_id=cluster.id,
                             syncset_map=self.construct_syncset(
                                 with_id=True,
-                                dt_api_url=dt_api_url,
-                                secrets=k8s_secrets,
+                                secrets=desired_secrets,
                             ),
                         )
                 except Exception as e:
                     _expose_errors_as_service_log(
                         ocm_client,
                         cluster.external_id,
-                        f"DTP can't create {token_spec.name=} {e.args!s}",
+                        f"DTP can't create {SYNCSET_AND_MANIFEST_ID} due to {e.args!s}",
                     )
-            logging.info(
-                f"{token_spec.name=} created in {dt_api_url} for {cluster.id=}."
-            )
             logging.info(f"{SYNCSET_AND_MANIFEST_ID} created for {cluster.id=}.")
-        else:
-            current_k8s_secrets: list[K8sSecret] = []
-            if cluster.is_hcp:
-                current_k8s_secrets = self.get_secrets_from_manifest(
-                    manifest=existing_data, token_spec=token_spec
-                )
-            else:
-                current_k8s_secrets = self.get_secrets_from_syncset(
-                    syncset=existing_data, token_spec=token_spec
-                )
-            has_diff, desired_secrets = self.generate_desired(
-                dry_run=dry_run,
-                current_k8s_secrets=current_k8s_secrets,
-                desired_spec=token_spec,
-                existing_dtp_tokens=existing_dtp_tokens,
-                dt_client=dt_client,
-                cluster_uuid=cluster.external_id,
-                dt_tenant_id=tenant_id,
-                ocm_env_name=ocm_env_name,
-            )
-            if has_diff:
-                if not dry_run:
-                    try:
-                        if cluster.is_hcp:
-                            ocm_client.patch_manifest(
-                                cluster_id=cluster.id,
-                                manifest_id=SYNCSET_AND_MANIFEST_ID,
-                                manifest_map=self.construct_manifest(
-                                    dt_api_url=dt_api_url,
-                                    secrets=desired_secrets,
-                                    with_id=False,
-                                ),
-                            )
-                        else:
-                            ocm_client.patch_syncset(
-                                cluster_id=cluster.id,
-                                syncset_id=SYNCSET_AND_MANIFEST_ID,
-                                syncset_map=self.construct_syncset(
-                                    dt_api_url=dt_api_url,
-                                    secrets=desired_secrets,
-                                    with_id=False,
-                                ),
-                            )
-                    except Exception as e:
-                        _expose_errors_as_service_log(
-                            ocm_client,
-                            cluster.external_id,
-                            f"DTP can't patch {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} due to {e.args!s}",
+        elif has_diff:
+            if not dry_run:
+                try:
+                    if cluster.is_hcp:
+                        ocm_client.patch_manifest(
+                            cluster_id=cluster.id,
+                            manifest_id=SYNCSET_AND_MANIFEST_ID,
+                            manifest_map=self.construct_manifest(
+                                secrets=desired_secrets,
+                                with_id=False,
+                            ),
                         )
-                logging.info(
-                    f"Patched {token_spec.name=} for {SYNCSET_AND_MANIFEST_ID} in {cluster.id=}."
-                )
+                    else:
+                        ocm_client.patch_syncset(
+                            cluster_id=cluster.id,
+                            syncset_id=SYNCSET_AND_MANIFEST_ID,
+                            syncset_map=self.construct_syncset(
+                                secrets=desired_secrets,
+                                with_id=False,
+                            ),
+                        )
+                except Exception as e:
+                    _expose_errors_as_service_log(
+                        ocm_client,
+                        cluster.external_id,
+                        f"DTP can't patch {SYNCSET_AND_MANIFEST_ID} due to {e.args!s}",
+                    )
+            logging.info(f"Patched {SYNCSET_AND_MANIFEST_ID} in {cluster.id=}.")
+
+    def dt_api_url(self, tenant_id: str) -> str:
+        return f"https://{tenant_id}.live.dynatrace.com/api"
 
     def scopes_hash(self, scopes: Iterable[str], length: int) -> str:
         m = hashlib.sha256()
@@ -428,7 +433,7 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
         dry_run: bool,
         current_k8s_secrets: Iterable[K8sSecret],
         desired_spec: DynatraceTokenProviderTokenSpecV1,
-        existing_dtp_tokens: MutableMapping[str, str],
+        existing_dtp_tokens: dict[str, str],
         dt_client: DynatraceClient,
         cluster_uuid: str,
         ocm_env_name: str,
@@ -475,6 +480,7 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                     secret_name=secret.name,
                     namespace_name=secret.namespace,
                     tokens=desired_tokens,
+                    dt_api_url=self.dt_api_url(tenant_id=dt_tenant_id),
                 )
             )
 
@@ -500,6 +506,7 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
         self,
         token_spec: DynatraceTokenProviderTokenSpecV1,
         dt_client: DynatraceClient,
+        dt_api_url: str,
         cluster_uuid: str,
     ) -> list[K8sSecret]:
         secrets: list[K8sSecret] = []
@@ -513,6 +520,7 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                     secret_name=secret.name,
                     namespace_name=secret.namespace,
                     tokens=new_tokens,
+                    dt_api_url=dt_api_url,
                 )
             )
         return secrets
@@ -562,11 +570,13 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
                             secret_key=token.key_name_in_secret,
                         )
                     )
+                dt_api_url = self.base64_decode(secret_data.get("apiUrl", ""))
                 secrets.append(
                     K8sSecret(
                         secret_name=secret.name,
                         namespace_name=secret.namespace,
                         tokens=tokens,
+                        dt_api_url=dt_api_url,
                     )
                 )
         return secrets
@@ -574,6 +584,8 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def get_secrets_from_syncset(
         self, syncset: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
     ) -> list[K8sSecret]:
+        if not syncset:
+            return []
         secret_data_by_name = {
             resource.get("metadata", {}).get("name"): resource.get("data", {})
             for resource in syncset.get("resources", [])
@@ -586,6 +598,8 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def get_secrets_from_manifest(
         self, manifest: Mapping[str, Any], token_spec: DynatraceTokenProviderTokenSpecV1
     ) -> list[K8sSecret]:
+        if not manifest:
+            return []
         secret_data_by_name = {
             resource.get("metadata", {}).get("name"): resource.get("data", {})
             for resource in manifest.get("workloads", [])
@@ -598,12 +612,11 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def construct_secrets_data(
         self,
         secrets: Iterable[K8sSecret],
-        dt_api_url: str,
     ) -> list[dict[str, Any]]:
         secrets_data: list[dict[str, Any]] = []
         for secret in secrets:
             data: dict[str, str] = {
-                "apiUrl": f"{self.base64_encode_str(dt_api_url)}",
+                "apiUrl": f"{self.base64_encode_str(secret.dt_api_url)}",
             }
             for token in secret.tokens:
                 data[token.secret_key] = f"{self.base64_encode_str(token.token)}"
@@ -626,25 +639,19 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def construct_base_syncset(
         self,
         secrets: Iterable[K8sSecret],
-        dt_api_url: str,
     ) -> dict[str, Any]:
         return {
             "kind": "SyncSet",
-            "resources": self.construct_secrets_data(
-                secrets=secrets, dt_api_url=dt_api_url
-            ),
+            "resources": self.construct_secrets_data(secrets=secrets),
         }
 
     def construct_base_manifest(
         self,
         secrets: Iterable[K8sSecret],
-        dt_api_url: str,
     ) -> dict[str, Any]:
         return {
             "kind": "Manifest",
-            "workloads": self.construct_secrets_data(
-                secrets=secrets, dt_api_url=dt_api_url
-            ),
+            "workloads": self.construct_secrets_data(secrets=secrets),
         }
 
     def base64_decode(self, encoded: str) -> str:
@@ -659,12 +666,10 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def construct_syncset(
         self,
         secrets: Iterable[K8sSecret],
-        dt_api_url: str,
         with_id: bool,
     ) -> dict[str, Any]:
         syncset = self.construct_base_syncset(
             secrets=secrets,
-            dt_api_url=dt_api_url,
         )
         if with_id:
             syncset["id"] = SYNCSET_AND_MANIFEST_ID
@@ -673,12 +678,10 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
     def construct_manifest(
         self,
         secrets: Iterable[K8sSecret],
-        dt_api_url: str,
         with_id: bool,
     ) -> dict[str, Any]:
         manifest = self.construct_base_manifest(
             secrets=secrets,
-            dt_api_url=dt_api_url,
         )
         if with_id:
             manifest["id"] = SYNCSET_AND_MANIFEST_ID

reconcile/dynatrace_token_provider/model.py

@@ -1,5 +1,9 @@
 from pydantic import BaseModel
 
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+)
+
 
 class DynatraceAPIToken(BaseModel):
     token: str
@@ -11,11 +15,12 @@ class DynatraceAPIToken(BaseModel):
 class K8sSecret(BaseModel):
     namespace_name: str
     secret_name: str
+    dt_api_url: str
     tokens: list[DynatraceAPIToken]
 
 
 class TokenSpecTenantBinding(BaseModel):
-    spec_name: str
+    spec: DynatraceTokenProviderTokenSpecV1
     tenant_id: str
 
 

reconcile/openshift_resources_base.py

@@ -62,7 +62,7 @@ from reconcile.utils.openshift_resource import (
 )
 from reconcile.utils.openshift_resource import OpenshiftResource as OR
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
-from reconcile.utils.secret_reader import SecretReader
+from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.sharding import is_in_shard
 from reconcile.utils.vault import (
@@ -402,9 +402,16 @@ def fetch_provider_vault_secret(
     validate_alertmanager_config: bool = False,
     alertmanager_config_key: str = "alertmanager.yaml",
     settings: Mapping[str, Any] | None = None,
+    secret_reader: SecretReaderBase | None = None,
 ) -> OR:
-    # get the fields from vault
-    secret_reader = SecretReader(settings)
+    if not secret_reader and not settings:
+        raise Exception(
+            "Parameter settings or secret_reader must be provided to run fetch_provider_vault_secret."
+        )
+
+    if not secret_reader:
+        # get the fields from vault
+        secret_reader = SecretReader(settings)
     raw_data = {
         k: v
         for k, v in secret_reader.read_all({"path": path, "version": version}).items()

reconcile/sql_query.py

@@ -19,6 +19,9 @@ from reconcile import (
 )
 from reconcile import openshift_resources_base as orb
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
 from reconcile.utils.defer import defer
 from reconcile.utils.external_resources import get_external_resource_specs
 from reconcile.utils.oc import (
@@ -30,7 +33,10 @@ from reconcile.utils.openshift_resource import (
     ResourceInventory,
 )
 from reconcile.utils.ruamel import create_ruamel_instance
-from reconcile.utils.secret_reader import SecretReader
+from reconcile.utils.secret_reader import (
+    SecretReaderBase,
+    create_secret_reader,
+)
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.smtp_client import (
     DEFAULT_SMTP_TIMEOUT,
@@ -187,14 +193,16 @@ def get_tf_resource_info(
 
 
 def collect_queries(
-    settings: dict[str, Any], smtp_client: SmtpClient, query_name: str | None = None
+    secret_reader: SecretReaderBase,
+    smtp_client: SmtpClient,
+    query_name: str | None = None,
 ) -> list[dict[str, Any]]:
     """
     Consults the app-interface and constructs the list of queries
     to be executed.
 
     :param query_name: (optional) query to look for
-    :param settings: App Interface settings
+    :param secret_reader: SecretReaderBase
 
     :return: List of queries dictionaries
     """
@@ -218,8 +226,8 @@ def collect_queries(
         "",
         1,
         accounts=[],
-        settings=settings,
         prefetch_resources_by_schemas=["/aws/rds-defaults-1.yml"],
+        secret_reader=secret_reader,
     )
 
     for sql_query in sql_queries:
@@ -499,11 +507,11 @@ def get_service_account(name: str, labels: dict) -> dict[str, Any]:
     }
 
 
-def split_long_query(q, size) -> list[str]:
+def split_long_query(q: str, size: int) -> list[str]:
     return [q[i : i + size] for i in range(0, len(q), size)]
 
 
-def merge_files_command(directory, file_glob, output_file):
+def merge_files_command(directory: str, file_glob: str, output_file: str) -> str:
     return f"cat ''{directory}''/{file_glob} > ''{output_file}''"
 
 
@@ -582,8 +590,8 @@ def _build_openshift_resources(
     query: dict[str, Any],
     image_repository: str,
     pull_secret: dict[str, Any] | None,
-    settings: dict[str, Any],
-):
+    secret_reader: SecretReaderBase,
+) -> list[OpenshiftResource]:
     query_name = query["name"]
     common_resource_labels = _build_common_resource_labels(query)
     openshift_resources: list[OpenshiftResource] = []
@@ -600,7 +608,7 @@ def _build_openshift_resources(
             type=pull_secret["type"],
             integration=QONTRACT_INTEGRATION,
             integration_version=QONTRACT_INTEGRATION_VERSION,
-            settings=settings,
+            secret_reader=secret_reader,
         )
         openshift_resources.append(secret_resource)
     # ConfigMap gpg
@@ -696,14 +704,14 @@ def _reconstruct_for_metrics(
     query: dict[str, Any],
     image_repository: str,
     pull_secret: dict[str, Any] | None,
-    settings: dict[str, Any],
     ri: ResourceInventory,
+    secret_reader: SecretReaderBase,
 ) -> None:
     openshift_resources = _build_openshift_resources(
         query,
         image_repository,
         pull_secret,
-        settings,
+        secret_reader,
     )
     cluster = query["cluster"]
     namespace = query["namespace"]["name"]
@@ -762,6 +770,7 @@ def _process_existing_query(
     image_repository: str,
     pull_secret: dict[str, Any],
     ri: ResourceInventory,
+    secret_reader: SecretReaderBase,
 ) -> None:
     match _get_query_status(query, state):
         case QueryStatus.ACTIVE:
@@ -769,8 +778,8 @@ def _process_existing_query(
                 query,
                 image_repository,
                 pull_secret,
-                settings,
                 ri,
+                secret_reader,
             )
         case QueryStatus.PENDING_DELETION:
             _delete_query_resources(
@@ -797,12 +806,10 @@ def _process_new_query(
     image_repository: str,
     pull_secret: dict[str, Any],
     ri: ResourceInventory,
-):
+    secret_reader: SecretReaderBase,
+) -> None:
     openshift_resources = _build_openshift_resources(
-        query,
-        image_repository,
-        pull_secret,
-        settings,
+        query, image_repository, pull_secret, secret_reader
     )
 
     cluster = query["cluster"]
@@ -835,12 +842,14 @@ def run(
 ) -> None:
     settings = queries.get_app_interface_settings()
     state = init_state(integration=QONTRACT_INTEGRATION)
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
     if defer:
         defer(state.cleanup)
     smtp_settings = typed_queries.smtp.settings()
     smtp_client = SmtpClient(
         server=get_smtp_server_connection(
-            secret_reader=SecretReader(settings=queries.get_secret_reader_settings()),
+            secret_reader=secret_reader,
             secret=smtp_settings.credentials,
         ),
         mail_address=smtp_settings.mail_address,
@@ -854,7 +863,7 @@ def run(
         image_repository = sql_query_settings["imageRepository"]
         pull_secret = sql_query_settings["pullSecret"]
 
-    queries_list = collect_queries(settings=settings, smtp_client=smtp_client)
+    queries_list = collect_queries(secret_reader=secret_reader, smtp_client=smtp_client)
     query_states = {s.lstrip("/") for s in state.ls()}
     ri = ResourceInventory()
     for query in queries_list:
@@ -868,6 +877,7 @@ def run(
                 image_repository,
                 pull_secret,
                 ri,
+                secret_reader,
             )
         else:
             _process_new_query(
@@ -878,6 +888,7 @@ def run(
                 image_repository,
                 pull_secret,
                 ri,
+                secret_reader,
             )
     openshift_base.publish_metrics(ri, QONTRACT_INTEGRATION)