qontract-reconcile 0.10.2.dev73__py3-none-any.whl → 0.10.2.dev75__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qontract-reconcile
3
- Version: 0.10.2.dev73
3
+ Version: 0.10.2.dev75
4
4
  Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
5
5
  Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
6
6
  Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=q96mwr2KeEQ5bpNniGlgIMZTxiuLSodcYfX-t
10
10
  reconcile/aws_support_cases_sos.py,sha256=hl_9L53yQYRQxKs3IWrd69Cc60XK067g_bJRM9B0udo,2975
11
11
  reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
12
12
  reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
13
- reconcile/cli.py,sha256=bLfpfCJqlpSOtcpFVJExYY9DYa4n156zj9SOCbPWfPw,107417
13
+ reconcile/cli.py,sha256=MRdXMNRMBk0e4kv-ctHyYtDD5kAKnT8j6_wiTlWkrQo,107736
14
14
  reconcile/closedbox_endpoint_monitoring_base.py,sha256=MvGKBqH9PdHWdMjhLuptze-dk0Tifhp3-0SZdI-7Fmo,4862
15
15
  reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
16
16
  reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
@@ -155,7 +155,7 @@ reconcile/aws_saml_idp/integration.py,sha256=Z2JtUx2YIbkn0KVrVa2CoAErPB8vTykOOkW
155
155
  reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
156
156
  reconcile/aws_saml_roles/integration.py,sha256=inU10Yu0lZpJw_00vVe2bytJrecjLtu2hR7kQQIAbx0,11234
157
157
  reconcile/aws_version_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
158
- reconcile/aws_version_sync/integration.py,sha256=BUVUC90cNVjFPVpGKxGY9b9x-TRVHDElc0kiec_5mDA,17284
158
+ reconcile/aws_version_sync/integration.py,sha256=7tfOZx9uY4bLcPFf0i6o9cu1Hmk9ByoGPTZ66Iv5fYQ,17885
159
159
  reconcile/aws_version_sync/utils.py,sha256=x-45QT7zAwdNvCg7w_qJNwLaksFcfz1_6KQoD_0IVuA,1727
160
160
  reconcile/aws_version_sync/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
161
161
  reconcile/aws_version_sync/merge_request_manager/merge_request.py,sha256=2FbqLLdqxycWNvX1eNbwMjWSVBb7q0p-8t5Db0m7b4Q,4842
@@ -206,6 +206,15 @@ reconcile/external_resources/model.py,sha256=dxwiyI3J9xyLeue8_W9NJoap-CkKLMAoY0S
206
206
  reconcile/external_resources/reconciler.py,sha256=-0trp1K-iUgOQn3mm1ZUSmfaReRrUT0eHzPkUhNPolQ,9583
207
207
  reconcile/external_resources/secrets_sync.py,sha256=50fK4fzgSz-K8uy5_DQQWA_ju_rTDYAC2HRymgfY7TA,16344
208
208
  reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
209
+ reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
210
+ reconcile/fleet_labeler/dependencies.py,sha256=Ta-SLnrHRN4OBAmhE_mTk1P7y1X7AInIiQsIYaY6hY0,2910
211
+ reconcile/fleet_labeler/integration.py,sha256=5etCwHDTOEeOXHPr9FEB5TaVPphI3sef5fiHq7BR-1Y,7253
212
+ reconcile/fleet_labeler/merge_request.py,sha256=j6RFAr5lujeL73fzhshoBe1JS4dK-EsfROKr9fENoAw,1131
213
+ reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
214
+ reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
215
+ reconcile/fleet_labeler/ocm.py,sha256=GGsz-bq1g8BJVVMCfI2kSwZCyngbQoZ3i3k8fO608KA,2506
216
+ reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
217
+ reconcile/fleet_labeler/vcs.py,sha256=v4e_3l8F6aquVfe-ItLv2WJtS0kjMiRZ6DQ4mzCLOlE,726
209
218
  reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
210
219
  reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
211
220
  reconcile/glitchtip/integration.py,sha256=vCyg8W4ZUGxjU8tB1Gkre_auSpzo83n05mmO8_-7al0,8263
@@ -215,7 +224,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
215
224
  reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
216
225
  reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
217
226
  reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
218
- reconcile/gql_definitions/introspection.json,sha256=p3gP25rfNSUn7J4tzfihZ8yyQQU0yutdB15uP9VLlGg,2239836
227
+ reconcile/gql_definitions/introspection.json,sha256=dhwQvjxcy9059f2opu1-Nv2KWO2givEsMzEcoAZhwB0,2238565
219
228
  reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
220
229
  reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
221
230
  reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -305,7 +314,7 @@ reconcile/gql_definitions/external_resources/external_resources_settings.py,sha2
305
314
  reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
306
315
  reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
307
316
  reconcile/gql_definitions/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
308
- reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=4echDp7gADwyr-c5aJm4SqnUbkjWCEpN430kBHwIr1Q,4168
317
+ reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=TGpc-NYm2qnURHigCppUZRY1WWaIqA3E_69BWyni1RQ,4323
309
318
  reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
310
319
  reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
311
320
  reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -526,7 +535,7 @@ reconcile/typed_queries/dynatrace.py,sha256=8vXDXDIDf9_vN_efYwysDr4gLN7SCx4I2bOo
526
535
  reconcile/typed_queries/dynatrace_environments.py,sha256=jnW1GwIopNjvssEkQMYB1pJR9moYe9hUo6SAGk_-tVA,404
527
536
  reconcile/typed_queries/dynatrace_token_provider_token_specs.py,sha256=51kbjfVwzz3n7zTLFi6Lrvotl_iRYKq21tREWTVUIXM,424
528
537
  reconcile/typed_queries/external_resources.py,sha256=AT5md8nH5gX56UrdWnU4T3_aVF_FanHorXNFkU6s6KY,1573
529
- reconcile/typed_queries/fleet_labels.py,sha256=1gTJPHywuRo6R4xZVzAKKvsYvyXlTIP7qhgw_DSjOyc,345
538
+ reconcile/typed_queries/fleet_labels.py,sha256=6yBuAEPKGLMrU-g0yVm8FIDgGAYP9DUh7H2GQxJaB8Q,364
530
539
  reconcile/typed_queries/get_state_aws_account.py,sha256=CSJjVPWsUZ2rkGIt8ehoQt7hokFqrUDgG9HFlg2lVD8,492
531
540
  reconcile/typed_queries/github_orgs.py,sha256=UZhoPl8qvA_tcO7CZlN8GuMKckt3ywd47Suu61rgHsc,258
532
541
  reconcile/typed_queries/gitlab_instances.py,sha256=ZVQHy2W9xIp53f5qYkjKLHLHgOVtQpxTfcmM1C2046g,291
@@ -777,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
777
786
  tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
778
787
  tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
779
788
  tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
780
- qontract_reconcile-0.10.2.dev73.dist-info/METADATA,sha256=Kw8mFt6cKF4tp764Q0g6r82MRSjY3jGMu7FTlJJr3k4,24565
781
- qontract_reconcile-0.10.2.dev73.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
782
- qontract_reconcile-0.10.2.dev73.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
783
- qontract_reconcile-0.10.2.dev73.dist-info/RECORD,,
789
+ qontract_reconcile-0.10.2.dev75.dist-info/METADATA,sha256=FT3VK_gLcrdE4RntPG5THjQp3zubRVTkGhO3IAUIXy8,24565
790
+ qontract_reconcile-0.10.2.dev75.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
791
+ qontract_reconcile-0.10.2.dev75.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
792
+ qontract_reconcile-0.10.2.dev75.dist-info/RECORD,,
@@ -75,11 +75,16 @@ class VersionFormat(StrEnum):
75
75
  MAJOR_MINOR_PATCH = "major_minor_patch"
76
76
 
77
77
 
78
+ class SupportedResourceProvider(StrEnum):
79
+ RDS = "rds"
80
+ ELASTICACHE = "elasticache"
81
+
82
+
78
83
  class ExternalResource(BaseModel):
79
84
  namespace_file: str | None = None
80
85
  provider: str = "aws"
81
86
  provisioner: ExternalResourceProvisioner
82
- resource_provider: str
87
+ resource_provider: SupportedResourceProvider
83
88
  resource_identifier: str
84
89
  resource_engine: str
85
90
  resource_engine_version: semver.VersionInfo
@@ -133,6 +138,15 @@ class ExternalResource(BaseModel):
133
138
  raise ValueError(
134
139
  f"Invalid version format: {resource_engine_version}"
135
140
  )
141
+ if values.get("resource_provider") == SupportedResourceProvider.ELASTICACHE:
142
+ if resource_engine_version.startswith("5"):
143
+ # AWS ElastiCache Redis 5 uses MAJOR_MINOR_PATCH format
144
+ values["resource_engine_version_format"] = (
145
+ VersionFormat.MAJOR_MINOR_PATCH
146
+ )
147
+ else:
148
+ # AWS ElastiCache Redis 6+ uses MAJOR_MINOR format
149
+ values["resource_engine_version_format"] = VersionFormat.MAJOR_MINOR
136
150
  return values
137
151
 
138
152
  @property
@@ -222,7 +236,7 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
222
236
  provisioner=ExternalResourceProvisioner(
223
237
  uid=m["aws_account_id"]
224
238
  ),
225
- resource_provider="rds",
239
+ resource_provider=SupportedResourceProvider.RDS,
226
240
  resource_identifier=m["dbinstance_identifier"],
227
241
  resource_engine=m["engine"],
228
242
  resource_engine_version=m["engine_version"],
@@ -241,7 +255,7 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
241
255
  provisioner=ExternalResourceProvisioner(
242
256
  uid=m["aws_account_id"]
243
257
  ),
244
- resource_provider="elasticache",
258
+ resource_provider=SupportedResourceProvider.ELASTICACHE,
245
259
  # replication_group_id != resource_identifier!
246
260
  resource_identifier=elasticache_replication_group_id_to_identifier.get(
247
261
  (
@@ -304,11 +318,11 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
304
318
  )
305
319
  defaults_cache[resource.defaults] = values
306
320
  values = override_values(values, resource.overrides)
307
- if resource.provider.lower() == "elasticache" and str(
308
- values["engine_version"]
309
- ).lower().endswith("x"):
310
- # see https://gitlab.cee.redhat.com/service/app-interface/-/blob/master/docs/app-sre/sop/upgrade-redis-minor-version.md
311
- # minor version not managed by app-interface anymore. skip it
321
+ if (
322
+ resource.provider == SupportedResourceProvider.ELASTICACHE
323
+ and str(values["engine_version"]).lower().endswith("x")
324
+ ):
325
+ # AWS ElastiCache Redis 6 could use a version like 6.x. Then, we don't need to manage it
312
326
  continue
313
327
 
314
328
  external_resources.append(
reconcile/cli.py CHANGED
@@ -3106,6 +3106,19 @@ def dynatrace_token_provider(ctx):
3106
3106
  )
3107
3107
 
3108
3108
 
3109
+ @integration.command(short_help="Manage labels across cluster fleets in OCM")
3110
+ @click.pass_context
3111
+ def fleet_labeler(ctx):
3112
+ from reconcile.fleet_labeler.integration import (
3113
+ FleetLabelerIntegration,
3114
+ )
3115
+
3116
+ run_class_integration(
3117
+ integration=FleetLabelerIntegration(),
3118
+ ctx=ctx.obj,
3119
+ )
3120
+
3121
+
3109
3122
  @integration.command(short_help="Manage additional routers in OCM.")
3110
3123
  @click.pass_context
3111
3124
  def ocm_additional_routers(ctx):
File without changes
@@ -0,0 +1,79 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Mapping
4
+
5
+ from reconcile.fleet_labeler.ocm import OCMClient, OCMClientConfig
6
+ from reconcile.fleet_labeler.vcs import VCS
7
+ from reconcile.gql_definitions.fleet_labeler.fleet_labels import FleetLabelsSpecV1
8
+ from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
9
+ from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
10
+ from reconcile.typed_queries.github_orgs import get_github_orgs
11
+ from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
12
+ from reconcile.utils.ocm_base_client import (
13
+ init_ocm_base_client,
14
+ )
15
+ from reconcile.utils.secret_reader import SecretReaderBase
16
+ from reconcile.utils.vcs import VCS as VCSBase
17
+
18
+
19
+ class Dependencies:
20
+ """
21
+ Depenedencies class to hold all the dependencies (API clients, Specs) for the Fleet Labeler.
22
+ Dependency inversion simplifies setting up tests.
23
+ """
24
+
25
+ def __init__(
26
+ self,
27
+ label_specs_by_name: Mapping[str, FleetLabelsSpecV1],
28
+ ocm_clients_by_label_spec_name: Mapping[str, OCMClient],
29
+ vcs: VCS,
30
+ ):
31
+ self.label_specs_by_name = label_specs_by_name
32
+ self.ocm_clients_by_label_spec_name = ocm_clients_by_label_spec_name
33
+ self.vcs = vcs
34
+
35
+ @classmethod
36
+ def create(
37
+ cls,
38
+ secret_reader: SecretReaderBase,
39
+ dry_run: bool = True,
40
+ ) -> Dependencies:
41
+ return Dependencies(
42
+ label_specs_by_name=_label_specs(),
43
+ ocm_clients_by_label_spec_name=_ocm_clients(secret_reader=secret_reader),
44
+ vcs=_vcs(secret_reader=secret_reader, dry_run=dry_run),
45
+ )
46
+
47
+
48
+ def _label_specs() -> dict[str, FleetLabelsSpecV1]:
49
+ return {spec.name: spec for spec in get_fleet_label_specs()}
50
+
51
+
52
+ def _ocm_clients(secret_reader: SecretReaderBase) -> dict[str, OCMClient]:
53
+ ocm_clients_by_label_spec_name: dict[str, OCMClient] = {}
54
+ for spec in get_fleet_label_specs():
55
+ ocm_base_client = init_ocm_base_client(
56
+ cfg=OCMClientConfig(
57
+ url=spec.ocm.environment.url,
58
+ access_token_client_id=spec.ocm.access_token_client_id,
59
+ access_token_url=spec.ocm.access_token_url,
60
+ access_token_client_secret=spec.ocm.access_token_client_secret,
61
+ ),
62
+ secret_reader=secret_reader,
63
+ )
64
+ ocm_clients_by_label_spec_name[spec.name] = OCMClient(ocm_base_client)
65
+ return ocm_clients_by_label_spec_name
66
+
67
+
68
+ def _vcs(secret_reader: SecretReaderBase, dry_run: bool = True) -> VCS:
69
+ return VCS(
70
+ vcs=VCSBase(
71
+ secret_reader=secret_reader,
72
+ github_orgs=get_github_orgs(),
73
+ gitlab_instances=get_gitlab_instances(),
74
+ app_interface_repo_url=get_app_interface_repo_url(),
75
+ dry_run=dry_run,
76
+ allow_deleting_mrs=False,
77
+ allow_opening_mrs=True,
78
+ )
79
+ )
@@ -0,0 +1,190 @@
1
+ import logging
2
+ from collections import defaultdict
3
+ from collections.abc import Iterable
4
+ from typing import Any
5
+
6
+ import yaml
7
+ from pydantic import BaseModel
8
+ from ruamel.yaml.compat import StringIO
9
+
10
+ from reconcile.fleet_labeler.dependencies import Dependencies
11
+ from reconcile.fleet_labeler.merge_request import YamlCluster
12
+ from reconcile.fleet_labeler.meta import (
13
+ QONTRACT_INTEGRATION,
14
+ QONTRACT_INTEGRATION_VERSION,
15
+ )
16
+ from reconcile.fleet_labeler.metrics import FleetLabelerDuplicateClusterMatchesGauge
17
+ from reconcile.fleet_labeler.ocm import OCMClient
18
+ from reconcile.fleet_labeler.validate import validate_label_specs
19
+ from reconcile.fleet_labeler.vcs import VCS
20
+ from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
21
+ FleetLabelDefaultV1,
22
+ FleetLabelsSpecV1,
23
+ FleetSubscriptionLabelTemplateV1,
24
+ )
25
+ from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
26
+ from reconcile.utils import (
27
+ metrics,
28
+ )
29
+ from reconcile.utils.jinja2.utils import process_jinja2_template
30
+ from reconcile.utils.ruamel import create_ruamel_instance
31
+ from reconcile.utils.runtime.integration import (
32
+ NoParams,
33
+ QontractReconcileIntegration,
34
+ )
35
+
36
+
37
+ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
38
+ def __init__(self) -> None:
39
+ super().__init__(NoParams())
40
+
41
+ @property
42
+ def name(self) -> str:
43
+ return QONTRACT_INTEGRATION
44
+
45
+ def get_early_exit_desired_state(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
46
+ """Return the desired state for early exit."""
47
+ return {
48
+ "version": QONTRACT_INTEGRATION_VERSION,
49
+ "specs": {spec.name: spec.dict() for spec in get_fleet_label_specs()},
50
+ }
51
+
52
+ def run(self, dry_run: bool) -> None:
53
+ dependencies = Dependencies.create(
54
+ secret_reader=self.secret_reader,
55
+ dry_run=dry_run,
56
+ )
57
+ self.reconcile(dependencies=dependencies)
58
+
59
+ def reconcile(self, dependencies: Dependencies) -> None:
60
+ validate_label_specs(specs=dependencies.label_specs_by_name)
61
+ for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
62
+ self._sync_cluster_inventory(
63
+ ocm, dependencies.label_specs_by_name[spec_name], dependencies.vcs
64
+ )
65
+
66
+ def _render_default_labels(
67
+ self,
68
+ template: FleetSubscriptionLabelTemplateV1,
69
+ labels: dict[str, str],
70
+ ) -> dict[str, Any]:
71
+ if not template.path:
72
+ # Make mypy happy
73
+ raise ValueError("path is required for subscription label template")
74
+ body = template.path.content
75
+ type = template.q_type or "jinja2"
76
+ extra_curly = type == "extracurlyjinja2"
77
+ vars = dict(template.variables or {})
78
+ vars["labels"] = labels
79
+ rendered = process_jinja2_template(
80
+ body,
81
+ vars,
82
+ extra_curly=extra_curly,
83
+ )
84
+ return yaml.safe_load(rendered)
85
+
86
+ def _render_yaml_file(
87
+ self,
88
+ current_content: str,
89
+ ids_to_delete: Iterable[str],
90
+ clusters_to_add: Iterable[YamlCluster],
91
+ ) -> str:
92
+ yml = create_ruamel_instance(pure=True)
93
+ content = yml.load(current_content)
94
+ current_clusters = content.get("clusters", [])
95
+ desired_clusters = [
96
+ cluster
97
+ for cluster in current_clusters
98
+ if cluster.get("clusterId") not in ids_to_delete
99
+ ]
100
+ desired_clusters.extend([
101
+ {
102
+ "name": cluster.name,
103
+ "clusterId": cluster.cluster_id,
104
+ "serverUrl": cluster.server_url,
105
+ "subscriptionLabels": cluster.subscription_labels_content,
106
+ }
107
+ for cluster in clusters_to_add
108
+ ])
109
+ content["clusters"] = desired_clusters
110
+ with StringIO() as stream:
111
+ yml.dump(content, stream)
112
+ return stream.getvalue()
113
+
114
+ def _sync_cluster_inventory(
115
+ self, ocm: OCMClient, spec: FleetLabelsSpecV1, vcs: VCS
116
+ ) -> None:
117
+ class ClusterData(BaseModel):
118
+ """
119
+ Helper structure for synching process
120
+ """
121
+
122
+ name: str
123
+ server_url: str
124
+ label_default: FleetLabelDefaultV1
125
+
126
+ all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
127
+ clusters: dict[str, list[ClusterData]] = defaultdict(list)
128
+ for label_default in spec.label_defaults:
129
+ match_subscription_labels = dict(label_default.match_subscription_labels)
130
+ for cluster in ocm.discover_clusters_by_labels(
131
+ labels=match_subscription_labels
132
+ ):
133
+ # TODO: ideally we filter on server side - see TODO in ocm.py
134
+ if (
135
+ match_subscription_labels.items()
136
+ <= cluster.subscription_labels.items()
137
+ ):
138
+ clusters[cluster.cluster_id].append(
139
+ ClusterData(
140
+ label_default=label_default,
141
+ name=cluster.name,
142
+ server_url=cluster.server_url,
143
+ )
144
+ )
145
+
146
+ cluster_with_duplicate_matches = {
147
+ k: v for k, v in clusters.items() if len(v) > 1
148
+ }
149
+ for cluster_id, matches in cluster_with_duplicate_matches.items():
150
+ label_matches = "\n".join(
151
+ str(m.label_default.match_subscription_labels) for m in matches
152
+ )
153
+ logging.error(
154
+ f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
155
+ )
156
+ metrics.set_gauge(
157
+ FleetLabelerDuplicateClusterMatchesGauge(
158
+ integration=self.name,
159
+ ocm_name=spec.ocm.name,
160
+ ),
161
+ len(cluster_with_duplicate_matches),
162
+ )
163
+
164
+ all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
165
+ clusters_to_add = [
166
+ YamlCluster(
167
+ cluster_id=cluster_id,
168
+ name=cluster_info.name,
169
+ server_url=cluster_info.server_url,
170
+ subscription_labels_content=self._render_default_labels(
171
+ template=cluster_info.label_default.subscription_label_template,
172
+ labels=ocm.get_cluster_labels(cluster_id=cluster_id),
173
+ ),
174
+ )
175
+ for cluster_id, cluster_info in all_desired_clusters.items()
176
+ if cluster_id not in all_current_cluster_ids
177
+ ]
178
+ cluster_ids_to_delete = all_current_cluster_ids - all_desired_clusters.keys()
179
+
180
+ if not (cluster_ids_to_delete or clusters_to_add):
181
+ return
182
+
183
+ current_content = vcs.get_file_content_from_main(path=spec.path)
184
+ # Lets make sure we are deterministic when adding new clusters
185
+ # The overhead is neglectable and it makes testing easier
186
+ sorted_clusters_to_add = sorted(clusters_to_add, key=lambda c: c.name)
187
+ desired_content = self._render_yaml_file(
188
+ current_content, cluster_ids_to_delete, sorted_clusters_to_add
189
+ )
190
+ vcs.open_merge_request(path=spec.path, content=desired_content)
@@ -0,0 +1,48 @@
1
+ from typing import Any
2
+
3
+ from pydantic import BaseModel
4
+
5
+ from reconcile.utils.gitlab_api import GitLabApi
6
+ from reconcile.utils.mr.base import MergeRequestBase
7
+ from reconcile.utils.mr.labels import AUTO_MERGE
8
+
9
+ FLEET_LABELER_LABEL = "FleetLabeler"
10
+
11
+
12
+ class YamlCluster(BaseModel):
13
+ name: str
14
+ server_url: str
15
+ cluster_id: str
16
+ subscription_labels_content: Any
17
+
18
+
19
+ class FleetLabelerUpdates(MergeRequestBase):
20
+ def __init__(
21
+ self,
22
+ path: str,
23
+ content: str,
24
+ ):
25
+ self._path = path
26
+ self._content = content
27
+ self.name = f"[Fleet Labeler] Update cluster inventory for {path}"
28
+
29
+ super().__init__()
30
+
31
+ self.labels = [AUTO_MERGE, FLEET_LABELER_LABEL]
32
+
33
+ @property
34
+ def title(self) -> str:
35
+ return self.name
36
+
37
+ @property
38
+ def description(self) -> str:
39
+ return self.name
40
+
41
+ def process(self, gitlab_cli: GitLabApi) -> None:
42
+ msg = "update cluster inventory"
43
+ gitlab_cli.update_file(
44
+ branch_name=self.branch,
45
+ file_path=self._path,
46
+ commit_message=msg,
47
+ content=self._content,
48
+ )
@@ -0,0 +1,4 @@
1
+ from reconcile.utils.semver_helper import make_semver
2
+
3
+ QONTRACT_INTEGRATION = "fleet-labeler"
4
+ QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -0,0 +1,23 @@
1
+ from pydantic import BaseModel
2
+
3
+ from reconcile.utils.metrics import (
4
+ GaugeMetric,
5
+ )
6
+
7
+
8
+ class FleetLabelerBaseMetric(BaseModel):
9
+ integration: str
10
+ ocm_name: str
11
+
12
+
13
+ class FleetLabelerDuplicateClusterMatchesGauge(FleetLabelerBaseMetric, GaugeMetric):
14
+ """
15
+ Gauge for the number of clusters that have duplicate matches. Clusters with
16
+ duplicate matches are being ignored by fleet labeler, as it cannot clearly
17
+ determine which default label to apply for the cluster. Check the logs to
18
+ identify the clusters with duplicate matches.
19
+ """
20
+
21
+ @classmethod
22
+ def name(cls) -> str:
23
+ return "fleet_labeler_duplicate_cluster_matches"
@@ -0,0 +1,83 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Mapping
4
+
5
+ from pydantic import BaseModel
6
+
7
+ from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
8
+ from reconcile.utils.ocm.clusters import (
9
+ ClusterDetails,
10
+ discover_clusters_by_labels,
11
+ )
12
+ from reconcile.utils.ocm.labels import (
13
+ get_cluster_labels_for_cluster_id,
14
+ )
15
+ from reconcile.utils.ocm.search_filters import Filter, FilterMode
16
+ from reconcile.utils.ocm_base_client import (
17
+ OCMBaseClient,
18
+ )
19
+
20
+ """
21
+ Thin abstractions of reconcile.ocm module to reduce coupling.
22
+ """
23
+
24
+
25
+ class Cluster(BaseModel):
26
+ cluster_id: str
27
+ server_url: str
28
+ name: str
29
+ subscription_labels: dict[str, str]
30
+
31
+ @staticmethod
32
+ def from_cluster_details(cluster: ClusterDetails) -> Cluster:
33
+ server_url = (
34
+ cluster.ocm_cluster.console.url if cluster.ocm_cluster.console else ""
35
+ )
36
+
37
+ return Cluster(
38
+ cluster_id=cluster.ocm_cluster.id,
39
+ server_url=server_url,
40
+ name=cluster.ocm_cluster.name,
41
+ subscription_labels={
42
+ label.key: label.value
43
+ for label in cluster.subscription_labels.labels.values()
44
+ },
45
+ )
46
+
47
+
48
+ class OCMClientConfig(BaseModel):
49
+ """
50
+ OCMOrg does not have the required structure to comply with OCMBaseClient Protocol.
51
+ This class provides a concrete implementation for the required Protocol.
52
+ """
53
+
54
+ url: str
55
+ access_token_client_id: str
56
+ access_token_url: str
57
+ access_token_client_secret: VaultSecret
58
+
59
+
60
+ class OCMClient:
61
+ """
62
+ Thin OOP wrapper around OCMBaseClient to avoid function mocking in tests and reduce coupling.
63
+ """
64
+
65
+ def __init__(self, ocm_client: OCMBaseClient):
66
+ self._ocm_client = ocm_client
67
+
68
+ def discover_clusters_by_labels(self, labels: Mapping[str, str]) -> list[Cluster]:
69
+ label_filter = Filter(mode=FilterMode.AND).eq("type", "Subscription")
70
+ for key in labels:
71
+ label_filter = label_filter.eq("Key", key)
72
+ # TODO: This throws 400 bad request
73
+ # for k, v in labels.items():
74
+ # label_filter = label_filter.eq(k, v)
75
+ return [
76
+ Cluster.from_cluster_details(cluster)
77
+ for cluster in discover_clusters_by_labels(
78
+ ocm_api=self._ocm_client, label_filter=label_filter
79
+ )
80
+ ]
81
+
82
+ def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
83
+ return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)
@@ -0,0 +1,83 @@
1
+ from collections import Counter
2
+ from collections.abc import Mapping
3
+
4
+ from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
5
+ FleetLabelsSpecV1,
6
+ OpenShiftClusterManagerV1,
7
+ )
8
+
9
+
10
+ class OCMAccessTokenClientIdMissing(Exception):
11
+ pass
12
+
13
+
14
+ class OCMAccessTokenClientSecretMissing(Exception):
15
+ pass
16
+
17
+
18
+ class OCMAccessTokenUrlMissing(Exception):
19
+ pass
20
+
21
+
22
+ class MatchLabelsNotUniqueError(Exception):
23
+ pass
24
+
25
+
26
+ def validate_label_specs(specs: Mapping[str, FleetLabelsSpecV1]) -> None:
27
+ """
28
+ We cannot catch all potential errors through json schema definition.
29
+ """
30
+ for spec in specs.values():
31
+ _validate_ocm_token_spec(spec.ocm)
32
+ _validate_match_labels(spec)
33
+ _validate_unique_ocm_managed_label_combo(spec)
34
+
35
+
36
+ def _validate_unique_ocm_managed_label_combo(spec: FleetLabelsSpecV1) -> None:
37
+ """
38
+ Every fleet labeler spec is pinned to one OCM client and manages a single
39
+ label prefix. We must be sure, that the label prefixes are not overlapping
40
+ for the same OCM client, as that would mean to default label specs will be
41
+ competing.
42
+ """
43
+ # TODO: implement
44
+ pass
45
+
46
+
47
+ def _validate_match_labels(spec: FleetLabelsSpecV1) -> None:
48
+ """
49
+ Match labels should be unique within the same spec.
50
+ """
51
+ for label_default in spec.label_defaults:
52
+ keys = (
53
+ ".".join(
54
+ f"{k}:{v}"
55
+ for k, v in sorted(
56
+ dict(label_default.match_subscription_labels).items()
57
+ )
58
+ )
59
+ for label_default in spec.label_defaults
60
+ )
61
+ duplicates = [key for key, count in Counter(keys).items() if count > 1]
62
+ if duplicates:
63
+ raise MatchLabelsNotUniqueError(
64
+ f"The 'matchSubscriptionLabels' combinations must be unique within a spec. Found duplicates in spec {spec.name} for matchers: {duplicates}"
65
+ )
66
+
67
+
68
+ def _validate_ocm_token_spec(ocm: OpenShiftClusterManagerV1) -> None:
69
+ """
70
+ OCM tokens are optional in the schema. Lets verify they exist.
71
+ """
72
+ if not ocm.access_token_client_id:
73
+ raise OCMAccessTokenClientIdMissing(
74
+ f"accessTokenClientId missing in ocm spec '{ocm.name}'"
75
+ )
76
+ if not ocm.access_token_client_secret:
77
+ raise OCMAccessTokenClientSecretMissing(
78
+ f"accessTokenClientSecret missing in ocm spec '{ocm.name}'"
79
+ )
80
+ if not ocm.access_token_url:
81
+ raise OCMAccessTokenUrlMissing(
82
+ f"accessTokenUrl missing in ocm spec '{ocm.name}'"
83
+ )
@@ -0,0 +1,21 @@
1
+ from reconcile.fleet_labeler.merge_request import FleetLabelerUpdates
2
+ from reconcile.utils.vcs import VCS as VCSBase
3
+
4
+
5
+ class VCS:
6
+ """
7
+ Thin abstractions of reconcile.utils.vcs module to reduce coupling and simplify tests.
8
+ """
9
+
10
+ def __init__(self, vcs: VCSBase):
11
+ self._vcs = vcs
12
+
13
+ def get_file_content_from_main(self, path: str) -> str:
14
+ return self._vcs.get_file_content_from_app_interface_ref(
15
+ file_path=path, ref="main"
16
+ )
17
+
18
+ def open_merge_request(self, path: str, content: str) -> None:
19
+ mr = FleetLabelerUpdates(path=path, content=content)
20
+ # Note, that VCS is initialized with dry-run flag already
21
+ self._vcs.open_app_interface_merge_request(mr)
@@ -28,11 +28,13 @@ fragment VaultSecret on VaultSecret_v1 {
28
28
  format
29
29
  }
30
30
 
31
- query FleetLabels {
32
- fleet_labels: fleet_labels_v1 {
31
+ query FleetLabelSpecs {
32
+ fleet_labels_specs: fleet_labels_specs_v1 {
33
33
  name
34
+ path
34
35
  managedSubscriptionLabelPrefix
35
- ocms {
36
+ ocm {
37
+ name
36
38
  environment {
37
39
  url
38
40
  }
@@ -75,6 +77,7 @@ class OpenShiftClusterManagerEnvironmentV1(ConfiguredBaseModel):
75
77
 
76
78
 
77
79
  class OpenShiftClusterManagerV1(ConfiguredBaseModel):
80
+ name: str = Field(..., alias="name")
78
81
  environment: OpenShiftClusterManagerEnvironmentV1 = Field(..., alias="environment")
79
82
  access_token_client_id: Optional[str] = Field(..., alias="accessTokenClientId")
80
83
  access_token_client_secret: Optional[VaultSecret] = Field(..., alias="accessTokenClientSecret")
@@ -104,19 +107,20 @@ class FleetClusterV1(ConfiguredBaseModel):
104
107
  subscription_labels: Json = Field(..., alias="subscriptionLabels")
105
108
 
106
109
 
107
- class FleetLabelsV1(ConfiguredBaseModel):
110
+ class FleetLabelsSpecV1(ConfiguredBaseModel):
108
111
  name: str = Field(..., alias="name")
112
+ path: str = Field(..., alias="path")
109
113
  managed_subscription_label_prefix: str = Field(..., alias="managedSubscriptionLabelPrefix")
110
- ocms: list[OpenShiftClusterManagerV1] = Field(..., alias="ocms")
114
+ ocm: OpenShiftClusterManagerV1 = Field(..., alias="ocm")
111
115
  label_defaults: list[FleetLabelDefaultV1] = Field(..., alias="labelDefaults")
112
116
  clusters: list[FleetClusterV1] = Field(..., alias="clusters")
113
117
 
114
118
 
115
- class FleetLabelsQueryData(ConfiguredBaseModel):
116
- fleet_labels: Optional[list[FleetLabelsV1]] = Field(..., alias="fleet_labels")
119
+ class FleetLabelSpecsQueryData(ConfiguredBaseModel):
120
+ fleet_labels_specs: Optional[list[FleetLabelsSpecV1]] = Field(..., alias="fleet_labels_specs")
117
121
 
118
122
 
119
- def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
123
+ def query(query_func: Callable, **kwargs: Any) -> FleetLabelSpecsQueryData:
120
124
  """
121
125
  This is a convenience function which queries and parses the data into
122
126
  concrete types. It should be compatible with most GQL clients.
@@ -129,7 +133,7 @@ def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
129
133
  kwargs: optional arguments that will be passed to the query function
130
134
 
131
135
  Returns:
132
- FleetLabelsQueryData: queried data parsed into generated classes
136
+ FleetLabelSpecsQueryData: queried data parsed into generated classes
133
137
  """
134
138
  raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
135
- return FleetLabelsQueryData(**raw_data)
139
+ return FleetLabelSpecsQueryData(**raw_data)
@@ -3757,7 +3757,7 @@
3757
3757
  "deprecationReason": null
3758
3758
  },
3759
3759
  {
3760
- "name": "fleet_labels_v1",
3760
+ "name": "fleet_labels_specs_v1",
3761
3761
  "description": null,
3762
3762
  "args": [
3763
3763
  {
@@ -3799,7 +3799,7 @@
3799
3799
  "name": null,
3800
3800
  "ofType": {
3801
3801
  "kind": "OBJECT",
3802
- "name": "FleetLabels_v1",
3802
+ "name": "FleetLabelsSpec_v1",
3803
3803
  "ofType": null
3804
3804
  }
3805
3805
  }
@@ -5621,7 +5621,7 @@
5621
5621
  },
5622
5622
  {
5623
5623
  "kind": "OBJECT",
5624
- "name": "FleetLabels_v1",
5624
+ "name": "FleetLabelsSpec_v1",
5625
5625
  "ofType": null
5626
5626
  },
5627
5627
  {
@@ -10426,6 +10426,26 @@
10426
10426
  },
10427
10427
  "isDeprecated": false,
10428
10428
  "deprecationReason": null
10429
+ },
10430
+ {
10431
+ "name": "namespaces",
10432
+ "description": null,
10433
+ "args": [],
10434
+ "type": {
10435
+ "kind": "LIST",
10436
+ "name": null,
10437
+ "ofType": {
10438
+ "kind": "NON_NULL",
10439
+ "name": null,
10440
+ "ofType": {
10441
+ "kind": "OBJECT",
10442
+ "name": "Namespace_v1",
10443
+ "ofType": null
10444
+ }
10445
+ }
10446
+ },
10447
+ "isDeprecated": false,
10448
+ "deprecationReason": null
10429
10449
  }
10430
10450
  ],
10431
10451
  "inputFields": null,
@@ -33880,22 +33900,6 @@
33880
33900
  "isDeprecated": false,
33881
33901
  "deprecationReason": null
33882
33902
  },
33883
- {
33884
- "name": "module_default_resources",
33885
- "description": null,
33886
- "args": [],
33887
- "type": {
33888
- "kind": "NON_NULL",
33889
- "name": null,
33890
- "ofType": {
33891
- "kind": "OBJECT",
33892
- "name": "DeployResources_v1",
33893
- "ofType": null
33894
- }
33895
- },
33896
- "isDeprecated": false,
33897
- "deprecationReason": null
33898
- },
33899
33903
  {
33900
33904
  "name": "module_default_resources",
33901
33905
  "description": null,
@@ -34113,18 +34117,6 @@
34113
34117
  "isDeprecated": false,
34114
34118
  "deprecationReason": null
34115
34119
  },
34116
- {
34117
- "name": "resources",
34118
- "description": null,
34119
- "args": [],
34120
- "type": {
34121
- "kind": "OBJECT",
34122
- "name": "DeployResources_v1",
34123
- "ofType": null
34124
- },
34125
- "isDeprecated": false,
34126
- "deprecationReason": null
34127
- },
34128
34120
  {
34129
34121
  "name": "resources",
34130
34122
  "description": null,
@@ -34338,7 +34330,7 @@
34338
34330
  },
34339
34331
  {
34340
34332
  "kind": "OBJECT",
34341
- "name": "FleetLabels_v1",
34333
+ "name": "FleetLabelsSpec_v1",
34342
34334
  "description": null,
34343
34335
  "fields": [
34344
34336
  {
@@ -34402,24 +34394,16 @@
34402
34394
  "deprecationReason": null
34403
34395
  },
34404
34396
  {
34405
- "name": "ocms",
34397
+ "name": "ocm",
34406
34398
  "description": null,
34407
34399
  "args": [],
34408
34400
  "type": {
34409
34401
  "kind": "NON_NULL",
34410
34402
  "name": null,
34411
34403
  "ofType": {
34412
- "kind": "LIST",
34413
- "name": null,
34414
- "ofType": {
34415
- "kind": "NON_NULL",
34416
- "name": null,
34417
- "ofType": {
34418
- "kind": "OBJECT",
34419
- "name": "OpenShiftClusterManager_v1",
34420
- "ofType": null
34421
- }
34422
- }
34404
+ "kind": "OBJECT",
34405
+ "name": "OpenShiftClusterManager_v1",
34406
+ "ofType": null
34423
34407
  }
34424
34408
  },
34425
34409
  "isDeprecated": false,
@@ -43406,18 +43390,6 @@
43406
43390
  "isDeprecated": false,
43407
43391
  "deprecationReason": null
43408
43392
  },
43409
- {
43410
- "name": "resources",
43411
- "description": null,
43412
- "args": [],
43413
- "type": {
43414
- "kind": "OBJECT",
43415
- "name": "DeployResources_v1",
43416
- "ofType": null
43417
- },
43418
- "isDeprecated": false,
43419
- "deprecationReason": null
43420
- },
43421
43393
  {
43422
43394
  "name": "resources",
43423
43395
  "description": null,
@@ -1,14 +1,14 @@
1
1
  from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
2
- FleetLabelsV1,
2
+ FleetLabelsSpecV1,
3
3
  query,
4
4
  )
5
5
  from reconcile.utils import gql
6
6
  from reconcile.utils.gql import GqlApi
7
7
 
8
8
 
9
- def get_fleet_labels(
9
+ def get_fleet_label_specs(
10
10
  api: GqlApi | None = None,
11
- ) -> list[FleetLabelsV1]:
11
+ ) -> list[FleetLabelsSpecV1]:
12
12
  api = api or gql.get_api()
13
13
  data = query(api.query)
14
- return data.fleet_labels or []
14
+ return data.fleet_labels_specs or []