qontract-reconcile 0.10.2.dev89__py3-none-any.whl → 0.10.2.dev91__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qontract-reconcile
3
- Version: 0.10.2.dev89
3
+ Version: 0.10.2.dev91
4
4
  Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
5
5
  Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
6
6
  Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -208,11 +208,11 @@ reconcile/external_resources/secrets_sync.py,sha256=ZDxzGZ6wC4zxLhA7-L39xDRH6rzU
208
208
  reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
209
209
  reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
210
210
  reconcile/fleet_labeler/dependencies.py,sha256=ZOpmtwgxEPBWU2yHRc6rhPwlvqhwYSsNMJQ_jVq1dLI,2993
211
- reconcile/fleet_labeler/integration.py,sha256=jm2wvkxsxqGLsX_vE874crgce_YsDWyggL3Q3olMNJM,8753
212
- reconcile/fleet_labeler/merge_request.py,sha256=VA_XSIob4LWYb02dRUr9coXjIa8_0prObI1v5Gqi_mY,1513
213
- reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
211
+ reconcile/fleet_labeler/integration.py,sha256=x-vfUcWRCI8-_2O1mdEdSbqGM51IwEBgUUKQPlu5_TM,12849
212
+ reconcile/fleet_labeler/merge_request.py,sha256=SfGxXInxeJzVnsTtO0ZC9-PesUJMdpKxKY9eCB6ms-g,1538
213
+ reconcile/fleet_labeler/meta.py,sha256=lWnpH2U0PHCPXu9Ok_CPmO494qQJQ5pOuqo28s0jzIQ,146
214
214
  reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
215
- reconcile/fleet_labeler/ocm.py,sha256=GGsz-bq1g8BJVVMCfI2kSwZCyngbQoZ3i3k8fO608KA,2506
215
+ reconcile/fleet_labeler/ocm.py,sha256=1VkUZtgcoqg-F9r--VtTg2LReumkOcRBIPhj-yTCKE0,3470
216
216
  reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
217
217
  reconcile/fleet_labeler/vcs.py,sha256=6UHUQ08AGAHXF7629I6X-T_E1pvx96LxjS66EeOzve4,1108
218
218
  reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
@@ -224,7 +224,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
224
224
  reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
225
225
  reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
226
226
  reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
227
- reconcile/gql_definitions/introspection.json,sha256=fhnOfg-0gFKOL6yrnI5JJE0w9khgA8ivmEzH_gmDajw,2238549
227
+ reconcile/gql_definitions/introspection.json,sha256=nBoWZM_ilrKlbCUF9cbRe7D1jsTqHqfGkLQW2gbleBM,2239254
228
228
  reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
229
229
  reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
230
230
  reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -314,7 +314,7 @@ reconcile/gql_definitions/external_resources/external_resources_settings.py,sha2
314
314
  reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
315
315
  reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
316
316
  reconcile/gql_definitions/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
317
- reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=TGpc-NYm2qnURHigCppUZRY1WWaIqA3E_69BWyni1RQ,4323
317
+ reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=rII_VzEfjGibY1lJXMBMKdCeV7kaNyRXk9-J9-neBoU,4412
318
318
  reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
319
319
  reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
320
320
  reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -786,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
786
786
  tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
787
787
  tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
788
788
  tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
789
- qontract_reconcile-0.10.2.dev89.dist-info/METADATA,sha256=qs4cK6oBZn2zwikJef7d6Z1khsXcc-k_Izwl1NWgO6o,24565
790
- qontract_reconcile-0.10.2.dev89.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
791
- qontract_reconcile-0.10.2.dev89.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
792
- qontract_reconcile-0.10.2.dev89.dist-info/RECORD,,
789
+ qontract_reconcile-0.10.2.dev91.dist-info/METADATA,sha256=Zb9JTF0crcCR37ioM_rENLPJWgXh1_1IyFQGxR8kqx8,24565
790
+ qontract_reconcile-0.10.2.dev91.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
791
+ qontract_reconcile-0.10.2.dev91.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
792
+ qontract_reconcile-0.10.2.dev91.dist-info/RECORD,,
@@ -26,6 +26,7 @@ from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
26
26
  from reconcile.utils import (
27
27
  metrics,
28
28
  )
29
+ from reconcile.utils.differ import diff_mappings
29
30
  from reconcile.utils.jinja2.utils import process_jinja2_template
30
31
  from reconcile.utils.ruamel import create_ruamel_instance
31
32
  from reconcile.utils.runtime.integration import (
@@ -34,6 +35,18 @@ from reconcile.utils.runtime.integration import (
34
35
  )
35
36
 
36
37
 
38
+ class ClusterData(BaseModel):
39
+ """
40
+ Helper structure for synching process
41
+ """
42
+
43
+ name: str
44
+ server_url: str
45
+ subscription_id: str
46
+ desired_label_default: FleetLabelDefaultV1
47
+ current_subscription_labels: dict[str, str]
48
+
49
+
37
50
  class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
38
51
  def __init__(self) -> None:
39
52
  super().__init__(NoParams())
@@ -59,13 +72,57 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
59
72
  def reconcile(self, dependencies: Dependencies) -> None:
60
73
  validate_label_specs(specs=dependencies.label_specs_by_name)
61
74
  for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
75
+ spec = dependencies.label_specs_by_name[spec_name]
76
+ discovered_clusters = self._discover_desired_clusters(
77
+ spec=spec,
78
+ ocm=ocm,
79
+ )
80
+ all_desired_clusters = {
81
+ k: v[0] for k, v in discovered_clusters.items() if len(v) == 1
82
+ }
83
+ clusters_with_duplicate_matches = {
84
+ k: v for k, v in discovered_clusters.items() if len(v) > 1
85
+ }
62
86
  self._sync_cluster_inventory(
63
87
  ocm=ocm,
64
88
  spec=dependencies.label_specs_by_name[spec_name],
65
89
  vcs=dependencies.vcs,
90
+ all_desired_clusters=all_desired_clusters,
91
+ clusters_with_duplicate_matches=clusters_with_duplicate_matches,
92
+ dry_run=dependencies.dry_run,
93
+ )
94
+ self._sync_subscription_labels(
95
+ spec=spec,
96
+ desired_clusters=all_desired_clusters,
97
+ ocm=ocm,
66
98
  dry_run=dependencies.dry_run,
67
99
  )
68
100
 
101
+ def _discover_desired_clusters(
102
+ self, spec: FleetLabelsSpecV1, ocm: OCMClient
103
+ ) -> dict[str, list[ClusterData]]:
104
+ clusters: dict[str, list[ClusterData]] = defaultdict(list)
105
+ for label_default in spec.label_defaults:
106
+ match_subscription_labels = dict(label_default.match_subscription_labels)
107
+ for cluster in ocm.discover_clusters_by_labels(
108
+ labels=match_subscription_labels
109
+ ):
110
+ # TODO: ideally we filter on server side - see TODO in ocm.py
111
+ if (
112
+ match_subscription_labels.items()
113
+ <= cluster.subscription_labels.items()
114
+ ):
115
+ clusters[cluster.cluster_id].append(
116
+ ClusterData(
117
+ subscription_id=cluster.subscription_id,
118
+ desired_label_default=label_default,
119
+ name=cluster.name,
120
+ server_url=cluster.server_url,
121
+ current_subscription_labels=cluster.subscription_labels,
122
+ )
123
+ )
124
+ return clusters
125
+
69
126
  def _render_default_labels(
70
127
  self,
71
128
  template: FleetSubscriptionLabelTemplateV1,
@@ -104,6 +161,7 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
104
161
  {
105
162
  "name": cluster.name,
106
163
  "clusterId": cluster.cluster_id,
164
+ "subscriptionId": cluster.subscription_id,
107
165
  "serverUrl": cluster.server_url,
108
166
  "subscriptionLabels": cluster.subscription_labels_content,
109
167
  }
@@ -114,68 +172,99 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
114
172
  yml.dump(content, stream)
115
173
  return stream.getvalue()
116
174
 
175
+ def _sync_subscription_labels(
176
+ self,
177
+ spec: FleetLabelsSpecV1,
178
+ desired_clusters: dict[str, ClusterData],
179
+ ocm: OCMClient,
180
+ dry_run: bool,
181
+ ) -> None:
182
+ """
183
+ Synchronize subscription labels for clusters in the spec's inventory.
184
+ Note, that we only update labels for clusters which are also part of the
185
+ discovered desired state.
186
+ I.e., we only operate on clusters that are in both, current state and desired state.
187
+ That way we ensure we do not work on deleted clusters and still synch only on
188
+ whats written in the rendered spec yet.
189
+ """
190
+ for cluster in spec.clusters:
191
+ if not desired_clusters.get(cluster.cluster_id):
192
+ # The cluster is not part of the desired inventory (will be updated with MR soon)
193
+ continue
194
+ # Ensure we only handle labels for our managed prefix
195
+ current_subscription_labels = {
196
+ k: v
197
+ for k, v in desired_clusters[
198
+ cluster.cluster_id
199
+ ].current_subscription_labels.items()
200
+ if k.startswith(spec.managed_subscription_label_prefix)
201
+ }
202
+ desired_subscription_labels = {
203
+ f"{spec.managed_subscription_label_prefix}.{k}": v
204
+ for k, v in dict(cluster.subscription_labels).items()
205
+ }
206
+ diff = diff_mappings(
207
+ current=current_subscription_labels,
208
+ desired=desired_subscription_labels,
209
+ )
210
+ for key in diff.add:
211
+ value = desired_subscription_labels[key]
212
+ logging.info(
213
+ f"[{spec.name}] Adding label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
214
+ )
215
+ if not dry_run:
216
+ ocm.add_subscription_label(
217
+ subscription_id=cluster.subscription_id,
218
+ key=key,
219
+ value=value,
220
+ )
221
+ for key in diff.change:
222
+ value = desired_subscription_labels[key]
223
+ logging.info(
224
+ f"[{spec.name}] Updating label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
225
+ )
226
+ if not dry_run:
227
+ ocm.update_subscription_label(
228
+ subscription_id=cluster.subscription_id,
229
+ key=key,
230
+ value=value,
231
+ )
232
+ # Note, we dont want to enable removal for now - its too dangerous on a broad managed prefix
233
+ # However, if it is needed in the future, we could easily add it here.
234
+
117
235
  def _sync_cluster_inventory(
118
236
  self,
119
237
  ocm: OCMClient,
120
238
  spec: FleetLabelsSpecV1,
121
239
  vcs: VCS,
240
+ clusters_with_duplicate_matches: dict[str, list[ClusterData]],
241
+ all_desired_clusters: dict[str, ClusterData],
122
242
  dry_run: bool,
123
243
  ) -> None:
124
- class ClusterData(BaseModel):
125
- """
126
- Helper structure for synching process
127
- """
128
-
129
- name: str
130
- server_url: str
131
- label_default: FleetLabelDefaultV1
132
-
133
244
  all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
134
- clusters: dict[str, list[ClusterData]] = defaultdict(list)
135
- for label_default in spec.label_defaults:
136
- match_subscription_labels = dict(label_default.match_subscription_labels)
137
- for cluster in ocm.discover_clusters_by_labels(
138
- labels=match_subscription_labels
139
- ):
140
- # TODO: ideally we filter on server side - see TODO in ocm.py
141
- if (
142
- match_subscription_labels.items()
143
- <= cluster.subscription_labels.items()
144
- ):
145
- clusters[cluster.cluster_id].append(
146
- ClusterData(
147
- label_default=label_default,
148
- name=cluster.name,
149
- server_url=cluster.server_url,
150
- )
151
- )
152
-
153
- cluster_with_duplicate_matches = {
154
- k: v for k, v in clusters.items() if len(v) > 1
155
- }
156
- for cluster_id, matches in cluster_with_duplicate_matches.items():
245
+ for cluster_id, matches in clusters_with_duplicate_matches.items():
157
246
  label_matches = "\n".join(
158
- str(m.label_default.match_subscription_labels) for m in matches
247
+ str(m.desired_label_default.match_subscription_labels) for m in matches
159
248
  )
160
249
  logging.error(
161
- f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
250
+ f"[{spec.name}] Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
162
251
  )
163
252
  metrics.set_gauge(
164
253
  FleetLabelerDuplicateClusterMatchesGauge(
165
254
  integration=self.name,
166
255
  ocm_name=spec.ocm.name,
167
256
  ),
168
- len(cluster_with_duplicate_matches),
257
+ len(clusters_with_duplicate_matches),
169
258
  )
170
259
 
171
- all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
172
260
  clusters_to_add = [
173
261
  YamlCluster(
174
262
  cluster_id=cluster_id,
263
+ subscription_id=cluster_info.subscription_id,
175
264
  name=cluster_info.name,
176
265
  server_url=cluster_info.server_url,
177
266
  subscription_labels_content=self._render_default_labels(
178
- template=cluster_info.label_default.subscription_label_template,
267
+ template=cluster_info.desired_label_default.subscription_label_template,
179
268
  labels=ocm.get_cluster_labels(cluster_id=cluster_id),
180
269
  ),
181
270
  )
@@ -219,4 +308,5 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
219
308
  desired_content = self._render_yaml_file(
220
309
  current_content, cluster_ids_to_delete, sorted_clusters_to_add
221
310
  )
222
- vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)
311
+ if not dry_run:
312
+ vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)
@@ -13,6 +13,7 @@ class YamlCluster(BaseModel):
13
13
  name: str
14
14
  server_url: str
15
15
  cluster_id: str
16
+ subscription_id: str
16
17
  subscription_labels_content: Any
17
18
 
18
19
 
@@ -1,4 +1,4 @@
1
1
  from reconcile.utils.semver_helper import make_semver
2
2
 
3
3
  QONTRACT_INTEGRATION = "fleet-labeler"
4
- QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
4
+ QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
@@ -10,7 +10,9 @@ from reconcile.utils.ocm.clusters import (
10
10
  discover_clusters_by_labels,
11
11
  )
12
12
  from reconcile.utils.ocm.labels import (
13
+ add_label,
13
14
  get_cluster_labels_for_cluster_id,
15
+ update_label,
14
16
  )
15
17
  from reconcile.utils.ocm.search_filters import Filter, FilterMode
16
18
  from reconcile.utils.ocm_base_client import (
@@ -24,18 +26,22 @@ Thin abstractions of reconcile.ocm module to reduce coupling.
24
26
 
25
27
  class Cluster(BaseModel):
26
28
  cluster_id: str
29
+ subscription_id: str
27
30
  server_url: str
28
31
  name: str
29
32
  subscription_labels: dict[str, str]
30
33
 
31
34
  @staticmethod
32
35
  def from_cluster_details(cluster: ClusterDetails) -> Cluster:
33
- server_url = (
36
+ console_url = (
34
37
  cluster.ocm_cluster.console.url if cluster.ocm_cluster.console else ""
35
38
  )
39
+ api_url = cluster.ocm_cluster.api_url or ""
40
+ server_url = api_url or console_url or ""
36
41
 
37
42
  return Cluster(
38
43
  cluster_id=cluster.ocm_cluster.id,
44
+ subscription_id=cluster.ocm_cluster.subscription.id,
39
45
  server_url=server_url,
40
46
  name=cluster.ocm_cluster.name,
41
47
  subscription_labels={
@@ -81,3 +87,25 @@ class OCMClient:
81
87
 
82
88
  def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
83
89
  return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)
90
+
91
+ def add_subscription_label(
92
+ self, subscription_id: str, key: str, value: str
93
+ ) -> None:
94
+ # TODO: move href into a utils function
95
+ add_label(
96
+ ocm_api=self._ocm_client,
97
+ label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
98
+ label=key,
99
+ value=value,
100
+ )
101
+
102
+ def update_subscription_label(
103
+ self, subscription_id: str, key: str, value: str
104
+ ) -> None:
105
+ # TODO: move href into a utils function
106
+ update_label(
107
+ ocm_api=self._ocm_client,
108
+ label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
109
+ label=key,
110
+ value=value,
111
+ )
@@ -58,6 +58,7 @@ query FleetLabelSpecs {
58
58
  clusters {
59
59
  name
60
60
  serverUrl
61
+ subscriptionId
61
62
  clusterId
62
63
  subscriptionLabels
63
64
  }
@@ -103,6 +104,7 @@ class FleetLabelDefaultV1(ConfiguredBaseModel):
103
104
  class FleetClusterV1(ConfiguredBaseModel):
104
105
  name: str = Field(..., alias="name")
105
106
  server_url: str = Field(..., alias="serverUrl")
107
+ subscription_id: str = Field(..., alias="subscriptionId")
106
108
  cluster_id: str = Field(..., alias="clusterId")
107
109
  subscription_labels: Json = Field(..., alias="subscriptionLabels")
108
110
 
@@ -34632,6 +34632,22 @@
34632
34632
  "isDeprecated": false,
34633
34633
  "deprecationReason": null
34634
34634
  },
34635
+ {
34636
+ "name": "subscriptionId",
34637
+ "description": null,
34638
+ "args": [],
34639
+ "type": {
34640
+ "kind": "NON_NULL",
34641
+ "name": null,
34642
+ "ofType": {
34643
+ "kind": "SCALAR",
34644
+ "name": "String",
34645
+ "ofType": null
34646
+ }
34647
+ },
34648
+ "isDeprecated": false,
34649
+ "deprecationReason": null
34650
+ },
34635
34651
  {
34636
34652
  "name": "subscriptionLabels",
34637
34653
  "description": null,