qontract-reconcile 0.10.2.dev257__py3-none-any.whl → 0.10.2.dev259__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qontract-reconcile
3
- Version: 0.10.2.dev257
3
+ Version: 0.10.2.dev259
4
4
  Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
5
5
  Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
6
6
  Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -8,14 +8,14 @@ reconcile/aws_iam_password_reset.py,sha256=FpAqAXngmLFqkOtOjrz_i90qteUyLHJL0GMjo
8
8
  reconcile/aws_support_cases_sos.py,sha256=PDhilxQ4TBxVnxUPIUdTbKEaNUI0wzPiEsB91oHT2fY,3384
9
9
  reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
10
10
  reconcile/checkpoint.py,sha256=gjtS8g6KIyKFYlHMSZjAqDUOlVh83nh4go-9yNrhWZU,5016
11
- reconcile/cli.py,sha256=2ZdX3b04aVYqpBYdF81dvbSRUjOxm17n-5ABNiZ8pT4,112157
11
+ reconcile/cli.py,sha256=GJ15k4DljB0aMHvtsX7zxPI-tY1j4DTHpd9FUBPSkKk,112289
12
12
  reconcile/closedbox_endpoint_monitoring_base.py,sha256=_OKz7K7HHw0-gzxeEma8PcUCtd70pRBy7JMoaAm8IVU,4940
13
13
  reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
14
14
  reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
15
15
  reconcile/dashdotdb_dora.py,sha256=UI83R8VsrX3vd2ocBakQDKPNE5Ym2a8pnAGUhfkDeR0,17771
16
16
  reconcile/dashdotdb_dvo.py,sha256=lCkZ0iby6HrNQb-3kYb6xrt8wCjVUZYxKzz9SiStfHU,8946
17
17
  reconcile/dashdotdb_slo.py,sha256=TvKdMOtUZcZP9QydcUJMKh0zURHgOMN_RTpQpCkD1Z8,3960
18
- reconcile/database_access_manager.py,sha256=HZQdrZ1b-cj0fTKgRZYDJ7HU9rrd35pPkPZufkdRaG0,25581
18
+ reconcile/database_access_manager.py,sha256=TOqJl46J98stl99iP_t-oXtsuWuBcvh7k0zVqPxfxv4,25700
19
19
  reconcile/deadmanssnitch.py,sha256=n-5W-djUgwzpmdDM4eQIZpkkDmHY0vndt-42LJXI4Y8,7491
20
20
  reconcile/email_sender.py,sha256=38Wvl6WHqCwlqLx4oxVJOIeDmoJsyitD3g1F4jTkAj8,4246
21
21
  reconcile/gabi_authorized_users.py,sha256=w0MSAqZnROzuf0_SZYRI9NU6RMxQnWquedGVl6oPC68,4842
@@ -88,7 +88,7 @@ reconcile/quay_mirror.py,sha256=PBooiA0ShZpWYfO6oeKFqYYT6Syi7Q8JJD9kj0wRRLg,1403
88
88
  reconcile/quay_mirror_org.py,sha256=I-tEqRHLL6uFqbSi7qCfPuDNoae7EAI2U68NbDOhmv8,10809
89
89
  reconcile/quay_permissions.py,sha256=9KOutS1w4RFQqkvMSy54VtsKNx56-phzP6yI_rEW-B8,4244
90
90
  reconcile/quay_repos.py,sha256=cuEYG0HUe0ut5yvLdEwOF5-CmccpXQHRb_wDazvDrvQ,6895
91
- reconcile/queries.py,sha256=PRlDXp-ViYjky3k4-Cvf6FptraOpzVpsZUluUEJW7II,49886
91
+ reconcile/queries.py,sha256=aaffpfMIlmHy73CUxos--NnUsTR5ms3VPeaUY3dnvIs,49879
92
92
  reconcile/query_validator.py,sha256=MSh5pKLBksws4AqfuvT8nrIGucIbqX-IOzYyPYTLO7k,1491
93
93
  reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
94
94
  reconcile/resource_scraper.py,sha256=znXCHrU7YwPfKuxGBiUrV7T1tYtn4vlz9qmZlfy6Flg,2307
@@ -100,7 +100,7 @@ reconcile/service_dependencies.py,sha256=G2qCuYFc8wQLpRxkdhmibxSAl3nUM3hcan4x50W
100
100
  reconcile/signalfx_endpoint_monitoring.py,sha256=Nqgsg1cflSd2nNnm89y_e8c--7xLUqTrKOHkDs-qADE,2868
101
101
  reconcile/slack_base.py,sha256=I-msunWxfgu5bSwXYulGbtLjxUB_tRmTCAUCU-3nabI,3484
102
102
  reconcile/slack_usergroups.py,sha256=3uQVZK0WeZfvE1g7xQwciKCcC3LifDa3NuE1ygQ0cRk,30174
103
- reconcile/sql_query.py,sha256=auZCWe6dytsDp83Imfo4zqkpMCLRXU007IUlPeUE3j4,26376
103
+ reconcile/sql_query.py,sha256=FVwANLPWjkUHqN2OXJ-vnX5hqqcO6rTdyLEO4HkmAgM,26397
104
104
  reconcile/status.py,sha256=cY4IJFXemhxptRJqR4qaaOWqei9e4jgLXuVSGajMsjg,544
105
105
  reconcile/status_board.py,sha256=0vTQzxrFPTmJtzNOC-iaJG_BmXbDe2vgBUe0LMUyfDE,15313
106
106
  reconcile/terraform_aws_route53.py,sha256=dQzzT46YhwRA902_H6pi-f7WlX4EaH187wXSdmJAUkQ,9958
@@ -213,7 +213,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=d3PMy-mQSbSZdIGAVaZCA2U
213
213
  reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
214
214
  reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
215
215
  reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
216
- reconcile/gql_definitions/introspection.json,sha256=_PKgzpmgWQJ8HF4irKjyxjOTXdw4YRgytoZQWJfigao,2346322
216
+ reconcile/gql_definitions/introspection.json,sha256=3-WcCYBV8GP1OH1_NLgu9yyXNSNZDJ5_1dZIiTCWhwM,2346833
217
217
  reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
218
218
  reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
219
219
  reconcile/gql_definitions/acs/acs_policies.py,sha256=Ygpfl2-VkYLSlJvHgp_dJBfb66K_Rwfdfpsa18w1v1s,4338
@@ -469,7 +469,8 @@ reconcile/rhidp/sso_client/base.py,sha256=EfQ2ewcOKh5idg46UKAkY6z0m_nGQfvnQKffa2
469
469
  reconcile/rhidp/sso_client/integration.py,sha256=kA8g7c38ZBSdrRtyfEqy_WgSreD1PbwY7ZIN-3tZRPc,2221
470
470
  reconcile/rhidp/sso_client/metrics.py,sha256=Tq7tSOsqL3XdcPUdozxqzSPIodUeOV87UCTqpuuqqhw,1013
471
471
  reconcile/saas_auto_promotions_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
472
- reconcile/saas_auto_promotions_manager/integration.py,sha256=PhngUNVQMdZ_7KlIOCokdYqQ2k-XdsvvfCb8BAdSiAA,6889
472
+ reconcile/saas_auto_promotions_manager/dependencies.py,sha256=9OdUadp9NHtXUeFj1da8Rw8NH-3w6AxAZeNpjHdaIqw,3600
473
+ reconcile/saas_auto_promotions_manager/integration.py,sha256=LE5-S0RTayYFvZRWn2OnAFLFqXNgemA5WZkTnr7dtmQ,4415
473
474
  reconcile/saas_auto_promotions_manager/meta.py,sha256=76Jp50r6Y_KyJoXFfSjrt5YrCtXyg_A4FXXxHYiS3TE,161
474
475
  reconcile/saas_auto_promotions_manager/publisher.py,sha256=5gphMxr2NUvyB7WDK4eAbgZeyeF30cZ3a2ZGrbFQgZk,2976
475
476
  reconcile/saas_auto_promotions_manager/s3_exporter.py,sha256=Y-r5R6viiAzglUHbYKItYSjT_axmLlPEJVmu_H6N170,2682
@@ -796,7 +797,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
796
797
  tools/saas_promotion_state/saas_promotion_state.py,sha256=oF7C4hpIgyMTwTRm3Aun3cDCHIjVar65JoLp6NcJHlU,3909
797
798
  tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
798
799
  tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
799
- qontract_reconcile-0.10.2.dev257.dist-info/METADATA,sha256=hzKI6s9NpHkpbiHIReBsUFBmMZW6-ZhXvLnjjJJIW0s,23827
800
- qontract_reconcile-0.10.2.dev257.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
801
- qontract_reconcile-0.10.2.dev257.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
802
- qontract_reconcile-0.10.2.dev257.dist-info/RECORD,,
800
+ qontract_reconcile-0.10.2.dev259.dist-info/METADATA,sha256=Set5kuKqo7t8Zg8eXwKelmzWOPVXIbcMV6U8d-MWVA0,23827
801
+ qontract_reconcile-0.10.2.dev259.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
802
+ qontract_reconcile-0.10.2.dev259.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
803
+ qontract_reconcile-0.10.2.dev259.dist-info/RECORD,,
reconcile/cli.py CHANGED
@@ -2375,14 +2375,18 @@ def saas_auto_promotions_manager(
2375
2375
  env_name: str | None,
2376
2376
  app_name: str | None,
2377
2377
  ) -> None:
2378
- import reconcile.saas_auto_promotions_manager.integration
2378
+ from reconcile.saas_auto_promotions_manager.integration import (
2379
+ SaasAutoPromotionsManager,
2380
+ )
2379
2381
 
2380
- run_integration(
2381
- reconcile.saas_auto_promotions_manager.integration,
2382
- ctx,
2383
- thread_pool_size,
2384
- env_name=env_name,
2385
- app_name=app_name,
2382
+ run_class_integration(
2383
+ integration=SaasAutoPromotionsManager.create(
2384
+ env_name=env_name,
2385
+ app_name=app_name,
2386
+ thread_pool_size=thread_pool_size,
2387
+ dry_run=ctx.obj["dry_run"],
2388
+ ),
2389
+ ctx=ctx,
2386
2390
  )
2387
2391
 
2388
2392
 
@@ -13,6 +13,7 @@ from typing import (
13
13
  )
14
14
 
15
15
  from pydantic import BaseModel
16
+ from sretoolbox.container.image import Image
16
17
 
17
18
  from reconcile import openshift_base, queries
18
19
  from reconcile import openshift_resources_base as orb
@@ -52,7 +53,6 @@ QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
52
53
  SUPPORTED_ENGINES = ["postgres"]
53
54
 
54
55
  JOB_DEADLINE_IN_SECONDS = 60
55
- JOB_PSQL_ENGINE_VERSION = "15.4-alpine"
56
56
 
57
57
 
58
58
  def get_database_access_namespaces(
@@ -227,9 +227,8 @@ def get_db_engine(resource: NamespaceTerraformResourceRDSV1) -> str:
227
227
 
228
228
  class JobData(BaseModel):
229
229
  engine: str
230
- engine_version: str
231
230
  name_suffix: str
232
- image_repository: str
231
+ image: str
233
232
  service_account_name: str
234
233
  rds_admin_secret_name: str
235
234
  script_secret_name: str
@@ -239,8 +238,11 @@ class JobData(BaseModel):
239
238
  def get_job_spec(job_data: JobData) -> OpenshiftResource:
240
239
  job_name = f"dbam-{job_data.name_suffix}"
241
240
 
241
+ image_tag = Image(job_data.image).tag
242
+ image_pull_policy = "Always" if image_tag == "latest" else "IfNotPresent"
243
+
242
244
  if job_data.engine == "postgres":
243
- command = "/usr/local/bin/psql"
245
+ command = "/usr/bin/psql"
244
246
 
245
247
  job = {
246
248
  "apiVersion": "batch/v1",
@@ -269,7 +271,8 @@ def get_job_spec(job_data: JobData) -> OpenshiftResource:
269
271
  "containers": [
270
272
  {
271
273
  "name": job_name,
272
- "image": f"{job_data.image_repository}/{job_data.engine}:{job_data.engine_version}",
274
+ "image": f"{job_data.image}",
275
+ "imagePullPolicy": image_pull_policy,
273
276
  "command": [
274
277
  command,
275
278
  ],
@@ -415,7 +418,7 @@ class JobStatus(BaseModel):
415
418
  def _populate_resources(
416
419
  db_access: DatabaseAccessV1,
417
420
  engine: str,
418
- image_repository: str,
421
+ job_image: str,
419
422
  pull_secret: dict[Any, Any],
420
423
  admin_secret_name: str,
421
424
  resource_prefix: str,
@@ -479,9 +482,8 @@ def _populate_resources(
479
482
  resource=get_job_spec(
480
483
  JobData(
481
484
  engine=engine,
482
- engine_version=JOB_PSQL_ENGINE_VERSION,
483
485
  name_suffix=db_access.name,
484
- image_repository=image_repository,
486
+ image=job_image,
485
487
  service_account_name=resource_prefix,
486
488
  rds_admin_secret_name=admin_secret_name,
487
489
  script_secret_name=script_secret_name,
@@ -620,10 +622,15 @@ def _process_db_access(
620
622
  if not sql_query_settings:
621
623
  raise KeyError("sqlQuery settings are required")
622
624
 
625
+ job_image = (
626
+ sql_query_settings.get("jobImage")
627
+ or "quay.io/app-sre/debug-container:latest"
628
+ )
629
+
623
630
  managed_resources = _populate_resources(
624
631
  db_access,
625
632
  engine,
626
- sql_query_settings["imageRepository"],
633
+ job_image,
627
634
  sql_query_settings["pullSecret"],
628
635
  admin_secret_name,
629
636
  resource_prefix,
@@ -25826,6 +25826,18 @@
25826
25826
  "isDeprecated": false,
25827
25827
  "deprecationReason": null
25828
25828
  },
25829
+ {
25830
+ "name": "jobImage",
25831
+ "description": null,
25832
+ "args": [],
25833
+ "type": {
25834
+ "kind": "SCALAR",
25835
+ "name": "String",
25836
+ "ofType": null
25837
+ },
25838
+ "isDeprecated": false,
25839
+ "deprecationReason": null
25840
+ },
25829
25841
  {
25830
25842
  "name": "pullSecret",
25831
25843
  "description": null,
reconcile/queries.py CHANGED
@@ -87,7 +87,7 @@ APP_INTERFACE_SETTINGS_QUERY = """
87
87
  }
88
88
  }
89
89
  sqlQuery {
90
- imageRepository
90
+ jobImage
91
91
  pullSecret {
92
92
  path
93
93
  version
@@ -0,0 +1,95 @@
1
+ from typing import Self
2
+
3
+ from reconcile.openshift_saas_deploy import (
4
+ QONTRACT_INTEGRATION as OPENSHIFT_SAAS_DEPLOY,
5
+ )
6
+ from reconcile.saas_auto_promotions_manager.meta import QONTRACT_INTEGRATION
7
+ from reconcile.saas_auto_promotions_manager.utils.saas_files_inventory import (
8
+ SaasFilesInventory,
9
+ )
10
+ from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
11
+ from reconcile.typed_queries.app_interface_vault_settings import (
12
+ get_app_interface_vault_settings,
13
+ )
14
+ from reconcile.typed_queries.github_orgs import get_github_orgs
15
+ from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
16
+ from reconcile.typed_queries.saas_files import get_saas_files
17
+ from reconcile.utils.promotion_state import PromotionState
18
+ from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
19
+ from reconcile.utils.state import State, init_state
20
+ from reconcile.utils.unleash import get_feature_toggle_state
21
+ from reconcile.utils.vcs import VCS
22
+
23
+
24
+ class Dependencies:
25
+ """
26
+ Dependencies class to hold all the external dependencies (API clients, state, etc.) for SAPM.
27
+ Dependency inversion simplifies setting up tests and centralizes dependency management.
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ secret_reader: SecretReaderBase,
33
+ deployment_state: PromotionState,
34
+ vcs: VCS,
35
+ saas_file_inventory: SaasFilesInventory,
36
+ saas_deploy_state: State,
37
+ sapm_state: State,
38
+ ):
39
+ self.secret_reader = secret_reader
40
+ self.deployment_state = deployment_state
41
+ self.vcs = vcs
42
+ self.saas_file_inventory = saas_file_inventory
43
+ self.saas_deploy_state = saas_deploy_state
44
+ self.sapm_state = sapm_state
45
+
46
+ @classmethod
47
+ def create(
48
+ cls,
49
+ dry_run: bool,
50
+ thread_pool_size: int,
51
+ env_name: str | None = None,
52
+ app_name: str | None = None,
53
+ ) -> Self:
54
+ vault_settings = get_app_interface_vault_settings()
55
+ allow_deleting_mrs = get_feature_toggle_state(
56
+ integration_name=f"{QONTRACT_INTEGRATION}-allow-deleting-mrs",
57
+ default=False,
58
+ )
59
+ allow_opening_mrs = get_feature_toggle_state(
60
+ integration_name=f"{QONTRACT_INTEGRATION}-allow-opening-mrs",
61
+ default=False,
62
+ )
63
+ secret_reader = create_secret_reader(use_vault=vault_settings.vault)
64
+ vcs = VCS(
65
+ secret_reader=secret_reader,
66
+ github_orgs=get_github_orgs(),
67
+ gitlab_instances=get_gitlab_instances(),
68
+ app_interface_repo_url=get_app_interface_repo_url(),
69
+ dry_run=dry_run,
70
+ allow_deleting_mrs=allow_deleting_mrs,
71
+ allow_opening_mrs=allow_opening_mrs,
72
+ )
73
+ saas_files = get_saas_files(env_name=env_name, app_name=app_name)
74
+ saas_inventory = SaasFilesInventory(
75
+ saas_files=saas_files,
76
+ secret_reader=secret_reader,
77
+ thread_pool_size=thread_pool_size,
78
+ )
79
+ saas_deploy_state = init_state(
80
+ integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
81
+ )
82
+ deployment_state = PromotionState(
83
+ state=saas_deploy_state,
84
+ )
85
+ sapm_state = init_state(
86
+ integration=QONTRACT_INTEGRATION, secret_reader=secret_reader
87
+ )
88
+ return cls(
89
+ secret_reader=secret_reader,
90
+ deployment_state=deployment_state,
91
+ vcs=vcs,
92
+ saas_file_inventory=saas_inventory,
93
+ saas_deploy_state=saas_deploy_state,
94
+ sapm_state=sapm_state,
95
+ )
@@ -1,13 +1,11 @@
1
+ from __future__ import annotations
2
+
1
3
  from collections.abc import Callable
2
4
 
3
5
  from sretoolbox.utils import threaded
4
6
 
5
- from reconcile.openshift_saas_deploy import (
6
- QONTRACT_INTEGRATION as OPENSHIFT_SAAS_DEPLOY,
7
- )
8
- from reconcile.saas_auto_promotions_manager.merge_request_manager.batcher import (
9
- Batcher,
10
- )
7
+ from reconcile.saas_auto_promotions_manager.dependencies import Dependencies
8
+ from reconcile.saas_auto_promotions_manager.merge_request_manager.batcher import Batcher
11
9
  from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager_v2 import (
12
10
  MergeRequestManagerV2,
13
11
  )
@@ -21,189 +19,111 @@ from reconcile.saas_auto_promotions_manager.meta import QONTRACT_INTEGRATION
21
19
  from reconcile.saas_auto_promotions_manager.publisher import Publisher
22
20
  from reconcile.saas_auto_promotions_manager.s3_exporter import S3Exporter
23
21
  from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
24
- from reconcile.saas_auto_promotions_manager.utils.saas_files_inventory import (
25
- SaasFilesInventory,
26
- )
27
- from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
28
- from reconcile.typed_queries.app_interface_vault_settings import (
29
- get_app_interface_vault_settings,
30
- )
31
- from reconcile.typed_queries.github_orgs import get_github_orgs
32
- from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
33
- from reconcile.typed_queries.saas_files import get_saas_files
34
22
  from reconcile.utils.defer import defer
35
- from reconcile.utils.promotion_state import PromotionState
36
- from reconcile.utils.secret_reader import create_secret_reader
37
- from reconcile.utils.state import State, init_state
38
- from reconcile.utils.unleash import get_feature_toggle_state
39
- from reconcile.utils.vcs import VCS
23
+ from reconcile.utils.runtime.integration import (
24
+ PydanticRunParams,
25
+ QontractReconcileIntegration,
26
+ )
40
27
 
41
28
 
42
- class SaasAutoPromotionsManager:
43
- def __init__(
44
- self,
45
- deployment_state: PromotionState,
46
- vcs: VCS,
47
- saas_file_inventory: SaasFilesInventory,
48
- merge_request_manager_v2: MergeRequestManagerV2,
49
- s3_exporter: S3Exporter,
50
- thread_pool_size: int,
29
+ class SaasAutoPromotionsManagerParams(PydanticRunParams):
30
+ thread_pool_size: int
31
+ env_name: str | None
32
+ app_name: str | None
33
+
34
+
35
+ class SaasAutoPromotionsManager(
36
+ QontractReconcileIntegration[SaasAutoPromotionsManagerParams]
37
+ ):
38
+ @classmethod
39
+ def create(
40
+ cls,
51
41
  dry_run: bool,
42
+ thread_pool_size: int,
43
+ env_name: str | None = None,
44
+ app_name: str | None = None,
45
+ ) -> SaasAutoPromotionsManager:
46
+ dependencies = Dependencies.create(
47
+ dry_run=dry_run,
48
+ thread_pool_size=thread_pool_size,
49
+ env_name=env_name,
50
+ app_name=app_name,
51
+ )
52
+ params = SaasAutoPromotionsManagerParams(
53
+ thread_pool_size=thread_pool_size,
54
+ env_name=env_name,
55
+ app_name=app_name,
56
+ )
57
+ return cls(dependencies=dependencies, params=params)
58
+
59
+ def __init__(
60
+ self, dependencies: Dependencies, params: SaasAutoPromotionsManagerParams
52
61
  ):
53
- self._deployment_state = deployment_state
54
- self._vcs = vcs
55
- self._saas_file_inventory = saas_file_inventory
56
- self._merge_request_manager_v2 = merge_request_manager_v2
57
- self._s3_exporter = s3_exporter
58
- self._thread_pool_size = thread_pool_size
59
- self._dry_run = dry_run
62
+ super().__init__(params)
63
+ self._dependencies = dependencies
60
64
 
61
65
  def _fetch_publisher_state(
62
66
  self,
63
67
  publisher: Publisher,
64
68
  ) -> None:
65
69
  publisher.fetch_commit_shas_and_deployment_info(
66
- vcs=self._vcs,
67
- deployment_state=self._deployment_state,
70
+ vcs=self._dependencies.vcs,
71
+ deployment_state=self._dependencies.deployment_state,
68
72
  )
69
73
 
70
- def _fetch_publisher_real_world_states(self) -> None:
74
+ def _fetch_publisher_real_world_states(self, thread_pool_size: int) -> None:
71
75
  threaded.run(
72
76
  self._fetch_publisher_state,
73
- self._saas_file_inventory.publishers,
74
- thread_pool_size=self._thread_pool_size,
77
+ self._dependencies.saas_file_inventory.publishers,
78
+ thread_pool_size=thread_pool_size,
75
79
  )
76
80
 
77
81
  def _compute_desired_subscriber_states(self) -> None:
78
- for subscriber in self._saas_file_inventory.subscribers:
82
+ for subscriber in self._dependencies.saas_file_inventory.subscribers:
79
83
  subscriber.compute_desired_state()
80
84
 
81
85
  def _get_subscribers_with_diff(self) -> list[Subscriber]:
82
- return [s for s in self._saas_file_inventory.subscribers if s.has_diff()]
86
+ return [
87
+ s
88
+ for s in self._dependencies.saas_file_inventory.subscribers
89
+ if s.has_diff()
90
+ ]
83
91
 
84
- def reconcile(self) -> None:
85
- self._deployment_state.cache_commit_shas_from_s3()
86
- self._fetch_publisher_real_world_states()
92
+ def reconcile(self, thread_pool_size: int, dry_run: bool) -> None:
93
+ self._dependencies.deployment_state.cache_commit_shas_from_s3()
94
+ self._fetch_publisher_real_world_states(thread_pool_size)
87
95
  self._compute_desired_subscriber_states()
88
96
  subscribers_with_diff = self._get_subscribers_with_diff()
89
- self._merge_request_manager_v2.reconcile(subscribers=subscribers_with_diff)
90
- self._s3_exporter.export_publisher_data(
91
- publishers=self._saas_file_inventory.publishers
97
+
98
+ mr_parser = MRParser(vcs=self._dependencies.vcs)
99
+ merge_request_manager_v2 = MergeRequestManagerV2(
100
+ vcs=self._dependencies.vcs,
101
+ reconciler=Batcher(),
102
+ mr_parser=mr_parser,
103
+ renderer=Renderer(),
104
+ )
105
+ merge_request_manager_v2.reconcile(subscribers=subscribers_with_diff)
106
+
107
+ s3_exporter = S3Exporter(
108
+ state=self._dependencies.sapm_state,
109
+ dry_run=dry_run,
110
+ )
111
+ s3_exporter.export_publisher_data(
112
+ publishers=self._dependencies.saas_file_inventory.publishers
92
113
  )
93
114
 
115
+ @property
116
+ def name(self) -> str:
117
+ return QONTRACT_INTEGRATION
94
118
 
95
- def init_external_dependencies(
96
- dry_run: bool,
97
- thread_pool_size: int,
98
- env_name: str | None = None,
99
- app_name: str | None = None,
100
- ) -> tuple[
101
- PromotionState,
102
- VCS,
103
- SaasFilesInventory,
104
- MergeRequestManagerV2,
105
- S3Exporter,
106
- State,
107
- State,
108
- ]:
109
- """
110
- Lets initialize everything that involves calls to external dependencies:
111
- - VCS -> Gitlab / Github queries
112
- - SaaSFileInventory -> qontract-server GQL query
113
- - DeploymentState -> S3 queries
114
- - MergeRequestManager -> Managing SAPM MRs with app-interface
115
- """
116
- vault_settings = get_app_interface_vault_settings()
117
- allow_deleting_mrs = get_feature_toggle_state(
118
- integration_name=f"{QONTRACT_INTEGRATION}-allow-deleting-mrs",
119
- default=False,
120
- )
121
- allow_opening_mrs = get_feature_toggle_state(
122
- integration_name=f"{QONTRACT_INTEGRATION}-allow-opening-mrs",
123
- default=False,
124
- )
125
- secret_reader = create_secret_reader(use_vault=vault_settings.vault)
126
- vcs = VCS(
127
- secret_reader=secret_reader,
128
- github_orgs=get_github_orgs(),
129
- gitlab_instances=get_gitlab_instances(),
130
- app_interface_repo_url=get_app_interface_repo_url(),
131
- dry_run=dry_run,
132
- allow_deleting_mrs=allow_deleting_mrs,
133
- allow_opening_mrs=allow_opening_mrs,
134
- )
135
- mr_parser = MRParser(vcs=vcs)
136
- merge_request_manager_v2 = MergeRequestManagerV2(
137
- vcs=vcs,
138
- reconciler=Batcher(),
139
- mr_parser=mr_parser,
140
- renderer=Renderer(),
141
- )
142
- saas_files = get_saas_files(env_name=env_name, app_name=app_name)
143
- saas_inventory = SaasFilesInventory(
144
- saas_files=saas_files,
145
- secret_reader=secret_reader,
146
- thread_pool_size=thread_pool_size,
147
- )
148
- saas_deploy_state = init_state(
149
- integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
150
- )
151
- deployment_state = PromotionState(
152
- state=saas_deploy_state,
153
- )
154
- sapm_state = init_state(
155
- integration=QONTRACT_INTEGRATION, secret_reader=secret_reader
156
- )
157
- s3_exporter = S3Exporter(
158
- state=sapm_state,
159
- dry_run=dry_run,
160
- )
161
- return (
162
- deployment_state,
163
- vcs,
164
- saas_inventory,
165
- merge_request_manager_v2,
166
- s3_exporter,
167
- saas_deploy_state,
168
- sapm_state,
169
- )
170
-
171
-
172
- @defer
173
- def run(
174
- dry_run: bool,
175
- thread_pool_size: int,
176
- env_name: str | None = None,
177
- app_name: str | None = None,
178
- defer: Callable | None = None,
179
- ) -> None:
180
- (
181
- deployment_state,
182
- vcs,
183
- saas_inventory,
184
- merge_request_manager_v2,
185
- s3_exporter,
186
- saas_deploy_state,
187
- sapm_state,
188
- ) = init_external_dependencies(
189
- dry_run=dry_run,
190
- env_name=env_name,
191
- app_name=app_name,
192
- thread_pool_size=thread_pool_size,
193
- )
194
- if defer:
195
- defer(vcs.cleanup)
196
- defer(saas_deploy_state.cleanup)
197
- defer(sapm_state.cleanup)
198
-
199
- integration = SaasAutoPromotionsManager(
200
- deployment_state=deployment_state,
201
- vcs=vcs,
202
- saas_file_inventory=saas_inventory,
203
- merge_request_manager_v2=merge_request_manager_v2,
204
- s3_exporter=s3_exporter,
205
- thread_pool_size=thread_pool_size,
206
- dry_run=dry_run,
207
- )
208
-
209
- integration.reconcile()
119
+ @defer
120
+ def run(
121
+ self,
122
+ dry_run: bool,
123
+ defer: Callable | None = None,
124
+ ) -> None:
125
+ if defer:
126
+ defer(self._dependencies.vcs.cleanup)
127
+ defer(self._dependencies.saas_deploy_state.cleanup)
128
+ defer(self._dependencies.sapm_state.cleanup)
129
+ self.reconcile(thread_pool_size=self.params.thread_pool_size, dry_run=dry_run)
reconcile/sql_query.py CHANGED
@@ -11,6 +11,7 @@ from textwrap import indent
11
11
  from typing import Any
12
12
 
13
13
  import jinja2
14
+ from sretoolbox.container.image import Image
14
15
 
15
16
  from reconcile import (
16
17
  openshift_base,
@@ -81,35 +82,36 @@ spec:
81
82
  metadata:
82
83
  name: {{ JOB_NAME }}
83
84
  spec:
84
- {% if PULL_SECRET is not none %}
85
+ {%- if PULL_SECRET is not none %}
85
86
  imagePullSecrets:
86
87
  - name: {{ PULL_SECRET }}
87
- {% endif %}
88
+ {%- endif %}
88
89
  restartPolicy: Never
89
90
  serviceAccountName: {{ SVC_NAME }}
90
91
  containers:
91
92
  - name: {{ JOB_NAME }}
92
- image: {{ IMAGE_REPOSITORY }}/{{ ENGINE }}:{{ENGINE_VERSION}}
93
+ image: {{ JOB_IMAGE }}
94
+ imagePullPolicy: {{ JOB_IMAGE_PULL_POLICY }}
93
95
  command:
94
96
  - /bin/bash
95
97
  args:
96
98
  - '-c'
97
99
  - '{{ COMMAND }}'
98
100
  env:
99
- {% for key, value in DB_CONN.items() %}
100
- {% if value is none %}
101
+ {%- for key, value in DB_CONN.items() %}
102
+ {%- if value is none %}
101
103
  # When value is not provided, we get it from the secret
102
104
  - name: {{ key }}
103
105
  valueFrom:
104
106
  secretKeyRef:
105
107
  name: {{ SECRET_NAME }}
106
108
  key: {{ key }}
107
- {% else %}
109
+ {%- else %}
108
110
  # When value is provided, we get just use it
109
111
  - name: {{ key }}
110
112
  value: {{ value }}
111
- {% endif %}
112
- {% endfor %}
113
+ {%- endif %}
114
+ {%- endfor %}
113
115
  resources:
114
116
  requests:
115
117
  memory: "{{ REQUESTS_MEM }}"
@@ -125,10 +127,10 @@ spec:
125
127
  - name: configs
126
128
  projected:
127
129
  sources:
128
- {% for cm in CONFIG_MAPS %}
130
+ {%- for cm in CONFIG_MAPS %}
129
131
  - configMap:
130
132
  name: {{ cm }}
131
- {% endfor %}
133
+ {%- endfor %}
132
134
  """
133
135
 
134
136
 
@@ -187,7 +189,6 @@ def get_tf_resource_info(
187
189
  "cluster": spec.cluster_name,
188
190
  "output_resource_name": spec.output_resource_name,
189
191
  "engine": values.get("engine", "postgres"),
190
- "engine_version": values.get("engine_version", "latest"),
191
192
  }
192
193
  return None
193
194
 
@@ -412,7 +413,7 @@ def encrypted_closing_message(recipient: str) -> list[str]:
412
413
 
413
414
  def process_template(
414
415
  query: dict[str, Any],
415
- image_repository: str,
416
+ job_image: str,
416
417
  use_pull_secret: bool,
417
418
  config_map_names: list[str],
418
419
  service_account_name: str,
@@ -422,7 +423,7 @@ def process_template(
422
423
 
423
424
  :param query: the query dictionary containing the parameters
424
425
  to be used in the Template
425
- :param image_repository: docker image repo url
426
+ :param job_image: docker image repo url
426
427
  :param use_pull_secret: add imagePullSecrets to Job
427
428
  :param config_map_names: ConfigMap names to mount in Job
428
429
 
@@ -451,16 +452,18 @@ def process_template(
451
452
  command += engine_cmd_map[engine](sqlqueries_file="/tmp/queries")
452
453
  command += make_output_cmd(output=output, recipient=query.get("recipient", ""))
453
454
 
455
+ job_image_tag = Image(job_image).tag
456
+ job_image_pull_policy = "Always" if job_image_tag == "latest" else "IfNotPresent"
457
+
454
458
  template_to_render = JOB_TEMPLATE
455
459
  render_kwargs = {
456
460
  "JOB_NAME": query["name"],
457
461
  "CONFIG_MAPS_MOUNT_PATH": CONFIG_MAPS_MOUNT_PATH,
458
462
  "QUERY_NAME": query["name"],
459
463
  "QONTRACT_INTEGRATION": QONTRACT_INTEGRATION,
460
- "IMAGE_REPOSITORY": image_repository,
464
+ "JOB_IMAGE": job_image,
465
+ "JOB_IMAGE_PULL_POLICY": job_image_pull_policy,
461
466
  "SECRET_NAME": query["output_resource_name"],
462
- "ENGINE": engine,
463
- "ENGINE_VERSION": query["engine_version"],
464
467
  "DB_CONN": query["db_conn"],
465
468
  "CONFIG_MAPS": config_map_names,
466
469
  "COMMAND": command,
@@ -588,7 +591,7 @@ def _build_common_resource_labels(query: dict[str, Any]) -> dict[str, str]:
588
591
 
589
592
  def _build_openshift_resources(
590
593
  query: dict[str, Any],
591
- image_repository: str,
594
+ job_image: str,
592
595
  pull_secret: dict[str, Any] | None,
593
596
  secret_reader: SecretReaderBase,
594
597
  ) -> list[OpenshiftResource]:
@@ -649,7 +652,7 @@ def _build_openshift_resources(
649
652
  # Job (sql executer)
650
653
  job_yaml = process_template(
651
654
  query,
652
- image_repository=image_repository,
655
+ job_image=job_image,
653
656
  use_pull_secret=bool(pull_secret),
654
657
  config_map_names=[cm["metadata"]["name"] for cm in config_map_resources],
655
658
  service_account_name=svc["metadata"]["name"],
@@ -702,14 +705,14 @@ def _initialize_resource_types(
702
705
 
703
706
  def _reconstruct_for_metrics(
704
707
  query: dict[str, Any],
705
- image_repository: str,
708
+ job_image: str,
706
709
  pull_secret: dict[str, Any] | None,
707
710
  ri: ResourceInventory,
708
711
  secret_reader: SecretReaderBase,
709
712
  ) -> None:
710
713
  openshift_resources = _build_openshift_resources(
711
714
  query,
712
- image_repository,
715
+ job_image,
713
716
  pull_secret,
714
717
  secret_reader,
715
718
  )
@@ -767,7 +770,7 @@ def _process_existing_query(
767
770
  enable_deletion: bool,
768
771
  settings: dict[str, Any],
769
772
  state: State,
770
- image_repository: str,
773
+ job_image: str,
771
774
  pull_secret: dict[str, Any],
772
775
  ri: ResourceInventory,
773
776
  secret_reader: SecretReaderBase,
@@ -776,7 +779,7 @@ def _process_existing_query(
776
779
  case QueryStatus.ACTIVE:
777
780
  _reconstruct_for_metrics(
778
781
  query,
779
- image_repository,
782
+ job_image,
780
783
  pull_secret,
781
784
  ri,
782
785
  secret_reader,
@@ -803,13 +806,13 @@ def _process_new_query(
803
806
  dry_run: bool,
804
807
  settings: dict[str, Any],
805
808
  state: State,
806
- image_repository: str,
809
+ job_image: str,
807
810
  pull_secret: dict[str, Any],
808
811
  ri: ResourceInventory,
809
812
  secret_reader: SecretReaderBase,
810
813
  ) -> None:
811
814
  openshift_resources = _build_openshift_resources(
812
- query, image_repository, pull_secret, secret_reader
815
+ query, job_image, pull_secret, secret_reader
813
816
  )
814
817
 
815
818
  cluster = query["cluster"]
@@ -855,12 +858,12 @@ def run(
855
858
  mail_address=smtp_settings.mail_address,
856
859
  timeout=smtp_settings.timeout or DEFAULT_SMTP_TIMEOUT,
857
860
  )
858
- image_repository = "quay.io/app-sre"
861
+ job_image = "quay.io/app-sre/debug-container:latest"
859
862
 
860
863
  sql_query_settings = settings.get("sqlQuery")
861
864
  pull_secret: dict[str, Any] = {}
862
865
  if sql_query_settings:
863
- image_repository = sql_query_settings["imageRepository"]
866
+ job_image = sql_query_settings.get("jobImage") or job_image
864
867
  pull_secret = sql_query_settings["pullSecret"]
865
868
 
866
869
  queries_list = collect_queries(secret_reader=secret_reader, smtp_client=smtp_client)
@@ -874,7 +877,7 @@ def run(
874
877
  enable_deletion,
875
878
  settings,
876
879
  state,
877
- image_repository,
880
+ job_image,
878
881
  pull_secret,
879
882
  ri,
880
883
  secret_reader,
@@ -885,7 +888,7 @@ def run(
885
888
  dry_run,
886
889
  settings,
887
890
  state,
888
- image_repository,
891
+ job_image,
889
892
  pull_secret,
890
893
  ri,
891
894
  secret_reader,