qontract-reconcile 0.10.2.dev456__py3-none-any.whl → 0.10.2.dev473__py3-none-any.whl

reconcile/quay_mirror_org.py

@@ -17,7 +17,7 @@ from sretoolbox.container.image import (
 )
 from sretoolbox.container.skopeo import SkopeoCmdError
 
-from reconcile.quay_base import get_quay_api_store
+from reconcile.quay_base import QuayApiStore
 from reconcile.quay_mirror import QuayMirror
 from reconcile.utils.quay_mirror import record_timestamp, sync_tag
 
@@ -45,7 +45,7 @@ class QuayMirrorOrg:
     ) -> None:
         self.dry_run = dry_run
         self.skopeo_cli = Skopeo(dry_run)
-        self.quay_api_store = get_quay_api_store()
+        self.quay_api_store = QuayApiStore()
         self.compare_tags = compare_tags
         self.compare_tags_interval = compare_tags_interval
         self.orgs = orgs
@@ -71,6 +71,7 @@ class QuayMirrorOrg:
 
     def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         self.session.close()
+        self.quay_api_store.cleanup()
 
     def run(self) -> None:
         sync_tasks = self.process_sync_tasks()
@@ -101,11 +102,9 @@ class QuayMirrorOrg:
             if self.orgs and org_key.org_name not in self.orgs:
                 continue
 
-            quay_api = org_info["api"]
             upstream_org_key = org_info["mirror"]
             assert upstream_org_key is not None
             upstream_org = self.quay_api_store[upstream_org_key]
-            upstream_quay_api = upstream_org["api"]
 
             push_token = upstream_org["push_token"]
 
@@ -114,7 +113,10 @@ class QuayMirrorOrg:
             username = push_token["user"]
             token = push_token["token"]
 
+            quay_api = org_info["api"]
             org_repos = [item["name"] for item in quay_api.list_images()]
+
+            upstream_quay_api = upstream_org["api"]
             for repo in upstream_quay_api.list_images():
                 if repo["name"] not in org_repos:
                     continue

reconcile/quay_permissions.py

@@ -2,7 +2,10 @@ import logging
 import sys
 from typing import Any
 
-from reconcile.quay_base import OrgKey, get_quay_api_store
+from reconcile.quay_base import (
+    OrgKey,
+    QuayApiStore,
+)
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 
@@ -57,85 +60,88 @@ def run(dry_run: bool) -> None:
         return
 
     apps: list[dict[str, Any]] = result.get("apps") or []
-    quay_api_store = get_quay_api_store()
     error = False
-    for app in apps:
-        quay_repo_configs = app.get("quayRepos")
-        if not quay_repo_configs:
-            continue
-        for quay_repo_config in quay_repo_configs:
-            instance_name = quay_repo_config["org"]["instance"]["name"]
-            org_name = quay_repo_config["org"]["name"]
-            org_key = OrgKey(instance_name, org_name)
-
-            if not quay_repo_config["org"]["managedRepos"]:
-                logging.error(
-                    f"[{app['name']}] Can not manage repo permissions in {org_name} "
-                    "since managedRepos is set to false."
-                )
-                error = True
-                continue
-
-            # processing quayRepos section
-            logging.debug(["app", app["name"], instance_name, org_name])
 
-            quay_api = quay_api_store[org_key]["api"]
-            teams = quay_repo_config.get("teams")
-            if not teams:
+    with QuayApiStore() as quay_api_store:
+        for app in apps:
+            quay_repo_configs = app.get("quayRepos")
+            if not quay_repo_configs:
                 continue
-            repos = quay_repo_config["items"]
-            for repo in repos:
-                repo_name = repo["name"]
-
-                # processing repo section
-                logging.debug(["repo", repo_name])
-
-                for team in teams:
-                    permissions = team["permissions"]
-                    role = team["role"]
-                    for permission in permissions:
-                        if permission["service"] != "quay-membership":
-                            logging.warning(
-                                "wrong service kind, should be quay-membership"
+            for quay_repo_config in quay_repo_configs:
+                instance_name = quay_repo_config["org"]["instance"]["name"]
+                org_name = quay_repo_config["org"]["name"]
+                org_key = OrgKey(instance_name, org_name)
+
+                if not quay_repo_config["org"]["managedRepos"]:
+                    logging.error(
+                        f"[{app['name']}] Can not manage repo permissions in {org_name} "
+                        "since managedRepos is set to false."
+                    )
+                    error = True
+                    continue
+
+                # processing quayRepos section
+                logging.debug(["app", app["name"], instance_name, org_name])
+
+                org_data = quay_api_store[org_key]
+                teams = quay_repo_config.get("teams")
+                if not teams:
+                    continue
+                repos = quay_repo_config["items"]
+                quay_api = org_data["api"]
+
+                for repo in repos:
+                    repo_name = repo["name"]
+
+                    # processing repo section
+                    logging.debug(["repo", repo_name])
+
+                    for team in teams:
+                        permissions = team["permissions"]
+                        role = team["role"]
+                        for permission in permissions:
+                            if permission["service"] != "quay-membership":
+                                logging.warning(
+                                    "wrong service kind, should be quay-membership"
+                                )
+                                continue
+
+                            perm_org_key = OrgKey(
+                                permission["quayOrg"]["instance"]["name"],
+                                permission["quayOrg"]["name"],
                             )
-                            continue
-
-                        perm_org_key = OrgKey(
-                            permission["quayOrg"]["instance"]["name"],
-                            permission["quayOrg"]["name"],
-                        )
-
-                        if perm_org_key != org_key:
-                            logging.warning(f"wrong org, should be {org_key}")
-                            continue
 
-                        team_name = permission["team"]
-
-                        # processing team section
-                        logging.debug(["team", team_name])
-                        try:
-                            current_role = quay_api.get_repo_team_permissions(
-                                repo_name, team_name
-                            )
-                            if current_role != role:
-                                logging.info([
-                                    "update_role",
-                                    org_key,
-                                    repo_name,
-                                    team_name,
-                                    role,
-                                ])
-                                if not dry_run:
-                                    quay_api.set_repo_team_permissions(
-                                        repo_name, team_name, role
-                                    )
-                        except Exception:
-                            error = True
-                            logging.exception(
-                                "could not manage repo permissions: "
-                                f"repo name: {repo_name}, "
-                                f"team name: {team_name}"
-                            )
+                            if perm_org_key != org_key:
+                                logging.warning(f"wrong org, should be {org_key}")
+                                continue
+
+                            team_name = permission["team"]
+
+                            # processing team section
+                            logging.debug(["team", team_name])
+                            try:
+                                current_role = quay_api.get_repo_team_permissions(
+                                    repo_name, team_name
+                                )
+                                if current_role != role:
+                                    logging.info([
+                                        "update_role",
+                                        org_key,
+                                        repo_name,
+                                        team_name,
+                                        role,
+                                    ])
+                                    if not dry_run:
+                                        quay_api.set_repo_team_permissions(
+                                            repo_name, team_name, role
+                                        )
+                            except Exception:
+                                error = True
+                                logging.exception(
+                                    "could not manage repo permissions: "
+                                    f"repo name: {repo_name}, "
+                                    f"team name: {team_name}"
+                                )
 
     if error:
         sys.exit(ExitCodes.ERROR)

reconcile/quay_repos.py

@@ -3,18 +3,14 @@ from __future__ import annotations
 import logging
 import sys
 from collections import namedtuple
-from typing import TYPE_CHECKING
 
 from reconcile.quay_base import (
     OrgKey,
-    get_quay_api_store,
+    QuayApiStore,
 )
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 
-if TYPE_CHECKING:
-    from reconcile.quay_base import QuayApiStore
-
 QUAY_REPOS_QUERY = """
 {
   apps: apps_v1 {
@@ -51,7 +47,6 @@ def fetch_current_state(quay_api_store: QuayApiStore) -> list[RepoInfo]:
             continue
 
         quay_api = org_info["api"]
-
         for repo in quay_api.list_images():
             name = repo["name"]
             public = repo["is_public"]
@@ -150,7 +145,8 @@ def act_delete(
         current_repo.name,
     ])
     if not dry_run:
-        api = quay_api_store[current_repo.org_key]["api"]
+        org_data = quay_api_store[current_repo.org_key]
+        api = org_data["api"]
         api.repo_delete(current_repo.name)
 
 
@@ -164,7 +160,8 @@ def act_create(
         desired_repo.name,
     ])
     if not dry_run:
-        api = quay_api_store[desired_repo.org_key]["api"]
+        org_data = quay_api_store[desired_repo.org_key]
+        api = org_data["api"]
         api.repo_create(
             desired_repo.name, desired_repo.description, desired_repo.public
         )
@@ -180,7 +177,8 @@ def act_description(
         desired_repo.description,
     ])
     if not dry_run:
-        api = quay_api_store[desired_repo.org_key]["api"]
+        org_data = quay_api_store[desired_repo.org_key]
+        api = org_data["api"]
         api.repo_update_description(desired_repo.name, desired_repo.description)
 
 
@@ -194,7 +192,8 @@ def act_public(
         desired_repo.name,
     ])
     if not dry_run:
-        api = quay_api_store[desired_repo.org_key]["api"]
+        org_data = quay_api_store[desired_repo.org_key]
+        api = org_data["api"]
         if desired_repo.public:
             api.repo_make_public(desired_repo.name)
         else:
@@ -223,32 +222,31 @@ def act(
 
 
 def run(dry_run: bool) -> None:
-    quay_api_store = get_quay_api_store()
-
-    # consistency checks
-    for org_key, org_info in quay_api_store.items():
-        if org_info.get("mirror"):
-            # ensure there are no circular mirror dependencies
-            mirror_org_key = org_info["mirror"]
-            assert mirror_org_key is not None
-            mirror_org = quay_api_store[mirror_org_key]
-            if mirror_org.get("mirror"):
-                logging.error(
-                    f"{mirror_org_key.instance}/"
-                    + f"{mirror_org_key.org_name} "
-                    + "can't have mirrors and be a mirror"
-                )
-                sys.exit(ExitCodes.ERROR)
+    with QuayApiStore() as quay_api_store:
+        # consistency checks
+        for org_key, org_info in quay_api_store.items():
+            if org_info.get("mirror"):
+                # ensure there are no circular mirror dependencies
+                mirror_org_key = org_info["mirror"]
+                assert mirror_org_key is not None
+                mirror_org = quay_api_store[mirror_org_key]
+                if mirror_org.get("mirror"):
+                    logging.error(
+                        f"{mirror_org_key.instance}/"
+                        + f"{mirror_org_key.org_name} "
+                        + "can't have mirrors and be a mirror"
+                    )
+                    sys.exit(ExitCodes.ERROR)
 
-            # ensure no org defines `managedRepos` and `mirror` at the same
-            if org_info.get("managedRepos"):
-                logging.error(
-                    f"{org_key.instance}/{org_key.org_name} "
-                    + "has defined mirror and managedRepos"
-                )
-                sys.exit(ExitCodes.ERROR)
+                # ensure no org defines `managedRepos` and `mirror` at the same
+                if org_info.get("managedRepos"):
+                    logging.error(
+                        f"{org_key.instance}/{org_key.org_name} "
+                        + "has defined mirror and managedRepos"
+                    )
+                    sys.exit(ExitCodes.ERROR)
 
-    # run integration
-    current_state = fetch_current_state(quay_api_store)
-    desired_state = fetch_desired_state(quay_api_store)
-    act(dry_run, quay_api_store, current_state, desired_state)
+        # run integration
+        current_state = fetch_current_state(quay_api_store)
+        desired_state = fetch_desired_state(quay_api_store)
+        act(dry_run, quay_api_store, current_state, desired_state)

reconcile/queries.py

@@ -1815,7 +1815,7 @@ def get_review_repos() -> list[dict[str, str]]:
     return [
         {"url": c["url"], "name": c["name"]}
         for c in code_components
-        if c["showInReviewQueue"] is not None
+        if c.get("showInReviewQueue", False)
     ]
 
 

reconcile/templating/validator.py

@@ -141,11 +141,11 @@ class TemplateValidatorIntegration(QontractReconcileIntegration):
         if diffs:
             for diff in diffs:
                 logging.error(f"template: {diff.template}, test: {diff.test}")
-                # This log should never be added except for local debugging.
-                # Credentials could be leaked, i.e. creating an MR with a diff,
-                # using a template, that uses the vault function.
-                # Use template-validator CLI instead.
                 # logging.debug(diff.diff)
+
+            logging.error(
+                "The diff is never logged to avoid accidental credential leaks. Use template-validator CLI locally for debugging templates."
+            )
             raise ValueError("Template validation failed")
 
     @property

reconcile/terraform_vpc_resources/merge_request.py

@@ -1,5 +1,6 @@
 import re
 import string
+from enum import StrEnum
 
 from pydantic import BaseModel
 
@@ -9,6 +10,12 @@ PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
 VERSION = "0.1.0"
 LABEL = "terraform-vpc-resources"
 
+
+class Action(StrEnum):
+    CREATE = "create"
+    UPDATE = "update"
+
+
 VERSION_REF = "tf_vpc_resources_version"
 ACCOUNT_REF = "account"
 COMPILED_REGEXES = {
@@ -53,5 +60,8 @@ class Renderer:
     def render_description(self, account: str) -> str:
         return DESC.safe_substitute(account=account)
 
-    def render_title(self, account: str) -> str:
-        return f"[auto] VPC data file creation to {account}"
+    def render_title(self, account: str, action: Action) -> str:
+        return f"[auto] {action} VPC data file for {account}"
+
+    def render_update_title(self, account: str) -> str:
+        return f"[auto] VPC data file update for {account}"

reconcile/terraform_vpc_resources/merge_request_manager.py

@@ -5,6 +5,7 @@ from pydantic import BaseModel
 
 from reconcile.terraform_vpc_resources.merge_request import (
     LABEL,
+    Action,
     Info,
     Renderer,
 )
@@ -28,6 +29,7 @@ class VPCRequestMR(MergeRequestBase):
         vpc_tmpl_file_path: str,
         vpc_tmpl_file_content: str,
         labels: list[str],
+        action: Action,
     ):
         super().__init__()
         self._title = title
@@ -35,6 +37,7 @@ class VPCRequestMR(MergeRequestBase):
         self._vpc_tmpl_file_path = vpc_tmpl_file_path
         self._vpc_tmpl_file_content = vpc_tmpl_file_content
         self.labels = labels
+        self._action = action
 
     @property
     def title(self) -> str:
@@ -45,12 +48,21 @@ class VPCRequestMR(MergeRequestBase):
         return self._description
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        gitlab_cli.create_file(
-            branch_name=self.branch,
-            file_path=self._vpc_tmpl_file_path,
-            commit_message="add vpc datafile",
-            content=self._vpc_tmpl_file_content,
-        )
+        # Create or update file based on whether it already exists
+        if self._action == Action.UPDATE:
+            gitlab_cli.update_file(
+                branch_name=self.branch,
+                file_path=self._vpc_tmpl_file_path,
+                commit_message="update vpc datafile",
+                content=self._vpc_tmpl_file_content,
+            )
+        else:
+            gitlab_cli.create_file(
+                branch_name=self.branch,
+                file_path=self._vpc_tmpl_file_path,
+                commit_message="add vpc datafile",
+                content=self._vpc_tmpl_file_content,
+            )
 
 
 class MrData(BaseModel):
@@ -73,26 +85,37 @@ class MergeRequestManager(MergeRequestManagerBase[Info]):
         self._renderer = renderer
         self._auto_merge_enabled = auto_merge_enabled
 
-    def create_merge_request(self, data: MrData) -> None:
-        """Open a new MR, if not already present, for a VPC datafile and close any outdated before."""
-        if not self._housekeeping_ran:
-            self.housekeeping()
-
+    def _create_action(self, data: MrData) -> Action | None:
         if self._merge_request_already_exists({"account": data.account}):
             logging.info("MR already exists for %s", data.account)
             return None
-
         try:
-            self._vcs.get_file_content_from_app_interface_ref(file_path=data.path)
-            # the file exists, nothing to do
-            return None
+            existing_content = self._vcs.get_file_content_from_app_interface_ref(
+                file_path=data.path
+            )
         except GitlabGetError as e:
-            if e.response_code != 404:
-                raise
+            if e.response_code == 404:
+                return Action.CREATE
+            raise
+
+        if existing_content.strip() != data.content.strip():
+            return Action.UPDATE
+
+        logging.info("VPC data file exists and is up-to-date for %s", data.account)
+        return None
+
+    def create_merge_request(self, data: MrData) -> None:
+        """Open a new MR for VPC datafile updates, or update existing if changed."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+        action = self._create_action(data)
+        if action is None:
+            return
 
         description = self._renderer.render_description(account=data.account)
-        title = self._renderer.render_title(account=data.account)
-        logging.info("Open MR for %s", data.account)
+        title = self._renderer.render_title(account=data.account, action=action)
+
+        logging.info("Open MR for %s (%s)", data.account, action)
         mr_labels = [LABEL]
         if self._auto_merge_enabled:
             mr_labels.append(AUTO_MERGE)
@@ -103,5 +126,6 @@ class MergeRequestManager(MergeRequestManagerBase[Info]):
                 description=description,
                 vpc_tmpl_file_content=data.content,
                 labels=mr_labels,
+                action=action,
             )
         )

reconcile/typed_queries/saas_files.py

@@ -83,7 +83,7 @@ class SaasResourceTemplateTarget(
         if used_for_security_is_enabled():
             # When USED_FOR_SECURITY is enabled, use blake2s without digest_size and truncate to 20 bytes
             # This is needed for FIPS compliance where digest_size parameter is not supported
-            return hashlib.blake2s(data).digest()[:20].hex()
+            return hashlib.sha256(data).digest()[:20].hex()
         else:
             # Default behavior: use blake2s with digest_size=20
             return hashlib.blake2s(data, digest_size=20).hexdigest()

reconcile/utils/external_resource_spec.py

@@ -157,6 +157,8 @@ class ExternalResourceSpec:
             tags["cost-center"] = cost_center
         if service_phase := self.namespace["environment"].get("servicePhase"):
             tags["service-phase"] = service_phase
+        if cost_center := self.namespace["environment"].get("costCenter"):
+            tags["cost-center"] = cost_center
 
         resource_tags_str = self.resource.get("tags")
         if resource_tags_str: