qontract-reconcile 0.10.2.dev430__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/terraform_vpc_resources/integration.py

@@ -24,6 +24,7 @@ from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
+from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.runtime.integration import (
     DesiredStateShardConfig,
     PydanticRunParams,
@@ -62,12 +63,14 @@ class TerraformVpcResources(QontractReconcileIntegration[TerraformVpcResourcesPa
     ) -> list[AWSAccountV1]:
         """Return a list of accounts extracted from the provided VPCRequests.
         If account_name is given returns the account object with that name."""
-        accounts = [vpc.account for vpc in data]
-
-        if account_name:
-            accounts = [account for account in accounts if account.name == account_name]
-
-        return accounts
+        return [
+            vpc.account
+            for vpc in data
+            if (
+                integration_is_enabled(self.name, vpc.account)
+                and (not account_name or vpc.account.name == account_name)
+            )
+        ]
 
     def _handle_outputs(
         self, requests: Iterable[VPCRequest], outputs: Mapping[str, Any]
@@ -155,7 +158,7 @@ class TerraformVpcResources(QontractReconcileIntegration[TerraformVpcResourcesPa
         if data:
             accounts = self._filter_accounts(data, account_name)
             if account_name and not accounts:
-                msg = f"The account {account_name} doesn't have any managed vpc. Verify your input"
+                msg = f"The account {account_name} doesn't have any managed vpcs or the {QONTRACT_INTEGRATION} integration is disabled for this account. Verify your input"
                 logging.debug(msg)
                 sys.exit(ExitCodes.SUCCESS)
         else:

reconcile/terraform_vpc_resources/merge_request.py

@@ -1,5 +1,6 @@
 import re
 import string
+from enum import StrEnum
 
 from pydantic import BaseModel
 
@@ -9,6 +10,12 @@ PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
 VERSION = "0.1.0"
 LABEL = "terraform-vpc-resources"
 
+
+class Action(StrEnum):
+    CREATE = "create"
+    UPDATE = "update"
+
+
 VERSION_REF = "tf_vpc_resources_version"
 ACCOUNT_REF = "account"
 COMPILED_REGEXES = {
@@ -53,5 +60,8 @@ class Renderer:
     def render_description(self, account: str) -> str:
         return DESC.safe_substitute(account=account)
 
-    def render_title(self, account: str) -> str:
-        return f"[auto] VPC data file creation to {account}"
+    def render_title(self, account: str, action: Action) -> str:
+        return f"[auto] {action} VPC data file for {account}"
+
+    def render_update_title(self, account: str) -> str:
+        return f"[auto] VPC data file update for {account}"

reconcile/terraform_vpc_resources/merge_request_manager.py

@@ -5,6 +5,7 @@ from pydantic import BaseModel
 
 from reconcile.terraform_vpc_resources.merge_request import (
     LABEL,
+    Action,
     Info,
     Renderer,
 )
@@ -28,6 +29,7 @@ class VPCRequestMR(MergeRequestBase):
         vpc_tmpl_file_path: str,
         vpc_tmpl_file_content: str,
         labels: list[str],
+        action: Action,
     ):
         super().__init__()
         self._title = title
@@ -35,6 +37,7 @@ class VPCRequestMR(MergeRequestBase):
         self._vpc_tmpl_file_path = vpc_tmpl_file_path
         self._vpc_tmpl_file_content = vpc_tmpl_file_content
         self.labels = labels
+        self._action = action
 
     @property
     def title(self) -> str:
@@ -45,12 +48,21 @@ class VPCRequestMR(MergeRequestBase):
         return self._description
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        gitlab_cli.create_file(
-            branch_name=self.branch,
-            file_path=self._vpc_tmpl_file_path,
-            commit_message="add vpc datafile",
-            content=self._vpc_tmpl_file_content,
-        )
+        # Create or update file based on whether it already exists
+        if self._action == Action.UPDATE:
+            gitlab_cli.update_file(
+                branch_name=self.branch,
+                file_path=self._vpc_tmpl_file_path,
+                commit_message="update vpc datafile",
+                content=self._vpc_tmpl_file_content,
+            )
+        else:
+            gitlab_cli.create_file(
+                branch_name=self.branch,
+                file_path=self._vpc_tmpl_file_path,
+                commit_message="add vpc datafile",
+                content=self._vpc_tmpl_file_content,
+            )
 
 
 class MrData(BaseModel):
@@ -73,26 +85,37 @@ class MergeRequestManager(MergeRequestManagerBase[Info]):
         self._renderer = renderer
         self._auto_merge_enabled = auto_merge_enabled
 
-    def create_merge_request(self, data: MrData) -> None:
-        """Open a new MR, if not already present, for a VPC datafile and close any outdated before."""
-        if not self._housekeeping_ran:
-            self.housekeeping()
-
+    def _create_action(self, data: MrData) -> Action | None:
         if self._merge_request_already_exists({"account": data.account}):
             logging.info("MR already exists for %s", data.account)
             return None
-
         try:
-            self._vcs.get_file_content_from_app_interface_ref(file_path=data.path)
-            # the file exists, nothing to do
-            return None
+            existing_content = self._vcs.get_file_content_from_app_interface_ref(
+                file_path=data.path
+            )
         except GitlabGetError as e:
-            if e.response_code != 404:
-                raise
+            if e.response_code == 404:
+                return Action.CREATE
+            raise
+
+        if existing_content.strip() != data.content.strip():
+            return Action.UPDATE
+
+        logging.info("VPC data file exists and is up-to-date for %s", data.account)
+        return None
+
+    def create_merge_request(self, data: MrData) -> None:
+        """Open a new MR for VPC datafile updates, or update existing if changed."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+        action = self._create_action(data)
+        if action is None:
+            return
 
         description = self._renderer.render_description(account=data.account)
-        title = self._renderer.render_title(account=data.account)
-        logging.info("Open MR for %s", data.account)
+        title = self._renderer.render_title(account=data.account, action=action)
+
+        logging.info("Open MR for %s (%s)", data.account, action)
         mr_labels = [LABEL]
         if self._auto_merge_enabled:
             mr_labels.append(AUTO_MERGE)
@@ -103,5 +126,6 @@ class MergeRequestManager(MergeRequestManagerBase[Info]):
                 description=description,
                 vpc_tmpl_file_content=data.content,
                 labels=mr_labels,
+                action=action,
             )
         )

reconcile/typed_queries/saas_files.py

@@ -42,6 +42,7 @@ from reconcile.gql_definitions.fragments.saas_target_namespace import (
     SaasTargetNamespace,
 )
 from reconcile.utils import gql
+from reconcile.utils.environ import used_for_security_is_enabled
 from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     ParameterError,
@@ -78,10 +79,14 @@ class SaasResourceTemplateTarget(
         self, parent_saas_file_name: str, parent_resource_template_name: str
     ) -> str:
         """Returns a unique identifier for a target."""
-        return hashlib.blake2s(
-            f"{parent_saas_file_name}:{parent_resource_template_name}:{self.name or 'default'}:{self.namespace.cluster.name}:{self.namespace.name}".encode(),
-            digest_size=20,
-        ).hexdigest()
+        data = f"{parent_saas_file_name}:{parent_resource_template_name}:{self.name or 'default'}:{self.namespace.cluster.name}:{self.namespace.name}".encode()
+        if used_for_security_is_enabled():
+            # When USED_FOR_SECURITY is enabled, use blake2s without digest_size and truncate to 20 bytes
+            # This is needed for FIPS compliance where digest_size parameter is not supported
+            return hashlib.sha256(data).digest()[:20].hex()
+        else:
+            # Default behavior: use blake2s with digest_size=20
+            return hashlib.blake2s(data, digest_size=20).hexdigest()
 
 
 class SaasResourceTemplate(ConfiguredBaseModel, validate_by_alias=True):

reconcile/utils/environ.py

@@ -4,6 +4,11 @@ from functools import wraps
 from typing import Any
 
 
+def used_for_security_is_enabled() -> bool:
+    used_for_security_env = os.getenv("USED_FOR_SECURITY", "false")
+    return used_for_security_env.lower() == "true"
+
+
 def environ(variables: Iterable[str] | None = None) -> Callable:
     """Check that environment variables are set before execution."""
     if variables is None:

reconcile/utils/external_resource_spec.py

@@ -157,6 +157,8 @@ class ExternalResourceSpec:
             tags["cost-center"] = cost_center
         if service_phase := self.namespace["environment"].get("servicePhase"):
             tags["service-phase"] = service_phase
+        if cost_center := self.namespace["environment"].get("costCenter"):
+            tags["cost-center"] = cost_center
 
         resource_tags_str = self.resource.get("tags")
         if resource_tags_str:

reconcile/utils/gitlab_api.py

@@ -444,6 +444,8 @@ class GitLabApi:
     def get_merge_request_comments(
         merge_request: ProjectMergeRequest,
         include_description: bool = False,
+        include_approvals: bool = False,
+        approval_body: str = "",
     ) -> list[Comment]:
         comments = []
         if include_description:
@@ -455,6 +457,16 @@ class GitLabApi:
                     created_at=merge_request.created_at,
                 )
             )
+        if include_approvals:
+            comments.extend(
+                Comment(
+                    id=approval["user"]["id"],
+                    username=approval["user"]["username"],
+                    body=approval_body,
+                    created_at=approval["approved_at"],
+                )
+                for approval in merge_request.approvals.get().approved_by
+            )
         comments.extend(
             Comment(
                 id=note.id,

reconcile/utils/jjb_client.py

@@ -33,6 +33,10 @@ from reconcile.utils.vcs import GITHUB_BASE_URL
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
 
+class MissingJobUrlError(Exception):
+    pass
+
+
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
@@ -335,7 +339,7 @@ class JJB:
                 job_name = job["name"]
                 try:
                     repos.add(self.get_repo_url(job))
-                except KeyError:
+                except MissingJobUrlError:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
@@ -355,7 +359,19 @@ class JJB:
 
     @staticmethod
     def get_repo_url(job: Mapping[str, Any]) -> str:
-        repo_url_raw = job["properties"][0]["github"]["url"]
+        repo_url_raw = job.get("properties", [{}])[0].get("github", {}).get("url")
+
+        # we may be in a Github Branch Source type of job
+        if not repo_url_raw:
+            gh_org = job.get("scm", [{}])[0].get("github", {}).get("repo-owner")
+            gh_repo = job.get("scm", [{}])[0].get("github", {}).get("repo")
+            if gh_org and gh_repo:
+                repo_url_raw = f"https://github.com/{gh_org}/{gh_repo}/"
+            else:
+                raise MissingJobUrlError(
+                    f"Cannot find job url for {job['display-name']}"
+                )
+
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
@@ -404,7 +420,7 @@ class JJB:
                 try:
                     if self.get_repo_url(job).lower() == repo_url.rstrip("/").lower():
                         return job
-                except KeyError:
+                except MissingJobUrlError:
                     # something wrong here. ignore this job
                     pass
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")

reconcile/utils/oc.py

@@ -651,9 +651,15 @@ class OCCli:
             raise e
         return True
 
+    def _use_oc_project(self, namespace: str) -> bool:
+        # Note, that openshift-* namespaces cannot be created via new-project
+        return self.is_kind_supported(PROJECT_KIND) and not namespace.startswith(
+            "openshift-"
+        )
+
     @OCDecorators.process_reconcile_time
     def new_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["new-project", namespace]
         else:
             cmd = ["create", "namespace", namespace]
@@ -669,7 +675,7 @@ class OCCli:
 
     @OCDecorators.process_reconcile_time
     def delete_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["delete", "project", namespace]
         else:
             cmd = ["delete", "namespace", namespace]

reconcile/utils/quay_api.py

@@ -1,13 +1,16 @@
+import contextlib
 from typing import Any
 
 import requests
 
+from reconcile.utils.rest_api_base import ApiBase, BearerTokenAuth
+
 
 class QuayTeamNotFoundError(Exception):
     pass
 
 
-class QuayApi:
+class QuayApi(ApiBase):
     LIMIT_FOLLOWS = 15
 
     def __init__(
@@ -17,14 +20,18 @@ class QuayApi:
         base_url: str = "quay.io",
         timeout: int = 60,
     ) -> None:
-        self.token = token
+        # Support both hostname (e.g., "quay.io") and full URLs (e.g., "http://localhost:12345")
+        if base_url.startswith(("http://", "https://")):
+            host = base_url
+        else:
+            host = f"https://{base_url}"
+        super().__init__(
+            host=host,
+            auth=BearerTokenAuth(token),
+            read_timeout=timeout,
+        )
         self.organization = organization
-        self.auth_header = {"Authorization": "Bearer %s" % (token,)}
         self.team_members: dict[str, Any] = {}
-        self.api_url = f"https://{base_url}/api/v1"
-
-        self._timeout = timeout
-        """Timeout to use for HTTP calls to Quay (seconds)."""
 
     def list_team_members(self, team: str, **kwargs: Any) -> list[dict]:
         """
@@ -38,19 +45,20 @@ class QuayApi:
             if cache_members:
                 return cache_members
 
-        url = f"{self.api_url}/organization/{self.organization}/team/{team}/members?includePending=true"
-
-        r = requests.get(url, headers=self.auth_header, timeout=self._timeout)
-        if r.status_code == 404:
-            raise QuayTeamNotFoundError(
-                f"team {team} is not found in "
-                f"org {self.organization}. "
-                f"contact org owner to create the "
-                f"team manually."
-            )
-        r.raise_for_status()
-
-        body = r.json()
+        url = f"/api/v1/organization/{self.organization}/team/{team}/members"
+        params = {"includePending": "true"}
+
+        try:
+            body = self._get(url, params=params)
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                raise QuayTeamNotFoundError(
+                    f"team {team} is not found in "
+                    f"org {self.organization}. "
+                    f"contact org owner to create the "
+                    f"team manually."
+                ) from e
+            raise
 
         # Using a set because members may be repeated
         members = {member["name"] for member in body["members"]}
@@ -61,30 +69,37 @@ class QuayApi:
         return members_list
 
     def user_exists(self, user: str) -> bool:
-        url = f"{self.api_url}/users/{user}"
-        r = requests.get(url, headers=self.auth_header, timeout=self._timeout)
-        return r.ok
+        url = f"/api/v1/users/{user}"
+        try:
+            self._get(url)
+            return True
+        except requests.exceptions.HTTPError:
+            return False
 
     def remove_user_from_team(self, user: str, team: str) -> bool:
         """Deletes an user from a team.
 
         :raises HTTPError if there are any problems with the request
         """
-        url_team = f"{self.api_url}/organization/{self.organization}/team/{team}/members/{user}"
+        url_team = (
+            f"/api/v1/organization/{self.organization}/team/{team}/members/{user}"
+        )
 
-        r = requests.delete(url_team, headers=self.auth_header, timeout=self._timeout)
-        if not r.ok:
-            message = r.json().get("message", "")
+        try:
+            self._delete(url_team)
+        except requests.exceptions.HTTPError as e:
+            message = ""
+            if e.response is not None:
+                with contextlib.suppress(ValueError, AttributeError):
+                    message = e.response.json().get("message", "")
 
             expected_message = f"User {user} does not belong to team {team}"
 
             if message != expected_message:
-                r.raise_for_status()
+                raise
 
-        url_org = f"{self.api_url}/organization/{self.organization}/members/{user}"
-
-        r = requests.delete(url_org, headers=self.auth_header, timeout=self._timeout)
-        r.raise_for_status()
+        url_org = f"/api/v1/organization/{self.organization}/members/{user}"
+        self._delete(url_org)
 
         return True
 
@@ -96,9 +111,8 @@ class QuayApi:
         if user in self.list_team_members(team, cache=True):
             return True
 
-        url = f"{self.api_url}/organization/{self.organization}/team/{team}/members/{user}"
-        r = requests.put(url, headers=self.auth_header, timeout=self._timeout)
-        r.raise_for_status()
+        url = f"/api/v1/organization/{self.organization}/team/{team}/members/{user}"
+        self._put(url)
         return True
 
     def create_or_update_team(
@@ -115,17 +129,14 @@ class QuayApi:
         :raises HTTPError: unsuccessful attempt to create the team
         """
 
-        url = f"{self.api_url}/organization/{self.organization}/team/{team}"
+        url = f"/api/v1/organization/{self.organization}/team/{team}"
 
         payload = {"role": role}
 
         if description:
             payload.update({"description": description})
 
-        r = requests.put(
-            url, headers=self.auth_header, json=payload, timeout=self._timeout
-        )
-        r.raise_for_status()
+        self._put(url, data=payload)
 
     def list_images(
         self, images: list | None = None, page: str | None = None, count: int = 0
@@ -140,7 +151,7 @@ class QuayApi:
         if count > self.LIMIT_FOLLOWS:
             raise ValueError("Too many page follows")
 
-        url = f"{self.api_url}/repository"
+        url = "/api/v1/repository"
 
         # params
         params = {"namespace": self.organization}
@@ -148,13 +159,7 @@ class QuayApi:
             params["next_page"] = page
 
         # perform request
-        r = requests.get(
-            url, params=params, headers=self.auth_header, timeout=self._timeout
-        )
-        r.raise_for_status()
-
-        # read body
-        body = r.json()
+        body = self._get(url, params=params)
         repositories = body.get("repositories", [])
         next_page = body.get("next_page")
 
@@ -176,7 +181,7 @@ class QuayApi:
         """
         visibility = "public" if public else "private"
 
-        url = f"{self.api_url}/repository"
+        url = "/repository"
 
         params = {
             "repo_kind": "image",
@@ -186,29 +191,16 @@ class QuayApi:
             "description": description,
         }
 
-        # perform request
-        r = requests.post(
-            url, json=params, headers=self.auth_header, timeout=self._timeout
-        )
-        r.raise_for_status()
+        self._post(url, data=params)
 
     def repo_delete(self, repo_name: str) -> None:
-        url = f"{self.api_url}/repository/{self.organization}/{repo_name}"
-
-        # perform request
-        r = requests.delete(url, headers=self.auth_header, timeout=self._timeout)
-        r.raise_for_status()
+        url = f"/api/v1/repository/{self.organization}/{repo_name}"
+        self._delete(url)
 
     def repo_update_description(self, repo_name: str, description: str) -> None:
-        url = f"{self.api_url}/repository/{self.organization}/{repo_name}"
-
+        url = f"/api/v1/repository/{self.organization}/{repo_name}"
         params = {"description": description}
-
-        # perform request
-        r = requests.put(
-            url, json=params, headers=self.auth_header, timeout=self._timeout
-        )
-        r.raise_for_status()
+        self._put(url, data=params)
 
     def repo_make_public(self, repo_name: str) -> None:
         self._repo_change_visibility(repo_name, "public")
@@ -217,39 +209,34 @@ class QuayApi:
         self._repo_change_visibility(repo_name, "private")
 
     def _repo_change_visibility(self, repo_name: str, visibility: str) -> None:
-        url = f"{self.api_url}/repository/{self.organization}/{repo_name}/changevisibility"
-
+        url = f"/api/v1/repository/{self.organization}/{repo_name}/changevisibility"
         params = {"visibility": visibility}
-
-        # perform request
-        r = requests.post(
-            url, json=params, headers=self.auth_header, timeout=self._timeout
-        )
-        r.raise_for_status()
+        self._post(url, data=params)
 
     def get_repo_team_permissions(self, repo_name: str, team: str) -> str | None:
         url = (
-            f"{self.api_url}/repository/{self.organization}/"
+            f"/api/v1/repository/{self.organization}/"
             + f"{repo_name}/permissions/team/{team}"
         )
-        r = requests.get(url, headers=self.auth_header, timeout=self._timeout)
-        if not r.ok:
-            message = r.json().get("message")
+        try:
+            body = self._get(url)
+            return body.get("role") or None
+        except requests.exceptions.HTTPError as e:
+            message = ""
+            if e.response is not None:
+                with contextlib.suppress(ValueError, AttributeError):
+                    message = e.response.json().get("message", "")
+
             expected_message = "Team does not have permission for repo."
             if message == expected_message:
                 return None
 
-            r.raise_for_status()
-
-        return r.json().get("role") or None
+            raise
 
     def set_repo_team_permissions(self, repo_name: str, team: str, role: str) -> None:
         url = (
-            f"{self.api_url}/repository/{self.organization}/"
+            f"/api/v1/repository/{self.organization}/"
             + f"{repo_name}/permissions/team/{team}"
         )
         body = {"role": role}
-        r = requests.put(
-            url, json=body, headers=self.auth_header, timeout=self._timeout
-        )
-        r.raise_for_status()
+        self._put(url, data=body)