qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/mr/glitchtip_access_reporter.py

@@ -1,8 +1,7 @@
 import logging
 from collections.abc import Sequence
-from datetime import date
+from datetime import UTC, date
 from datetime import datetime as dt
-from datetime import timezone
 from pathlib import Path
 
 from jinja2 import Template
@@ -47,7 +46,6 @@ class GlitchtipAccessReportUser(BaseModel):
 
 
 class UpdateGlitchtipAccessReport(MergeRequestBase):
-
     name = "glitchtip_access_report_mr"
 
     def __init__(
@@ -62,7 +60,7 @@ class UpdateGlitchtipAccessReport(MergeRequestBase):
         self._glitchtip_access_revalidation_workbook = str(
             glitchtip_access_revalidation_workbook
         )
-        self._isodate = dt.now(tz=timezone.utc).isoformat()
+        self._isodate = dt.now(tz=UTC).isoformat()
         self._dry_run = dry_run
 
     @property

reconcile/utils/mr/labels.py

@@ -8,10 +8,23 @@ DO_NOT_MERGE_PENDING_REVIEW = "do-not-merge/pending-review"
 HOLD = "bot/hold"
 LGTM = "lgtm"
 SAAS_FILE_UPDATE = "saas-file-update"
-
+NEEDS_REBASE = "needs-rebase"
 SELF_SERVICEABLE = "self-serviceable"
+RESTRICTED = "restricted"
 NOT_SELF_SERVICEABLE = "not-self-serviceable"
+ONBOARDING = "onboarding"
 
 
 def prioritized_approval_label(priority: str) -> str:
     return f"{APPROVED}: {priority}"
+
+
+def change_owner_label(label: str) -> str:
+    return f"change-owner/{label}"
+
+
+SHOW_SELF_SERVICEABLE_IN_REVIEW_QUEUE = change_owner_label(
+    "show-self-serviceable-in-review-queue"
+)
+# aws-version-sync
+AVS = "AVS"

reconcile/utils/mr/notificator.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel
 
@@ -27,13 +26,12 @@ class Notification(BaseModel):
 
 
 class CreateAppInterfaceNotificator(MergeRequestBase):
-
     name = "create_app_interface_notificator_mr"
 
     def __init__(
         self,
         notification: Notification,
-        labels: Optional[list[str]] = None,
+        labels: list[str] | None = None,
         email_base_path: Path = Path("data") / "app-interface" / "emails",
         dry_run: bool = False,
     ):

reconcile/utils/mr/ocm_update_recommended_version.py

@@ -6,7 +6,6 @@ from reconcile.utils.mr.labels import AUTO_MERGE
 
 
 class CreateOCMUpdateRecommendedVersion(MergeRequestBase):
-
     name = "create_ocm_update_recommended_version_mr"
 
     def __init__(
@@ -30,7 +29,7 @@ class CreateOCMUpdateRecommendedVersion(MergeRequestBase):
 
     def process(self, gitlab_cli: GitLabApi) -> None:
         raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=self.main_branch
+            file_path=self.path, ref=gitlab_cli.main_branch
         )
         content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
 

reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py

@@ -1,11 +1,11 @@
 from ruamel import yaml
 
+from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
 
 
 class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
-
     name = "create_ocm_upgrade_scheduler_org_updates_mr"
 
     def __init__(self, updates_info):
@@ -23,13 +23,13 @@ class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
     def description(self) -> str:
         return f'ocm upgrade scheduler org updates for {self.updates_info["name"]}'
 
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         changes = False
         ocm_path = self.updates_info["path"]
         ocm_name = self.updates_info["name"]
 
         raw_file = gitlab_cli.project.files.get(
-            file_path=ocm_path, ref=self.main_branch
+            file_path=ocm_path, ref=gitlab_cli.main_branch
         )
         content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
         upgrade_policy_clusters = content["upgradePolicyClusters"]
@@ -37,6 +37,8 @@ class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
         for update in self.updates_info["updates"]:
             action = update["action"]
             cluster_name = update["cluster"]
+            cluster_id = update.get("id")
+            cluster_server_url = update.get("url")
             upgrade_policy = update.get("policy")
 
             if action == "add":
@@ -47,6 +49,8 @@ class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
                     continue
                 item = {
                     "name": cluster_name,
+                    "serverUrl": cluster_server_url,
+                    "spec": {"id": cluster_id},
                     "upgradePolicy": upgrade_policy,
                 }
                 upgrade_policy_clusters.append(item)

reconcile/utils/mr/promote_qontract.py

@@ -0,0 +1,203 @@
+from jsonpath_ng.ext import parser
+from ruamel.yaml.compat import StringIO
+
+from reconcile.typed_queries.users import get_users
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.ruamel import create_ruamel_instance
+
+
+class PromoteQontractSchemas(MergeRequestBase):
+    name = "promote_qontract_schemas"
+
+    def __init__(self, version: str, author_email: str | None = None):
+        self.path = ".env"
+        self.version = version
+        self.author_email = author_email
+
+        super().__init__()
+
+        self.labels = []
+
+    @property
+    def title(self) -> str:
+        author = self.infer_author(
+            author_email=self.author_email, all_users=get_users()
+        )
+        return f"[{self.name}] promote qontract-schemas to version {self.version}" + (
+            f" by @{author}"
+            if author
+            else f" by {self.author_email}"
+            if self.author_email
+            else ""
+        )
+
+    @property
+    def description(self) -> str:
+        return f"promote qontract-schemas to version {self.version}"
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        raw_file = gitlab_cli.project.files.get(
+            file_path=self.path, ref=gitlab_cli.main_branch
+        )
+        content = raw_file.decode().decode("utf-8")
+        lines = content.splitlines()
+        for index, text in enumerate(lines):
+            if text.startswith("export SCHEMAS_IMAGE_TAG="):
+                lines[index] = f"export SCHEMAS_IMAGE_TAG={self.version}"
+
+        new_content = "\n".join(lines) + "\n"
+
+        gitlab_cli.update_file(
+            branch_name=self.branch,
+            file_path=self.path,
+            commit_message=self.description,
+            content=new_content,
+        )
+
+
+class PromoteQontractReconcileCommercial(MergeRequestBase):
+    name = "promote_qontract_reconcile"
+
+    def __init__(self, version: str, commit_sha: str, author_email: str | None = None):
+        self.version = version
+        self.commit_sha = commit_sha
+        self.author_email = author_email
+
+        super().__init__()
+
+        self.labels = []
+
+    @property
+    def title(self) -> str:
+        author = self.infer_author(
+            author_email=self.author_email, all_users=get_users()
+        )
+        return f"[{self.name}] promote qontract-reconcile to version {self.version}" + (
+            f" by @{author}"
+            if author
+            else f" by {self.author_email}"
+            if self.author_email
+            else ""
+        )
+
+    @property
+    def description(self) -> str:
+        return f"promote qontract-reconcile to version {self.version}"
+
+    def _process_by_line_search(
+        self, raw_file: bytes, search_text: str, replace_text: str
+    ) -> str:
+        content = raw_file.decode("utf-8")
+        lines = content.splitlines()
+        for index, text in enumerate(lines):
+            if text.startswith(search_text):
+                lines[index] = replace_text
+        new_content = "\n".join(lines) + "\n"
+
+        return new_content
+
+    def _process_by_json_path(
+        self, raw_file: bytes, search_text: str, replace_text: str
+    ) -> str:
+        yml = create_ruamel_instance()
+        content = yml.load(raw_file)
+        for match in parser.parse(search_text).find(content):
+            parent = match.context.value
+            key = match.path.fields[0]
+            parent[key] = replace_text
+        new_content = "---\n"
+        with StringIO() as stream:
+            yml.dump(content, stream)
+            new_content += stream.getvalue() or ""
+
+        return new_content
+
+    def _process_by(
+        self,
+        method: str,
+        gitlab_cli: GitLabApi,
+        path: str,
+        search_text: str,
+        replace_text: str,
+    ) -> None:
+        raw_file = gitlab_cli.project.files.get(
+            file_path=path, ref=gitlab_cli.main_branch
+        ).decode()
+        match method:
+            case "line_search":
+                new_content = self._process_by_line_search(
+                    raw_file=raw_file,
+                    search_text=search_text,
+                    replace_text=replace_text,
+                )
+            case "json_path":
+                new_content = self._process_by_json_path(
+                    raw_file=raw_file,
+                    search_text=search_text,
+                    replace_text=replace_text,
+                )
+            case _:
+                raise NotImplementedError(method)
+
+        gitlab_cli.update_file(
+            branch_name=self.branch,
+            file_path=path,
+            commit_message=self.description,
+            content=new_content,
+        )
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        # .env
+        self._process_by(
+            "line_search",
+            gitlab_cli=gitlab_cli,
+            path=".env",
+            search_text="export RECONCILE_IMAGE_TAG=",
+            replace_text=f"export RECONCILE_IMAGE_TAG={self.version}",
+        )
+
+        # resources/jenkins/global/defaults.yaml
+        self._process_by(
+            "line_search",
+            gitlab_cli=gitlab_cli,
+            path="resources/jenkins/global/defaults.yaml",
+            search_text="    qontract_reconcile_image_tag: ",
+            replace_text=f"    qontract_reconcile_image_tag: '{self.version}'",
+        )
+
+        # data/services/app-interface/cicd/ci-ext/saas-qontract-dashboards.yaml
+        self._process_by(
+            "json_path",
+            gitlab_cli=gitlab_cli,
+            path="data/services/app-interface/cicd/ci-ext/saas-qontract-dashboards.yaml",
+            search_text="$.resourceTemplates[?(@.url == 'https://github.com/app-sre/qontract-reconcile')].targets[?(@.name == 'app-sre-observability-production')].ref",
+            replace_text=self.commit_sha,
+        )
+
+        # data/services/app-interface/cicd/ci-int/saas-qontract-manager-int.yaml
+        self._process_by(
+            "json_path",
+            gitlab_cli=gitlab_cli,
+            path="data/services/app-interface/cicd/ci-int/saas-qontract-manager-int.yaml",
+            search_text="$.resourceTemplates[?(@.url == 'https://github.com/app-sre/qontract-reconcile')].targets[?(@.name == 'app-interface-production')].ref",
+            replace_text=self.commit_sha,
+        )
+
+        # data/services/app-interface/terraform-repo/cicd/ci-int/saas-terraform-repo.yaml
+        self._process_by(
+            "json_path",
+            gitlab_cli=gitlab_cli,
+            path="data/services/app-interface/terraform-repo/cicd/ci-int/saas-terraform-repo.yaml",
+            search_text="$.resourceTemplates[?(@.name == 'terraform-repo')].targets[?(@.name == 'tf-repo-prod')].parameters.QR_IMAGE_TAG",
+            replace_text=self.version,
+        )
+
+        # data/pipelines/tekton-provider-global-defaults.yaml
+        self._process_by(
+            "json_path",
+            gitlab_cli=gitlab_cli,
+            path="data/pipelines/tekton-provider-global-defaults.yaml",
+            search_text="$.taskTemplates[?(@.name == 'openshift-saas-deploy')].variables.qontract_reconcile_image_tag",
+            replace_text=self.version,
+        )

reconcile/utils/mr/user_maintenance.py

@@ -1,5 +1,8 @@
+from pathlib import Path
+
 from ruamel import yaml
 
+from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
 
@@ -10,10 +13,10 @@ class PathTypes:
     QUERY = 2
     GABI = 3
     AWS_ACCOUNTS = 4
+    SCHEDULE = 5
 
 
 class CreateDeleteUserAppInterface(MergeRequestBase):
-
     name = "create_delete_user_mr"
 
     def __init__(self, username, paths):
@@ -32,11 +35,11 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
     def description(self) -> str:
         return f"delete user {self.username}"
 
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         for path_spec in self.paths:
             path_type = path_spec["type"]
             path = path_spec["path"]
-            if path_type in [PathTypes.USER, PathTypes.REQUEST, PathTypes.QUERY]:
+            if path_type in {PathTypes.USER, PathTypes.REQUEST, PathTypes.QUERY}:
                 gitlab_cli.delete_file(
                     branch_name=self.branch, file_path=path, commit_message=self.title
                 )
@@ -68,10 +71,27 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                             commit_message=self.title,
                             content=new_content,
                         )
+            elif path_type == PathTypes.SCHEDULE:
+                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
+                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                delete_indexes: list[tuple[int, int]] = []
+                for schedule_index, schedule_record in enumerate(content["schedule"]):
+                    for user_index, user in enumerate(schedule_record["users"]):
+                        if self.username == Path(user["$ref"]).stem:
+                            delete_indexes.append((schedule_index, user_index))
+                for schedule_index, user_index in reversed(delete_indexes):
+                    del content["schedule"][schedule_index]["users"][user_index]
+                new_content = "---\n"
+                new_content += yaml.dump(content, Dumper=yaml.RoundTripDumper)
+                gitlab_cli.update_file(
+                    branch_name=self.branch,
+                    file_path=path,
+                    commit_message=self.title,
+                    content=new_content,
+                )
 
 
 class CreateDeleteUserInfra(MergeRequestBase):
-
     PLAYBOOK = "ansible/playbooks/bastion-accounts.yml"
 
     name = "create_ssh_key_mr"

reconcile/utils/oauth2_backend_application_session.py

@@ -0,0 +1,132 @@
+import threading
+from collections.abc import Mapping, MutableMapping
+from typing import Any, Self
+
+from oauthlib.oauth2 import BackendApplicationClient, TokenExpiredError
+from requests import Response
+from requests.adapters import BaseAdapter
+from requests_oauthlib import OAuth2Session
+
+FETCH_TOKEN_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+    "Connection": "close",  # Close connection to avoid RemoteDisconnected ConnectionError
+}
+
+
+class OAuth2BackendApplicationSession:
+    """
+    OAuth2 session using Backend Application flow with auto and thread-safe token fetch.
+    """
+
+    def __init__(
+        self,
+        client_id: str,
+        client_secret: str,
+        token_url: str,
+        scope: list[str] | None = None,
+    ) -> None:
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.token_url = token_url
+        self.scope = scope
+        client = BackendApplicationClient(client_id=client_id)
+        self.session = OAuth2Session(client=client, scope=scope)
+        self.token_lock = threading.Lock()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.close()
+
+    def fetch_token(self) -> dict:
+        """
+        Fetch token from token_url and store it in the session.
+        Thread-safe method to avoid multiple threads fetching the token at the same time.
+        """
+        token = self.session.token
+        with self.token_lock:
+            # Check if token is already fetched by another thread
+            if token is not self.session.token:
+                return self.session.token
+            return self.session.fetch_token(
+                token_url=self.token_url,
+                client_id=self.client_id,
+                client_secret=self.client_secret,
+                scope=self.scope,
+                headers=FETCH_TOKEN_HEADERS,
+            )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        data: Any = None,
+        headers: Mapping[str, str] | None = None,
+        withhold_token: bool = False,
+        client_id: str | None = None,
+        client_secret: str | None = None,
+        **kwargs: Any,
+    ) -> Response:
+        """
+        Make a request using OAuth2 session, compatible with the OAuth2Session.request method.
+        Auto fetch token if never fetched before or if the token is expired.
+        """
+        if not self.session.authorized:
+            self.fetch_token()
+        try:
+            return self.session.request(
+                method=method,
+                url=url,
+                data=data,
+                headers=headers,
+                withhold_token=withhold_token,
+                client_id=client_id,
+                client_secret=client_secret,
+                **kwargs,
+            )
+        except TokenExpiredError:
+            self.fetch_token()
+            return self.session.request(
+                method=method,
+                url=url,
+                data=data,
+                headers=headers,
+                withhold_token=withhold_token,
+                client_id=client_id,
+                client_secret=client_secret,
+                **kwargs,
+            )
+
+    # Delegates for ApiBase
+
+    def close(self) -> None:
+        self.session.close()
+
+    def mount(self, prefix: str, adapter: BaseAdapter) -> None:
+        self.session.mount(prefix, adapter)
+
+    @property
+    def headers(self) -> MutableMapping:
+        return self.session.headers
+
+    @property
+    def auth(self) -> Any:
+        return self.session.auth
+
+    @auth.setter
+    def auth(self, value: Any) -> None:
+        self.session.auth = value
+
+    def get(self, url: str, **kwargs: Any) -> Response:
+        return self.session.get(url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> Response:
+        return self.session.post(url, **kwargs)
+
+    def put(self, url: str, **kwargs: Any) -> Response:
+        return self.session.put(url, **kwargs)
+
+    def delete(self, url: str, **kwargs: Any) -> Response:
+        return self.session.delete(url, **kwargs)