qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/promotion_state.py

@@ -1,6 +1,5 @@
 import logging
 from collections import defaultdict
-from typing import Optional
 
 from pydantic import (
     BaseModel,
@@ -19,9 +18,15 @@ class PromotionData(BaseModel):
     requirements.
     """
 
+    # The success is primarily used for SAPM auto-promotions
     success: bool
-    target_config_hash: Optional[str]
-    saas_file: Optional[str]
+    target_config_hash: str | None
+    saas_file: str | None
+    check_in: str | None
+    # Whether this promotion has ever succeeded
+    # Note, this shouldnt be overridden on subsequent promotions of same ref
+    # This attribute is primarily used by saasherder validations
+    has_succeeded_once: bool | None
 
     class Config:
         smart_union = True
@@ -33,11 +38,20 @@ class PromotionState:
     A wrapper around a reconcile.utils.state.State object.
     This is dedicated to storing and retrieving information
     about promotions on S3.
+
+    Note, that PromotionsState holds 2 caches.
+    One cache for the promotion data that has already been fetched.
+    Another cache for commit sha lookup, i.e., checking if a commit sha
+    exists in S3 before making any API calls to it.
     """
 
     def __init__(self, state: State):
         self._state = state
         self._commits_by_channel: dict[str, set[str]] = defaultdict(set)
+        self._promotion_data_cache: dict[str, PromotionData | None] = {}
+
+    def _target_key(self, channel: str, target_uid: str) -> str:
+        return f"{channel}/{target_uid}"
 
     def cache_commit_shas_from_s3(self) -> None:
         """
@@ -47,28 +61,51 @@ class PromotionState:
         """
         all_keys = self._state.ls()
         for commit in all_keys:
-            # Format: /promotions/{channel}/{commit-sha}
-            if not commit.startswith("/promotions/"):
+            # Format: /promotions_v2/{channel}/{publisher-target-uid}/{commit-sha}
+            if not commit.startswith("/promotions_v2/"):
                 continue
-            _, _, channel_name, commit_sha = commit.split("/")
-            self._commits_by_channel[channel_name].add(commit_sha)
+            _, _, channel_name, publisher_uid, commit_sha = commit.split("/")
+            key = self._target_key(channel=channel_name, target_uid=publisher_uid)
+            self._commits_by_channel[key].add(commit_sha)
 
     def get_promotion_data(
-        self, sha: str, channel: str, local_lookup: bool = True
-    ) -> Optional[PromotionData]:
-        if local_lookup and sha not in self._commits_by_channel[channel]:
+        self,
+        sha: str,
+        channel: str,
+        target_uid: str = "",
+        pre_check_sha_exists: bool = True,
+        use_cache: bool = False,
+    ) -> PromotionData | None:
+        """
+        Fetch promotion data from S3.
+
+        @param use_cache: Each fetched promotion data is cached locally. Setting this
+        flag to True will use the cache if the data is already fetched.
+
+        @param pre_check_sha_exists: If set to True, we will check if the commit sha exists
+        in local cache and if not will exit before making any API calls. Note, that this requires
+        a prior call to cache_commit_shas_from_s3 to populate the local commit cache.
+        """
+        cache_key_v2 = self._target_key(channel=channel, target_uid=target_uid)
+        if pre_check_sha_exists and sha not in self._commits_by_channel[cache_key_v2]:
             # Lets reduce unecessary calls to S3
             return None
-        key = f"promotions/{channel}/{sha}"
-        try:
-            data = self._state.get(key)
-        except KeyError:
+
+        path_v2 = f"promotions_v2/{channel}/{target_uid}/{sha}"
+        if use_cache and path_v2 in self._promotion_data_cache:
+            return self._promotion_data_cache[path_v2]
+
+        data = self._state.get(path_v2, None)
+        if not data:
             return None
-        return PromotionData(**data)
+
+        promotion_data = PromotionData(**data)
+        self._promotion_data_cache[path_v2] = promotion_data
+        return promotion_data
 
     def publish_promotion_data(
-        self, sha: str, channel: str, data: PromotionData
+        self, sha: str, channel: str, target_uid: str, data: PromotionData
     ) -> None:
-        state_key = f"promotions/{channel}/{sha}"
-        self._state.add(state_key, data.dict(), force=True)
-        logging.info("Uploaded %s to %s", data, state_key)
+        state_key_v2 = f"promotions_v2/{channel}/{target_uid}/{sha}"
+        self._state.add(state_key_v2, data.dict(), force=True)
+        logging.info("Uploaded %s to %s", data, state_key_v2)

reconcile/utils/promtool.py

@@ -2,17 +2,16 @@ import copy
 import os
 import subprocess
 import tempfile
-from typing import (
-    Iterable,
-    Mapping,
-    MutableMapping,
-)
+from collections.abc import Iterable, Mapping, MutableMapping
 
 import yaml
 
 from reconcile.utils.defer import defer
 from reconcile.utils.structs import CommandExecutionResult
 
+PROMTOOL_VERSION = ["2.33.3"]
+PROMTOOL_VERSION_REGEX = r"^promtool,\sversion\s([\d]+\.[\d]+\.[\d]+).+$"
+
 
 def check_rule(yaml_spec: Mapping) -> CommandExecutionResult:
     """Run promtool check rules on the given yaml spec given as dict"""
@@ -67,9 +66,7 @@ def _run_yaml_spec_cmd(cmd: list[str], yaml_spec: Mapping) -> CommandExecutionRe
             return CommandExecutionResult(False, f"Error creating temporary file: {e}")
 
         try:
-            result = subprocess.run(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
-            )
+            result = subprocess.run(cmd, capture_output=True, check=True)
         except subprocess.CalledProcessError as e:
             msg = f'Error running promtool command [{" ".join(cmd)}]'
             if e.stdout:

reconcile/utils/quay_api.py

@@ -30,9 +30,7 @@ class QuayApi:
             if cache_members:
                 return cache_members
 
-        url = "{}/organization/{}/team/{}/members?includePending=true".format(
-            self.api_url, self.organization, team
-        )
+        url = f"{self.api_url}/organization/{self.organization}/team/{team}/members?includePending=true"
 
         r = requests.get(url, headers=self.auth_header, timeout=self._timeout)
         if r.status_code == 404:
@@ -57,33 +55,27 @@ class QuayApi:
         return members_list
 
     def user_exists(self, user):
-        url = "{}/users/{}".format(self.api_url, user)
+        url = f"{self.api_url}/users/{user}"
         r = requests.get(url, headers=self.auth_header, timeout=self._timeout)
-        if not r.ok:
-            return False
-        return True
+        return r.ok
 
     def remove_user_from_team(self, user, team):
         """Deletes an user from a team.
 
         :raises HTTPError if there are any problems with the request
         """
-        url_team = "{}/organization/{}/team/{}/members/{}".format(
-            self.api_url, self.organization, team, user
-        )
+        url_team = f"{self.api_url}/organization/{self.organization}/team/{team}/members/{user}"
 
         r = requests.delete(url_team, headers=self.auth_header, timeout=self._timeout)
         if not r.ok:
             message = r.json().get("message", "")
 
-            expected_message = "User {} does not belong to team {}".format(user, team)
+            expected_message = f"User {user} does not belong to team {team}"
 
             if message != expected_message:
                 r.raise_for_status()
 
-        url_org = "{}/organization/{}/members/{}".format(
-            self.api_url, self.organization, user
-        )
+        url_org = f"{self.api_url}/organization/{self.organization}/members/{user}"
 
         r = requests.delete(url_org, headers=self.auth_header, timeout=self._timeout)
         r.raise_for_status()
@@ -98,9 +90,7 @@ class QuayApi:
         if user in self.list_team_members(team, cache=True):
             return True
 
-        url = "{}/organization/{}/team/{}/members/{}".format(
-            self.api_url, self.organization, team, user
-        )
+        url = f"{self.api_url}/organization/{self.organization}/team/{team}/members/{user}"
         r = requests.put(url, headers=self.auth_header, timeout=self._timeout)
         r.raise_for_status()
         return True
@@ -117,7 +107,7 @@ class QuayApi:
         :raises HTTPError: unsuccessful attempt to create the team
         """
 
-        url = "{}/organization/{}/team/{}".format(self.api_url, self.organization, team)
+        url = f"{self.api_url}/organization/{self.organization}/team/{team}"
 
         payload = {"role": role}
 
@@ -140,7 +130,7 @@ class QuayApi:
         if count > self.LIMIT_FOLLOWS:
             raise ValueError("Too many page follows")
 
-        url = "{}/repository".format(self.api_url)
+        url = f"{self.api_url}/repository"
 
         # params
         params = {"namespace": self.organization}
@@ -176,7 +166,7 @@ class QuayApi:
         """
         visibility = "public" if public else "private"
 
-        url = "{}/repository".format(self.api_url)
+        url = f"{self.api_url}/repository"
 
         params = {
             "repo_kind": "image",
@@ -193,14 +183,14 @@ class QuayApi:
         r.raise_for_status()
 
     def repo_delete(self, repo_name):
-        url = "{}/repository/{}/{}".format(self.api_url, self.organization, repo_name)
+        url = f"{self.api_url}/repository/{self.organization}/{repo_name}"
 
         # perform request
         r = requests.delete(url, headers=self.auth_header, timeout=self._timeout)
         r.raise_for_status()
 
     def repo_update_description(self, repo_name, description):
-        url = "{}/repository/{}/{}".format(self.api_url, self.organization, repo_name)
+        url = f"{self.api_url}/repository/{self.organization}/{repo_name}"
 
         params = {"description": description}
 
@@ -217,9 +207,7 @@ class QuayApi:
         self._repo_change_visibility(repo_name, "private")
 
     def _repo_change_visibility(self, repo_name, visibility):
-        url = "{}/repository/{}/{}/changevisibility".format(
-            self.api_url, self.organization, repo_name
-        )
+        url = f"{self.api_url}/repository/{self.organization}/{repo_name}/changevisibility"
 
         params = {"visibility": visibility}
 

reconcile/utils/raw_github_api.py

@@ -72,7 +72,7 @@ class RawGithubApi:
         return result
 
     def org_invitations(self, org):
-        invitations = self.query("/orgs/{}/invitations".format(org))
+        invitations = self.query(f"/orgs/{org}/invitations")
 
         return [
             login
@@ -81,9 +81,7 @@ class RawGithubApi:
         ]
 
     def team_invitations(self, org_id, team_id):
-        invitations = self.query(
-            "/organizations/{}/team/{}/invitations".format(org_id, team_id)
-        )
+        invitations = self.query(f"/organizations/{org_id}/team/{team_id}/invitations")
 
         return [
             login
@@ -95,6 +93,6 @@ class RawGithubApi:
         return self.query("/user/repository_invitations")
 
     def accept_repo_invitation(self, invitation_id):
-        url = self.BASE_URL + "/user/repository_invitations/{}".format(invitation_id)
+        url = self.BASE_URL + f"/user/repository_invitations/{invitation_id}"
         res = self.patch(url)
         res.raise_for_status()

reconcile/utils/repo_owners.py

@@ -194,15 +194,9 @@ class RepoOwners:
     @staticmethod
     def _set_to_sorted_list(owners):
         approvers = owners["approvers"]
-        if approvers:
-            sorted_approvers = sorted(approvers)
-        else:
-            sorted_approvers = []
+        sorted_approvers = sorted(approvers) if approvers else []
 
         reviewers = owners["reviewers"]
-        if reviewers:
-            sorted_reviewers = sorted(reviewers)
-        else:
-            sorted_reviewers = []
+        sorted_reviewers = sorted(reviewers) if reviewers else []
 
         return {"approvers": sorted_approvers, "reviewers": sorted_reviewers}

reconcile/utils/rest_api_base.py

@@ -0,0 +1,126 @@
+import logging
+from typing import Any, Self
+from urllib.parse import urljoin
+
+import requests
+from urllib3 import Retry
+
+from reconcile.utils.oauth2_backend_application_session import (
+    OAuth2BackendApplicationSession,
+)
+
+
+def get_next_url(links: dict[str, dict[str, str]]) -> str | None:
+    """Parse response header 'Link' attribute and return the next page url if exists.
+
+    See
+    * https://gitlab.com/glitchtip/glitchtip-backend/-/blob/master/glitchtip/pagination.py#L34
+    * https://requests.readthedocs.io/en/latest/api/?highlight=links#requests.Response.links
+    """
+    if links.get("next", {}).get("results", "false") == "true":
+        return links["next"]["url"]
+    return None
+
+
+class BearerTokenAuth(requests.auth.AuthBase):
+    """Use this class to add a Bearer token to the request headers."""
+
+    def __init__(self, token: str):
+        self.token = token
+
+    def __eq__(self, other: Any) -> bool:
+        return self.token == getattr(other, "token", None)
+
+    def __ne__(self, other: Any) -> bool:
+        return not self == other
+
+    def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
+        r.headers["Authorization"] = f"Bearer {self.token}"
+        return r
+
+
+class ApiBase:
+    """This class provides a common standard for REST API clients."""
+
+    def __init__(
+        self,
+        host: str,
+        auth: requests.auth.AuthBase | None = None,
+        max_retries: int | Retry | None = None,
+        read_timeout: float | None = None,
+        session: requests.Session | OAuth2BackendApplicationSession | None = None,
+    ) -> None:
+        self.host = host
+        self.max_retries = max_retries if max_retries is not None else 3
+        self.read_timeout = read_timeout if read_timeout is not None else 30
+        self.session = session or requests.Session()
+        if auth:
+            self.session.auth = auth
+        for prefix in ["http://", "https://"]:
+            self.session.mount(
+                prefix,
+                requests.adapters.HTTPAdapter(max_retries=self.max_retries),
+            )
+        self.session.headers.update({
+            "Content-Type": "application/json",
+        })
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        self.session.close()
+
+    def _get(self, url: str) -> dict[str, Any]:
+        response = self.session.get(urljoin(self.host, url), timeout=self.read_timeout)
+        response.raise_for_status()
+        try:
+            return response.json()
+        except requests.exceptions.JSONDecodeError:
+            logging.error(
+                f"Failed to decode JSON response from {url}"
+                f"Response: {response.text}"
+            )
+            raise
+
+    def _list(
+        self, url: str, params: dict | None = None, attribute: str | None = None
+    ) -> list[dict[str, Any]]:
+        response = self.session.get(
+            urljoin(self.host, url), params=params, timeout=self.read_timeout
+        )
+        response.raise_for_status()
+        results = response.json()
+        if response.links:
+            # handle pagination
+            while next_url := get_next_url(response.links):
+                response = self.session.get(next_url)
+                results += response.json()
+        if attribute:
+            return results[attribute]
+        return results
+
+    def _post(self, url: str, data: dict | None = None) -> dict[str, Any]:
+        response = self.session.post(
+            urljoin(self.host, url), json=data, timeout=self.read_timeout
+        )
+        response.raise_for_status()
+        if response.status_code == 204 or not response.text:
+            return {}
+        return response.json()
+
+    def _put(self, url: str, data: dict | None = None) -> dict[str, Any]:
+        response = self.session.put(
+            urljoin(self.host, url), json=data, timeout=self.read_timeout
+        )
+        response.raise_for_status()
+        if response.status_code == 204 or not response.text:
+            return {}
+        return response.json()
+
+    def _delete(self, url: str) -> None:
+        response = self.session.delete(urljoin(self.host, url), timeout=None)
+        response.raise_for_status()