qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/internal_groups/client.py

@@ -0,0 +1,160 @@
+from typing import (
+    Any,
+    Self,
+)
+
+import requests
+from oauthlib.oauth2 import (
+    BackendApplicationClient,
+    TokenExpiredError,
+)
+from requests import Response
+from requests_oauthlib import OAuth2Session
+from sretoolbox.utils import retry
+
+from reconcile.utils.internal_groups.models import Group
+
+REQUEST_TIMEOUT = 30
+
+
+class NotFound(Exception):
+    """Not found exception."""
+
+
+class InternalGroupsApi:
+    """Internal groups API client."""
+
+    def __init__(
+        self, api_url: str, issuer_url: str, client_id: str, client_secret: str
+    ):
+        self.api_url = api_url
+        self.issuer_url = issuer_url
+        self.client_id = client_id
+        self.client_secret = client_secret
+        client = BackendApplicationClient(client_id=self.client_id)
+        self._client = OAuth2Session(self.client_id, client=client)
+
+    def _fetch_token(self) -> dict:
+        self._client.token = {}
+        return self._client.fetch_token(
+            token_url=f"{self.issuer_url}/protocol/openid-connect/token",
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+        )
+
+    def _check_response(self, resp: requests.Response) -> None:
+        """Check response."""
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code == 404:
+                raise NotFound(e.response.text) from e
+            raise
+
+    def __enter__(self) -> Self:
+        """Fetch token."""
+        if not self._client.token:
+            self._client.token = self._fetch_token()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        pass
+
+    @retry(exceptions=(TokenExpiredError,), max_attempts=2)
+    def _request(
+        self,
+        method: str,
+        url: str,
+        json: dict[Any, Any] | None = None,
+        timeout: int = REQUEST_TIMEOUT,
+    ) -> Response:
+        try:
+            return self._client.request(
+                method=method,
+                url=url,
+                json=json,
+                # Content-Type is required for GET too :(
+                headers={"Content-Type": "application/json"},
+                timeout=timeout,
+            )
+        except TokenExpiredError:
+            self._client.token = self._fetch_token()
+            raise
+
+    def close(self) -> None:
+        """Close client session."""
+        self._client.close()
+
+    def group(self, name: str) -> dict:
+        """Get a group by name."""
+        resp = self._request("GET", f"{self.api_url}/v1/groups/{name}")
+        self._check_response(resp)
+        return resp.json()
+
+    def delete_group(self, name: str) -> None:
+        """Delete a group by name."""
+        resp = self._request("DELETE", f"{self.api_url}/v1/groups/{name}")
+        self._check_response(resp)
+
+    def create_group(self, data: dict) -> dict:
+        """Create a group."""
+        resp = self._request("POST", f"{self.api_url}/v1/groups/", json=data)
+        self._check_response(resp)
+        return resp.json()
+
+    def update_group(self, name: str, data: dict) -> dict:
+        """Update a group."""
+        resp = self._request(
+            "PATCH",
+            f"{self.api_url}/v1/groups/{name}",
+            json=data,
+        )
+        self._check_response(resp)
+        return resp.json()
+
+
+class InternalGroupsClient:
+    """High level Internal groups client."""
+
+    def __init__(
+        self,
+        api_url: str,
+        issuer_url: str,
+        client_id: str,
+        client_secret: str,
+        api_class: type[InternalGroupsApi] = InternalGroupsApi,
+    ):
+        self._api = api_class(api_url, issuer_url, client_id, client_secret)
+
+    def close(self) -> None:
+        """Close client session."""
+        self._api.close()
+
+    def group(self, name: str) -> Group:
+        """Get group by name."""
+        with self._api as api:
+            return Group(**api.group(name))
+
+    def create_group(self, group: Group) -> Group:
+        """Create group."""
+        with self._api as api:
+            return Group(
+                **api.create_group(
+                    data=group.dict(by_alias=True),
+                )
+            )
+
+    def delete_group(self, name: str) -> None:
+        """Delete group."""
+        with self._api as api:
+            api.delete_group(name)
+
+    def update_group(self, group: Group) -> Group:
+        """Update group."""
+        with self._api as api:
+            return Group(
+                **api.update_group(
+                    name=group.name,
+                    data=group.dict(by_alias=True),
+                )
+            )

reconcile/utils/internal_groups/models.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import (
+    BaseModel,
+    Field,
+)
+
+
+class EntityType(StrEnum):
+    USER = "user"
+    SERVICE_ACCOUNT = "serviceaccount"
+    DELETED_USER = "deleteduser"
+
+
+class Entity(BaseModel):
+    type: EntityType
+    id: str
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Entity):
+            raise NotImplementedError("Cannot compare to non Entity objects.")
+        return self.id == other.id
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+
+class Group(BaseModel):
+    name: str
+    description: str
+    member_approval_type: str = Field("self-service", alias="memberApprovalType")
+    contact_list: str = Field(..., alias="contactList")
+    owners: list[Entity]
+    display_name: str = Field(..., alias="displayName")
+    notes: str | None = None
+    rover_group_member_query: str | None = Field(None, alias="roverGroupMemberQuery")
+    rover_group_inclusions: list[Entity] | None = Field(
+        None, alias="roverGroupInclusions"
+    )
+    rover_group_exclusions: list[Entity] | None = Field(
+        None, alias="roverGroupExclusions"
+    )
+    members: list[Entity] = []
+    member_of: list[str] | None = Field(None, alias="memberOf")
+    namespace: str | None = None
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Group):
+            raise NotImplementedError("Cannot compare to non Group objects.")
+        return (
+            self.description == other.description
+            and self.member_approval_type == other.member_approval_type
+            and self.contact_list == other.contact_list
+            and set(self.owners) == set(other.owners)
+            and self.display_name == other.display_name
+            and self.notes == other.notes
+            and set(self.members) == set(other.members)
+        )
+
+    class Config:
+        allow_population_by_field_name = True
+        # exclude read-only fields in the json/dict dumps
+        fields = {
+            "rover_group_member_query": {"exclude": True},
+            "rover_group_inclusions": {"exclude": True},
+            "rover_group_exclusions": {"exclude": True},
+            "member_of": {"exclude": True},
+            "namespace": {"exclude": True},
+        }

reconcile/utils/jenkins_api.py

@@ -66,7 +66,7 @@ class JenkinsApi:
 
     @retry()
     def get_jobs_state(self):
-        url = f"{self.url}/api/json?tree=jobs[name,builds[number,result,changeSet[items[commitId]]]]"
+        url = f"{self.url}/api/json?tree=jobs[name,builds[number,result,actions[lastBuiltRevision[SHA1]]]]"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
         )
@@ -77,9 +77,11 @@ class JenkinsApi:
             job_name = r["name"]
             builds = r.get("builds", [])
             for b in builds:
-                change_set_items = b.get("changeSet", {}).get("items")
-                if change_set_items:
-                    b["commit_sha"] = change_set_items[0]["commitId"]
+                actions = b.get("actions", [])
+                for a in actions:
+                    revision = a.get("lastBuiltRevision")
+                    if revision:
+                        b["commit_sha"] = revision["SHA1"]
             jobs_state[job_name] = builds
 
         return jobs_state
@@ -106,7 +108,7 @@ class JenkinsApi:
         res.raise_for_status()
 
     def get_all_roles(self):
-        url = "{}/role-strategy/strategy/getAllRoles".format(self.url)
+        url = f"{self.url}/role-strategy/strategy/getAllRoles"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
         )
@@ -115,7 +117,7 @@ class JenkinsApi:
         return res.json()
 
     def assign_role_to_user(self, role, user):
-        url = "{}/role-strategy/strategy/assignRole".format(self.url)
+        url = f"{self.url}/role-strategy/strategy/assignRole"
         data = {"type": "globalRoles", "roleName": role, "sid": user}
         res = requests.post(
             url,
@@ -128,7 +130,7 @@ class JenkinsApi:
         res.raise_for_status()
 
     def unassign_role_from_user(self, role, user):
-        url = "{}/role-strategy/strategy/unassignRole".format(self.url)
+        url = f"{self.url}/role-strategy/strategy/unassignRole"
         data = {"type": "globalRoles", "roleName": role, "sid": user}
         res = requests.post(
             url,
@@ -140,34 +142,8 @@ class JenkinsApi:
 
         res.raise_for_status()
 
-    def list_plugins(self):
-        url = "{}/pluginManager/api/json?depth=1".format(self.url)
-
-        res = requests.get(
-            url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
-        )
-
-        res.raise_for_status()
-        return res.json()["plugins"]
-
-    def install_plugin(self, name):
-        self.should_restart = True
-        header = {"Content-Type": "text/xml"}
-        url = "{}/pluginManager/installNecessaryPlugins".format(self.url)
-        data = '<jenkins><install plugin="{}@current" /></jenkins>'.format(name)
-        res = requests.post(
-            url,
-            verify=self.ssl_verify,
-            data=data,
-            headers=header,
-            auth=(self.user, self.password),
-            timeout=60,
-        )
-
-        res.raise_for_status()
-
     def safe_restart(self, force_restart=False):
-        url = "{}/safeRestart".format(self.url)
+        url = f"{self.url}/safeRestart"
         if self.should_restart or force_restart:
             logging.debug(
                 "performing safe restart. "

reconcile/utils/{jinja2_ext.py → jinja2/extensions.py}

@@ -1,15 +1,17 @@
 import base64
 import textwrap
+from collections.abc import Callable
 
 from jinja2 import nodes
 from jinja2.exceptions import TemplateRuntimeError
 from jinja2.ext import Extension
+from jinja2.parser import Parser
 
 
 class B64EncodeExtension(Extension):
     tags = {"b64encode"}
 
-    def parse(self, parser):
+    def parse(self, parser: Parser) -> nodes.CallBlock:
         lineno = next(parser.stream).lineno
 
         body = parser.parse_statements(["name:endb64encode"], drop_needle=True)
@@ -19,7 +21,7 @@ class B64EncodeExtension(Extension):
         ).set_lineno(lineno)
 
     @staticmethod
-    def _b64encode(caller):
+    def _b64encode(caller: Callable) -> str:
         content = caller()
         content = textwrap.dedent(content)
         return base64.b64encode(content.encode()).decode("utf-8")
@@ -28,7 +30,7 @@ class B64EncodeExtension(Extension):
 class RaiseErrorExtension(Extension):
     tags = {"raise_error"}
 
-    def parse(self, parser):
+    def parse(self, parser: Parser) -> nodes.CallBlock:
         lineno = next(parser.stream).lineno
 
         msg = parser.parse_expression()
@@ -42,5 +44,5 @@ class RaiseErrorExtension(Extension):
         )
 
     @staticmethod
-    def _raise_error(msg, caller):
+    def _raise_error(msg: str, caller: Callable) -> None:
         raise TemplateRuntimeError(msg)

reconcile/utils/jinja2/filters.py

@@ -0,0 +1,142 @@
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from typing import Any
+from urllib import parse
+
+import jinja2
+import yaml
+
+from reconcile.utils.jsonpath import parse_jsonpath
+
+
+def json_to_dict(input: str) -> Any:
+    """Jinja2 filter to parse JSON strings into dictionaries.
+       This becomes useful to access Graphql queries data (labels)
+    :param input: json string
+    :return: dict with the parsed inputs contents
+    """
+    data = json.loads(input)
+    return data
+
+
+def yaml_to_dict(input: str) -> Any:
+    """Jinja2 filter to parse YAML strings into dictionaries.
+    :param input: yaml string
+    :return: dict with the parsed inputs contents
+    """
+    data = yaml.safe_load(input)
+    return data
+
+
+def urlescape(string: str, safe: str = "/", encoding: str | None = None) -> str:
+    """Jinja2 filter that is a simple wrapper around urllib's URL quoting
+    functions that takes a string value and makes it safe for use as URL
+    components escaping any reserved characters using URL encoding. See:
+    urllib.parse.quote() and urllib.parse.quote_plus() for reference.
+
+    :param str string: String value to escape.
+    :param str safe: Optional characters that should not be escaped.
+    :param encoding: Encoding to apply to the string to be escaped. Defaults
+        to UTF-8. Unsupported characters raise a UnicodeEncodeError error.
+    :type encoding: typing.Optional[str]
+    :returns: A string with reserved characters escaped.
+    :rtype: str
+    """
+    return parse.quote(string, safe=safe, encoding=encoding)
+
+
+def urlunescape(string: str, encoding: str | None = None) -> str:
+    """Jinja2 filter that is a simple wrapper around urllib's URL unquoting
+    functions that takes an URL-encoded string value and unescapes it
+    replacing any URL-encoded values with their character equivalent. See:
+    urllib.parse.unquote() and urllib.parse.unquote_plus() for reference.
+
+    :param str string: String value to unescape.
+    :param encoding: Encoding to apply to the string to be unescaped. Defaults
+        to UTF-8. Unsupported characters are replaced by placeholder values.
+    :type encoding: typing.Optional[str]
+    :returns: A string with URL-encoded sequences unescaped.
+    :rtype: str
+    """
+    if encoding is None:
+        encoding = "utf-8"
+    return parse.unquote(string, encoding=encoding)
+
+
+def eval_filter(input: str, **kwargs: dict[str, Any]) -> str:
+    """Jinja2 filter be used when the string
+    is in itself a jinja2 template that must be
+    evaluated with kwargs. For example in the case
+    of the slo-document expression fields.
+    :param input: template string
+    :kwargs: variables that will be used to evaluate the
+             input string
+    :return: rendered string
+    """
+    return jinja2.Template(input).render(**kwargs)
+
+
+def hash_list(input: Iterable) -> str:
+    """
+    Deterministic hash of a list for jinja2 templates.
+    The order of the list doesn't matter as it is sorted
+    before hashing. Note, that the list elements
+    must be flat primitives (no dicts/lists).
+    """
+    lst = list(input)
+    str_lst = []
+    for el in lst:
+        if isinstance(el, list | dict):
+            raise RuntimeError(
+                f"jinja2 hash_list function received non-primitive value {el}. All values received {lst}"
+            )
+        str_lst.append(str(el))
+    msg = "a"  # keep non-empty for hashing empty list
+    msg += "".join(sorted(str_lst))
+    m = hashlib.sha256()
+    m.update(msg.encode("utf-8"))
+    return m.hexdigest()
+
+
+def _find_jsonpath(input: Any, jsonpath: str | None) -> Any:
+    assert jsonpath is not None and len(jsonpath) > 0
+    return parse_jsonpath(jsonpath).find(input)
+
+
+def extract_jsonpath(input: Any, jsonpath: str) -> Any:
+    """
+    Extracts data from the input using jsonpath.
+    The result is a list of matching elements.
+    The list will be empty if nothing matches.
+    """
+    return [i.value for i in _find_jsonpath(input, jsonpath)]
+
+
+def matches_jsonpath(input: Any, jsonpath: str | None) -> bool:
+    """
+    Returns True if the input matches the provided jsonpath
+    """
+    return len(_find_jsonpath(input, jsonpath)) > 0
+
+
+def _convert_pointer(pointer: str) -> str:
+    """
+    Converts a jsonpath_ng pointer (eg "items.[2].type.[3]")
+    to a rfc6901 one (https://www.rfc-editor.org/rfc/rfc6901)
+    "/items/2/type/3"
+    """
+    elems = [e[1:-1] if re.match(r"\[\d\]", e) else e for e in pointer.split(".")]
+    return "/" + "/".join(elems)
+
+
+def json_pointers(input: Any, jsonpath: str) -> list[str]:
+    """
+    Finds the RFC6901 JSON pointers of the input elements matching the given jsonpath
+    """
+    return [_convert_pointer(str(i.full_path)) for i in _find_jsonpath(input, jsonpath)]
+
+
+def str_format(value: str, format: str) -> str:
+    return format % value