qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/gql.py

@@ -1,13 +1,12 @@
+import contextlib
 import logging
 import textwrap
+import threading
 from datetime import (
+    UTC,
     datetime,
-    timezone,
-)
-from typing import (
-    Any,
-    Optional,
 )
+from typing import Any
 from urllib.parse import urlparse
 
 import requests
@@ -18,6 +17,8 @@ from gql import (
 from gql.transport.exceptions import TransportQueryError
 from gql.transport.requests import RequestsHTTPTransport
 from gql.transport.requests import log as requests_logger
+from requests.auth import AuthBase
+from requests.cookies import RequestsCookieJar
 from sentry_sdk import capture_exception
 from sretoolbox.utils import retry
 
@@ -44,10 +45,8 @@ def capture_and_forget(error):
     :type error: Exception
     """
 
-    try:
+    with contextlib.suppress(Exception):
         capture_exception(error)
-    except Exception:
-        pass
 
 
 class GqlApiError(Exception):
@@ -78,9 +77,7 @@ class GqlApiErrorForbiddenSchema(Exception):
 
 class GqlGetResourceError(Exception):
     def __init__(self, path, msg):
-        super().__init__(
-            "Error getting resource from path {}: {}".format(path, str(msg))
-        )
+        super().__init__(f"Error getting resource from path {path}: {msg!s}")
 
 
 class GqlApi:
@@ -90,11 +87,11 @@ class GqlApi:
     def __init__(
         self,
         url: str,
-        token: Optional[str] = None,
+        token: str | None = None,
         int_name=None,
         validate_schemas=False,
-        commit: Optional[str] = None,
-        commit_timestamp: Optional[str] = None,
+        commit: str | None = None,
+        commit_timestamp: str | None = None,
     ) -> None:
         self.url = url
         self.token = token
@@ -102,6 +99,7 @@ class GqlApi:
         self.validate_schemas = validate_schemas
         self.commit = commit
         self.commit_timestamp = commit_timestamp
+        self.client = self._init_gql_client()
 
         if validate_schemas and not int_name:
             raise Exception(
@@ -116,31 +114,40 @@ class GqlApi:
                     self._valid_schemas = integration["schemas"]
                     break
 
-            if not self._valid_schemas:
+            if validate_schemas and not self._valid_schemas:
                 raise GqlApiIntegrationNotFound(int_name)
 
+    def _init_gql_client(self) -> Client:
+        req_headers = None
+        if self.token:
+            # The token stored in vault is already in the format 'Basic ...'
+            req_headers = {"Authorization": self.token}
+        transport = PersistentRequestsHTTPTransport(
+            requests.Session(), self.url, headers=req_headers, timeout=30
+        )
+        return Client(transport=transport)
+
+    def close(self):
+        logging.debug("Closing GqlApi client")
+        if self.client.transport.session:
+            self.client.transport.session.close()
+
     @retry(exceptions=GqlApiError, max_attempts=5, hook=capture_and_forget)
     def query(
         self, query: str, variables=None, skip_validation=False
-    ) -> Optional[dict[str, Any]]:
-
-        # Here we are recreating client on purpose as some integrations such as `openshift-resource` require multiple
-        # queries in separate threads. With RequestsHTTPTransport that is currently not possible as it expects
-        # session to be reused and thus throws `Transport is already connected` error.
-        # Adding a synchronization mechanism would lead to increase wait time given some of our integrations are not
-        # sharded and thread pool capacity is also large for those.
-        # Hence, we are recreating client for every query.
-        client = _init_gql_client(self.url, self.token)
+    ) -> dict[str, Any] | None:
         try:
-            result = client.execute(
+            result = self.client.execute(
                 gql(query), variables, get_execution_result=True
             ).formatted
         except requests.exceptions.ConnectionError as e:
-            raise GqlApiError("Could not connect to GraphQL server ({})".format(e))
+            raise GqlApiError(f"Could not connect to GraphQL server ({e})") from None
         except TransportQueryError as e:
-            raise GqlApiError("`error` returned with GraphQL response {}".format(e))
+            raise GqlApiError(f"`error` returned with GraphQL response {e}") from None
         except AssertionError:
-            raise GqlApiError("`data` field missing from GraphQL response payload")
+            raise GqlApiError(
+                "`data` field missing from GraphQL response payload"
+            ) from None
         except Exception as e:
             raise GqlApiError("Unexpected error occurred") from e
 
@@ -165,6 +172,29 @@ class GqlApi:
 
         return result["data"]
 
+    def get_template(self, path: str) -> dict[str, str]:
+        query = """
+        query Template($path: String) {
+          templates: template_v1(path: $path) {
+            path
+            template
+          }
+        }
+        """
+
+        try:
+            templates = []
+            q_result = self.query(query, {"path": path})
+            if q_result:
+                templates = q_result["templates"]
+        except GqlApiError:
+            raise GqlGetResourceError(path, "Template not found.") from None
+
+        if len(templates) != 1:
+            raise GqlGetResourceError(path, "Expecting one and only one template.")
+
+        return templates[0]
+
     def get_resource(self, path: str) -> dict[str, Any]:
         query = """
         query Resource($path: String) {
@@ -183,7 +213,7 @@ class GqlApi:
                 "resources"
             ]
         except GqlApiError:
-            raise GqlGetResourceError(path, "Resource not found.")
+            raise GqlGetResourceError(path, "Resource not found.") from None
 
         if len(resources) != 1:
             raise GqlGetResourceError(path, "Expecting one and only one resource.")
@@ -211,27 +241,52 @@ class GqlApi:
         return list(self._queried_schemas)
 
     @property
-    def commit_timestamp_utc(self) -> Optional[str]:
+    def commit_timestamp_utc(self) -> str | None:
         if self.commit_timestamp:
-            return datetime.fromtimestamp(
-                int(self.commit_timestamp), timezone.utc
-            ).isoformat()
+            return datetime.fromtimestamp(int(self.commit_timestamp), UTC).isoformat()
         return None
 
 
-_gqlapi: Optional[GqlApi] = None
+class GqlApiSingleton:
+    gql_api: GqlApi | None = None
+    gqlapi_lock = threading.Lock()
+
+    @classmethod
+    def create(cls, *args, **kwargs) -> GqlApi:
+        with cls.gqlapi_lock:
+            if cls.gql_api:
+                logging.debug("Resestting GqlApi instance")
+                cls.close_gqlapi()
+            cls.gql_api = GqlApi(*args, **kwargs)
+        return cls.gql_api
+
+    @classmethod
+    def close_gqlapi(cls):
+        cls.gql_api.close()
+
+    @classmethod
+    def instance(cls) -> GqlApi:
+        if not cls.gql_api:
+            raise GqlApiError("gql module has not been initialized.")
+        return cls.gql_api
+
+    @classmethod
+    def close(cls) -> None:
+        with cls.gqlapi_lock:
+            if cls.gql_api:
+                cls.close_gqlapi()
+                cls.gql_api = None
 
 
 def init(
     url: str,
-    token: Optional[str] = None,
+    token: str | None = None,
     integration=None,
     validate_schemas=False,
-    commit: Optional[str] = None,
-    commit_timestamp: Optional[str] = None,
+    commit: str | None = None,
+    commit_timestamp: str | None = None,
 ):
-    global _gqlapi
-    _gqlapi = GqlApi(
+    return GqlApiSingleton.create(
         url,
         token,
         integration,
@@ -239,20 +294,53 @@ def init(
         commit=commit,
         commit_timestamp=commit_timestamp,
     )
-    return _gqlapi
 
 
 def get_resource(path: str) -> dict[str, Any]:
     return get_api().get_resource(path)
 
 
-def _init_gql_client(url: str, token: Optional[str]) -> Client:
-    req_headers = None
-    if token:
-        # The token stored in vault is already in the format 'Basic ...'
-        req_headers = {"Authorization": token}
-    # Here we are explicitly using sync strategy
-    return Client(transport=RequestsHTTPTransport(url, headers=req_headers, timeout=30))
+class PersistentRequestsHTTPTransport(RequestsHTTPTransport):
+    """A transport for the GQL Client that uses an existing.
+    Is a reduced version of the RequestsHTTPTransport class from gql library
+    with the connect and close methods removed, cause they are implemented
+    to disconnect after each query.
+    """
+
+    def __init__(
+        self,
+        session: requests.Session,
+        url: str,
+        headers: dict[str, Any] | None = None,
+        cookies: dict[str, Any] | RequestsCookieJar | None = None,
+        auth: AuthBase | None = None,
+        use_json: bool = True,
+        timeout: int | None = None,
+        verify: bool | str = True,
+        retries: int = 0,
+        method: str = "POST",
+        **kwargs: Any,
+    ):
+        super().__init__(
+            url,
+            headers,
+            cookies,
+            auth,
+            use_json,
+            timeout,
+            verify,
+            retries,
+            method,
+            **kwargs,
+        )
+        # can't directly assign, due to mypy type checking
+        self.session = session  # type: ignore
+
+    def connect(self):
+        pass
+
+    def close(self) -> None:
+        pass
 
 
 @retry(exceptions=requests.exceptions.HTTPError, max_attempts=5)
@@ -302,8 +390,8 @@ def init_from_config(
 
 
 def _get_gql_server_and_token(
-    autodetect_sha: bool = False, sha: Optional[str] = None
-) -> tuple[str, str, Optional[str], Optional[str]]:
+    autodetect_sha: bool = False, sha: str | None = None
+) -> tuple[str, str, str | None, str | None]:
     config = get_config()
 
     server_url = urlparse(config["graphql"]["server"])
@@ -325,16 +413,11 @@ def _get_gql_server_and_token(
 
 
 def get_api() -> GqlApi:
-    global _gqlapi
-
-    if _gqlapi is None:
-        raise GqlApiError("gql module has not been initialized.")
-
-    return _gqlapi
+    return GqlApiSingleton.instance()
 
 
 def get_api_for_sha(
-    sha: str, integration: Optional[str] = None, validate_schemas: bool = True
+    sha: str, integration: str | None = None, validate_schemas: bool = True
 ) -> GqlApi:
     server, token, commit, timestamp = _get_gql_server_and_token(
         autodetect_sha=False, sha=sha
@@ -349,10 +432,26 @@ def get_api_for_sha(
     )
 
 
+def get_api_for_server(
+    server: str,
+    token: str | None,
+    integration: str | None = None,
+    validate_schemas: bool = True,
+) -> GqlApi:
+    return GqlApi(
+        server,
+        token,
+        integration,
+        validate_schemas,
+        commit=None,
+        commit_timestamp=None,
+    )
+
+
 @retry(exceptions=requests.exceptions.HTTPError, max_attempts=5)
 def get_diff(
-    old_sha: str, file_type: Optional[str] = None, file_path: Optional[str] = None
-):
+    old_sha: str, file_type: str | None = None, file_path: str | None = None
+) -> dict[str, Any]:
     config = get_config()
 
     server_url = urlparse(config["graphql"]["server"])

reconcile/utils/grouping.py

@@ -0,0 +1,17 @@
+from collections import defaultdict
+from collections.abc import Callable, Hashable, Iterable
+from typing import (
+    TypeVar,
+)
+
+ElementType = TypeVar("ElementType")
+KeyType = TypeVar("KeyType", bound=Hashable)
+
+
+def group_by(
+    iterable: Iterable[ElementType], key: Callable[[ElementType], KeyType]
+) -> dict[KeyType, list[ElementType]]:
+    groups = defaultdict(list)
+    for item in iterable:
+        groups[key(item)].append(item)
+    return groups

reconcile/utils/helm.py

@@ -1,8 +1,8 @@
 import json
+import os
 import tempfile
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from subprocess import (
-    PIPE,
     CalledProcessError,
     run,
 )
@@ -10,6 +10,7 @@ from typing import Any
 
 import yaml
 
+from reconcile.utils import git
 from reconcile.utils.runtime.sharding import ShardSpec
 
 
@@ -24,27 +25,97 @@ class JSONEncoder(json.JSONEncoder):
         return super().default(o)
 
 
-def template(values: Mapping[str, Any]) -> Mapping[str, Any]:
+def do_template(
+    values: Mapping[str, Any],
+    path: str,
+    namespace: str,
+) -> str:
     try:
-        with tempfile.NamedTemporaryFile(mode="w+") as values_file:
-            values_file.write(json.dumps(values, cls=JSONEncoder))
-            values_file.flush()
-            cmd = [
-                "helm",
-                "template",
-                "./helm/qontract-reconcile",
-                "-n",
-                "qontract-reconcile",
-                "-f",
-                values_file.name,
-            ]
-            result = run(cmd, stdout=PIPE, stderr=PIPE, check=True)
+        with (
+            tempfile.NamedTemporaryFile(
+                mode="w+", encoding="locale"
+            ) as repository_config_file,
+            tempfile.TemporaryDirectory() as repository_cache_dir,
+        ):
+            with open(
+                os.path.join(path, "Chart.yaml"), encoding="locale"
+            ) as chart_file:
+                chart = yaml.safe_load(chart_file)
+                if dependencies := chart.get("dependencies"):
+                    for dep in dependencies:
+                        if repo := dep.get("repository"):
+                            cmd = [
+                                "helm",
+                                "repo",
+                                "add",
+                                dep["name"],
+                                repo,
+                                "--repository-config",
+                                repository_config_file.name,
+                                "--repository-cache",
+                                repository_cache_dir,
+                            ]
+                            run(cmd, capture_output=True, check=True)
+                    cmd = [
+                        "helm",
+                        "dependency",
+                        "build",
+                        path,
+                        "--repository-config",
+                        repository_config_file.name,
+                        "--repository-cache",
+                        repository_cache_dir,
+                    ]
+                    run(cmd, capture_output=True, check=True)
+            with tempfile.NamedTemporaryFile(
+                mode="w+", encoding="locale"
+            ) as values_file:
+                values_file.write(json.dumps(values, cls=JSONEncoder))
+                values_file.flush()
+                cmd = [
+                    "helm",
+                    "template",
+                    path,
+                    "--namespace",
+                    namespace,
+                    "--values",
+                    values_file.name,
+                    "--repository-config",
+                    repository_config_file.name,
+                    "--repository-cache",
+                    repository_cache_dir,
+                ]
+                result = run(cmd, capture_output=True, check=True)
     except CalledProcessError as e:
         msg = f'Error running helm template [{" ".join(cmd)}]'
         if e.stdout:
             msg += f" {e.stdout.decode()}"
         if e.stderr:
             msg += f" {e.stderr.decode()}"
-        raise HelmTemplateError(msg)
+        raise HelmTemplateError(msg) from None
 
-    return yaml.safe_load(result.stdout.decode())
+    return result.stdout.decode()
+
+
+def template(
+    values: Mapping[str, Any],
+    path: str = "./helm/qontract-reconcile",
+    namespace: str = "qontract-reconcile",
+) -> Mapping[str, Any]:
+    return yaml.safe_load(do_template(values=values, path=path, namespace=namespace))
+
+
+def template_all(
+    url: str,
+    path: str,
+    ref: str,
+    namespace: str,
+    values: Mapping[str, Any],
+    ssl_verify: bool = True,
+) -> Iterable[Mapping[str, Any]]:
+    with tempfile.TemporaryDirectory() as wd:
+        git.clone(url, wd, depth=1, verify=ssl_verify)
+        git.checkout(ref, wd, verify=ssl_verify)
+        return yaml.safe_load_all(
+            do_template(values=values, path=f"{wd}{path}", namespace=namespace)
+        )

reconcile/utils/helpers.py

@@ -1,5 +1,17 @@
 import logging
+import random
+import re
+import string
+from collections import Counter
+from collections.abc import (
+    Iterable,
+    Mapping,
+)
 from contextlib import contextmanager
+from typing import (
+    Any,
+    TypeVar,
+)
 
 DEFAULT_TOGGLE_LEVEL = logging.ERROR
 
@@ -12,3 +24,42 @@ def toggle_logger(log_level: int = DEFAULT_TOGGLE_LEVEL):
         yield logger.setLevel(log_level)  # type: ignore[func-returns-value]
     finally:
         logger.setLevel(default_level)
+
+
+# Copied with love from https://stackoverflow.com/questions/6027558
+def flatten(
+    d: Mapping[str, Any], parent_key: str = "", sep: str = "."
+) -> dict[str, str]:
+    items: list[tuple[str, Any]] = []
+    for k, v in d.items():
+        if v is None:
+            continue
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, Mapping):
+            items.extend(flatten(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, str(v)))
+    return dict(items)
+
+
+Item = TypeVar("Item")
+
+
+def find_duplicates(items: Iterable[Item]) -> list[Item]:
+    return [item for item, count in Counter(items).items() if count > 1]
+
+
+def generate_random_password(string_length: int = 20) -> str:
+    """Generate a random string of letters and digits"""
+    letters_and_digits = string.ascii_letters + string.digits
+    return "".join(random.choices(letters_and_digits, k=string_length))
+
+
+def match_patterns(patterns: Iterable[str], s: str) -> bool:
+    """
+    Check if any pattern matches the string.
+    :param patterns: patterns to match
+    :param s: string to check
+    :return: True if any pattern matches, False otherwise
+    """
+    return any(re.match(p, s) for p in patterns)

reconcile/utils/imap_client.py

@@ -1,8 +1,5 @@
 import imaplib
-from typing import (
-    Any,
-    Union,
-)
+from typing import Any
 
 from reconcile.utils.secret_reader import SecretReader
 
@@ -15,7 +12,7 @@ class ImapClient:
         self.password: str = imap_config["password"]
         self.port: int = int(imap_config["port"])
         self.timeout: int = settings["imap"].get("timeout", 30)
-        self._server: Union[imaplib.IMAP4_SSL, None] = None
+        self._server: imaplib.IMAP4_SSL | None = None
 
     def __enter__(self) -> "ImapClient":
         self._server = imaplib.IMAP4_SSL(
@@ -36,7 +33,9 @@ class ImapClient:
         try:
             config = {k: data[k] for k in required_keys}
         except KeyError as e:
-            raise Exception(f"Missing expected IMAP config key in vault secret: {e}")
+            raise Exception(
+                f"Missing expected IMAP config key in vault secret: {e}"
+            ) from None
         return config
 
     def get_mails(