qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/jinja2/utils.py

@@ -0,0 +1,278 @@
+import datetime
+from functools import cache
+from typing import Any, Self
+
+import jinja2
+from jinja2.sandbox import SandboxedEnvironment
+from pydantic import BaseModel
+from sretoolbox.utils import retry
+
+from reconcile import queries
+from reconcile.checkpoint import url_makes_sense
+from reconcile.github_users import init_github
+from reconcile.utils import gql
+from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.helpers import flatten
+from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
+from reconcile.utils.jinja2.filters import (
+    eval_filter,
+    extract_jsonpath,
+    hash_list,
+    json_pointers,
+    json_to_dict,
+    matches_jsonpath,
+    str_format,
+    urlescape,
+    urlunescape,
+    yaml_to_dict,
+)
+from reconcile.utils.secret_reader import SecretNotFound, SecretReader, SecretReaderBase
+from reconcile.utils.vault import SecretFieldNotFound
+
+
+class Jinja2TemplateError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("error processing jinja2 template: " + str(msg))
+
+
+class TemplateRenderOptions(BaseModel):
+    trim_blocks: bool
+    lstrip_blocks: bool
+    keep_trailing_newline: bool
+
+    class Config:
+        frozen = True
+
+    @classmethod
+    def create(
+        cls,
+        trim_blocks: bool | None = None,
+        lstrip_blocks: bool | None = None,
+        keep_trailing_newline: bool | None = None,
+    ) -> Self:
+        return cls(
+            trim_blocks=trim_blocks or False,
+            lstrip_blocks=lstrip_blocks or False,
+            keep_trailing_newline=keep_trailing_newline or False,
+        )
+
+
+@cache
+def compile_jinja2_template(
+    body: str,
+    extra_curly: bool = False,
+    template_render_options: TemplateRenderOptions | None = None,
+) -> Any:
+    if not template_render_options:
+        template_render_options = TemplateRenderOptions.create()
+    env: dict[str, Any] = template_render_options.dict()
+    if extra_curly:
+        env.update({
+            "block_start_string": "{{%",
+            "block_end_string": "%}}",
+            "variable_start_string": "{{{",
+            "variable_end_string": "}}}",
+            "comment_start_string": "{{#",
+            "comment_end_string": "#}}",
+        })
+
+    jinja_env = SandboxedEnvironment(
+        extensions=[B64EncodeExtension, RaiseErrorExtension],
+        undefined=jinja2.StrictUndefined,
+        **env,
+    )
+    jinja_env.filters.update({
+        "json_to_dict": json_to_dict,
+        "yaml_to_dict": yaml_to_dict,
+        "urlescape": urlescape,
+        "urlunescape": urlunescape,
+        "eval": eval_filter,
+        "extract_jsonpath": extract_jsonpath,
+        "matches_jsonpath": matches_jsonpath,
+        "json_pointers": json_pointers,
+        "str_format": str_format,
+    })
+
+    return jinja_env.from_string(body)
+
+
+def lookup_github_file_content(
+    repo: str,
+    path: str,
+    ref: str,
+    tvars: dict[str, Any] | None = None,
+    settings: dict[str, Any] | None = None,
+    secret_reader: SecretReaderBase | None = None,
+) -> str:
+    if tvars is not None:
+        repo = process_jinja2_template(
+            body=repo, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        path = process_jinja2_template(
+            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        ref = process_jinja2_template(
+            body=ref, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+
+    gh = init_github()
+    c = gh.get_repo(repo).get_contents(path, ref).decoded_content
+    return c.decode("utf-8")
+
+
+def lookup_graphql_query_results(query: str, **kwargs: dict[str, Any]) -> list[Any]:
+    gqlapi = gql.get_api()
+    resource = gqlapi.get_resource(query)["content"]
+    rendered_resource = jinja2.Template(resource).render(**kwargs)
+    results = next(iter(gqlapi.query(rendered_resource).values()))
+    return results
+
+
+def lookup_s3_object(
+    account_name: str,
+    bucket_name: str,
+    path: str,
+    region_name: str | None = None,
+) -> str:
+    settings = queries.get_app_interface_settings()
+    accounts = queries.get_aws_accounts(name=account_name)
+    if not accounts:
+        raise Exception(f"aws account not found: {account_name}")
+
+    with AWSApi(1, accounts, settings=settings, init_users=False) as aws_api:
+        return aws_api.get_s3_object_content(
+            account_name,
+            bucket_name,
+            path,
+            region_name=region_name,
+        )
+
+
+def list_s3_objects(
+    account_name: str,
+    bucket_name: str,
+    path: str,
+    region_name: str | None = None,
+) -> list[str]:
+    settings = queries.get_app_interface_settings()
+    accounts = queries.get_aws_accounts(name=account_name)
+    if not accounts:
+        raise Exception(f"aws account not found: {account_name}")
+
+    with AWSApi(1, accounts, settings=settings, init_users=False) as aws_api:
+        return aws_api.list_s3_objects(
+            account_name,
+            bucket_name,
+            path,
+            region_name=region_name,
+        )
+
+
+@retry()
+def lookup_secret(
+    path: str,
+    key: str,
+    version: str | None = None,
+    tvars: dict[str, Any] | None = None,
+    allow_not_found: bool = False,
+    settings: dict[str, Any] | None = None,
+    secret_reader: SecretReaderBase | None = None,
+) -> str | None:
+    if tvars is not None:
+        path = process_jinja2_template(
+            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        key = process_jinja2_template(
+            body=key, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        if version and not isinstance(version, int):
+            version = process_jinja2_template(
+                body=version, vars=tvars, settings=settings, secret_reader=secret_reader
+            )
+    secret = {"path": path, "field": key, "version": version}
+    try:
+        if not secret_reader:
+            secret_reader = SecretReader(settings)
+        return secret_reader.read(secret)
+    except (SecretNotFound, SecretFieldNotFound) as e:
+        if allow_not_found:
+            return None
+        raise FetchSecretError(e) from None
+    except Exception as e:
+        raise FetchSecretError(e) from e
+
+
+def process_jinja2_template(
+    body: str,
+    vars: dict[str, Any] | None = None,
+    extra_curly: bool = False,
+    settings: dict[str, Any] | None = None,
+    secret_reader: SecretReaderBase | None = None,
+    template_render_options: TemplateRenderOptions | None = None,
+) -> Any:
+    if vars is None:
+        vars = {}
+    vars.update({
+        "vault": lambda p, k, v=None, allow_not_found=False: lookup_secret(
+            path=p,
+            key=k,
+            version=v,
+            tvars=vars,
+            allow_not_found=allow_not_found,
+            settings=settings,
+            secret_reader=secret_reader,
+        ),
+        "github": lambda u, p, r, v=None: lookup_github_file_content(
+            repo=u,
+            path=p,
+            ref=r,
+            tvars=vars,
+            settings=settings,
+            secret_reader=secret_reader,
+        ),
+        "urlescape": lambda u, s="/", e=None: urlescape(string=u, safe=s, encoding=e),
+        "urlunescape": lambda u, e=None: urlunescape(string=u, encoding=e),
+        "hash_list": hash_list,
+        "query": lookup_graphql_query_results,
+        "url": url_makes_sense,
+        "s3": lookup_s3_object,
+        "s3_ls": list_s3_objects,
+        "flatten_dict": flatten,
+        "yesterday": lambda: (datetime.datetime.now() - datetime.timedelta(1)).strftime(
+            "%Y-%m-%d"
+        ),
+    })
+    if "_template_mocks" in vars:
+        for k, v in vars["_template_mocks"].items():
+            vars[k] = lambda *args, **kwargs: v  # noqa: B023
+    try:
+        template = compile_jinja2_template(body, extra_curly, template_render_options)
+        r = template.render(vars)
+    except Exception as e:
+        raise Jinja2TemplateError(e) from None
+    return r
+
+
+def process_extracurlyjinja2_template(
+    body: str,
+    vars: dict[str, Any] | None = None,
+    extra_curly: bool = True,  # ignored. Just to be compatible with process_jinja2_template
+    settings: dict[str, Any] | None = None,
+    secret_reader: SecretReaderBase | None = None,
+    template_render_options: TemplateRenderOptions | None = None,
+) -> Any:
+    if vars is None:
+        vars = {}
+    return process_jinja2_template(
+        body,
+        vars=vars,
+        extra_curly=True,
+        settings=settings,
+        secret_reader=secret_reader,
+        template_render_options=template_render_options,
+    )
+
+
+class FetchSecretError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("error fetching secret: " + str(msg))

reconcile/utils/jira_client.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import functools
 import logging
 from collections.abc import (
     Iterable,
@@ -5,7 +8,7 @@ from collections.abc import (
 )
 from typing import (
     Any,
-    Optional,
+    Protocol,
 )
 
 from jira import (
@@ -13,38 +16,131 @@ from jira import (
     Issue,
 )
 from jira.client import ResultList
+from pydantic import BaseModel
 
 from reconcile.utils.secret_reader import SecretReader
 
 
+class JiraWatcherSettings(Protocol):
+    read_timeout: int
+    connect_timeout: int
+
+
+class SecurityLevel(BaseModel):
+    """Jira security level."""
+
+    id: str
+    name: str
+
+
+class Priority(BaseModel):
+    """Jira priority."""
+
+    id: str
+    name: str
+
+
+class IssueType(BaseModel):
+    """Jira issue type."""
+
+    id: str
+    name: str
+    statuses: list[str]
+
+
 class JiraClient:
     """Wrapper around Jira client."""
 
+    DEFAULT_CONNECT_TIMEOUT = 60
+    DEFAULT_READ_TIMEOUT = 60
+
     def __init__(
-        self, jira_board: Mapping[str, Any], settings: Optional[Mapping] = None
+        self,
+        jira_board: Mapping[str, Any] | None = None,
+        settings: Mapping | None = None,
+        jira_api: JIRA | None = None,
+        project: str | None = None,
+        server: str | None = None,
     ):
-        self.secret_reader = SecretReader(settings=settings)
+        """
+        Note: jira_board and settings is to be deprecated. Use JiraClient.create() instead.
+        """
+        if jira_api and jira_board:
+            raise RuntimeError(
+                "jira_board parameter is deprecated. Use JiraClient.create() instead."
+            )
+        if not (jira_api and project):
+            # kept for backwards-compatibility
+            if not jira_board:
+                raise RuntimeError(
+                    "JiraClient needs jira_api and project or jira_board."
+                )
+            self._deprecated_init(jira_board=jira_board, settings=settings)
+            return
+
+        self.server = server
+        self.project = project
+        self.jira = jira_api
+
+        # some caches
+        self.priorities = functools.lru_cache(maxsize=None)(self._priorities)
+        self.public_projects = functools.lru_cache(maxsize=None)(self._public_projects)
+        self.my_permissions = functools.lru_cache(maxsize=None)(self._my_permissions)
+
+    def _deprecated_init(
+        self, jira_board: Mapping[str, Any], settings: Mapping | None
+    ) -> None:
+        secret_reader = SecretReader(settings=settings)
         self.project = jira_board["name"]
         jira_server = jira_board["server"]
         self.server = jira_server["serverUrl"]
         token = jira_server["token"]
-        token_auth = self.secret_reader.read(token)
+        token_auth = secret_reader.read(token)
         read_timeout = 60
         connect_timeout = 60
         if settings and settings["jiraWatcher"]:
             read_timeout = settings["jiraWatcher"]["readTimeout"]
             connect_timeout = settings["jiraWatcher"]["connectTimeout"]
+        if not self.server:
+            raise RuntimeError("JiraClient.server is not set.")
 
         self.jira = JIRA(
-            self.server, token_auth=token_auth, timeout=(read_timeout, connect_timeout)
+            self.server,
+            token_auth=token_auth,
+            timeout=(read_timeout, connect_timeout),
+            logging=False,
         )
 
-    def get_issues(self, fields: Optional[Mapping] = None) -> list[Issue]:
+    @staticmethod
+    def create(
+        project_name: str,
+        token: str,
+        server_url: str,
+        jira_watcher_settings: JiraWatcherSettings | None = None,
+    ) -> JiraClient:
+        read_timeout = JiraClient.DEFAULT_READ_TIMEOUT
+        connect_timeout = JiraClient.DEFAULT_CONNECT_TIMEOUT
+        if jira_watcher_settings:
+            read_timeout = jira_watcher_settings.read_timeout
+            connect_timeout = jira_watcher_settings.connect_timeout
+        jira_api = JIRA(
+            server=server_url,
+            token_auth=token,
+            timeout=(read_timeout, connect_timeout),
+            logging=False,
+        )
+        return JiraClient(
+            jira_api=jira_api,
+            project=project_name,
+            server=server_url,
+        )
+
+    def get_issues(self, fields: Mapping | None = None) -> list[Issue]:
         block_size = 100
         block_num = 0
 
         all_issues: list[Issue] = []
-        jql = "project={}".format(self.project)
+        jql = f"project={self.project}"
         kwargs: dict[str, Any] = {}
         if fields:
             kwargs["fields"] = ",".join(fields)
@@ -72,7 +168,7 @@ class JiraClient:
         self,
         summary: str,
         body: str,
-        labels: Optional[Iterable[str]] = None,
+        labels: Iterable[str] | None = None,
         links: Iterable[str] = (),
     ) -> Issue:
         """Create an issue in our project with the given labels."""
@@ -88,3 +184,64 @@ class JiraClient:
                 type="is caused by", inwardIssue=issue.key, outwardIssue=ln
             )
         return issue
+
+    def _my_permissions(self, project: str) -> dict[str, Any]:
+        return self.jira.my_permissions(projectKey=project)["permissions"]
+
+    def can_i(self, permission: str) -> bool:
+        return bool(
+            self.my_permissions(project=self.project)[permission]["havePermission"]
+        )
+
+    def can_create_issues(self) -> bool:
+        return self.can_i("CREATE_ISSUES")
+
+    def can_transition_issues(self) -> bool:
+        return self.can_i("TRANSITION_ISSUES")
+
+    def project_issue_types(self) -> list[IssueType]:
+        return [
+            IssueType(id=t.id, name=t.name, statuses=[s.name for s in t.statuses])
+            for t in self.jira.issue_types_for_project(self.project)
+        ]
+
+    def security_levels(self) -> list[SecurityLevel]:
+        """Return a list of all available security levels for the project.
+
+        This API endpoint needs admin/owner project permissions.
+        """
+        scheme = self.jira.project_issue_security_level_scheme(self.project)
+        return [SecurityLevel(id=level.id, name=level.name) for level in scheme.levels]
+
+    def _priorities(self) -> list[Priority]:
+        """Return a list of all available Jira priorities."""
+        return [Priority(id=p.id, name=p.name) for p in self.jira.priorities()]
+
+    def project_priority_scheme(self) -> list[str]:
+        """Return a list of all priority IDs for the project."""
+        scheme = self.jira.project_priority_scheme(self.project)
+        return scheme.optionIds
+
+    def _public_projects(self) -> list[str]:
+        """Return a list of all public available projects."""
+        if not self.server:
+            raise RuntimeError("JiraClient.server is not set.")
+
+        # use anonymous access to get public projects
+        jira_api_anon = JIRA(
+            server=self.server,
+            timeout=(
+                JiraClient.DEFAULT_READ_TIMEOUT,
+                JiraClient.DEFAULT_CONNECT_TIMEOUT,
+            ),
+            logging=False,
+        )
+        return [project.key for project in jira_api_anon.projects()]
+
+    def components(self) -> list[str]:
+        """Return a list of all components for the project."""
+        return [c.name for c in self.jira.project_components(self.project)]
+
+    @property
+    def is_archived(self) -> bool:
+        return self.jira.project(self.project).archived

reconcile/utils/jjb_client.py

@@ -14,10 +14,7 @@ from subprocess import (
     STDOUT,
     CalledProcessError,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import yaml
 from jenkins_jobs.builder import JenkinsManager
@@ -63,8 +60,8 @@ class JJB:  # pylint: disable=too-many-public-methods
                 ini = self.secret_reader.read(token)
                 ini = ini.replace('"', "")
                 ini = ini.replace("false", "False")
-            ini_file_path = "{}/{}.ini".format(wd, name)
-            with open(ini_file_path, "w") as f:
+            ini_file_path = f"{wd}/{name}.ini"
+            with open(ini_file_path, "w", encoding="locale") as f:
                 f.write(ini)
                 f.write("\n")
             working_dirs[name] = wd
@@ -75,14 +72,18 @@ class JJB:  # pylint: disable=too-many-public-methods
         for c in configs:
             instance_name = c["instance"]["name"]
             config = c["config"]
-            config_file_path = "{}/config.yaml".format(working_dirs[instance_name])
+            config_file_path = f"{working_dirs[instance_name]}/config.yaml"
             if config:
-                with open(config_file_path, "a") as f:
-                    yaml.dump(yaml.load(config, Loader=yaml.FullLoader), f)
+                content = yaml.load(config, Loader=yaml.FullLoader)
+                if c["type"] == "jobs":
+                    for item in content:
+                        item["project"]["app_name"] = c["app"]["name"]
+                with open(config_file_path, "a", encoding="locale") as f:
+                    yaml.dump(content, f)
                     f.write("\n")
             else:
                 config = c["config_path"]["content"]
-                with open(config_file_path, "a") as f:
+                with open(config_file_path, "a", encoding="locale") as f:
                     f.write(config)
                     f.write("\n")
 
@@ -95,8 +96,8 @@ class JJB:  # pylint: disable=too-many-public-methods
         config files in the working directories with
         the supplied configs"""
         for name, wd in self.working_dirs.items():
-            config_path = "{}/config.yaml".format(wd)
-            with open(config_path, "w") as f:
+            config_path = f"{wd}/config.yaml"
+            with open(config_path, "w", encoding="locale") as f:
                 f.write(configs[name])
 
     def sort(self, configs):
@@ -131,8 +132,8 @@ class JJB:  # pylint: disable=too-many-public-methods
         working directories"""
         configs = {}
         for name, wd in self.working_dirs.items():
-            config_path = "{}/config.yaml".format(wd)
-            with open(config_path, "r") as f:
+            config_path = f"{wd}/config.yaml"
+            with open(config_path, encoding="locale") as f:
                 configs[name] = f.read()
 
         return configs
@@ -145,8 +146,8 @@ class JJB:  # pylint: disable=too-many-public-methods
         :param fetch_state: subdirectory to use ('desired' or 'current')
         """
         for name, wd in self.working_dirs.items():
-            ini_path = "{}/{}.ini".format(wd, name)
-            config_path = "{}/config.yaml".format(wd)
+            ini_path = f"{wd}/{name}.ini"
+            config_path = f"{wd}/config.yaml"
 
             output_dir = path.join(io_dir, "jjb", fetch_state, name)
             args = [
@@ -192,7 +193,7 @@ class JJB:  # pylint: disable=too-many-public-methods
             logging.info([action, item_type, instance, item])
 
             if action == "update":
-                with open(ft) as c, open(f) as d:
+                with open(ft, encoding="locale") as c, open(f, encoding="locale") as d:
                     clines = c.readlines()
                     dlines = d.readlines()
 
@@ -223,8 +224,8 @@ class JJB:  # pylint: disable=too-many-public-methods
 
     def update(self) -> None:
         for name, wd in self.working_dirs.items():
-            ini_path = "{}/{}.ini".format(wd, name)
-            config_path = "{}/config.yaml".format(wd)
+            ini_path = f"{wd}/{name}.ini"
+            config_path = f"{wd}/config.yaml"
 
             os.environ["PYTHONHTTPSVERIFY"] = self.python_https_verify
             cmd = ["jenkins-jobs", "--conf", ini_path, "update", config_path]
@@ -243,7 +244,7 @@ class JJB:  # pylint: disable=too-many-public-methods
 
     @staticmethod
     def get_jjb(args):
-        from jenkins_jobs.cli.entry import JenkinsJobs
+        from jenkins_jobs.cli.entry import JenkinsJobs  # noqa: PLC0415
 
         return JenkinsJobs(args)
 
@@ -264,8 +265,8 @@ class JJB:  # pylint: disable=too-many-public-methods
 
     @retry(exceptions=(JenkinsJobsException))
     def get_jobs(self, wd, name):
-        ini_path = "{}/{}.ini".format(wd, name)
-        config_path = "{}/config.yaml".format(wd)
+        ini_path = f"{wd}/{name}.ini"
+        config_path = f"{wd}/config.yaml"
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
@@ -287,13 +288,15 @@ class JJB:  # pylint: disable=too-many-public-methods
                     project_url_raw = job["properties"][0]["github"]["url"]
                     if "https://github.com" in project_url_raw:
                         continue
+                    if str(job.get("disabled")).lower() == "true":
+                        continue
                     job_url = "{}/project/{}".format(
                         self.instance_urls[name], job["name"]
                     )
                     project_url = project_url_raw.strip("/").replace(".git", "")
-                    gitlab_triggers = job["triggers"][0]["gitlab"]
-                    mr_trigger = gitlab_triggers["trigger-merge-request"]
-                    trigger = "mr" if mr_trigger else "push"
+                    trigger = self.get_gitlab_webhook_trigger(job)
+                    if not trigger:
+                        continue
                     hook = {
                         "job_url": job_url,
                         "trigger": trigger,
@@ -314,7 +317,7 @@ class JJB:  # pylint: disable=too-many-public-methods
                 try:
                     repos.add(self.get_repo_url(job))
                 except KeyError:
-                    logging.debug("missing github url: {}".format(job_name))
+                    logging.debug(f"missing github url: {job_name}")
         return repos
 
     def get_admins(self):
@@ -340,9 +343,7 @@ class JJB:  # pylint: disable=too-many-public-methods
     def get_ref(job: dict) -> str:
         return job["scm"][0]["git"]["branches"][0]
 
-    def get_all_jobs(
-        self, job_types=None, instance_name=None, include_test=False
-    ) -> dict[str, list[dict]]:
+    def get_all_jobs(self, job_types=None, instance_name=None) -> dict[str, list[dict]]:
         if job_types is None:
             job_types = []
         all_jobs: dict[str, list[dict]] = {}
@@ -356,11 +357,6 @@ class JJB:  # pylint: disable=too-many-public-methods
                 job_name = job["name"]
                 if not any(job_type in job_name for job_type in job_types):
                     continue
-                if not include_test and "test" in job_name:
-                    continue
-                # temporarily ignore openshift-saas-deploy jobs
-                if job_name.startswith("openshift-saas-deploy"):
-                    continue
                 all_jobs[name].append(job)
 
         return all_jobs
@@ -393,10 +389,26 @@ class JJB:  # pylint: disable=too-many-public-methods
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")
 
     @staticmethod
-    def get_trigger_phrases_regex(job: dict) -> Optional[str]:
+    def get_trigger_phrases_regex(job: dict) -> str | None:
         for trigger in job["triggers"]:
             if "gitlab" in trigger:
                 return trigger["gitlab"].get("note-regex")
             if "github-pull-request" in trigger:
                 return trigger["github-pull-request"].get("trigger-phrase")
         return None
+
+    @staticmethod
+    def get_gitlab_webhook_trigger(job: dict) -> list[str]:
+        gitlab_triggers = job["triggers"][0]["gitlab"]
+        # pr-check job should be triggered by merge request events
+        # and certain comments: [test]|/retest|/lgtm|/lgtm cancel|/hold|/hold cancel
+        if gitlab_triggers.get("trigger-merge-request"):
+            return ["mr", "note"]
+        # build main/master job should be triggered by push events
+        elif gitlab_triggers.get("trigger-push"):
+            return ["push"]
+        # On-demand test job should be triggered by special comment
+        elif gitlab_triggers.get("trigger-note"):
+            return ["note"]
+        else:
+            return []