qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/jira_client.py

@@ -17,10 +17,8 @@ from jira.resources import CustomFieldOption as JiraCustomFieldOption
 from jira.resources import Resource
 from pydantic import BaseModel
 
-from reconcile.utils.secret_reader import SecretReader
-
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Iterable
 
 
 class JiraWatcherSettings(Protocol):
@@ -87,36 +85,18 @@ class IssueField(BaseModel):
     options: list[FieldOption | CustomFieldOption]
 
 
+CREATE_ISSUES = "CREATE_ISSUES"
+TRANSITION_ISSUES = "TRANSITION_ISSUES"
+PERMISSIONS = [CREATE_ISSUES, TRANSITION_ISSUES]
+
+
 class JiraClient:
     """Wrapper around Jira client."""
 
     DEFAULT_CONNECT_TIMEOUT = 60
     DEFAULT_READ_TIMEOUT = 60
 
-    def __init__(
-        self,
-        jira_board: Mapping[str, Any] | None = None,
-        settings: Mapping | None = None,
-        jira_api: JIRA | None = None,
-        project: str | None = None,
-        server: str | None = None,
-    ):
-        """
-        Note: jira_board and settings is to be deprecated. Use JiraClient.create() instead.
-        """
-        if jira_api and jira_board:
-            raise RuntimeError(
-                "jira_board parameter is deprecated. Use JiraClient.create() instead."
-            )
-        if not (jira_api and project):
-            # kept for backwards-compatibility
-            if not jira_board:
-                raise RuntimeError(
-                    "JiraClient needs jira_api and project or jira_board."
-                )
-            self._deprecated_init(jira_board=jira_board, settings=settings)
-            return
-
+    def __init__(self, jira_api: JIRA, project: str, server: str):
         self.server = server
         self.project = project
         self.jira = jira_api
@@ -128,47 +108,31 @@ class JiraClient:
         self.project_issue_types = functools.cache(self._project_issue_types)
         self.project_issue_fields = functools.cache(self._project_issue_fields)
 
-    def _deprecated_init(
-        self, jira_board: Mapping[str, Any], settings: Mapping | None
-    ) -> None:
-        secret_reader = SecretReader(settings=settings)
-        self.project = jira_board["name"]
-        jira_server = jira_board["server"]
-        self.server = jira_server["serverUrl"]
-        token = jira_server["token"]
-        token_auth = secret_reader.read(token)
-        read_timeout = 60
-        connect_timeout = 60
-        if settings and settings["jiraWatcher"]:
-            read_timeout = settings["jiraWatcher"]["readTimeout"]
-            connect_timeout = settings["jiraWatcher"]["connectTimeout"]
-        if not self.server:
-            raise RuntimeError("JiraClient.server is not set.")
-
-        self.jira = JIRA(
-            self.server,
-            token_auth=token_auth,
-            timeout=(read_timeout, connect_timeout),
-            logging=False,
-        )
-
     @staticmethod
     def create(
         project_name: str,
         token: str,
+        email: str | None,
         server_url: str,
         jira_watcher_settings: JiraWatcherSettings | None = None,
     ) -> JiraClient:
+        """Create a Jira client for the given project."""
         read_timeout = JiraClient.DEFAULT_READ_TIMEOUT
         connect_timeout = JiraClient.DEFAULT_CONNECT_TIMEOUT
         if jira_watcher_settings:
             read_timeout = jira_watcher_settings.read_timeout
             connect_timeout = jira_watcher_settings.connect_timeout
+
+        # Jira Cloud uses email+API token for basic auth
+        # Jira Server/Data Center can use token auth (personal access token)
+        auth_params: dict[str, Any] = (
+            {"basic_auth": (email, token)} if email else {"token_auth": token}
+        )
         jira_api = JIRA(
             server=server_url,
-            token_auth=token,
             timeout=(read_timeout, connect_timeout),
             logging=False,
+            **auth_params,
         )
         return JiraClient(
             jira_api=jira_api,
@@ -176,7 +140,13 @@ class JiraClient:
             server=server_url,
         )
 
+    @property
+    def is_cloud(self) -> bool:
+        """Return whether we are on a Cloud based Jira instance."""
+        return self.jira.deploymentType == "Cloud"
+
     def get_issues(self, fields: Iterable | None = None) -> list[Issue]:
+        """Return all issues for our project."""
         block_size = 100
         block_num = 0
 
@@ -227,20 +197,34 @@ class JiraClient:
         return issue
 
     def _my_permissions(self, project: str) -> dict[str, Any]:
+        """Return my permissions for the given project.
+
+        Don't use this function directly, use self.my_permissions which is cached."""
+        if self.is_cloud:
+            return self.jira.my_permissions(
+                projectKey=project, permissions=",".join(PERMISSIONS)
+            )["permissions"]
         return self.jira.my_permissions(projectKey=project)["permissions"]
 
     def can_i(self, permission: str) -> bool:
+        """Return whether I have the given permission in the project."""
         return bool(
             self.my_permissions(project=self.project)[permission]["havePermission"]
         )
 
     def can_create_issues(self) -> bool:
-        return self.can_i("CREATE_ISSUES")
+        """Return whether I can create issues in the project."""
+        return self.can_i(CREATE_ISSUES)
 
     def can_transition_issues(self) -> bool:
-        return self.can_i("TRANSITION_ISSUES")
+        """Return whether I can transition issues in the project."""
+        return self.can_i(TRANSITION_ISSUES)
 
     def _project_issue_types(self, project: str) -> list[IssueType]:
+        """Return all available issue types (e.g. Task, Bug) for the project.
+
+        Don't use this function directly, use self.project_issue_types which is cached.
+        """
         # Don't use self.project here, because of function.cache usage
         return [
             IssueType(id=t.id, name=t.name, statuses=[s.name for s in t.statuses])
@@ -248,6 +232,7 @@ class JiraClient:
         ]
 
     def get_issue_type(self, issue_type: str) -> IssueType | None:
+        """Return a issue type (e.g. Task) for the project if it exists."""
         for _issue_type in self.project_issue_types(self.project):
             if _issue_type.name == issue_type:
                 return _issue_type
@@ -255,15 +240,23 @@ class JiraClient:
 
     @staticmethod
     def _get_allowed_issue_field_options(
-        allowed_values: list[Resource],
+        allowed_values: list[Resource] | list[dict[str, str]],
     ) -> list[FieldOption | CustomFieldOption]:
-        """Return a list of allowed values for a field."""
-        return [
-            CustomFieldOption(value=v.value)
-            if isinstance(v, JiraCustomFieldOption)
-            else FieldOption(name=v.name)
-            for v in allowed_values
-        ]
+        """Return a list of allowed values for a field. E.g. Minor, Major ... for Priority in a Task."""
+        items: list[FieldOption | CustomFieldOption] = []
+        for v in allowed_values:
+            match v:
+                case dict() if "value" in v:
+                    items.append(CustomFieldOption(value=v["value"]))
+                case dict() if "name" in v:
+                    items.append(FieldOption(name=v["name"]))
+                case JiraCustomFieldOption():
+                    items.append(CustomFieldOption(value=v.value))
+                case Resource():
+                    items.append(FieldOption(name=v.name))
+                case _:
+                    logging.warning(f"Unknown allowed value type: {type(v)}")
+        return items
 
     def _project_issue_fields(
         self, project: str, issue_type_id: str
@@ -273,6 +266,27 @@ class JiraClient:
         This API endpoint needs createIssue project permissions.
         """
         # Don't use self.project here, because of function.cache usage
+        if self.is_cloud:
+            metadata = self.jira.createmeta(
+                projectKeys=self.project,
+                issuetypeIds=[issue_type_id],
+                expand="projects.issuetypes.fields",
+            )
+            if not metadata["projects"] or not metadata["projects"][0]["issuetypes"]:
+                return []
+            return [
+                IssueField(
+                    name=field["name"],
+                    id=field_id,
+                    options=self._get_allowed_issue_field_options(
+                        field.get("allowedValues", [])
+                    ),
+                )
+                for field_id, field in metadata["projects"][0]["issuetypes"][0][
+                    "fields"
+                ].items()
+            ]
+
         return [
             IssueField(
                 name=field.name,
@@ -304,6 +318,10 @@ class JiraClient:
 
     def project_priority_scheme(self) -> list[str]:
         """Return a list of all priority IDs for the project."""
+        if self.is_cloud:
+            # Cloud does not have a way to retrieve project specific priority schemes
+            return []
+
         scheme = self.jira.project_priority_scheme(self.project)
         return scheme.optionIds
 
@@ -329,4 +347,5 @@ class JiraClient:
 
     @property
     def is_archived(self) -> bool:
-        return self.jira.project(self.project).archived
+        """Return whether the project is archived."""
+        return getattr(self.jira.project(self.project), "archived", False)

reconcile/utils/jjb_client.py

@@ -1,6 +1,5 @@
 import difflib
 import filecmp
-import json
 import logging
 import os
 import re
@@ -8,7 +7,9 @@ import shutil
 import subprocess
 import tempfile
 import xml.etree.ElementTree as ET
+from collections.abc import Iterable, Mapping
 from os import path
+from pathlib import Path
 from subprocess import (
     PIPE,
     STDOUT,
@@ -17,30 +18,44 @@ from subprocess import (
 from typing import Any
 
 import yaml
-from jenkins_jobs.builder import JenkinsManager
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.parser import YamlParser
-from jenkins_jobs.registry import ModuleRegistry
+from jenkins_jobs.loader import load_files
+from jenkins_jobs.roots import Roots
 from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.json import json_dumps
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.state import State
 from reconcile.utils.vcs import GITHUB_BASE_URL
 
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
 
+class MissingJobUrlError(Exception):
+    pass
+
+
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
-    def __init__(self, configs, ssl_verify=True, secret_reader=None, print_only=False):
+    def __init__(
+        self,
+        configs: list[dict[str, Any]],
+        ssl_verify: bool = True,
+        secret_reader: SecretReaderBase | None = None,
+        print_only: bool = False,
+    ) -> None:
         self.print_only = print_only
         self.secret_reader = secret_reader
+        if not self.print_only and self.secret_reader is None:
+            raise ValueError("secret_reader must be provided if print_only is False")
         self.collect_configs(configs)
         self.modify_logger()
         self.python_https_verify = str(int(ssl_verify))
 
-    def collect_configs(self, configs):
+    def collect_configs(self, configs: list[dict[str, Any]]) -> None:
         instances = {
             c["instance"]["name"]: {
                 "serverUrl": c["instance"]["serverUrl"],
@@ -57,7 +72,7 @@ class JJB:
             server_url = data["serverUrl"]
             wd = tempfile.mkdtemp()
             ini = JJB_INI
-            if not self.print_only:
+            if not self.print_only and self.secret_reader:
                 ini = self.secret_reader.read(token)
                 ini = ini.replace('"', "")
                 ini = ini.replace("false", "False")
@@ -92,7 +107,7 @@ class JJB:
         self.instance_urls = instance_urls
         self.working_dirs = working_dirs
 
-    def overwrite_configs(self, configs):
+    def overwrite_configs(self, configs: Mapping[str, str] | State) -> None:
         """This function will override the existing
         config files in the working directories with
         the supplied configs"""
@@ -101,12 +116,12 @@ class JJB:
             with open(config_path, "w", encoding="locale") as f:
                 f.write(configs[name])
 
-    def sort(self, configs):
+    def sort(self, configs: list[dict[str, Any]]) -> None:
         configs.sort(key=self.sort_by_name)
-        configs.sort(key=self.sort_by_type)
+        configs.sort(key=lambda x: self.sort_by_type(x) or 0)
 
     @staticmethod
-    def sort_by_type(config):
+    def sort_by_type(config: Mapping[str, Any]) -> int:
         if config["type"] == "defaults":
             return 0
         if config["type"] == "global-defaults":
@@ -123,12 +138,13 @@ class JJB:
             return 40
         if config["type"] == "jobs":
             return 50
+        return 100
 
     @staticmethod
-    def sort_by_name(config):
+    def sort_by_name(config: Mapping[str, Any]) -> str:
         return config["name"]
 
-    def get_configs(self):
+    def get_configs(self) -> dict[str, str]:
         """This function gets the configs from the
         working directories"""
         configs = {}
@@ -139,7 +155,7 @@ class JJB:
 
         return configs
 
-    def generate(self, io_dir, fetch_state):
+    def generate(self, io_dir: str, fetch_state: str) -> None:
         """
         Generates job definitions from JJB configs
 
@@ -163,7 +179,7 @@ class JJB:
             self.execute(args)
             throughput.change_files_ownership(io_dir)
 
-    def print_diffs(self, io_dir, instance_name=None):
+    def print_diffs(self, io_dir: str, instance_name: str | None = None) -> None:
         """Print the diffs between the current and
         the desired job definitions"""
         current_path = path.join(io_dir, "jjb", "current")
@@ -179,7 +195,7 @@ class JJB:
         self.print_diff(delete, current_path, "delete")
         self.print_diff(common, desired_path, "update")
 
-    def print_diff(self, files, replace_path, action):
+    def print_diff(self, files: Iterable[str], replace_path: str, action: str) -> None:
         for f in files:
             if action == "update":
                 ft = self.toggle_cd(f)
@@ -210,11 +226,16 @@ class JJB:
                     ]
                     logging.debug("DIFF:\n" + "".join(diff))
 
-    def compare_files(self, from_files, subtract_files, in_op=False):
+    def compare_files(
+        self,
+        from_files: Iterable[str],
+        subtract_files: Iterable[str],
+        in_op: bool = False,
+    ) -> list[str]:
         return [f for f in from_files if (self.toggle_cd(f) in subtract_files) is in_op]
 
     @staticmethod
-    def get_files(search_path, instance_name=None):
+    def get_files(search_path: str, instance_name: str | None = None) -> list[str]:
         if instance_name is not None:
             search_path = path.join(search_path, instance_name)
         return [
@@ -222,7 +243,7 @@ class JJB:
         ]
 
     @staticmethod
-    def toggle_cd(file_name):
+    def toggle_cd(file_name: str) -> str:
         if "desired" in file_name:
             return file_name.replace("desired", "current")
         return file_name.replace("current", "desired")
@@ -248,43 +269,40 @@ class JJB:
                 raise
 
     @staticmethod
-    def get_jjb(args):
+    def get_jjb(args: Iterable[str]) -> Any:
         from jenkins_jobs.cli.entry import JenkinsJobs  # noqa: PLC0415
 
         return JenkinsJobs(args)
 
-    def execute(self, args):
+    def execute(self, args: Iterable[str]) -> None:
         jjb = self.get_jjb(args)
         with toggle_logger():
             jjb.execute()
 
-    def modify_logger(self):
+    def modify_logger(self) -> None:
         yaml.warnings({"YAMLLoadWarning": False})
         formatter = logging.Formatter("%(levelname)s: %(message)s")
         logger = logging.getLogger()
         logger.handlers[0].setFormatter(formatter)
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         for wd in self.working_dirs.values():
             shutil.rmtree(wd)
 
     @retry(exceptions=(JenkinsJobsException))
-    def get_jobs(self, wd, name):
+    def get_jobs(self, wd: str, name: str) -> list[dict[str, Any]]:
         ini_path = f"{wd}/{name}.ini"
         config_path = f"{wd}/config.yaml"
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
-        builder = JenkinsManager(jjb.jjb_config)
-        registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
-        parser = YamlParser(jjb.jjb_config)
-        parser.load_files(jjb.options.path)
-        jobs, _ = parser.expandYaml(registry, jjb.options.names)
+        roots = Roots(jjb.jjb_config)
+        load_files(jjb.jjb_config, roots, [Path(config_path)])
+        job_view_data_list = roots.generate_jobs()
+        return [job.data for job in job_view_data_list]
 
-        return jobs
-
-    def get_job_webhooks_data(self):
-        job_webhooks_data = {}
+    def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
+        job_webhooks_data: dict[str, list[dict[str, Any]]] = {}
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
 
@@ -313,7 +331,7 @@ class JJB:
 
         return job_webhooks_data
 
-    def get_repos(self):
+    def get_repos(self) -> set[str]:
         repos = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -321,11 +339,11 @@ class JJB:
                 job_name = job["name"]
                 try:
                     repos.add(self.get_repo_url(job))
-                except KeyError:
+                except MissingJobUrlError:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
-    def get_admins(self):
+    def get_admins(self) -> set[str]:
         admins = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -340,15 +358,29 @@ class JJB:
         return admins
 
     @staticmethod
-    def get_repo_url(job):
-        repo_url_raw = job["properties"][0]["github"]["url"]
+    def get_repo_url(job: Mapping[str, Any]) -> str:
+        repo_url_raw = job.get("properties", [{}])[0].get("github", {}).get("url")
+
+        # we may be in a Github Branch Source type of job
+        if not repo_url_raw:
+            gh_org = job.get("scm", [{}])[0].get("github", {}).get("repo-owner")
+            gh_repo = job.get("scm", [{}])[0].get("github", {}).get("repo")
+            if gh_org and gh_repo:
+                repo_url_raw = f"https://github.com/{gh_org}/{gh_repo}/"
+            else:
+                raise MissingJobUrlError(
+                    f"Cannot find job url for {job['display-name']}"
+                )
+
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
-    def get_ref(job: dict) -> str:
+    def get_ref(job: Mapping[str, Any]) -> str:
         return job["scm"][0]["git"]["branches"][0]
 
-    def get_all_jobs(self, job_types=None, instance_name=None) -> dict[str, list[dict]]:
+    def get_all_jobs(
+        self, job_types: Iterable[str] | None = None, instance_name: str | None = None
+    ) -> dict[str, list[dict[str, Any]]]:
         if job_types is None:
             job_types = []
         all_jobs: dict[str, list[dict]] = {}
@@ -366,8 +398,8 @@ class JJB:
 
         return all_jobs
 
-    def print_jobs(self, job_name=None):
-        all_jobs = {}
+    def print_jobs(self, job_name: str | None = None) -> None:
+        all_jobs: dict[str, list[dict[str, Any]]] = {}
         found = False
         for name, wd in self.working_dirs.items():
             logging.debug(f"getting jobs from {name}")
@@ -380,7 +412,7 @@ class JJB:
                 found = True
         if not found:
             raise ValueError(f"job name {job_name} is not found")
-        print(json.dumps(all_jobs, indent=2))
+        print(json_dumps(all_jobs, indent=2))
 
     def get_job_by_repo_url(self, repo_url: str, job_type: str) -> dict[str, Any]:
         for jobs in self.get_all_jobs(job_types=[job_type]).values():
@@ -388,13 +420,13 @@ class JJB:
                 try:
                     if self.get_repo_url(job).lower() == repo_url.rstrip("/").lower():
                         return job
-                except KeyError:
+                except MissingJobUrlError:
                     # something wrong here. ignore this job
                     pass
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")
 
     @staticmethod
-    def get_trigger_phrases_regex(job: dict) -> str | None:
+    def get_trigger_phrases_regex(job: Mapping[str, Any]) -> str | None:
         for trigger in job["triggers"]:
             if "gitlab" in trigger:
                 return trigger["gitlab"].get("note-regex")
@@ -403,7 +435,7 @@ class JJB:
         return None
 
     @staticmethod
-    def get_gitlab_webhook_trigger(job: dict) -> list[str]:
+    def get_gitlab_webhook_trigger(job: Mapping[str, Any]) -> list[str]:
         gitlab_triggers = job["triggers"][0]["gitlab"]
         # pr-check job should be triggered by merge request events
         # and certain comments: [test]|/retest|/lgtm|/lgtm cancel|/hold|/hold cancel

reconcile/utils/jobcontroller/controller.py

@@ -3,7 +3,7 @@ import time
 from datetime import datetime
 from typing import Protocol, TextIO
 
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1Job,
     V1ObjectMeta,
@@ -100,7 +100,7 @@ class K8sJobController:
         """
         new_cache = {}
         for item in self.oc.get_items(
-            kind="Job",
+            kind="Job.batch",
             namespace=self.namespace,
         ):
             openshift_resource = OpenshiftResource(

reconcile/utils/jobcontroller/models.py

@@ -38,6 +38,8 @@ class JobValidationError(Exception):
 
 
 JOB_GENERATION_ANNOTATION = "qontract-reconcile/job.generation"
+MAX_JOB_NAME_LENGTH = 63
+UNIT_OF_WORK_DIGEST_LENGTH = 10
 
 
 class K8sJob(ABC):
@@ -72,7 +74,21 @@ class K8sJob(ABC):
     """
 
     def name(self) -> str:
-        return f"{self.name_prefix()}-{self.unit_of_work_digest()}"
+        """
+        Generate the full job name by combining the name prefix with a digest.
+
+        The name is constructed from the name_prefix (truncated to ensure total
+        length compliance) and the unit_of_work_digest. The total length is
+        limited to MAX_JOB_NAME_LENGTH (63 characters) to comply with Kubernetes
+        naming constraints.
+
+        Returns:
+            A unique job name in the format: {name_prefix}-{digest}
+        """
+        prefix = self.name_prefix()[
+            : MAX_JOB_NAME_LENGTH - UNIT_OF_WORK_DIGEST_LENGTH - 1
+        ]
+        return f"{prefix}-{self.unit_of_work_digest(UNIT_OF_WORK_DIGEST_LENGTH)}"
 
     @abstractmethod
     def name_prefix(self) -> str: