qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/utils/jira_client.py

@@ -17,10 +17,8 @@ from jira.resources import CustomFieldOption as JiraCustomFieldOption
 from jira.resources import Resource
 from pydantic import BaseModel
 
-from reconcile.utils.secret_reader import SecretReader
-
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Iterable
 
 
 class JiraWatcherSettings(Protocol):
@@ -87,36 +85,18 @@ class IssueField(BaseModel):
     options: list[FieldOption | CustomFieldOption]
 
 
+CREATE_ISSUES = "CREATE_ISSUES"
+TRANSITION_ISSUES = "TRANSITION_ISSUES"
+PERMISSIONS = [CREATE_ISSUES, TRANSITION_ISSUES]
+
+
 class JiraClient:
     """Wrapper around Jira client."""
 
     DEFAULT_CONNECT_TIMEOUT = 60
     DEFAULT_READ_TIMEOUT = 60
 
-    def __init__(
-        self,
-        jira_board: Mapping[str, Any] | None = None,
-        settings: Mapping | None = None,
-        jira_api: JIRA | None = None,
-        project: str | None = None,
-        server: str | None = None,
-    ):
-        """
-        Note: jira_board and settings is to be deprecated. Use JiraClient.create() instead.
-        """
-        if jira_api and jira_board:
-            raise RuntimeError(
-                "jira_board parameter is deprecated. Use JiraClient.create() instead."
-            )
-        if not (jira_api and project):
-            # kept for backwards-compatibility
-            if not jira_board:
-                raise RuntimeError(
-                    "JiraClient needs jira_api and project or jira_board."
-                )
-            self._deprecated_init(jira_board=jira_board, settings=settings)
-            return
-
+    def __init__(self, jira_api: JIRA, project: str, server: str):
         self.server = server
         self.project = project
         self.jira = jira_api
@@ -128,47 +108,31 @@ class JiraClient:
         self.project_issue_types = functools.cache(self._project_issue_types)
         self.project_issue_fields = functools.cache(self._project_issue_fields)
 
-    def _deprecated_init(
-        self, jira_board: Mapping[str, Any], settings: Mapping | None
-    ) -> None:
-        secret_reader = SecretReader(settings=settings)
-        self.project = jira_board["name"]
-        jira_server = jira_board["server"]
-        self.server = jira_server["serverUrl"]
-        token = jira_server["token"]
-        token_auth = secret_reader.read(token)
-        read_timeout = 60
-        connect_timeout = 60
-        if settings and settings["jiraWatcher"]:
-            read_timeout = settings["jiraWatcher"]["readTimeout"]
-            connect_timeout = settings["jiraWatcher"]["connectTimeout"]
-        if not self.server:
-            raise RuntimeError("JiraClient.server is not set.")
-
-        self.jira = JIRA(
-            self.server,
-            token_auth=token_auth,
-            timeout=(read_timeout, connect_timeout),
-            logging=False,
-        )
-
     @staticmethod
     def create(
         project_name: str,
         token: str,
+        email: str | None,
         server_url: str,
         jira_watcher_settings: JiraWatcherSettings | None = None,
     ) -> JiraClient:
+        """Create a Jira client for the given project."""
         read_timeout = JiraClient.DEFAULT_READ_TIMEOUT
         connect_timeout = JiraClient.DEFAULT_CONNECT_TIMEOUT
         if jira_watcher_settings:
             read_timeout = jira_watcher_settings.read_timeout
             connect_timeout = jira_watcher_settings.connect_timeout
+
+        # Jira Cloud uses email+API token for basic auth
+        # Jira Server/Data Center can use token auth (personal access token)
+        auth_params: dict[str, Any] = (
+            {"basic_auth": (email, token)} if email else {"token_auth": token}
+        )
         jira_api = JIRA(
             server=server_url,
-            token_auth=token,
             timeout=(read_timeout, connect_timeout),
             logging=False,
+            **auth_params,
         )
         return JiraClient(
             jira_api=jira_api,
@@ -176,7 +140,13 @@ class JiraClient:
             server=server_url,
         )
 
+    @property
+    def is_cloud(self) -> bool:
+        """Return whether we are on a Cloud based Jira instance."""
+        return self.jira.deploymentType == "Cloud"
+
     def get_issues(self, fields: Iterable | None = None) -> list[Issue]:
+        """Return all issues for our project."""
         block_size = 100
         block_num = 0
 
@@ -227,20 +197,34 @@ class JiraClient:
         return issue
 
     def _my_permissions(self, project: str) -> dict[str, Any]:
+        """Return my permissions for the given project.
+
+        Don't use this function directly, use self.my_permissions which is cached."""
+        if self.is_cloud:
+            return self.jira.my_permissions(
+                projectKey=project, permissions=",".join(PERMISSIONS)
+            )["permissions"]
         return self.jira.my_permissions(projectKey=project)["permissions"]
 
     def can_i(self, permission: str) -> bool:
+        """Return whether I have the given permission in the project."""
         return bool(
             self.my_permissions(project=self.project)[permission]["havePermission"]
         )
 
     def can_create_issues(self) -> bool:
-        return self.can_i("CREATE_ISSUES")
+        """Return whether I can create issues in the project."""
+        return self.can_i(CREATE_ISSUES)
 
     def can_transition_issues(self) -> bool:
-        return self.can_i("TRANSITION_ISSUES")
+        """Return whether I can transition issues in the project."""
+        return self.can_i(TRANSITION_ISSUES)
 
     def _project_issue_types(self, project: str) -> list[IssueType]:
+        """Return all available issue types (e.g. Task, Bug) for the project.
+
+        Don't use this function directly, use self.project_issue_types which is cached.
+        """
         # Don't use self.project here, because of function.cache usage
         return [
             IssueType(id=t.id, name=t.name, statuses=[s.name for s in t.statuses])
@@ -248,6 +232,7 @@ class JiraClient:
         ]
 
     def get_issue_type(self, issue_type: str) -> IssueType | None:
+        """Return a issue type (e.g. Task) for the project if it exists."""
         for _issue_type in self.project_issue_types(self.project):
             if _issue_type.name == issue_type:
                 return _issue_type
@@ -255,15 +240,23 @@ class JiraClient:
 
     @staticmethod
     def _get_allowed_issue_field_options(
-        allowed_values: list[Resource],
+        allowed_values: list[Resource] | list[dict[str, str]],
     ) -> list[FieldOption | CustomFieldOption]:
-        """Return a list of allowed values for a field."""
-        return [
-            CustomFieldOption(value=v.value)
-            if isinstance(v, JiraCustomFieldOption)
-            else FieldOption(name=v.name)
-            for v in allowed_values
-        ]
+        """Return a list of allowed values for a field. E.g. Minor, Major ... for Priority in a Task."""
+        items: list[FieldOption | CustomFieldOption] = []
+        for v in allowed_values:
+            match v:
+                case dict() if "value" in v:
+                    items.append(CustomFieldOption(value=v["value"]))
+                case dict() if "name" in v:
+                    items.append(FieldOption(name=v["name"]))
+                case JiraCustomFieldOption():
+                    items.append(CustomFieldOption(value=v.value))
+                case Resource():
+                    items.append(FieldOption(name=v.name))
+                case _:
+                    logging.warning(f"Unknown allowed value type: {type(v)}")
+        return items
 
     def _project_issue_fields(
         self, project: str, issue_type_id: str
@@ -273,6 +266,27 @@ class JiraClient:
         This API endpoint needs createIssue project permissions.
         """
         # Don't use self.project here, because of function.cache usage
+        if self.is_cloud:
+            metadata = self.jira.createmeta(
+                projectKeys=self.project,
+                issuetypeIds=[issue_type_id],
+                expand="projects.issuetypes.fields",
+            )
+            if not metadata["projects"] or not metadata["projects"][0]["issuetypes"]:
+                return []
+            return [
+                IssueField(
+                    name=field["name"],
+                    id=field_id,
+                    options=self._get_allowed_issue_field_options(
+                        field.get("allowedValues", [])
+                    ),
+                )
+                for field_id, field in metadata["projects"][0]["issuetypes"][0][
+                    "fields"
+                ].items()
+            ]
+
         return [
             IssueField(
                 name=field.name,
@@ -304,6 +318,10 @@ class JiraClient:
 
     def project_priority_scheme(self) -> list[str]:
         """Return a list of all priority IDs for the project."""
+        if self.is_cloud:
+            # Cloud does not have a way to retrieve project specific priority schemes
+            return []
+
         scheme = self.jira.project_priority_scheme(self.project)
         return scheme.optionIds
 
@@ -329,4 +347,5 @@ class JiraClient:
 
     @property
     def is_archived(self) -> bool:
-        return self.jira.project(self.project).archived
+        """Return whether the project is archived."""
+        return getattr(self.jira.project(self.project), "archived", False)

reconcile/utils/jjb_client.py

@@ -9,6 +9,7 @@ import tempfile
 import xml.etree.ElementTree as ET
 from collections.abc import Iterable, Mapping
 from os import path
+from pathlib import Path
 from subprocess import (
     PIPE,
     STDOUT,
@@ -17,10 +18,9 @@ from subprocess import (
 from typing import Any
 
 import yaml
-from jenkins_jobs.builder import JenkinsManager
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.parser import YamlParser
-from jenkins_jobs.registry import ModuleRegistry
+from jenkins_jobs.loader import load_files
+from jenkins_jobs.roots import Roots
 from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
@@ -33,6 +33,10 @@ from reconcile.utils.vcs import GITHUB_BASE_URL
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
 
+class MissingJobUrlError(Exception):
+    pass
+
+
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
@@ -292,13 +296,10 @@ class JJB:
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
-        builder = JenkinsManager(jjb.jjb_config)
-        registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
-        parser = YamlParser(jjb.jjb_config)
-        parser.load_files(jjb.options.path)
-        jobs, _ = parser.expandYaml(registry, jjb.options.names)
-
-        return jobs
+        roots = Roots(jjb.jjb_config)
+        load_files(jjb.jjb_config, roots, [Path(config_path)])
+        job_view_data_list = roots.generate_jobs()
+        return [job.data for job in job_view_data_list]
 
     def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
         job_webhooks_data: dict[str, list[dict[str, Any]]] = {}
@@ -338,7 +339,7 @@ class JJB:
                 job_name = job["name"]
                 try:
                     repos.add(self.get_repo_url(job))
-                except KeyError:
+                except MissingJobUrlError:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
@@ -358,7 +359,19 @@ class JJB:
 
     @staticmethod
     def get_repo_url(job: Mapping[str, Any]) -> str:
-        repo_url_raw = job["properties"][0]["github"]["url"]
+        repo_url_raw = job.get("properties", [{}])[0].get("github", {}).get("url")
+
+        # we may be in a Github Branch Source type of job
+        if not repo_url_raw:
+            gh_org = job.get("scm", [{}])[0].get("github", {}).get("repo-owner")
+            gh_repo = job.get("scm", [{}])[0].get("github", {}).get("repo")
+            if gh_org and gh_repo:
+                repo_url_raw = f"https://github.com/{gh_org}/{gh_repo}/"
+            else:
+                raise MissingJobUrlError(
+                    f"Cannot find job url for {job['display-name']}"
+                )
+
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
@@ -407,7 +420,7 @@ class JJB:
                 try:
                     if self.get_repo_url(job).lower() == repo_url.rstrip("/").lower():
                         return job
-                except KeyError:
+                except MissingJobUrlError:
                     # something wrong here. ignore this job
                     pass
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")

reconcile/utils/jobcontroller/controller.py

@@ -3,7 +3,7 @@ import time
 from datetime import datetime
 from typing import Protocol, TextIO
 
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1Job,
     V1ObjectMeta,
@@ -100,7 +100,7 @@ class K8sJobController:
         """
         new_cache = {}
         for item in self.oc.get_items(
-            kind="Job",
+            kind="Job.batch",
             namespace=self.namespace,
         ):
             openshift_resource = OpenshiftResource(

reconcile/utils/jobcontroller/models.py

@@ -38,6 +38,8 @@ class JobValidationError(Exception):
 
 
 JOB_GENERATION_ANNOTATION = "qontract-reconcile/job.generation"
+MAX_JOB_NAME_LENGTH = 63
+UNIT_OF_WORK_DIGEST_LENGTH = 10
 
 
 class K8sJob(ABC):
@@ -72,7 +74,21 @@ class K8sJob(ABC):
     """
 
     def name(self) -> str:
-        return f"{self.name_prefix()}-{self.unit_of_work_digest()}"
+        """
+        Generate the full job name by combining the name prefix with a digest.
+
+        The name is constructed from the name_prefix (truncated to ensure total
+        length compliance) and the unit_of_work_digest. The total length is
+        limited to MAX_JOB_NAME_LENGTH (63 characters) to comply with Kubernetes
+        naming constraints.
+
+        Returns:
+            A unique job name in the format: {name_prefix}-{digest}
+        """
+        prefix = self.name_prefix()[
+            : MAX_JOB_NAME_LENGTH - UNIT_OF_WORK_DIGEST_LENGTH - 1
+        ]
+        return f"{prefix}-{self.unit_of_work_digest(UNIT_OF_WORK_DIGEST_LENGTH)}"
 
     @abstractmethod
     def name_prefix(self) -> str:

reconcile/utils/json.py

@@ -1,15 +1,50 @@
 import json
-from typing import Any
+from collections.abc import Callable
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime
+from decimal import Decimal
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel
+from pydantic.main import IncEx
 
 JSON_COMPACT_SEPARATORS = (",", ":")
 
 
+def pydantic_encoder(obj: Any) -> Any:
+    if isinstance(obj, BaseModel):
+        return obj.model_dump()
+
+    if is_dataclass(obj):
+        return asdict(obj)  # type: ignore
+
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+
+    if isinstance(obj, Enum):
+        return obj.value
+
+    if isinstance(obj, Decimal):
+        return float(obj)
+
+    raise TypeError(
+        f"Object of type '{obj.__class__.__name__}' is not JSON serializable"
+    )
+
+
 def json_dumps(
     data: Any,
     *,
     compact: bool = False,
     indent: int | None = None,
     cls: type[json.JSONEncoder] | None = None,
+    defaults: Callable | None = None,
+    # BaseModel dump parameters
+    by_alias: bool = True,
+    exclude_none: bool = False,
+    exclude: IncEx | None = None,
+    mode: Literal["json", "python"] = "json",
 ) -> str:
     """
     Serialize `data` to a consistent JSON formatted `str` with dict keys sorted.
@@ -22,6 +57,12 @@ def json_dumps(
     Returns:
         A JSON formatted string.
     """
+    if isinstance(data, BaseModel):
+        data = data.model_dump(
+            mode=mode, by_alias=by_alias, exclude_none=exclude_none, exclude=exclude
+        )
+        if mode == "python":
+            defaults = pydantic_encoder
     separators = JSON_COMPACT_SEPARATORS if compact else None
     return json.dumps(
         data,
@@ -29,4 +70,5 @@ def json_dumps(
         separators=separators,
         sort_keys=True,
         cls=cls,
+        default=defaults,
     )

reconcile/utils/membershipsources/app_interface_resolver.py

@@ -55,6 +55,8 @@ def resolve_app_interface_membership_source(
 
 def build_member_list(role: RoleV1) -> list[RoleMember]:
     members: list[RoleMember] = []
-    members.extend([RoleUser(**u.dict()) for u in role.users or []])
-    members.extend([RoleBot(**b.dict()) for b in role.bots or [] if b.org_username])
+    members.extend([RoleUser(**u.model_dump()) for u in role.users or []])
+    members.extend([
+        RoleBot(**b.model_dump()) for b in role.bots or [] if b.org_username
+    ])
     return members

reconcile/utils/membershipsources/models.py

@@ -7,7 +7,6 @@ from typing import (
 
 from pydantic import (
     BaseModel,
-    Extra,
 )
 
 from reconcile.gql_definitions.fragments.membership_source import (
@@ -23,7 +22,7 @@ class User(Protocol):
     @property
     def org_username(self) -> str: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class Bot(Protocol):
@@ -33,7 +32,7 @@ class Bot(Protocol):
     @property
     def org_username(self) -> str | None: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class RoleWithMemberships(Protocol):
@@ -50,33 +49,27 @@ class RoleWithMemberships(Protocol):
     def member_sources(self) -> Sequence[RoleMembershipSource] | None: ...
 
 
-class RoleUser(BaseModel):
+class RoleUser(BaseModel, extra="ignore"):
     name: str
     org_username: str
-    github_username: str | None
-    quay_username: str | None
-    pagerduty_username: str | None
-    aws_username: str | None
-    cloudflare_user: str | None
-    public_gpg_key: str | None
+    github_username: str | None = None
+    quay_username: str | None = None
+    pagerduty_username: str | None = None
+    aws_username: str | None = None
+    cloudflare_user: str | None = None
+    public_gpg_key: str | None = None
     tag_on_cluster_updates: bool | None = False
     tag_on_merge_requests: bool | None = False
 
-    class Config:
-        extra = Extra.ignore
 
-
-class RoleBot(BaseModel):
+class RoleBot(BaseModel, extra="ignore"):
     name: str
-    description: str | None
-    org_username: str | None
-    github_username: str | None
-    gitlab_username: str | None
-    openshift_serviceaccount: str | None
-    quay_username: str | None
-
-    class Config:
-        extra = Extra.ignore
+    description: str | None = None
+    org_username: str | None = None
+    github_username: str | None = None
+    gitlab_username: str | None = None
+    openshift_serviceaccount: str | None = None
+    quay_username: str | None = None
 
 
 RoleMember = RoleUser | RoleBot

reconcile/utils/membershipsources/resolver.py

@@ -98,8 +98,10 @@ def resolve_role_members(
         members: list[RoleMember] = []
 
         # bring in the local users and bots ...
-        members.extend(RoleUser(**u.dict()) for u in r.users or [])
-        members.extend(RoleBot(**b.dict()) for b in r.bots or [] if b.org_username)
+        members.extend(RoleUser(**u.model_dump()) for u in r.users or [])
+        members.extend(
+            RoleBot(**b.model_dump()) for b in r.bots or [] if b.org_username
+        )
 
         # ... and enhance with the ones from member sources
         for ms in r.member_sources or []:

reconcile/utils/merge_request_manager/merge_request_manager.py

@@ -1,7 +1,7 @@
 import logging
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Generic, TypeVar
+from typing import Any, TypeVar
 
 from gitlab.v4.objects import ProjectMergeRequest
 from pydantic import BaseModel
@@ -17,12 +17,12 @@ T = TypeVar("T", bound=BaseModel)
 
 
 @dataclass
-class OpenMergeRequest(Generic[T]):
+class OpenMergeRequest[T: BaseModel]:
     raw: ProjectMergeRequest
     mr_info: T
 
 
-class MergeRequestManagerBase(Generic[T]):
+class MergeRequestManagerBase[T: BaseModel]:
     """ """
 
     def __init__(self, vcs: VCS, parser: Parser, mr_label: str):
@@ -42,7 +42,7 @@ class MergeRequestManagerBase(Generic[T]):
         expected_data: dict[str, Any],
     ) -> OpenMergeRequest | None:
         for mr in self._open_mrs:
-            mr_info_dict = mr.mr_info.dict()
+            mr_info_dict = mr.mr_info.model_dump()
             if all(mr_info_dict.get(k) == expected_data.get(k) for k in expected_data):
                 return mr
 

reconcile/utils/merge_request_manager/parser.py

@@ -1,5 +1,5 @@
 import re
-from typing import Generic, TypeVar
+from typing import TypeVar
 
 from pydantic import BaseModel
 
@@ -17,7 +17,7 @@ class ParserVersionError(Exception):
 T = TypeVar("T", bound=BaseModel)
 
 
-class Parser(Generic[T]):
+class Parser[T: BaseModel]:
     """This class is only concerned with parsing an MR description rendered by the Renderer."""
 
     def __init__(
@@ -60,8 +60,8 @@ class Parser(Generic[T]):
 
         if self.expected_version != self._find_by_name(self.version_ref, parts[1]):
             raise ParserVersionError("Version is outdated")
-        return self.klass(
-            **data_default_none(
-                self.klass, self._data_from_description(parts[1]), use_defaults=False
-            )
+        data = data_default_none(
+            self.klass, self._data_from_description(parts[1]), use_defaults=False
         )
+        assert isinstance(data, dict)
+        return self.klass(**data)

reconcile/utils/metrics.py

@@ -144,7 +144,7 @@ class GaugeMetric(BaseMetric):
 
     @classmethod
     def metric_family(cls) -> GaugeMetricFamily:
-        labels = [f.alias for f in cls.__fields__.values()]
+        labels = [f.alias or name for name, f in cls.model_fields.items()]
         return GaugeMetricFamily(cls.name(), cls.__doc__ or "", labels=labels)
 
     @classmethod
@@ -167,7 +167,7 @@ class CounterMetric(BaseMetric):
 
     @classmethod
     def metric_family(cls) -> CounterMetricFamily:
-        labels = [f.alias for f in cls.__fields__.values()]
+        labels = [f.alias or name for name, f in cls.model_fields.items()]
         return CounterMetricFamily(cls.name(), cls.__doc__ or "", labels=labels)
 
     @classmethod
@@ -198,7 +198,7 @@ class MetricsContainer:
         """
         Sets the value of the given gauge metric to the given value.
         """
-        label_values = tuple(metric.dict(by_alias=True).values())
+        label_values = tuple(metric.model_dump(by_alias=True).values())
         self._gauges[metric.__class__][label_values] = value
 
     def set_info(self, metric: InfoMetric) -> None:
@@ -213,7 +213,7 @@ class MetricsContainer:
         Increases the value of the given counter by the given amount.
         """
         # all label values need to be strings, so lets convert them
-        label_values = tuple(str(v) for v in counter.dict(by_alias=True).values())
+        label_values = tuple(str(v) for v in counter.model_dump(by_alias=True).values())
         current_value = self._counters[counter.__class__].get(label_values) or 0
         self._counters[counter.__class__][label_values] = current_value + by
 
@@ -270,7 +270,7 @@ class MetricsContainer:
             (
                 metric_class(**{
                     key: labels[i]
-                    for i, key in enumerate(metric_class.__fields__.keys())
+                    for i, key in enumerate(metric_class.model_fields.keys())
                 }),
                 value,
             )