qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

reconcile/utils/jinja2/utils.py

@@ -1,13 +1,10 @@
 import datetime
 import os
-import subprocess
-import tempfile
 from collections.abc import Mapping
 from functools import cache
 from typing import Any, Self
 
 import jinja2
-import yaml
 from github import Github
 from jinja2.sandbox import SandboxedEnvironment
 from pydantic import BaseModel
@@ -18,6 +15,7 @@ from reconcile.checkpoint import url_makes_sense
 from reconcile.github_org import get_default_config
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.helpers import flatten
 from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
@@ -38,7 +36,7 @@ from reconcile.utils.secret_reader import (
     SecretReader,
     SecretReaderBase,
 )
-from reconcile.utils.sloth import process_sloth_output
+from reconcile.utils.sloth import generate_sloth_rules
 from reconcile.utils.vault import SecretFieldNotFoundError
 
 
@@ -192,89 +190,6 @@ def list_s3_objects(
         )
 
 
-def sloth_alerts(
-    service: str,
-    slo_name: str,
-    objective: float,
-    error_query: str,
-    total_query: str,
-    version: str = "prometheus/v1",
-) -> str:
-    """Generate Prometheus rules using sloth: https://sloth.dev
-
-    Args:
-        service: Service name identifier
-        slo_name: Name of the SLO
-        objective: Target percentage (e.g. 99.9)
-        error_query: Prometheus query for error events
-        total_query: Prometheus query for total events
-        version: Spec version (default: "prometheus/v1")
-
-    Returns:
-        Generated Prometheus rules as YAML string
-    """
-    # Build the SLO definition
-    slo = {
-        "name": slo_name,
-        "objective": objective,
-        "description": f"{slo_name} SLO for {service}",
-        "sli": {
-            "events": {
-                "error_query": error_query.replace("{{window}}", "{{.window}}"),
-                "total_query": total_query.replace("{{window}}", "{{.window}}"),
-            }
-        },
-        "alerting": {
-            "name": f"{service.title()}{slo_name.title()}",
-            "annotations": {
-                "summary": f"High error rate on '{service}' {slo_name}",
-                "message": f"High error rate on '{service}' {slo_name}",
-            },
-            "page_alert": {
-                "labels": {
-                    "severity": "critical",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-            "ticket_alert": {
-                "labels": {
-                    "severity": "medium",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-        },
-    }
-
-    spec = {
-        "version": version,
-        "service": service,
-        "slos": [slo],
-    }
-
-    with (
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as input_file,
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as output_file,
-    ):
-        yaml.dump(spec, input_file, allow_unicode=True)
-        cmd = ["sloth", "generate", "-i", input_file.name, "-o", output_file.name]
-        try:
-            subprocess.run(cmd, capture_output=True, check=True, text=True)
-        except subprocess.CalledProcessError as e:
-            error_msg = f"{e}"
-            if e.stdout:
-                error_msg += f"\nstdout: {e.stdout}"
-            if e.stderr:
-                error_msg += f"\nstderr: {e.stderr}"
-            raise SlothGenerateError(error_msg) from e
-        return process_sloth_output(output_file.name)
-
-
 @retry()
 def lookup_secret(
     path: str,
@@ -345,10 +260,8 @@ def process_jinja2_template(
         "s3": lookup_s3_object,
         "s3_ls": list_s3_objects,
         "flatten_dict": flatten,
-        "yesterday": lambda: (datetime.datetime.now() - datetime.timedelta(1)).strftime(
-            "%Y-%m-%d"
-        ),
-        "sloth_alerts": sloth_alerts,
+        "yesterday": lambda: (utc_now() - datetime.timedelta(1)).strftime("%Y-%m-%d"),
+        "sloth_alerts": generate_sloth_rules,
     })
     if "_template_mocks" in vars:
         for k, v in vars["_template_mocks"].items():
@@ -381,11 +294,6 @@ def process_extracurlyjinja2_template(
     )
 
 
-class SlothGenerateError(Exception):
-    def __init__(self, msg: Any):
-        super().__init__("sloth generate failed: " + str(msg))
-
-
 class FetchSecretError(Exception):
     def __init__(self, msg: Any):
         super().__init__("error fetching secret: " + str(msg))

reconcile/utils/jira_client.py

@@ -17,10 +17,8 @@ from jira.resources import CustomFieldOption as JiraCustomFieldOption
 from jira.resources import Resource
 from pydantic import BaseModel
 
-from reconcile.utils.secret_reader import SecretReader
-
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Iterable
 
 
 class JiraWatcherSettings(Protocol):
@@ -87,36 +85,18 @@ class IssueField(BaseModel):
     options: list[FieldOption | CustomFieldOption]
 
 
+CREATE_ISSUES = "CREATE_ISSUES"
+TRANSITION_ISSUES = "TRANSITION_ISSUES"
+PERMISSIONS = [CREATE_ISSUES, TRANSITION_ISSUES]
+
+
 class JiraClient:
     """Wrapper around Jira client."""
 
     DEFAULT_CONNECT_TIMEOUT = 60
     DEFAULT_READ_TIMEOUT = 60
 
-    def __init__(
-        self,
-        jira_board: Mapping[str, Any] | None = None,
-        settings: Mapping | None = None,
-        jira_api: JIRA | None = None,
-        project: str | None = None,
-        server: str | None = None,
-    ):
-        """
-        Note: jira_board and settings is to be deprecated. Use JiraClient.create() instead.
-        """
-        if jira_api and jira_board:
-            raise RuntimeError(
-                "jira_board parameter is deprecated. Use JiraClient.create() instead."
-            )
-        if not (jira_api and project):
-            # kept for backwards-compatibility
-            if not jira_board:
-                raise RuntimeError(
-                    "JiraClient needs jira_api and project or jira_board."
-                )
-            self._deprecated_init(jira_board=jira_board, settings=settings)
-            return
-
+    def __init__(self, jira_api: JIRA, project: str, server: str):
         self.server = server
         self.project = project
         self.jira = jira_api
@@ -128,47 +108,31 @@ class JiraClient:
         self.project_issue_types = functools.cache(self._project_issue_types)
         self.project_issue_fields = functools.cache(self._project_issue_fields)
 
-    def _deprecated_init(
-        self, jira_board: Mapping[str, Any], settings: Mapping | None
-    ) -> None:
-        secret_reader = SecretReader(settings=settings)
-        self.project = jira_board["name"]
-        jira_server = jira_board["server"]
-        self.server = jira_server["serverUrl"]
-        token = jira_server["token"]
-        token_auth = secret_reader.read(token)
-        read_timeout = 60
-        connect_timeout = 60
-        if settings and settings["jiraWatcher"]:
-            read_timeout = settings["jiraWatcher"]["readTimeout"]
-            connect_timeout = settings["jiraWatcher"]["connectTimeout"]
-        if not self.server:
-            raise RuntimeError("JiraClient.server is not set.")
-
-        self.jira = JIRA(
-            self.server,
-            token_auth=token_auth,
-            timeout=(read_timeout, connect_timeout),
-            logging=False,
-        )
-
     @staticmethod
     def create(
         project_name: str,
         token: str,
+        email: str | None,
         server_url: str,
         jira_watcher_settings: JiraWatcherSettings | None = None,
     ) -> JiraClient:
+        """Create a Jira client for the given project."""
         read_timeout = JiraClient.DEFAULT_READ_TIMEOUT
         connect_timeout = JiraClient.DEFAULT_CONNECT_TIMEOUT
         if jira_watcher_settings:
             read_timeout = jira_watcher_settings.read_timeout
             connect_timeout = jira_watcher_settings.connect_timeout
+
+        # Jira Cloud uses email+API token for basic auth
+        # Jira Server/Data Center can use token auth (personal access token)
+        auth_params: dict[str, Any] = (
+            {"basic_auth": (email, token)} if email else {"token_auth": token}
+        )
         jira_api = JIRA(
             server=server_url,
-            token_auth=token,
             timeout=(read_timeout, connect_timeout),
             logging=False,
+            **auth_params,
         )
         return JiraClient(
             jira_api=jira_api,
@@ -176,7 +140,13 @@ class JiraClient:
             server=server_url,
         )
 
+    @property
+    def is_cloud(self) -> bool:
+        """Return whether we are on a Cloud based Jira instance."""
+        return self.jira.deploymentType == "Cloud"
+
     def get_issues(self, fields: Iterable | None = None) -> list[Issue]:
+        """Return all issues for our project."""
         block_size = 100
         block_num = 0
 
@@ -227,20 +197,34 @@ class JiraClient:
         return issue
 
     def _my_permissions(self, project: str) -> dict[str, Any]:
+        """Return my permissions for the given project.
+
+        Don't use this function directly, use self.my_permissions which is cached."""
+        if self.is_cloud:
+            return self.jira.my_permissions(
+                projectKey=project, permissions=",".join(PERMISSIONS)
+            )["permissions"]
         return self.jira.my_permissions(projectKey=project)["permissions"]
 
     def can_i(self, permission: str) -> bool:
+        """Return whether I have the given permission in the project."""
         return bool(
             self.my_permissions(project=self.project)[permission]["havePermission"]
         )
 
     def can_create_issues(self) -> bool:
-        return self.can_i("CREATE_ISSUES")
+        """Return whether I can create issues in the project."""
+        return self.can_i(CREATE_ISSUES)
 
     def can_transition_issues(self) -> bool:
-        return self.can_i("TRANSITION_ISSUES")
+        """Return whether I can transition issues in the project."""
+        return self.can_i(TRANSITION_ISSUES)
 
     def _project_issue_types(self, project: str) -> list[IssueType]:
+        """Return all available issue types (e.g. Task, Bug) for the project.
+
+        Don't use this function directly, use self.project_issue_types which is cached.
+        """
         # Don't use self.project here, because of function.cache usage
         return [
             IssueType(id=t.id, name=t.name, statuses=[s.name for s in t.statuses])
@@ -248,6 +232,7 @@ class JiraClient:
         ]
 
     def get_issue_type(self, issue_type: str) -> IssueType | None:
+        """Return a issue type (e.g. Task) for the project if it exists."""
         for _issue_type in self.project_issue_types(self.project):
             if _issue_type.name == issue_type:
                 return _issue_type
@@ -255,15 +240,23 @@ class JiraClient:
 
     @staticmethod
     def _get_allowed_issue_field_options(
-        allowed_values: list[Resource],
+        allowed_values: list[Resource] | list[dict[str, str]],
     ) -> list[FieldOption | CustomFieldOption]:
-        """Return a list of allowed values for a field."""
-        return [
-            CustomFieldOption(value=v.value)
-            if isinstance(v, JiraCustomFieldOption)
-            else FieldOption(name=v.name)
-            for v in allowed_values
-        ]
+        """Return a list of allowed values for a field. E.g. Minor, Major ... for Priority in a Task."""
+        items: list[FieldOption | CustomFieldOption] = []
+        for v in allowed_values:
+            match v:
+                case dict() if "value" in v:
+                    items.append(CustomFieldOption(value=v["value"]))
+                case dict() if "name" in v:
+                    items.append(FieldOption(name=v["name"]))
+                case JiraCustomFieldOption():
+                    items.append(CustomFieldOption(value=v.value))
+                case Resource():
+                    items.append(FieldOption(name=v.name))
+                case _:
+                    logging.warning(f"Unknown allowed value type: {type(v)}")
+        return items
 
     def _project_issue_fields(
         self, project: str, issue_type_id: str
@@ -273,6 +266,27 @@ class JiraClient:
         This API endpoint needs createIssue project permissions.
         """
         # Don't use self.project here, because of function.cache usage
+        if self.is_cloud:
+            metadata = self.jira.createmeta(
+                projectKeys=self.project,
+                issuetypeIds=[issue_type_id],
+                expand="projects.issuetypes.fields",
+            )
+            if not metadata["projects"] or not metadata["projects"][0]["issuetypes"]:
+                return []
+            return [
+                IssueField(
+                    name=field["name"],
+                    id=field_id,
+                    options=self._get_allowed_issue_field_options(
+                        field.get("allowedValues", [])
+                    ),
+                )
+                for field_id, field in metadata["projects"][0]["issuetypes"][0][
+                    "fields"
+                ].items()
+            ]
+
         return [
             IssueField(
                 name=field.name,
@@ -304,6 +318,10 @@ class JiraClient:
 
     def project_priority_scheme(self) -> list[str]:
         """Return a list of all priority IDs for the project."""
+        if self.is_cloud:
+            # Cloud does not have a way to retrieve project specific priority schemes
+            return []
+
         scheme = self.jira.project_priority_scheme(self.project)
         return scheme.optionIds
 
@@ -329,4 +347,5 @@ class JiraClient:
 
     @property
     def is_archived(self) -> bool:
-        return self.jira.project(self.project).archived
+        """Return whether the project is archived."""
+        return getattr(self.jira.project(self.project), "archived", False)

reconcile/utils/jjb_client.py

@@ -1,6 +1,5 @@
 import difflib
 import filecmp
-import json
 import logging
 import os
 import re
@@ -10,6 +9,7 @@ import tempfile
 import xml.etree.ElementTree as ET
 from collections.abc import Iterable, Mapping
 from os import path
+from pathlib import Path
 from subprocess import (
     PIPE,
     STDOUT,
@@ -18,14 +18,14 @@ from subprocess import (
 from typing import Any
 
 import yaml
-from jenkins_jobs.builder import JenkinsManager
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.parser import YamlParser
-from jenkins_jobs.registry import ModuleRegistry
+from jenkins_jobs.loader import load_files
+from jenkins_jobs.roots import Roots
 from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.json import json_dumps
 from reconcile.utils.secret_reader import SecretReaderBase
 from reconcile.utils.state import State
 from reconcile.utils.vcs import GITHUB_BASE_URL
@@ -292,13 +292,10 @@ class JJB:
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
-        builder = JenkinsManager(jjb.jjb_config)
-        registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
-        parser = YamlParser(jjb.jjb_config)
-        parser.load_files(jjb.options.path)
-        jobs, _ = parser.expandYaml(registry, jjb.options.names)
-
-        return jobs
+        roots = Roots(jjb.jjb_config)
+        load_files(jjb.jjb_config, roots, [Path(config_path)])
+        job_view_data_list = roots.generate_jobs()
+        return [job.data for job in job_view_data_list]
 
     def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
         job_webhooks_data: dict[str, list[dict[str, Any]]] = {}
@@ -399,7 +396,7 @@ class JJB:
                 found = True
         if not found:
             raise ValueError(f"job name {job_name} is not found")
-        print(json.dumps(all_jobs, indent=2))
+        print(json_dumps(all_jobs, indent=2))
 
     def get_job_by_repo_url(self, repo_url: str, job_type: str) -> dict[str, Any]:
         for jobs in self.get_all_jobs(job_types=[job_type]).values():

reconcile/utils/jobcontroller/controller.py

@@ -100,7 +100,7 @@ class K8sJobController:
         """
         new_cache = {}
         for item in self.oc.get_items(
-            kind="Job",
+            kind="Job.batch",
             namespace=self.namespace,
         ):
             openshift_resource = OpenshiftResource(

reconcile/utils/jobcontroller/models.py

@@ -38,6 +38,8 @@ class JobValidationError(Exception):
 
 
 JOB_GENERATION_ANNOTATION = "qontract-reconcile/job.generation"
+MAX_JOB_NAME_LENGTH = 63
+UNIT_OF_WORK_DIGEST_LENGTH = 10
 
 
 class K8sJob(ABC):
@@ -72,7 +74,21 @@ class K8sJob(ABC):
     """
 
     def name(self) -> str:
-        return f"{self.name_prefix()}-{self.unit_of_work_digest()}"
+        """
+        Generate the full job name by combining the name prefix with a digest.
+
+        The name is constructed from the name_prefix (truncated to ensure total
+        length compliance) and the unit_of_work_digest. The total length is
+        limited to MAX_JOB_NAME_LENGTH (63 characters) to comply with Kubernetes
+        naming constraints.
+
+        Returns:
+            A unique job name in the format: {name_prefix}-{digest}
+        """
+        prefix = self.name_prefix()[
+            : MAX_JOB_NAME_LENGTH - UNIT_OF_WORK_DIGEST_LENGTH - 1
+        ]
+        return f"{prefix}-{self.unit_of_work_digest(UNIT_OF_WORK_DIGEST_LENGTH)}"
 
     @abstractmethod
     def name_prefix(self) -> str:

reconcile/utils/json.py

@@ -0,0 +1,32 @@
+import json
+from typing import Any
+
+JSON_COMPACT_SEPARATORS = (",", ":")
+
+
+def json_dumps(
+    data: Any,
+    *,
+    compact: bool = False,
+    indent: int | None = None,
+    cls: type[json.JSONEncoder] | None = None,
+) -> str:
+    """
+    Serialize `data` to a consistent JSON formatted `str` with dict keys sorted.
+
+    Args:
+        data: The data to serialize.
+        compact: If True, use compact separators (no spaces after commas or colons).
+        indent: If specified, pretty-print the JSON with this many spaces of indentation.
+        cls: A custom JSONEncoder subclass to use for serialization.
+    Returns:
+        A JSON formatted string.
+    """
+    separators = JSON_COMPACT_SEPARATORS if compact else None
+    return json.dumps(
+        data,
+        indent=indent,
+        separators=separators,
+        sort_keys=True,
+        cls=cls,
+    )

reconcile/utils/merge_request_manager/merge_request_manager.py

@@ -1,7 +1,7 @@
 import logging
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Generic, TypeVar
+from typing import Any, TypeVar
 
 from gitlab.v4.objects import ProjectMergeRequest
 from pydantic import BaseModel
@@ -17,12 +17,12 @@ T = TypeVar("T", bound=BaseModel)
 
 
 @dataclass
-class OpenMergeRequest(Generic[T]):
+class OpenMergeRequest[T: BaseModel]:
     raw: ProjectMergeRequest
     mr_info: T
 
 
-class MergeRequestManagerBase(Generic[T]):
+class MergeRequestManagerBase[T: BaseModel]:
     """ """
 
     def __init__(self, vcs: VCS, parser: Parser, mr_label: str):

reconcile/utils/merge_request_manager/parser.py

@@ -1,5 +1,5 @@
 import re
-from typing import Generic, TypeVar
+from typing import TypeVar
 
 from pydantic import BaseModel
 
@@ -17,7 +17,7 @@ class ParserVersionError(Exception):
 T = TypeVar("T", bound=BaseModel)
 
 
-class Parser(Generic[T]):
+class Parser[T: BaseModel]:
     """This class is only concerned with parsing an MR description rendered by the Renderer."""
 
     def __init__(

reconcile/utils/mr/app_interface_reporter.py

@@ -1,9 +1,9 @@
 from collections.abc import Iterable
-from datetime import datetime
 from pathlib import Path
 
 from ruamel.yaml.scalarstring import PreservedScalarString
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -26,7 +26,7 @@ class CreateAppInterfaceReporter(MergeRequestBase):
 
         self.labels = [AUTO_MERGE]
 
-        now = datetime.now()
+        now = utc_now()
         self.isodate = now.isoformat()
         self.ts = now.strftime("%Y%m%d%H%M%S")
 

reconcile/utils/mr/base.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from abc import (
     ABC,
@@ -14,6 +13,7 @@ from jinja2 import Template
 from reconcile.gql_definitions.fragments.user import User
 from reconcile.utils.constants import PROJ_ROOT
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.mr.labels import DO_NOT_MERGE_HOLD
 from reconcile.utils.sqs_gateway import SQSGateway
 
@@ -193,7 +193,7 @@ class MergeRequestBase(ABC):
             # logging this exception
             raise MergeRequestProcessingError(
                 f"error processing {self.name} changes "
-                f"{json.dumps(self.sqs_msg_data)} "
+                f"{json_dumps(self.sqs_msg_data)} "
                 f"into temporary branch {self.branch}. "
                 f"Reason: {err}"
             ) from err

reconcile/utils/mr/notificator.py

@@ -1,9 +1,9 @@
 import logging
-from datetime import datetime
 from pathlib import Path
 
 from pydantic import BaseModel
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -58,7 +58,7 @@ class CreateAppInterfaceNotificator(MergeRequestBase):
         )
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        now = datetime.now()
+        now = utc_now()
         ts = now.strftime("%Y%m%d%H%M%S")
         short_date = now.strftime("%Y-%m-%d")
 

reconcile/utils/mr/update_access_report_base.py

@@ -1,14 +1,13 @@
 import logging
 from abc import abstractmethod
 from collections.abc import Sequence
-from datetime import UTC, date
-from datetime import datetime as dt
 from pathlib import Path
 from typing import TypeVar
 
 from jinja2 import Template
 from pydantic import BaseModel
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
@@ -27,7 +26,7 @@ class UpdateAccessReportBase(MergeRequestBase):
         self.labels = [AUTO_MERGE]
         self._users = users
         self._workbook_file_name = str(workbook_path)
-        self._isodate = dt.now(tz=UTC).isoformat()
+        self._isodate = utc_now().isoformat()
         self._dry_run = dry_run
 
     @property
@@ -65,7 +64,7 @@ class UpdateAccessReportBase(MergeRequestBase):
 
     def _render_tracking_table_row(self, old_number_of_users: int) -> str:
         # | Date Reviewed | Number of Current Users | +/- Red Hat Users |
-        return f"| {date.today()} | {len(self._users)} | {len(self._users) - old_number_of_users} |\n"
+        return f"| {utc_now().date()} | {len(self._users)} | {len(self._users) - old_number_of_users} |\n"
 
     def _update_workbook(self, workbook_md: str) -> str:
         new_workbook_md = ""