qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev430__py3-none-any.whl

reconcile/utils/gql.py

@@ -109,7 +109,7 @@ class GqlApi:
         if int_name:
             integrations = self.query(INTEGRATIONS_QUERY, skip_validation=True)
 
-            for integration in integrations["integrations"]:
+            for integration in integrations["integrations"] if integrations else []:
                 if integration["name"] == int_name:
                     self._valid_schemas = integration["schemas"]
                     break
@@ -142,7 +142,7 @@ class GqlApi:
         query: str,
         variables: dict[str, Any] | None = None,
         skip_validation: bool = False,
-    ) -> dict[str, Any] | None:
+    ) -> dict[str, Any]:
         try:
             result = self.client.execute(
                 gql(query), variables, get_execution_result=True
@@ -172,11 +172,8 @@ class GqlApi:
             if forbidden_schemas:
                 raise GqlApiErrorForbiddenSchemaError(forbidden_schemas)
 
-        # This is to appease mypy. This exception won't be thrown as this condition
-        # is already handled above with AssertionError
-        if result["data"] is None:
-            raise GqlApiError("`data` not received in GraphQL payload")
-
+        # make mypy happy
+        assert "data" in result and result["data"] is not None
         return result["data"]
 
     def get_template(self, path: str) -> dict[str, str]:

reconcile/utils/helpers.py

@@ -47,7 +47,7 @@ def flatten(
 Item = TypeVar("Item")
 
 
-def find_duplicates(items: Iterable[Item]) -> list[Item]:
+def find_duplicates[Item](items: Iterable[Item]) -> list[Item]:
     return [item for item, count in Counter(items).items() if count > 1]
 
 

reconcile/utils/instrumented_wrappers.py

@@ -40,7 +40,7 @@ class InstrumentedImage(Image):
 
 
 class InstrumentedSkopeo(Skopeo):
-    def copy(self, *args: Any, **kwargs: Any) -> bytes | str:
+    def copy(self, *args: Any, **kwargs: Any) -> None:
         metrics.copy_count.labels(
             integration=INTEGRATION_NAME, shard=SHARDS, shard_id=SHARD_ID
         ).inc()

reconcile/utils/internal_groups/client.py

@@ -140,7 +140,7 @@ class InternalGroupsClient:
         with self._api as api:
             return Group(
                 **api.create_group(
-                    data=group.dict(by_alias=True),
+                    data=group.model_dump(by_alias=True),
                 )
             )
 
@@ -155,6 +155,6 @@ class InternalGroupsClient:
             return Group(
                 **api.update_group(
                     name=group.name,
-                    data=group.dict(by_alias=True),
+                    data=group.model_dump(by_alias=True),
                 )
             )

reconcile/utils/internal_groups/models.py

@@ -27,7 +27,7 @@ class Entity(BaseModel):
         return hash(self.id)
 
 
-class Group(BaseModel):
+class Group(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     description: str
     member_approval_type: str = Field("self-service", alias="memberApprovalType")
@@ -35,16 +35,18 @@ class Group(BaseModel):
     owners: list[Entity]
     display_name: str = Field(..., alias="displayName")
     notes: str | None = None
-    rover_group_member_query: str | None = Field(None, alias="roverGroupMemberQuery")
+    rover_group_member_query: str | None = Field(
+        None, alias="roverGroupMemberQuery", exclude=True
+    )
     rover_group_inclusions: list[Entity] | None = Field(
-        None, alias="roverGroupInclusions"
+        None, alias="roverGroupInclusions", exclude=True
     )
     rover_group_exclusions: list[Entity] | None = Field(
-        None, alias="roverGroupExclusions"
+        None, alias="roverGroupExclusions", exclude=True
     )
     members: list[Entity] = []
-    member_of: list[str] | None = Field(None, alias="memberOf")
-    namespace: str | None = None
+    member_of: list[str] | None = Field(None, alias="memberOf", exclude=True)
+    namespace: str | None = Field(None, exclude=True)
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Group):
@@ -58,14 +60,3 @@ class Group(BaseModel):
             and self.notes == other.notes
             and set(self.members) == set(other.members)
         )
-
-    class Config:
-        allow_population_by_field_name = True
-        # exclude read-only fields in the json/dict dumps
-        fields = {
-            "rover_group_member_query": {"exclude": True},
-            "rover_group_inclusions": {"exclude": True},
-            "rover_group_exclusions": {"exclude": True},
-            "member_of": {"exclude": True},
-            "namespace": {"exclude": True},
-        }

reconcile/utils/jinja2/utils.py

@@ -1,13 +1,10 @@
 import datetime
 import os
-import subprocess
-import tempfile
 from collections.abc import Mapping
 from functools import cache
 from typing import Any, Self
 
 import jinja2
-import yaml
 from github import Github
 from jinja2.sandbox import SandboxedEnvironment
 from pydantic import BaseModel
@@ -18,6 +15,7 @@ from reconcile.checkpoint import url_makes_sense
 from reconcile.github_org import get_default_config
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.helpers import flatten
 from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
@@ -38,7 +36,7 @@ from reconcile.utils.secret_reader import (
     SecretReader,
     SecretReaderBase,
 )
-from reconcile.utils.sloth import process_sloth_output
+from reconcile.utils.sloth import generate_sloth_rules
 from reconcile.utils.vault import SecretFieldNotFoundError
 
 
@@ -47,14 +45,11 @@ class Jinja2TemplateError(Exception):
         super().__init__("error processing jinja2 template: " + str(msg))
 
 
-class TemplateRenderOptions(BaseModel):
+class TemplateRenderOptions(BaseModel, frozen=True):
     trim_blocks: bool
     lstrip_blocks: bool
     keep_trailing_newline: bool
 
-    class Config:
-        frozen = True
-
     @classmethod
     def create(
         cls,
@@ -77,7 +72,7 @@ def compile_jinja2_template(
 ) -> Any:
     if not template_render_options:
         template_render_options = TemplateRenderOptions.create()
-    env: dict[str, Any] = template_render_options.dict()
+    env: dict[str, Any] = template_render_options.model_dump()
     if extra_curly:
         env.update({
             "block_start_string": "{{%",
@@ -192,89 +187,6 @@ def list_s3_objects(
         )
 
 
-def sloth_alerts(
-    service: str,
-    slo_name: str,
-    objective: float,
-    error_query: str,
-    total_query: str,
-    version: str = "prometheus/v1",
-) -> str:
-    """Generate Prometheus rules using sloth: https://sloth.dev
-
-    Args:
-        service: Service name identifier
-        slo_name: Name of the SLO
-        objective: Target percentage (e.g. 99.9)
-        error_query: Prometheus query for error events
-        total_query: Prometheus query for total events
-        version: Spec version (default: "prometheus/v1")
-
-    Returns:
-        Generated Prometheus rules as YAML string
-    """
-    # Build the SLO definition
-    slo = {
-        "name": slo_name,
-        "objective": objective,
-        "description": f"{slo_name} SLO for {service}",
-        "sli": {
-            "events": {
-                "error_query": error_query.replace("{{window}}", "{{.window}}"),
-                "total_query": total_query.replace("{{window}}", "{{.window}}"),
-            }
-        },
-        "alerting": {
-            "name": f"{service.title()}{slo_name.title()}",
-            "annotations": {
-                "summary": f"High error rate on '{service}' {slo_name}",
-                "message": f"High error rate on '{service}' {slo_name}",
-            },
-            "page_alert": {
-                "labels": {
-                    "severity": "critical",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-            "ticket_alert": {
-                "labels": {
-                    "severity": "medium",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-        },
-    }
-
-    spec = {
-        "version": version,
-        "service": service,
-        "slos": [slo],
-    }
-
-    with (
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as input_file,
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as output_file,
-    ):
-        yaml.dump(spec, input_file, allow_unicode=True)
-        cmd = ["sloth", "generate", "-i", input_file.name, "-o", output_file.name]
-        try:
-            subprocess.run(cmd, capture_output=True, check=True, text=True)
-        except subprocess.CalledProcessError as e:
-            error_msg = f"{e}"
-            if e.stdout:
-                error_msg += f"\nstdout: {e.stdout}"
-            if e.stderr:
-                error_msg += f"\nstderr: {e.stderr}"
-            raise SlothGenerateError(error_msg) from e
-        return process_sloth_output(output_file.name)
-
-
 @retry()
 def lookup_secret(
     path: str,
@@ -345,10 +257,8 @@ def process_jinja2_template(
         "s3": lookup_s3_object,
         "s3_ls": list_s3_objects,
         "flatten_dict": flatten,
-        "yesterday": lambda: (datetime.datetime.now() - datetime.timedelta(1)).strftime(
-            "%Y-%m-%d"
-        ),
-        "sloth_alerts": sloth_alerts,
+        "yesterday": lambda: (utc_now() - datetime.timedelta(1)).strftime("%Y-%m-%d"),
+        "sloth_alerts": generate_sloth_rules,
     })
     if "_template_mocks" in vars:
         for k, v in vars["_template_mocks"].items():
@@ -381,11 +291,6 @@ def process_extracurlyjinja2_template(
     )
 
 
-class SlothGenerateError(Exception):
-    def __init__(self, msg: Any):
-        super().__init__("sloth generate failed: " + str(msg))
-
-
 class FetchSecretError(Exception):
     def __init__(self, msg: Any):
         super().__init__("error fetching secret: " + str(msg))

reconcile/utils/jira_client.py

@@ -17,10 +17,8 @@ from jira.resources import CustomFieldOption as JiraCustomFieldOption
 from jira.resources import Resource
 from pydantic import BaseModel
 
-from reconcile.utils.secret_reader import SecretReader
-
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Iterable
 
 
 class JiraWatcherSettings(Protocol):
@@ -87,36 +85,18 @@ class IssueField(BaseModel):
     options: list[FieldOption | CustomFieldOption]
 
 
+CREATE_ISSUES = "CREATE_ISSUES"
+TRANSITION_ISSUES = "TRANSITION_ISSUES"
+PERMISSIONS = [CREATE_ISSUES, TRANSITION_ISSUES]
+
+
 class JiraClient:
     """Wrapper around Jira client."""
 
     DEFAULT_CONNECT_TIMEOUT = 60
     DEFAULT_READ_TIMEOUT = 60
 
-    def __init__(
-        self,
-        jira_board: Mapping[str, Any] | None = None,
-        settings: Mapping | None = None,
-        jira_api: JIRA | None = None,
-        project: str | None = None,
-        server: str | None = None,
-    ):
-        """
-        Note: jira_board and settings is to be deprecated. Use JiraClient.create() instead.
-        """
-        if jira_api and jira_board:
-            raise RuntimeError(
-                "jira_board parameter is deprecated. Use JiraClient.create() instead."
-            )
-        if not (jira_api and project):
-            # kept for backwards-compatibility
-            if not jira_board:
-                raise RuntimeError(
-                    "JiraClient needs jira_api and project or jira_board."
-                )
-            self._deprecated_init(jira_board=jira_board, settings=settings)
-            return
-
+    def __init__(self, jira_api: JIRA, project: str, server: str):
         self.server = server
         self.project = project
         self.jira = jira_api
@@ -128,47 +108,31 @@ class JiraClient:
         self.project_issue_types = functools.cache(self._project_issue_types)
         self.project_issue_fields = functools.cache(self._project_issue_fields)
 
-    def _deprecated_init(
-        self, jira_board: Mapping[str, Any], settings: Mapping | None
-    ) -> None:
-        secret_reader = SecretReader(settings=settings)
-        self.project = jira_board["name"]
-        jira_server = jira_board["server"]
-        self.server = jira_server["serverUrl"]
-        token = jira_server["token"]
-        token_auth = secret_reader.read(token)
-        read_timeout = 60
-        connect_timeout = 60
-        if settings and settings["jiraWatcher"]:
-            read_timeout = settings["jiraWatcher"]["readTimeout"]
-            connect_timeout = settings["jiraWatcher"]["connectTimeout"]
-        if not self.server:
-            raise RuntimeError("JiraClient.server is not set.")
-
-        self.jira = JIRA(
-            self.server,
-            token_auth=token_auth,
-            timeout=(read_timeout, connect_timeout),
-            logging=False,
-        )
-
     @staticmethod
     def create(
         project_name: str,
         token: str,
+        email: str | None,
         server_url: str,
         jira_watcher_settings: JiraWatcherSettings | None = None,
     ) -> JiraClient:
+        """Create a Jira client for the given project."""
         read_timeout = JiraClient.DEFAULT_READ_TIMEOUT
         connect_timeout = JiraClient.DEFAULT_CONNECT_TIMEOUT
         if jira_watcher_settings:
             read_timeout = jira_watcher_settings.read_timeout
             connect_timeout = jira_watcher_settings.connect_timeout
+
+        # Jira Cloud uses email+API token for basic auth
+        # Jira Server/Data Center can use token auth (personal access token)
+        auth_params: dict[str, Any] = (
+            {"basic_auth": (email, token)} if email else {"token_auth": token}
+        )
         jira_api = JIRA(
             server=server_url,
-            token_auth=token,
             timeout=(read_timeout, connect_timeout),
             logging=False,
+            **auth_params,
         )
         return JiraClient(
             jira_api=jira_api,
@@ -176,7 +140,13 @@ class JiraClient:
             server=server_url,
         )
 
+    @property
+    def is_cloud(self) -> bool:
+        """Return whether we are on a Cloud based Jira instance."""
+        return self.jira.deploymentType == "Cloud"
+
     def get_issues(self, fields: Iterable | None = None) -> list[Issue]:
+        """Return all issues for our project."""
         block_size = 100
         block_num = 0
 
@@ -227,20 +197,34 @@ class JiraClient:
         return issue
 
     def _my_permissions(self, project: str) -> dict[str, Any]:
+        """Return my permissions for the given project.
+
+        Don't use this function directly, use self.my_permissions which is cached."""
+        if self.is_cloud:
+            return self.jira.my_permissions(
+                projectKey=project, permissions=",".join(PERMISSIONS)
+            )["permissions"]
         return self.jira.my_permissions(projectKey=project)["permissions"]
 
     def can_i(self, permission: str) -> bool:
+        """Return whether I have the given permission in the project."""
         return bool(
             self.my_permissions(project=self.project)[permission]["havePermission"]
         )
 
     def can_create_issues(self) -> bool:
-        return self.can_i("CREATE_ISSUES")
+        """Return whether I can create issues in the project."""
+        return self.can_i(CREATE_ISSUES)
 
     def can_transition_issues(self) -> bool:
-        return self.can_i("TRANSITION_ISSUES")
+        """Return whether I can transition issues in the project."""
+        return self.can_i(TRANSITION_ISSUES)
 
     def _project_issue_types(self, project: str) -> list[IssueType]:
+        """Return all available issue types (e.g. Task, Bug) for the project.
+
+        Don't use this function directly, use self.project_issue_types which is cached.
+        """
         # Don't use self.project here, because of function.cache usage
         return [
             IssueType(id=t.id, name=t.name, statuses=[s.name for s in t.statuses])
@@ -248,6 +232,7 @@ class JiraClient:
         ]
 
     def get_issue_type(self, issue_type: str) -> IssueType | None:
+        """Return a issue type (e.g. Task) for the project if it exists."""
         for _issue_type in self.project_issue_types(self.project):
             if _issue_type.name == issue_type:
                 return _issue_type
@@ -255,15 +240,23 @@ class JiraClient:
 
     @staticmethod
     def _get_allowed_issue_field_options(
-        allowed_values: list[Resource],
+        allowed_values: list[Resource] | list[dict[str, str]],
     ) -> list[FieldOption | CustomFieldOption]:
-        """Return a list of allowed values for a field."""
-        return [
-            CustomFieldOption(value=v.value)
-            if isinstance(v, JiraCustomFieldOption)
-            else FieldOption(name=v.name)
-            for v in allowed_values
-        ]
+        """Return a list of allowed values for a field. E.g. Minor, Major ... for Priority in a Task."""
+        items: list[FieldOption | CustomFieldOption] = []
+        for v in allowed_values:
+            match v:
+                case dict() if "value" in v:
+                    items.append(CustomFieldOption(value=v["value"]))
+                case dict() if "name" in v:
+                    items.append(FieldOption(name=v["name"]))
+                case JiraCustomFieldOption():
+                    items.append(CustomFieldOption(value=v.value))
+                case Resource():
+                    items.append(FieldOption(name=v.name))
+                case _:
+                    logging.warning(f"Unknown allowed value type: {type(v)}")
+        return items
 
     def _project_issue_fields(
         self, project: str, issue_type_id: str
@@ -273,6 +266,27 @@ class JiraClient:
         This API endpoint needs createIssue project permissions.
         """
         # Don't use self.project here, because of function.cache usage
+        if self.is_cloud:
+            metadata = self.jira.createmeta(
+                projectKeys=self.project,
+                issuetypeIds=[issue_type_id],
+                expand="projects.issuetypes.fields",
+            )
+            if not metadata["projects"] or not metadata["projects"][0]["issuetypes"]:
+                return []
+            return [
+                IssueField(
+                    name=field["name"],
+                    id=field_id,
+                    options=self._get_allowed_issue_field_options(
+                        field.get("allowedValues", [])
+                    ),
+                )
+                for field_id, field in metadata["projects"][0]["issuetypes"][0][
+                    "fields"
+                ].items()
+            ]
+
         return [
             IssueField(
                 name=field.name,
@@ -304,6 +318,10 @@ class JiraClient:
 
     def project_priority_scheme(self) -> list[str]:
         """Return a list of all priority IDs for the project."""
+        if self.is_cloud:
+            # Cloud does not have a way to retrieve project specific priority schemes
+            return []
+
         scheme = self.jira.project_priority_scheme(self.project)
         return scheme.optionIds
 
@@ -329,4 +347,5 @@ class JiraClient:
 
     @property
     def is_archived(self) -> bool:
-        return self.jira.project(self.project).archived
+        """Return whether the project is archived."""
+        return getattr(self.jira.project(self.project), "archived", False)

reconcile/utils/jjb_client.py

@@ -9,6 +9,7 @@ import tempfile
 import xml.etree.ElementTree as ET
 from collections.abc import Iterable, Mapping
 from os import path
+from pathlib import Path
 from subprocess import (
     PIPE,
     STDOUT,
@@ -17,10 +18,9 @@ from subprocess import (
 from typing import Any
 
 import yaml
-from jenkins_jobs.builder import JenkinsManager
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.parser import YamlParser
-from jenkins_jobs.registry import ModuleRegistry
+from jenkins_jobs.loader import load_files
+from jenkins_jobs.roots import Roots
 from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
@@ -292,13 +292,10 @@ class JJB:
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
-        builder = JenkinsManager(jjb.jjb_config)
-        registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
-        parser = YamlParser(jjb.jjb_config)
-        parser.load_files(jjb.options.path)
-        jobs, _ = parser.expandYaml(registry, jjb.options.names)
-
-        return jobs
+        roots = Roots(jjb.jjb_config)
+        load_files(jjb.jjb_config, roots, [Path(config_path)])
+        job_view_data_list = roots.generate_jobs()
+        return [job.data for job in job_view_data_list]
 
     def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
         job_webhooks_data: dict[str, list[dict[str, Any]]] = {}

reconcile/utils/jobcontroller/controller.py

@@ -3,7 +3,7 @@ import time
 from datetime import datetime
 from typing import Protocol, TextIO
 
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1Job,
     V1ObjectMeta,
@@ -100,7 +100,7 @@ class K8sJobController:
         """
         new_cache = {}
         for item in self.oc.get_items(
-            kind="Job",
+            kind="Job.batch",
             namespace=self.namespace,
         ):
             openshift_resource = OpenshiftResource(

reconcile/utils/jobcontroller/models.py

@@ -38,6 +38,8 @@ class JobValidationError(Exception):
 
 
 JOB_GENERATION_ANNOTATION = "qontract-reconcile/job.generation"
+MAX_JOB_NAME_LENGTH = 63
+UNIT_OF_WORK_DIGEST_LENGTH = 10
 
 
 class K8sJob(ABC):
@@ -72,7 +74,21 @@ class K8sJob(ABC):
     """
 
     def name(self) -> str:
-        return f"{self.name_prefix()}-{self.unit_of_work_digest()}"
+        """
+        Generate the full job name by combining the name prefix with a digest.
+
+        The name is constructed from the name_prefix (truncated to ensure total
+        length compliance) and the unit_of_work_digest. The total length is
+        limited to MAX_JOB_NAME_LENGTH (63 characters) to comply with Kubernetes
+        naming constraints.
+
+        Returns:
+            A unique job name in the format: {name_prefix}-{digest}
+        """
+        prefix = self.name_prefix()[
+            : MAX_JOB_NAME_LENGTH - UNIT_OF_WORK_DIGEST_LENGTH - 1
+        ]
+        return f"{prefix}-{self.unit_of_work_digest(UNIT_OF_WORK_DIGEST_LENGTH)}"
 
     @abstractmethod
     def name_prefix(self) -> str: