qontract-reconcile 0.10.2.dev299__py3-none-any.whl → 0.10.2.dev430__py3-none-any.whl

reconcile/utils/jjb_client.py

@@ -1,6 +1,5 @@
 import difflib
 import filecmp
-import json
 import logging
 import os
 import re
@@ -8,7 +7,9 @@ import shutil
 import subprocess
 import tempfile
 import xml.etree.ElementTree as ET
+from collections.abc import Iterable, Mapping
 from os import path
+from pathlib import Path
 from subprocess import (
     PIPE,
     STDOUT,
@@ -17,14 +18,16 @@ from subprocess import (
 from typing import Any
 
 import yaml
-from jenkins_jobs.builder import JenkinsManager
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.parser import YamlParser
-from jenkins_jobs.registry import ModuleRegistry
+from jenkins_jobs.loader import load_files
+from jenkins_jobs.roots import Roots
 from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.json import json_dumps
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.state import State
 from reconcile.utils.vcs import GITHUB_BASE_URL
 
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
@@ -33,14 +36,22 @@ JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
-    def __init__(self, configs, ssl_verify=True, secret_reader=None, print_only=False):
+    def __init__(
+        self,
+        configs: list[dict[str, Any]],
+        ssl_verify: bool = True,
+        secret_reader: SecretReaderBase | None = None,
+        print_only: bool = False,
+    ) -> None:
         self.print_only = print_only
         self.secret_reader = secret_reader
+        if not self.print_only and self.secret_reader is None:
+            raise ValueError("secret_reader must be provided if print_only is False")
         self.collect_configs(configs)
         self.modify_logger()
         self.python_https_verify = str(int(ssl_verify))
 
-    def collect_configs(self, configs):
+    def collect_configs(self, configs: list[dict[str, Any]]) -> None:
         instances = {
             c["instance"]["name"]: {
                 "serverUrl": c["instance"]["serverUrl"],
@@ -57,7 +68,7 @@ class JJB:
             server_url = data["serverUrl"]
             wd = tempfile.mkdtemp()
             ini = JJB_INI
-            if not self.print_only:
+            if not self.print_only and self.secret_reader:
                 ini = self.secret_reader.read(token)
                 ini = ini.replace('"', "")
                 ini = ini.replace("false", "False")
@@ -92,7 +103,7 @@ class JJB:
         self.instance_urls = instance_urls
         self.working_dirs = working_dirs
 
-    def overwrite_configs(self, configs):
+    def overwrite_configs(self, configs: Mapping[str, str] | State) -> None:
         """This function will override the existing
         config files in the working directories with
         the supplied configs"""
@@ -101,12 +112,12 @@ class JJB:
             with open(config_path, "w", encoding="locale") as f:
                 f.write(configs[name])
 
-    def sort(self, configs):
+    def sort(self, configs: list[dict[str, Any]]) -> None:
         configs.sort(key=self.sort_by_name)
-        configs.sort(key=self.sort_by_type)
+        configs.sort(key=lambda x: self.sort_by_type(x) or 0)
 
     @staticmethod
-    def sort_by_type(config):
+    def sort_by_type(config: Mapping[str, Any]) -> int:
         if config["type"] == "defaults":
             return 0
         if config["type"] == "global-defaults":
@@ -123,12 +134,13 @@ class JJB:
             return 40
         if config["type"] == "jobs":
             return 50
+        return 100
 
     @staticmethod
-    def sort_by_name(config):
+    def sort_by_name(config: Mapping[str, Any]) -> str:
         return config["name"]
 
-    def get_configs(self):
+    def get_configs(self) -> dict[str, str]:
         """This function gets the configs from the
         working directories"""
         configs = {}
@@ -139,7 +151,7 @@ class JJB:
 
         return configs
 
-    def generate(self, io_dir, fetch_state):
+    def generate(self, io_dir: str, fetch_state: str) -> None:
         """
         Generates job definitions from JJB configs
 
@@ -163,7 +175,7 @@ class JJB:
             self.execute(args)
             throughput.change_files_ownership(io_dir)
 
-    def print_diffs(self, io_dir, instance_name=None):
+    def print_diffs(self, io_dir: str, instance_name: str | None = None) -> None:
         """Print the diffs between the current and
         the desired job definitions"""
         current_path = path.join(io_dir, "jjb", "current")
@@ -179,7 +191,7 @@ class JJB:
         self.print_diff(delete, current_path, "delete")
         self.print_diff(common, desired_path, "update")
 
-    def print_diff(self, files, replace_path, action):
+    def print_diff(self, files: Iterable[str], replace_path: str, action: str) -> None:
         for f in files:
             if action == "update":
                 ft = self.toggle_cd(f)
@@ -210,11 +222,16 @@ class JJB:
                     ]
                     logging.debug("DIFF:\n" + "".join(diff))
 
-    def compare_files(self, from_files, subtract_files, in_op=False):
+    def compare_files(
+        self,
+        from_files: Iterable[str],
+        subtract_files: Iterable[str],
+        in_op: bool = False,
+    ) -> list[str]:
         return [f for f in from_files if (self.toggle_cd(f) in subtract_files) is in_op]
 
     @staticmethod
-    def get_files(search_path, instance_name=None):
+    def get_files(search_path: str, instance_name: str | None = None) -> list[str]:
         if instance_name is not None:
             search_path = path.join(search_path, instance_name)
         return [
@@ -222,7 +239,7 @@ class JJB:
         ]
 
     @staticmethod
-    def toggle_cd(file_name):
+    def toggle_cd(file_name: str) -> str:
         if "desired" in file_name:
             return file_name.replace("desired", "current")
         return file_name.replace("current", "desired")
@@ -248,43 +265,40 @@ class JJB:
                 raise
 
     @staticmethod
-    def get_jjb(args):
+    def get_jjb(args: Iterable[str]) -> Any:
         from jenkins_jobs.cli.entry import JenkinsJobs  # noqa: PLC0415
 
         return JenkinsJobs(args)
 
-    def execute(self, args):
+    def execute(self, args: Iterable[str]) -> None:
         jjb = self.get_jjb(args)
         with toggle_logger():
             jjb.execute()
 
-    def modify_logger(self):
+    def modify_logger(self) -> None:
         yaml.warnings({"YAMLLoadWarning": False})
         formatter = logging.Formatter("%(levelname)s: %(message)s")
         logger = logging.getLogger()
         logger.handlers[0].setFormatter(formatter)
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         for wd in self.working_dirs.values():
             shutil.rmtree(wd)
 
     @retry(exceptions=(JenkinsJobsException))
-    def get_jobs(self, wd, name):
+    def get_jobs(self, wd: str, name: str) -> list[dict[str, Any]]:
         ini_path = f"{wd}/{name}.ini"
         config_path = f"{wd}/config.yaml"
 
         args = ["--conf", ini_path, "test", config_path]
         jjb = self.get_jjb(args)
-        builder = JenkinsManager(jjb.jjb_config)
-        registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
-        parser = YamlParser(jjb.jjb_config)
-        parser.load_files(jjb.options.path)
-        jobs, _ = parser.expandYaml(registry, jjb.options.names)
+        roots = Roots(jjb.jjb_config)
+        load_files(jjb.jjb_config, roots, [Path(config_path)])
+        job_view_data_list = roots.generate_jobs()
+        return [job.data for job in job_view_data_list]
 
-        return jobs
-
-    def get_job_webhooks_data(self):
-        job_webhooks_data = {}
+    def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
+        job_webhooks_data: dict[str, list[dict[str, Any]]] = {}
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
 
@@ -313,7 +327,7 @@ class JJB:
 
         return job_webhooks_data
 
-    def get_repos(self):
+    def get_repos(self) -> set[str]:
         repos = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -325,7 +339,7 @@ class JJB:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
-    def get_admins(self):
+    def get_admins(self) -> set[str]:
         admins = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -340,15 +354,17 @@ class JJB:
         return admins
 
     @staticmethod
-    def get_repo_url(job):
+    def get_repo_url(job: Mapping[str, Any]) -> str:
         repo_url_raw = job["properties"][0]["github"]["url"]
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
-    def get_ref(job: dict) -> str:
+    def get_ref(job: Mapping[str, Any]) -> str:
         return job["scm"][0]["git"]["branches"][0]
 
-    def get_all_jobs(self, job_types=None, instance_name=None) -> dict[str, list[dict]]:
+    def get_all_jobs(
+        self, job_types: Iterable[str] | None = None, instance_name: str | None = None
+    ) -> dict[str, list[dict[str, Any]]]:
         if job_types is None:
             job_types = []
         all_jobs: dict[str, list[dict]] = {}
@@ -366,8 +382,8 @@ class JJB:
 
         return all_jobs
 
-    def print_jobs(self, job_name=None):
-        all_jobs = {}
+    def print_jobs(self, job_name: str | None = None) -> None:
+        all_jobs: dict[str, list[dict[str, Any]]] = {}
         found = False
         for name, wd in self.working_dirs.items():
             logging.debug(f"getting jobs from {name}")
@@ -380,7 +396,7 @@ class JJB:
                 found = True
         if not found:
             raise ValueError(f"job name {job_name} is not found")
-        print(json.dumps(all_jobs, indent=2))
+        print(json_dumps(all_jobs, indent=2))
 
     def get_job_by_repo_url(self, repo_url: str, job_type: str) -> dict[str, Any]:
         for jobs in self.get_all_jobs(job_types=[job_type]).values():
@@ -394,7 +410,7 @@ class JJB:
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")
 
     @staticmethod
-    def get_trigger_phrases_regex(job: dict) -> str | None:
+    def get_trigger_phrases_regex(job: Mapping[str, Any]) -> str | None:
         for trigger in job["triggers"]:
             if "gitlab" in trigger:
                 return trigger["gitlab"].get("note-regex")
@@ -403,7 +419,7 @@ class JJB:
         return None
 
     @staticmethod
-    def get_gitlab_webhook_trigger(job: dict) -> list[str]:
+    def get_gitlab_webhook_trigger(job: Mapping[str, Any]) -> list[str]:
         gitlab_triggers = job["triggers"][0]["gitlab"]
         # pr-check job should be triggered by merge request events
         # and certain comments: [test]|/retest|/lgtm|/lgtm cancel|/hold|/hold cancel

reconcile/utils/jobcontroller/controller.py

@@ -3,7 +3,7 @@ import time
 from datetime import datetime
 from typing import Protocol, TextIO
 
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1Job,
     V1ObjectMeta,
@@ -100,7 +100,7 @@ class K8sJobController:
         """
         new_cache = {}
         for item in self.oc.get_items(
-            kind="Job",
+            kind="Job.batch",
             namespace=self.namespace,
         ):
             openshift_resource = OpenshiftResource(

reconcile/utils/jobcontroller/models.py

@@ -38,6 +38,8 @@ class JobValidationError(Exception):
 
 
 JOB_GENERATION_ANNOTATION = "qontract-reconcile/job.generation"
+MAX_JOB_NAME_LENGTH = 63
+UNIT_OF_WORK_DIGEST_LENGTH = 10
 
 
 class K8sJob(ABC):
@@ -72,7 +74,21 @@ class K8sJob(ABC):
     """
 
     def name(self) -> str:
-        return f"{self.name_prefix()}-{self.unit_of_work_digest()}"
+        """
+        Generate the full job name by combining the name prefix with a digest.
+
+        The name is constructed from the name_prefix (truncated to ensure total
+        length compliance) and the unit_of_work_digest. The total length is
+        limited to MAX_JOB_NAME_LENGTH (63 characters) to comply with Kubernetes
+        naming constraints.
+
+        Returns:
+            A unique job name in the format: {name_prefix}-{digest}
+        """
+        prefix = self.name_prefix()[
+            : MAX_JOB_NAME_LENGTH - UNIT_OF_WORK_DIGEST_LENGTH - 1
+        ]
+        return f"{prefix}-{self.unit_of_work_digest(UNIT_OF_WORK_DIGEST_LENGTH)}"
 
     @abstractmethod
     def name_prefix(self) -> str:

reconcile/utils/json.py

@@ -0,0 +1,74 @@
+import json
+from collections.abc import Callable
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime
+from decimal import Decimal
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel
+from pydantic.main import IncEx
+
+JSON_COMPACT_SEPARATORS = (",", ":")
+
+
+def pydantic_encoder(obj: Any) -> Any:
+    if isinstance(obj, BaseModel):
+        return obj.model_dump()
+
+    if is_dataclass(obj):
+        return asdict(obj)  # type: ignore
+
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+
+    if isinstance(obj, Enum):
+        return obj.value
+
+    if isinstance(obj, Decimal):
+        return float(obj)
+
+    raise TypeError(
+        f"Object of type '{obj.__class__.__name__}' is not JSON serializable"
+    )
+
+
+def json_dumps(
+    data: Any,
+    *,
+    compact: bool = False,
+    indent: int | None = None,
+    cls: type[json.JSONEncoder] | None = None,
+    defaults: Callable | None = None,
+    # BaseModel dump parameters
+    by_alias: bool = True,
+    exclude_none: bool = False,
+    exclude: IncEx | None = None,
+    mode: Literal["json", "python"] = "json",
+) -> str:
+    """
+    Serialize `data` to a consistent JSON formatted `str` with dict keys sorted.
+
+    Args:
+        data: The data to serialize.
+        compact: If True, use compact separators (no spaces after commas or colons).
+        indent: If specified, pretty-print the JSON with this many spaces of indentation.
+        cls: A custom JSONEncoder subclass to use for serialization.
+    Returns:
+        A JSON formatted string.
+    """
+    if isinstance(data, BaseModel):
+        data = data.model_dump(
+            mode=mode, by_alias=by_alias, exclude_none=exclude_none, exclude=exclude
+        )
+        if mode == "python":
+            defaults = pydantic_encoder
+    separators = JSON_COMPACT_SEPARATORS if compact else None
+    return json.dumps(
+        data,
+        indent=indent,
+        separators=separators,
+        sort_keys=True,
+        cls=cls,
+        default=defaults,
+    )

reconcile/utils/ldap_client.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 from collections.abc import Iterable
+from typing import Any, Self
 
 from ldap3 import (
     ALL,
@@ -17,15 +18,15 @@ class LdapClient:
     appropriately.
     """
 
-    def __init__(self, base_dn: str, connection: Connection):
+    def __init__(self, base_dn: str, connection: Connection) -> None:
         self.base_dn = base_dn
         self.connection = connection
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         self.connection.bind()
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         self.connection.unbind()
 
     def get_users(self, uids: Iterable[str]) -> set[str]:

reconcile/utils/lean_terraform_client.py

@@ -6,7 +6,9 @@ from collections.abc import Mapping
 from typing import Any
 
 
-def state_rm_access_key(working_dirs, account, user):
+def state_rm_access_key(
+    working_dirs: Mapping[str, str], account: str, user: str
+) -> bool:
     wd = working_dirs[account]
     init_result = subprocess.run(["terraform", "init"], check=False, cwd=wd)
     if init_result.returncode != 0:

reconcile/utils/membershipsources/app_interface_resolver.py

@@ -55,6 +55,8 @@ def resolve_app_interface_membership_source(
 
 def build_member_list(role: RoleV1) -> list[RoleMember]:
     members: list[RoleMember] = []
-    members.extend([RoleUser(**u.dict()) for u in role.users or []])
-    members.extend([RoleBot(**b.dict()) for b in role.bots or [] if b.org_username])
+    members.extend([RoleUser(**u.model_dump()) for u in role.users or []])
+    members.extend([
+        RoleBot(**b.model_dump()) for b in role.bots or [] if b.org_username
+    ])
     return members

reconcile/utils/membershipsources/models.py

@@ -7,7 +7,6 @@ from typing import (
 
 from pydantic import (
     BaseModel,
-    Extra,
 )
 
 from reconcile.gql_definitions.fragments.membership_source import (
@@ -23,7 +22,7 @@ class User(Protocol):
     @property
     def org_username(self) -> str: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class Bot(Protocol):
@@ -33,7 +32,7 @@ class Bot(Protocol):
     @property
     def org_username(self) -> str | None: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class RoleWithMemberships(Protocol):
@@ -50,33 +49,27 @@ class RoleWithMemberships(Protocol):
     def member_sources(self) -> Sequence[RoleMembershipSource] | None: ...
 
 
-class RoleUser(BaseModel):
+class RoleUser(BaseModel, extra="ignore"):
     name: str
     org_username: str
-    github_username: str | None
-    quay_username: str | None
-    pagerduty_username: str | None
-    aws_username: str | None
-    cloudflare_user: str | None
-    public_gpg_key: str | None
+    github_username: str | None = None
+    quay_username: str | None = None
+    pagerduty_username: str | None = None
+    aws_username: str | None = None
+    cloudflare_user: str | None = None
+    public_gpg_key: str | None = None
     tag_on_cluster_updates: bool | None = False
     tag_on_merge_requests: bool | None = False
 
-    class Config:
-        extra = Extra.ignore
 
-
-class RoleBot(BaseModel):
+class RoleBot(BaseModel, extra="ignore"):
     name: str
-    description: str | None
-    org_username: str | None
-    github_username: str | None
-    gitlab_username: str | None
-    openshift_serviceaccount: str | None
-    quay_username: str | None
-
-    class Config:
-        extra = Extra.ignore
+    description: str | None = None
+    org_username: str | None = None
+    github_username: str | None = None
+    gitlab_username: str | None = None
+    openshift_serviceaccount: str | None = None
+    quay_username: str | None = None
 
 
 RoleMember = RoleUser | RoleBot

reconcile/utils/membershipsources/resolver.py

@@ -98,8 +98,10 @@ def resolve_role_members(
         members: list[RoleMember] = []
 
         # bring in the local users and bots ...
-        members.extend(RoleUser(**u.dict()) for u in r.users or [])
-        members.extend(RoleBot(**b.dict()) for b in r.bots or [] if b.org_username)
+        members.extend(RoleUser(**u.model_dump()) for u in r.users or [])
+        members.extend(
+            RoleBot(**b.model_dump()) for b in r.bots or [] if b.org_username
+        )
 
         # ... and enhance with the ones from member sources
         for ms in r.member_sources or []:

reconcile/utils/merge_request_manager/merge_request_manager.py

@@ -1,7 +1,7 @@
 import logging
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Generic, TypeVar
+from typing import Any, TypeVar
 
 from gitlab.v4.objects import ProjectMergeRequest
 from pydantic import BaseModel
@@ -17,12 +17,12 @@ T = TypeVar("T", bound=BaseModel)
 
 
 @dataclass
-class OpenMergeRequest(Generic[T]):
+class OpenMergeRequest[T: BaseModel]:
     raw: ProjectMergeRequest
     mr_info: T
 
 
-class MergeRequestManagerBase(Generic[T]):
+class MergeRequestManagerBase[T: BaseModel]:
     """ """
 
     def __init__(self, vcs: VCS, parser: Parser, mr_label: str):
@@ -42,7 +42,7 @@ class MergeRequestManagerBase(Generic[T]):
         expected_data: dict[str, Any],
     ) -> OpenMergeRequest | None:
         for mr in self._open_mrs:
-            mr_info_dict = mr.mr_info.dict()
+            mr_info_dict = mr.mr_info.model_dump()
             if all(mr_info_dict.get(k) == expected_data.get(k) for k in expected_data):
                 return mr
 

reconcile/utils/merge_request_manager/parser.py

@@ -1,5 +1,5 @@
 import re
-from typing import Generic, TypeVar
+from typing import TypeVar
 
 from pydantic import BaseModel
 
@@ -17,7 +17,7 @@ class ParserVersionError(Exception):
 T = TypeVar("T", bound=BaseModel)
 
 
-class Parser(Generic[T]):
+class Parser[T: BaseModel]:
     """This class is only concerned with parsing an MR description rendered by the Renderer."""
 
     def __init__(
@@ -60,8 +60,8 @@ class Parser(Generic[T]):
 
         if self.expected_version != self._find_by_name(self.version_ref, parts[1]):
             raise ParserVersionError("Version is outdated")
-        return self.klass(
-            **data_default_none(
-                self.klass, self._data_from_description(parts[1]), use_defaults=False
-            )
+        data = data_default_none(
+            self.klass, self._data_from_description(parts[1]), use_defaults=False
         )
+        assert isinstance(data, dict)
+        return self.klass(**data)

reconcile/utils/metrics.py

@@ -144,7 +144,7 @@ class GaugeMetric(BaseMetric):
 
     @classmethod
     def metric_family(cls) -> GaugeMetricFamily:
-        labels = [f.alias for f in cls.__fields__.values()]
+        labels = [f.alias or name for name, f in cls.model_fields.items()]
         return GaugeMetricFamily(cls.name(), cls.__doc__ or "", labels=labels)
 
     @classmethod
@@ -167,7 +167,7 @@ class CounterMetric(BaseMetric):
 
     @classmethod
     def metric_family(cls) -> CounterMetricFamily:
-        labels = [f.alias for f in cls.__fields__.values()]
+        labels = [f.alias or name for name, f in cls.model_fields.items()]
         return CounterMetricFamily(cls.name(), cls.__doc__ or "", labels=labels)
 
     @classmethod
@@ -198,7 +198,7 @@ class MetricsContainer:
         """
         Sets the value of the given gauge metric to the given value.
         """
-        label_values = tuple(metric.dict(by_alias=True).values())
+        label_values = tuple(metric.model_dump(by_alias=True).values())
         self._gauges[metric.__class__][label_values] = value
 
     def set_info(self, metric: InfoMetric) -> None:
@@ -213,7 +213,7 @@ class MetricsContainer:
         Increases the value of the given counter by the given amount.
         """
         # all label values need to be strings, so lets convert them
-        label_values = tuple(str(v) for v in counter.dict(by_alias=True).values())
+        label_values = tuple(str(v) for v in counter.model_dump(by_alias=True).values())
         current_value = self._counters[counter.__class__].get(label_values) or 0
         self._counters[counter.__class__][label_values] = current_value + by
 
@@ -270,7 +270,7 @@ class MetricsContainer:
             (
                 metric_class(**{
                     key: labels[i]
-                    for i, key in enumerate(metric_class.__fields__.keys())
+                    for i, key in enumerate(metric_class.model_fields.keys())
                 }),
                 value,
             )