qontract-reconcile 0.10.2.dev245__tar.gz → 0.10.2.dev247__tar.gz

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev245
+Version: 0.10.2.dev247
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/pyproject.toml

@@ -290,7 +290,6 @@ module = [
     "reconcile.utils.gpg",
     "reconcile.utils.gql",
     "reconcile.utils.imap_client",
-    "reconcile.utils.jenkins_api",
     "reconcile.utils.jjb_client",
     "reconcile.utils.ldap_client",
     "reconcile.utils.lean_terraform_client",

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/reconcile/utils/jenkins_api.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Any
+from collections.abc import Mapping
+from typing import Any, NotRequired, Self, TypedDict
 
 import requests
 import toml
@@ -9,23 +10,40 @@ from sretoolbox.utils import retry
 from reconcile.utils.secret_reader import SecretReaderBase
 
 
+class JobBuildState(TypedDict):
+    _class: NotRequired[str]
+    number: int
+    result: NotRequired[str | None]
+    actions: NotRequired[list]
+    commit_sha: NotRequired[str]
+
+
 class JenkinsApi:
     """Wrapper around Jenkins API calls"""
 
-    @staticmethod
+    @classmethod
     def init_jenkins_from_secret(
-        secret_reader: SecretReaderBase, secret, ssl_verify=True
-    ) -> "JenkinsApi":
+        cls,
+        secret_reader: SecretReaderBase,
+        secret: Mapping[str, Any],
+        ssl_verify: bool = True,
+    ) -> Self:
         token_config = secret_reader.read(secret)
         config = toml.loads(token_config)
-        return JenkinsApi(
+        return cls(
             config["jenkins"]["url"],
             config["jenkins"]["user"],
             config["jenkins"]["password"],
             ssl_verify=ssl_verify,
         )
 
-    def __init__(self, url: str, user: str, password: str, ssl_verify=True):
+    def __init__(
+        self,
+        url: str,
+        user: str,
+        password: str,
+        ssl_verify: bool = True,
+    ):
         self.url = url
         self.user = user
         self.password = password
@@ -43,7 +61,7 @@ class JenkinsApi:
         res.raise_for_status()
         return yaml.safe_load(res.text)
 
-    def apply_jcasc_config(self, config: dict[str, Any]):
+    def apply_jcasc_config(self, config: dict[str, Any]) -> None:
         url = f"{self.url}/manage/configuration-as-code/apply"
         res = requests.post(
             url,
@@ -54,7 +72,7 @@ class JenkinsApi:
         )
         res.raise_for_status()
 
-    def get_job_names(self):
+    def get_job_names(self) -> list[str]:
         url = f"{self.url}/api/json?tree=jobs[name]"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
@@ -64,36 +82,52 @@ class JenkinsApi:
         job_names = [r["name"] for r in res.json()["jobs"]]
         return job_names
 
+    @staticmethod
+    def _get_commit_sha_from_build(build: Mapping[str, Any]) -> str | None:
+        for action in reversed(build.get("actions", [])):
+            if revision := action.get("lastBuiltRevision"):
+                return revision["SHA1"]
+        return None
+
+    def _build_job_build_state(self, build: Mapping) -> JobBuildState:
+        job_build_state = JobBuildState(number=build["number"])
+        if "_class" in build:
+            job_build_state["_class"] = build["_class"]
+        if "actions" in build:
+            job_build_state["actions"] = build["actions"]
+        if "result" in build:
+            job_build_state["result"] = build["result"]
+        if commit_sha := self._get_commit_sha_from_build(build):
+            job_build_state["commit_sha"] = commit_sha
+        return job_build_state
+
     @retry()
-    def get_jobs_state(self):
+    def get_jobs_state(self) -> dict[str, list[JobBuildState]]:
         url = f"{self.url}/api/json?tree=jobs[name,builds[number,result,actions[lastBuiltRevision[SHA1]]]]"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
         )
 
         res.raise_for_status()
-        jobs_state = {}
-        for r in res.json()["jobs"]:
-            job_name = r["name"]
-            builds = r.get("builds", [])
-            for b in builds:
-                actions = b.get("actions", [])
-                for a in actions:
-                    revision = a.get("lastBuiltRevision")
-                    if revision:
-                        b["commit_sha"] = revision["SHA1"]
-            jobs_state[job_name] = builds
-
-        return jobs_state
-
-    def delete_build(self, job_name, build_id):
+        jobs = res.json().get("jobs") or []
+        return {
+            job["name"]: list(
+                map(
+                    self._build_job_build_state,
+                    job.get("builds", []),
+                )
+            )
+            for job in jobs
+        }
+
+    def delete_build(self, job_name: str, build_id: str) -> None:
         url = f"{self.url}/job/{job_name}/{build_id}/doDelete"
         res = requests.post(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
         )
         res.raise_for_status()
 
-    def get_all_roles(self):
+    def get_all_roles(self) -> dict[str, Any]:
         url = f"{self.url}/role-strategy/strategy/getAllRoles"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
@@ -102,7 +136,7 @@ class JenkinsApi:
         res.raise_for_status()
         return res.json()
 
-    def assign_role_to_user(self, role, user):
+    def assign_role_to_user(self, role: str, user: str) -> None:
         url = f"{self.url}/role-strategy/strategy/assignRole"
         data = {"type": "globalRoles", "roleName": role, "sid": user}
         res = requests.post(
@@ -115,7 +149,7 @@ class JenkinsApi:
 
         res.raise_for_status()
 
-    def unassign_role_from_user(self, role, user):
+    def unassign_role_from_user(self, role: str, user: str) -> None:
         url = f"{self.url}/role-strategy/strategy/unassignRole"
         data = {"type": "globalRoles", "roleName": role, "sid": user}
         res = requests.post(
@@ -128,7 +162,7 @@ class JenkinsApi:
 
         res.raise_for_status()
 
-    def safe_restart(self, force_restart=False):
+    def safe_restart(self, force_restart: bool = False) -> None:
         url = f"{self.url}/safeRestart"
         if self.should_restart or force_restart:
             logging.debug(
@@ -142,7 +176,7 @@ class JenkinsApi:
 
             res.raise_for_status()
 
-    def get_builds(self, job_name):
+    def get_builds(self, job_name: str) -> list[dict[str, Any]]:
         url = (
             f"{self.url}/job/{job_name}/api/json"
             + "?tree=allBuilds[timestamp,result,id]"
@@ -157,14 +191,14 @@ class JenkinsApi:
         except KeyError:
             return []
 
-    def get_build_history(self, job_name, time_limit):
+    def get_build_history(self, job_name: str, time_limit: int) -> list[str]:
         return [
             b["result"]
             for b in self.get_builds(job_name)
             if time_limit < self.timestamp_seconds(b["timestamp"])
         ]
 
-    def is_job_running(self, job_name):
+    def is_job_running(self, job_name: str) -> bool:
         url = f"{self.url}/job/{job_name}/lastBuild/api/json"
         res = requests.get(
             url, verify=self.ssl_verify, auth=(self.user, self.password), timeout=60
@@ -178,7 +212,7 @@ class JenkinsApi:
         res.raise_for_status()
         return res.json()["building"] is True
 
-    def get_crumb_kwargs(self):
+    def get_crumb_kwargs(self) -> dict[str, Any]:
         try:
             crumb_url = f"{self.url}/crumbIssuer/api/json"
             res = requests.get(
@@ -197,7 +231,7 @@ class JenkinsApi:
 
         return kwargs
 
-    def trigger_job(self, job_name):
+    def trigger_job(self, job_name: str) -> None:
         kwargs = self.get_crumb_kwargs()
 
         url = f"{self.url}/job/{job_name}/build"
@@ -212,5 +246,5 @@ class JenkinsApi:
         res.raise_for_status()
 
     @staticmethod
-    def timestamp_seconds(timestamp):
+    def timestamp_seconds(timestamp: float) -> int:
         return int(timestamp / 1000)

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/reconcile/utils/saasherder/interfaces.py

@@ -245,6 +245,8 @@ class SaasResourceTemplateTargetPromotion(Protocol):
     @property
     def promotion_data(self) -> Sequence[SaasPromotionData] | None: ...
 
+    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+
 
 class Channel(Protocol):
     name: str
@@ -289,6 +291,8 @@ class SaasResourceTemplateTargetUpstream(Protocol):
     @property
     def instance(self) -> SaasJenkinsInstance: ...
 
+    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+
 
 class SaasQuayInstance(Protocol):
     url: str
@@ -307,6 +311,8 @@ class SaasResourceTemplateTargetImage(Protocol):
     @property
     def org(self) -> SaasQuayOrg: ...
 
+    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+
 
 class SaasResourceTemplateTarget(HasParameters, HasSecretParameters, Protocol):
     path: str | None

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/reconcile/utils/saasherder/models.py

@@ -4,7 +4,7 @@ import logging
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from enum import Enum
-from typing import Any
+from typing import Any, NotRequired, TypedDict
 
 from github import Github
 from pydantic import (
@@ -15,6 +15,7 @@ from pydantic import (
 from reconcile.gql_definitions.fragments.saas_slo_document import (
     SLODocument,
 )
+from reconcile.utils.jenkins_api import JobBuildState
 from reconcile.utils.oc_connection_parameters import Cluster
 from reconcile.utils.saasherder.interfaces import (
     HasParameters,
@@ -78,8 +79,53 @@ class SLOKey:
     cluster_name: str
 
 
+class TriggerSpecConfigStateContentNamespaceApp(TypedDict):
+    name: str
+
+
+class TriggerSpecConfigStateContentNamespaceCluster(TypedDict):
+    name: str
+    serverUrl: str
+
+
+class TriggerSpecConfigStateContentNamespace(TypedDict):
+    name: str
+    cluster: TriggerSpecConfigStateContentNamespaceCluster
+    app: TriggerSpecConfigStateContentNamespaceApp
+
+
+class TriggerSpecConfigStateContent(TypedDict):
+    """
+    dict representation of reconcile.typed_queries.saas_files.SaasResourceTemplateTarget
+    with some additional fields.
+    """
+
+    path: str | None
+    name: str | None
+    namespace: TriggerSpecConfigStateContentNamespace
+    ref: str | None
+    promotion: dict | None
+    parameters: str | None
+    secretParameters: list[dict] | None
+    slos: list[Any] | None
+    upstream: Any | None
+    images: list[Any] | None
+    disable: bool | None
+    delete: bool | None
+
+    # additional fields
+    saas_file_parameters: str | None
+    saas_file_managed_resource_types: list[str]
+    saas_file_managed_resource_names: NotRequired[list[Any]]
+    url: str
+    rt_parameters: str | None
+    rt_secretparameters: NotRequired[list[dict]]
+    saas_file_secretparameters: NotRequired[list[dict]]
+
+
 @dataclass(frozen=True)
 class TriggerSpecConfig(TriggerSpecBase):
+    state_content: TriggerSpecConfigStateContent | None
     resource_template_url: str
     slos: list[SLODocument] | None = None
     target_name: str | None = None
@@ -110,6 +156,7 @@ class TriggerSpecConfig(TriggerSpecBase):
 
 @dataclass(frozen=True)
 class TriggerSpecMovingCommit(TriggerSpecBase):
+    state_content: str
     ref: str
 
     @property
@@ -123,6 +170,7 @@ class TriggerSpecMovingCommit(TriggerSpecBase):
 
 @dataclass(frozen=True)
 class TriggerSpecUpstreamJob(TriggerSpecBase):
+    state_content: JobBuildState
     instance_name: str
     job_name: str
 
@@ -137,6 +185,7 @@ class TriggerSpecUpstreamJob(TriggerSpecBase):
 
 @dataclass(frozen=True)
 class TriggerSpecContainerImage(TriggerSpecBase):
+    state_content: str
     images: Sequence[str]
 
     @property

{qontract_reconcile-0.10.2.dev245 → qontract_reconcile-0.10.2.dev247}/reconcile/utils/saasherder/saasherder.py

@@ -6,7 +6,6 @@ import logging
 import os
 import re
 from collections import (
-    ChainMap,
     defaultdict,
 )
 from collections.abc import (
@@ -40,7 +39,7 @@ from reconcile.status import RunningState
 from reconcile.utils import helm
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
-from reconcile.utils.jenkins_api import JenkinsApi
+from reconcile.utils.jenkins_api import JenkinsApi, JobBuildState
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.oc import (
     OCLocal,
@@ -74,6 +73,10 @@ from reconcile.utils.saasherder.models import (
     SLOKey,
     TargetSpec,
     TriggerSpecConfig,
+    TriggerSpecConfigStateContent,
+    TriggerSpecConfigStateContentNamespace,
+    TriggerSpecConfigStateContentNamespaceApp,
+    TriggerSpecConfigStateContentNamespaceCluster,
     TriggerSpecContainerImage,
     TriggerSpecMovingCommit,
     TriggerSpecUnion,
@@ -1516,8 +1519,10 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         )
         return list(itertools.chain.from_iterable(results)), error
 
-    def _get_upstream_jobs_current_state(self) -> tuple[dict[str, Any], bool]:
-        current_state: dict[str, Any] = {}
+    def _get_upstream_jobs_current_state(
+        self,
+    ) -> tuple[dict[str, dict[str, list[JobBuildState]]], bool]:
+        current_state: dict[str, dict[str, list[JobBuildState]]] = {}
         error = False
         if not self.jenkins_map:
             raise Exception("jenkins_map is not initialized")
@@ -1534,7 +1539,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
 
     def _build_trigger_spec_upstream_job_reason(
         self,
-        last_build_result: Any,
+        last_build_result: JobBuildState,
         server_url: str,
         job_name: str,
         url: str,
@@ -1551,7 +1556,10 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         return f"{prefix}{server_url}/job/{job_name}/{last_build_result_number}"
 
     def get_upstream_jobs_diff_saas_file(
-        self, saas_file: SaasFile, dry_run: bool, current_state: dict[str, Any]
+        self,
+        saas_file: SaasFile,
+        dry_run: bool,
+        current_state: dict[str, dict[str, list[JobBuildState]]],
     ) -> list[TriggerSpecUpstreamJob]:
         trigger_specs = []
         for rt in saas_file.resource_templates:
@@ -1816,7 +1824,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         return True
 
     @staticmethod
-    def remove_none_values(d: dict[Any, Any] | None) -> dict[Any, Any]:
+    def remove_none_values(d: Mapping[Any, Any] | None) -> dict[Any, Any]:
         if d is None:
             return {}
         new = {}
@@ -1859,7 +1867,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
 
     def _build_trigger_spec_config_reason(
         self,
-        state_content: dict,
+        state_content: TriggerSpecConfigStateContent,
     ) -> str | None:
         if not self.include_trigger_trace:
             return None
@@ -1869,76 +1877,95 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         # to reduce false-positives.
         auto_promotion_suffix = (
             " [auto-promotion]"
-            if (state_content.get("promotion") or {}).get("auto", False)
+            if (state_content["promotion"] or {}).get("auto", False)
             else ""
         )
         return f"{self.repo_url}/commit/{RunningState().commit}{auto_promotion_suffix}"
 
+    def _build_trigger_spec_config_state_content(
+        self,
+        target: SaasResourceTemplateTarget,
+        saas_file: SaasFile,
+        resource_template: SaasResourceTemplate,
+    ) -> TriggerSpecConfigStateContent:
+        state_content = TriggerSpecConfigStateContent(
+            name=target.name,
+            ref=target.ref,
+            promotion=(
+                target.promotion.dict(by_alias=True) if target.promotion else None
+            ),
+            secretParameters=(
+                [p.dict(by_alias=True) for p in target.secret_parameters]
+                if target.secret_parameters
+                else None
+            ),
+            slos=(
+                [slo.dict(by_alias=True) for slo in target.slos]
+                if target.slos
+                else None
+            ),
+            upstream=(target.upstream.dict(by_alias=True) if target.upstream else None),
+            images=(
+                [i.dict(by_alias=True) for i in target.images]
+                if target.images
+                else None
+            ),
+            disable=target.disable,
+            delete=target.delete,
+            namespace=self.sanitize_namespace(target.namespace),
+            # add parent parameters to target config
+            # before the GQL classes are introduced, the parameters attribute
+            # was a json string. Keep it that way to be backwards compatible.
+            saas_file_parameters=(
+                json.dumps(saas_file.parameters, separators=(",", ":"))
+                if saas_file.parameters is not None
+                else None
+            ),
+            # before the GQL classes are introduced, the parameters attribute
+            # was a json string. Keep it that way to be backwards compatible.
+            parameters=(
+                json.dumps(target.parameters, separators=(",", ":"))
+                if target.parameters is not None
+                else None
+            ),
+            # add managed resource types to target config
+            saas_file_managed_resource_types=saas_file.managed_resource_types,
+            url=resource_template.url,
+            path=resource_template.path,
+            # before the GQL classes are introduced, the parameters attribute
+            # was a json string. Keep it that way to be backwards compatible.
+            rt_parameters=(
+                json.dumps(resource_template.parameters, separators=(",", ":"))
+                if resource_template.parameters is not None
+                else None
+            ),
+        )
+        if saas_file.managed_resource_names:
+            state_content["saas_file_managed_resource_names"] = [
+                m.dict() for m in saas_file.managed_resource_names
+            ]
+        # include secret parameters from resource template and saas file
+        if resource_template.secret_parameters:
+            state_content["rt_secretparameters"] = [
+                p.dict() for p in resource_template.secret_parameters
+            ]
+        if saas_file.secret_parameters:
+            state_content["saas_file_secretparameters"] = [
+                p.dict() for p in saas_file.secret_parameters
+            ]
+        return state_content
+
     def get_saas_targets_config_trigger_specs(
         self, saas_file: SaasFile
     ) -> dict[str, TriggerSpecConfig]:
         configs = {}
         for rt in saas_file.resource_templates:
             for target in rt.targets:
-                # ChainMap will store modifications avoiding a deep copy
-                desired_target_config = ChainMap(target.dict(by_alias=True))
-                # This will add the namespace key/value to the chainMap, but
-                # the target will remain with the original value
-                # When the namespace key is looked up, the chainmap will
-                # return the modified attribute (set in the first mapping)
-                desired_target_config["namespace"] = self.sanitize_namespace(
-                    target.namespace
-                )
-                # add parent parameters to target config
-                # before the GQL classes are introduced, the parameters attribute
-                # was a json string. Keep it that way to be backwards compatible.
-                desired_target_config["saas_file_parameters"] = (
-                    json.dumps(saas_file.parameters, separators=(",", ":"))
-                    if saas_file.parameters is not None
-                    else None
+                desired_target_config = self._build_trigger_spec_config_state_content(
+                    target=target,
+                    saas_file=saas_file,
+                    resource_template=rt,
                 )
-
-                # before the GQL classes are introduced, the parameters attribute
-                # was a json string. Keep it that way to be backwards compatible.
-                desired_target_config["parameters"] = (
-                    json.dumps(target.parameters, separators=(",", ":"))
-                    if target.parameters is not None
-                    else None
-                )
-
-                # add managed resource types to target config
-                desired_target_config["saas_file_managed_resource_types"] = (
-                    saas_file.managed_resource_types
-                )
-                if saas_file.managed_resource_names:
-                    desired_target_config["saas_file_managed_resource_names"] = [
-                        m.dict() for m in saas_file.managed_resource_names
-                    ]
-
-                desired_target_config["url"] = rt.url
-                desired_target_config["path"] = rt.path
-                # before the GQL classes are introduced, the parameters attribute
-                # was a json string. Keep it that way to be backwards compatible.
-                desired_target_config["rt_parameters"] = (
-                    json.dumps(rt.parameters, separators=(",", ":"))
-                    if rt.parameters is not None
-                    else None
-                )
-
-                # include secret parameters from resource template and saas file
-                if rt.secret_parameters:
-                    desired_target_config["rt_secretparameters"] = [
-                        p.dict() for p in rt.secret_parameters
-                    ]
-                if saas_file.secret_parameters:
-                    desired_target_config["saas_file_secretparameters"] = [
-                        p.dict() for p in saas_file.secret_parameters
-                    ]
-
-                # Convert to dict, ChainMap is not JSON serializable
-                # desired_target_config needs to be serialized to generate
-                # its config hash and to be stored in S3
-                serializable_target_config = dict(desired_target_config)
                 trigger_spec = TriggerSpecConfig(
                     saas_file_name=saas_file.name,
                     env_name=target.namespace.environment.name,
@@ -1948,12 +1975,12 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     cluster_name=target.namespace.cluster.name,
                     namespace_name=target.namespace.name,
                     target_name=target.name,
-                    state_content=serializable_target_config,
+                    state_content=desired_target_config,
                     resource_template_url=rt.url,
                     target_ref=target.ref,
                     slos=target.slos or None,
                     reason=self._build_trigger_spec_config_reason(
-                        state_content=serializable_target_config
+                        state_content=desired_target_config
                     ),
                 )
                 configs[trigger_spec.state_key] = trigger_spec
@@ -1963,15 +1990,17 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     @staticmethod
     def sanitize_namespace(
         namespace: SaasResourceTemplateTargetNamespace,
-    ) -> dict[str, dict[str, str]]:
+    ) -> TriggerSpecConfigStateContentNamespace:
         """Only keep fields that should trigger a new job."""
-        return namespace.dict(
-            by_alias=True,
-            include={
-                "name": True,
-                "cluster": {"name": True, "server_url": True},
-                "app": {"name": True},
-            },
+        return TriggerSpecConfigStateContentNamespace(
+            name=namespace.name,
+            cluster=TriggerSpecConfigStateContentNamespaceCluster(
+                name=namespace.cluster.name,
+                serverUrl=namespace.cluster.server_url,
+            ),
+            app=TriggerSpecConfigStateContentNamespaceApp(
+                name=namespace.app.name,
+            ),
             # TODO: add environment.parameters to the include list!?!?
         )