qontract-reconcile 0.10.2.dev414__py3-none-any.whl → 0.10.2.dev456__py3-none-any.whl

reconcile/utils/jjb_client.py

@@ -33,6 +33,10 @@ from reconcile.utils.vcs import GITHUB_BASE_URL
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
 
+class MissingJobUrlError(Exception):
+    pass
+
+
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
@@ -335,7 +339,7 @@ class JJB:
                 job_name = job["name"]
                 try:
                     repos.add(self.get_repo_url(job))
-                except KeyError:
+                except MissingJobUrlError:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
@@ -355,7 +359,19 @@ class JJB:
 
     @staticmethod
     def get_repo_url(job: Mapping[str, Any]) -> str:
-        repo_url_raw = job["properties"][0]["github"]["url"]
+        repo_url_raw = job.get("properties", [{}])[0].get("github", {}).get("url")
+
+        # we may be in a Github Branch Source type of job
+        if not repo_url_raw:
+            gh_org = job.get("scm", [{}])[0].get("github", {}).get("repo-owner")
+            gh_repo = job.get("scm", [{}])[0].get("github", {}).get("repo")
+            if gh_org and gh_repo:
+                repo_url_raw = f"https://github.com/{gh_org}/{gh_repo}/"
+            else:
+                raise MissingJobUrlError(
+                    f"Cannot find job url for {job['display-name']}"
+                )
+
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
@@ -404,7 +420,7 @@ class JJB:
                 try:
                     if self.get_repo_url(job).lower() == repo_url.rstrip("/").lower():
                         return job
-                except KeyError:
+                except MissingJobUrlError:
                     # something wrong here. ignore this job
                     pass
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")

reconcile/utils/jobcontroller/controller.py

@@ -3,7 +3,7 @@ import time
 from datetime import datetime
 from typing import Protocol, TextIO
 
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1Job,
     V1ObjectMeta,

reconcile/utils/json.py

@@ -7,6 +7,7 @@ from enum import Enum
 from typing import Any, Literal
 
 from pydantic import BaseModel
+from pydantic.main import IncEx
 
 JSON_COMPACT_SEPARATORS = (",", ":")
 
@@ -42,6 +43,7 @@ def json_dumps(
     # BaseModel dump parameters
     by_alias: bool = True,
     exclude_none: bool = False,
+    exclude: IncEx | None = None,
     mode: Literal["json", "python"] = "json",
 ) -> str:
     """
@@ -56,7 +58,9 @@ def json_dumps(
         A JSON formatted string.
     """
     if isinstance(data, BaseModel):
-        data = data.model_dump(mode=mode, by_alias=by_alias, exclude_none=exclude_none)
+        data = data.model_dump(
+            mode=mode, by_alias=by_alias, exclude_none=exclude_none, exclude=exclude
+        )
         if mode == "python":
             defaults = pydantic_encoder
     separators = JSON_COMPACT_SEPARATORS if compact else None

reconcile/utils/oc.py

@@ -10,6 +10,7 @@ import re
 import subprocess
 import threading
 import time
+from collections import defaultdict
 from contextlib import suppress
 from dataclasses import dataclass
 from functools import cache, wraps
@@ -18,7 +19,7 @@ from threading import Lock
 from typing import TYPE_CHECKING, Any, TextIO, cast
 
 import urllib3
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     Configuration,
 )
@@ -46,7 +47,6 @@ from sretoolbox.utils import (
 )
 
 from reconcile.status import RunningState
-from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.json import json_dumps
 from reconcile.utils.jump_host import (
     JumphostParameters,
@@ -70,6 +70,16 @@ urllib3.disable_warnings()
 GET_REPLICASET_MAX_ATTEMPTS = 20
 DEFAULT_GROUP = ""
 PROJECT_KIND = "Project.project.openshift.io"
+POD_RECYCLE_SUPPORTED_TRIGGER_KINDS = [
+    "ConfigMap",
+    "Secret",
+]
+POD_RECYCLE_SUPPORTED_OWNER_KINDS = [
+    "DaemonSet",
+    "Deployment",
+    "DeploymentConfig",
+    "StatefulSet",
+]
 
 oc_run_execution_counter = Counter(
     name="oc_run_execution_counter",
@@ -126,14 +136,6 @@ class JSONParsingError(Exception):
     pass
 
 
-class RecyclePodsUnsupportedKindError(Exception):
-    pass
-
-
-class RecyclePodsInvalidAnnotationValueError(Exception):
-    pass
-
-
 class PodNotReadyError(Exception):
     pass
 
@@ -649,9 +651,15 @@ class OCCli:
             raise e
         return True
 
+    def _use_oc_project(self, namespace: str) -> bool:
+        # Note, that openshift-* namespaces cannot be created via new-project
+        return self.is_kind_supported(PROJECT_KIND) and not namespace.startswith(
+            "openshift-"
+        )
+
     @OCDecorators.process_reconcile_time
     def new_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["new-project", namespace]
         else:
             cmd = ["create", "namespace", namespace]
@@ -667,7 +675,7 @@ class OCCli:
 
     @OCDecorators.process_reconcile_time
     def delete_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported(PROJECT_KIND):
+        if self._use_oc_project(namespace=namespace):
             cmd = ["delete", "project", namespace]
         else:
             cmd = ["delete", "namespace", namespace]
@@ -922,108 +930,105 @@ class OCCli:
             if not status["ready"]:
                 raise PodNotReadyError(name)
 
-    def recycle_pods(
-        self, dry_run: bool, namespace: str, dep_kind: str, dep_resource: OR
-    ) -> None:
-        """recycles pods which are using the specified resources.
-        will only act on Secrets containing the 'qontract.recycle' annotation.
-        dry_run: simulate pods recycle.
-        namespace: namespace in which dependant resource is applied.
-        dep_kind: dependant resource kind. currently only supports Secret.
-        dep_resource: dependant resource."""
-
-        supported_kinds = ["Secret", "ConfigMap"]
-        if dep_kind not in supported_kinds:
+    def _is_resource_supported_to_trigger_recycle(
+        self,
+        namespace: str,
+        resource: OR,
+    ) -> bool:
+        if resource.kind not in POD_RECYCLE_SUPPORTED_TRIGGER_KINDS:
             logging.debug([
                 "skipping_pod_recycle_unsupported",
                 self.cluster_name,
                 namespace,
-                dep_kind,
+                resource.kind,
+                resource.name,
             ])
-            return
+            return False
 
-        dep_annotations = dep_resource.body["metadata"].get("annotations", {})
         # Note, that annotations might have been set to None explicitly
-        dep_annotations = dep_resource.body["metadata"].get("annotations") or {}
-        qontract_recycle = dep_annotations.get("qontract.recycle")
-        if qontract_recycle is True:
-            raise RecyclePodsInvalidAnnotationValueError('should be "true"')
+        annotations = resource.body["metadata"].get("annotations") or {}
+        qontract_recycle = annotations.get("qontract.recycle")
         if qontract_recycle != "true":
             logging.debug([
                 "skipping_pod_recycle_no_annotation",
                 self.cluster_name,
                 namespace,
-                dep_kind,
+                resource.kind,
+                resource.name,
             ])
+            return False
+        return True
+
+    def recycle_pods(
+        self,
+        dry_run: bool,
+        namespace: str,
+        resource: OR,
+    ) -> None:
+        """
+        recycles pods which are using the specified resources.
+        will only act on Secret or ConfigMap containing the 'qontract.recycle' annotation.
+
+        Args:
+            dry_run (bool): if True, will only log the recycle action without executing it
+            namespace (str): namespace of the resource
+            resource (OR): resource object (Secret or ConfigMap) to check for pod usage
+        """
+
+        if not self._is_resource_supported_to_trigger_recycle(namespace, resource):
             return
 
-        dep_name = dep_resource.name
         pods = self.get(namespace, "Pod")["items"]
-
-        if dep_kind == "Secret":
-            pods_to_recycle = [
-                pod for pod in pods if self.secret_used_in_pod(dep_name, pod)
-            ]
-        elif dep_kind == "ConfigMap":
-            pods_to_recycle = [
-                pod for pod in pods if self.configmap_used_in_pod(dep_name, pod)
-            ]
-        else:
-            raise RecyclePodsUnsupportedKindError(dep_kind)
-
-        recyclables: dict[str, list[dict[str, Any]]] = {}
-        supported_recyclables = [
-            "Deployment",
-            "DeploymentConfig",
-            "StatefulSet",
-            "DaemonSet",
+        pods_to_recycle = [
+            pod
+            for pod in pods
+            if self.is_resource_used_in_pod(
+                name=resource.name,
+                kind=resource.kind,
+                pod=pod,
+            )
         ]
+
+        recycle_names_by_kind = defaultdict(set)
         for pod in pods_to_recycle:
             owner = self.get_obj_root_owner(namespace, pod, allow_not_found=True)
             kind = owner["kind"]
-            if kind not in supported_recyclables:
-                continue
-            recyclables.setdefault(kind, [])
-            exists = False
-            for obj in recyclables[kind]:
-                owner_name = owner["metadata"]["name"]
-                if obj["metadata"]["name"] == owner_name:
-                    exists = True
-                    break
-            if not exists:
-                recyclables[kind].append(owner)
-
-        for kind, objs in recyclables.items():
-            for obj in objs:
-                self.recycle(dry_run, namespace, kind, obj)
-
-    @retry(exceptions=ObjectHasBeenModifiedError)
+            if kind in POD_RECYCLE_SUPPORTED_OWNER_KINDS:
+                recycle_names_by_kind[kind].add(owner["metadata"]["name"])
+
+        for kind, names in recycle_names_by_kind.items():
+            for name in names:
+                self.recycle(
+                    dry_run=dry_run,
+                    namespace=namespace,
+                    kind=kind,
+                    name=name,
+                )
+
     def recycle(
-        self, dry_run: bool, namespace: str, kind: str, obj: MutableMapping[str, Any]
+        self,
+        dry_run: bool,
+        namespace: str,
+        kind: str,
+        name: str,
     ) -> None:
-        """Recycles an object by adding a recycle.time annotation
+        """
+        Recycles an object using oc rollout restart, which will add an annotation
+        kubectl.kubernetes.io/restartedAt with the current timestamp to the pod
+        template, triggering a rolling restart.
 
-        :param dry_run: Is this a dry run
-        :param namespace: Namespace to work in
-        :param kind: Object kind
-        :param obj: Object to recycle
+        Args:
+            dry_run (bool): if True, will only log the recycle action without executing it
+            namespace (str): namespace of the object to recycle
+            kind (str): kind of the object to recycle
+            name (str): name of the object to recycle
         """
-        name = obj["metadata"]["name"]
         logging.info([f"recycle_{kind.lower()}", self.cluster_name, namespace, name])
         if not dry_run:
-            now = utc_now()
-            recycle_time = now.strftime("%d/%m/%Y %H:%M:%S")
-
-            # get the object in case it was modified
-            obj = self.get(namespace, kind, name)
-            # honor update strategy by setting annotations to force
-            # a new rollout
-            a = obj["spec"]["template"]["metadata"].get("annotations", {})
-            a["recycle.time"] = recycle_time
-            obj["spec"]["template"]["metadata"]["annotations"] = a
-            cmd = ["apply", "-n", namespace, "-f", "-"]
-            stdin = json_dumps(obj)
-            self._run(cmd, stdin=stdin, apply=True)
+            self._run(
+                ["rollout", "restart", f"{kind}/{name}", "-n", namespace],
+                apply=True,
+            )
 
     def get_obj_root_owner(
         self,
@@ -1065,12 +1070,24 @@ class OCCli:
                     )
         return obj
 
-    def secret_used_in_pod(self, name: str, pod: Mapping[str, Any]) -> bool:
-        used_resources = self.get_resources_used_in_pod_spec(pod["spec"], "Secret")
-        return name in used_resources
+    def is_resource_used_in_pod(
+        self,
+        name: str,
+        kind: str,
+        pod: Mapping[str, Any],
+    ) -> bool:
+        """
+        Check if a resource (Secret or ConfigMap) is used in a Pod.
+
+        Args:
+            name: Name of the resource
+            kind: "Secret" or "ConfigMap"
+            pod: Pod object
 
-    def configmap_used_in_pod(self, name: str, pod: Mapping[str, Any]) -> bool:
-        used_resources = self.get_resources_used_in_pod_spec(pod["spec"], "ConfigMap")
+        Returns:
+            True if the resource is used in the Pod, False otherwise.
+        """
+        used_resources = self.get_resources_used_in_pod_spec(pod["spec"], kind)
         return name in used_resources
 
     @staticmethod
@@ -1079,25 +1096,39 @@ class OCCli:
         kind: str,
         include_optional: bool = True,
     ) -> dict[str, set[str]]:
-        if kind not in {"Secret", "ConfigMap"}:
-            raise KeyError(f"unsupported resource kind: {kind}")
+        """
+        Get resources (Secrets or ConfigMaps) used in a Pod spec.
+        Returns a dictionary where keys are resource names and values are sets of keys used from that resource.
+
+        Args:
+            spec: Pod spec
+            kind: "Secret" or "ConfigMap"
+            include_optional: Whether to include optional resources
+
+        Returns:
+            A dictionary mapping resource names to sets of keys used.
+        """
+        match kind:
+            case "Secret":
+                volume_kind, volume_kind_ref, env_from_kind, env_kind, env_ref = (
+                    "secret",
+                    "secretName",
+                    "secretRef",
+                    "secretKeyRef",
+                    "name",
+                )
+            case "ConfigMap":
+                volume_kind, volume_kind_ref, env_from_kind, env_kind, env_ref = (
+                    "configMap",
+                    "name",
+                    "configMapRef",
+                    "configMapKeyRef",
+                    "name",
+                )
+            case _:
+                raise KeyError(f"unsupported resource kind: {kind}")
+
         optional = "optional"
-        if kind == "Secret":
-            volume_kind, volume_kind_ref, env_from_kind, env_kind, env_ref = (
-                "secret",
-                "secretName",
-                "secretRef",
-                "secretKeyRef",
-                "name",
-            )
-        elif kind == "ConfigMap":
-            volume_kind, volume_kind_ref, env_from_kind, env_kind, env_ref = (
-                "configMap",
-                "name",
-                "configMapRef",
-                "configMapKeyRef",
-                "name",
-            )
 
         resources: dict[str, set[str]] = {}
         for v in spec.get("volumes") or []:
@@ -1126,8 +1157,8 @@ class OCCli:
                         continue
                     resource_name = resource_ref[env_ref]
                     resources.setdefault(resource_name, set())
-                    secret_key = resource_ref["key"]
-                    resources[resource_name].add(secret_key)
+                    key = resource_ref["key"]
+                    resources[resource_name].add(key)
                 except (KeyError, TypeError):
                     continue
 

reconcile/utils/rhcsv2_certs.py

@@ -1,21 +1,35 @@
+import base64
 import re
 from datetime import UTC
+from enum import StrEnum
 
 import requests
 from cryptography import x509
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives.serialization import pkcs12
 from cryptography.x509.oid import NameOID
 from pydantic import BaseModel, Field
 
 
-class RhcsV2Cert(BaseModel, validate_by_name=True, validate_by_alias=True):
+class CertificateFormat(StrEnum):
+    PEM = "PEM"
+    PKCS12 = "PKCS12"
+
+
+class RhcsV2CertPem(BaseModel, validate_by_name=True, validate_by_alias=True):
     certificate: str = Field(alias="tls.crt")
     private_key: str = Field(alias="tls.key")
     ca_cert: str = Field(alias="ca.crt")
     expiration_timestamp: int
 
 
+class RhcsV2CertPkcs12(BaseModel, validate_by_name=True, validate_by_alias=True):
+    pkcs12_keystore: str = Field(alias="keystore.pkcs12.b64")
+    pkcs12_truststore: str = Field(alias="truststore.pkcs12.b64")
+    expiration_timestamp: int
+
+
 def extract_cert(text: str) -> re.Match:
     # The CA webform returns an HTML page with inline JS that builds an array of “outputList”
     # objects. Each object looks roughly like:
@@ -67,7 +81,66 @@ def get_cert_expiry_timestamp(js_escaped_pem: str) -> int:
     return int(dt_expiry.timestamp())
 
 
-def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cert:
+def _format_pem(
+    private_key: rsa.RSAPrivateKey,
+    cert_pem: str,
+    ca_pem: str,
+    cert_expiry_timestamp: int,
+) -> RhcsV2CertPem:
+    """Generate RhcsV2Cert with PEM components."""
+    private_key_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    ).decode()
+    return RhcsV2CertPem(
+        private_key=private_key_pem,
+        certificate=cert_pem.encode().decode("unicode_escape").replace("\\/", "/"),
+        ca_cert=ca_pem,
+        expiration_timestamp=cert_expiry_timestamp,
+    )
+
+
+def _format_pkcs12(
+    private_key: rsa.RSAPrivateKey,
+    cert_pem: str,
+    ca_pem: str,
+    uid: str,
+    pwd: str,
+    cert_expiry_timestamp: int,
+) -> RhcsV2CertPkcs12:
+    """Generate PKCS#12 keystore and truststore components, returns base64-encoded strings."""
+    clean_cert_pem = cert_pem.encode().decode("unicode_escape").replace("\\/", "/")
+    cert_obj = x509.load_pem_x509_certificate(clean_cert_pem.encode())
+    ca_obj = x509.load_pem_x509_certificate(ca_pem.encode())
+    keystore_p12 = pkcs12.serialize_key_and_certificates(
+        name=uid.encode("utf-8"),
+        key=private_key,
+        cert=cert_obj,
+        cas=[ca_obj],
+        encryption_algorithm=serialization.BestAvailableEncryption(pwd.encode("utf-8")),
+    )
+    truststore_p12 = pkcs12.serialize_key_and_certificates(
+        name=b"ca-trust",
+        key=None,
+        cert=None,
+        cas=[ca_obj],
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    return RhcsV2CertPkcs12(
+        pkcs12_keystore=base64.b64encode(keystore_p12).decode("utf-8"),
+        pkcs12_truststore=base64.b64encode(truststore_p12).decode("utf-8"),
+        expiration_timestamp=cert_expiry_timestamp,
+    )
+
+
+def generate_cert(
+    issuer_url: str,
+    uid: str,
+    pwd: str,
+    ca_url: str,
+    cert_format: CertificateFormat = CertificateFormat.PEM,
+) -> RhcsV2CertPem | RhcsV2CertPkcs12:
     private_key = rsa.generate_private_key(65537, 4096)
     csr = (
         x509.CertificateSigningRequestBuilder()
@@ -78,6 +151,7 @@ def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cer
         )
         .sign(private_key, hashes.SHA256())
     )
+
     data = {
         "uid": uid,
         "pwd": pwd,
@@ -87,27 +161,19 @@ def generate_cert(issuer_url: str, uid: str, pwd: str, ca_url: str) -> RhcsV2Cer
         "renewal": "false",
         "xmlOutput": "false",
     }
-    response = requests.post(issuer_url, data=data)
+    response = requests.post(issuer_url, data=data, timeout=30)
     response.raise_for_status()
+    cert_pem = extract_cert(response.text).group(1)
+    cert_expiry_timestamp = get_cert_expiry_timestamp(cert_pem)
 
-    cert_pem = extract_cert(response.text)
-    cert_expiry_timestamp = get_cert_expiry_timestamp(cert_pem.group(1))
-    private_key_pem = private_key.private_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PrivateFormat.TraditionalOpenSSL,
-        encryption_algorithm=serialization.NoEncryption(),
-    ).decode()
-
-    response = requests.get(ca_url)
+    response = requests.get(ca_url, timeout=30)
     response.raise_for_status()
     ca_pem = response.text
 
-    return RhcsV2Cert(
-        private_key=private_key_pem,
-        certificate=cert_pem.group(1)
-        .encode()
-        .decode("unicode_escape")
-        .replace("\\/", "/"),
-        ca_cert=ca_pem,
-        expiration_timestamp=cert_expiry_timestamp,
-    )
+    match cert_format:
+        case CertificateFormat.PKCS12:
+            return _format_pkcs12(
+                private_key, cert_pem, ca_pem, uid, pwd, cert_expiry_timestamp
+            )
+        case CertificateFormat.PEM:
+            return _format_pem(private_key, cert_pem, ca_pem, cert_expiry_timestamp)

reconcile/utils/rosa/session.py

@@ -178,6 +178,22 @@ class RosaSession:
             )
             result.write_logs_to_logger(logging.info)
 
+    def upgrade_rosa_roles(
+        self,
+        cluster_name: str,
+        upgrade_version: str,
+        policy_version: str,
+        dry_run: bool,
+    ) -> None:
+        logging.info(
+            f"Upgrade  roles in AWS account {self.aws_account_id} to {upgrade_version}"
+        )
+        if not dry_run:
+            result = self.cli_execute(
+                f"rosa upgrade roles  -c {cluster_name} --cluster-version {upgrade_version}  --policy-version {policy_version} -y -m=auto"
+            )
+            result.write_logs_to_logger(logging.info)
+
 
 def generate_rosa_creation_script(
     cluster_name: str, cluster: OCMSpec, dry_run: bool

reconcile/utils/saasherder/saasherder.py

@@ -92,8 +92,7 @@ from reconcile.utils.state import State
 from reconcile.utils.vcs import VCS
 
 TARGET_CONFIG_HASH = "target_config_hash"
-
-
+TEMPLATE_API_VERSION = "template.openshift.io/v1"
 UNIQUE_SAAS_FILE_ENV_COMBO_LEN = 56
 REQUEST_TIMEOUT = 60
 
@@ -874,10 +873,23 @@ class SaasHerder:
         """
         if parameter_name in consolidated_parameters:
             return False
-        for template_parameter in template.get("parameters", {}):
-            if template_parameter["name"] == parameter_name:
-                return True
-        return False
+        return any(
+            template_parameter["name"] == parameter_name
+            for template_parameter in template.get("parameters") or []
+        )
+
+    @staticmethod
+    def _pre_process_template(template: dict[str, Any]) -> dict[str, Any]:
+        """
+        The only supported apiVersion for OpenShift Template is "template.openshift.io/v1".
+        There are examples of templates using "v1", it can't pass validation on 4.19+ oc versions.
+
+        Args:
+            template (dict): The OpenShift template dictionary.
+        Returns:
+            dict: The OpenShift template dictionary with the correct apiVersion.
+        """
+        return template | {"apiVersion": TEMPLATE_API_VERSION}
 
     def _process_template(
         self, spec: TargetSpec
@@ -967,7 +979,8 @@ class SaasHerder:
             oc = OCLocal("cluster", None, None, local=True)
             try:
                 resources: Iterable[Mapping[str, Any]] = oc.process(
-                    template, consolidated_parameters
+                    template=self._pre_process_template(template),
+                    parameters=consolidated_parameters,
                 )
             except StatusCodeError as e:
                 logging.error(f"{error_prefix} error processing template: {e!s}")