qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/utils/saasherder/models.py

@@ -7,10 +7,7 @@ from enum import Enum
 from typing import Any, NotRequired, TypedDict
 
 from github import Github
-from pydantic import (
-    BaseModel,
-    Field,
-)
+from pydantic import BaseModel, Field, model_validator
 
 from reconcile.gql_definitions.fragments.saas_slo_document import (
     SLODocument,
@@ -207,7 +204,12 @@ TriggerSpecUnion = (
 )
 
 
-class Namespace(BaseModel):
+class Namespace(
+    BaseModel,
+    validate_by_name=True,
+    validate_by_alias=True,
+    arbitrary_types_allowed=True,
+):
     name: str
     environment: SaasEnvironment
     app: SaasApp
@@ -217,29 +219,19 @@ class Namespace(BaseModel):
         ..., alias="managedResourceNames"
     )
 
-    class Config:
-        arbitrary_types_allowed = True
-        allow_population_by_field_name = True
-
 
-class PromotionChannelData(BaseModel):
+class PromotionChannelData(BaseModel, validate_by_name=True, validate_by_alias=True):
     q_type: str = Field(..., alias="type")
 
-    class Config:
-        allow_population_by_field_name = True
 
-
-class ParentSaasPromotion(BaseModel):
+class ParentSaasPromotion(BaseModel, validate_by_name=True, validate_by_alias=True):
     q_type: str = Field(..., alias="type")
-    parent_saas: str | None
-    target_config_hash: str | None
-
-    class Config:
-        allow_population_by_field_name = True
+    parent_saas: str | None = None
+    target_config_hash: str | None = None
 
 
 class PromotionData(BaseModel):
-    channel: str | None
+    channel: str | None = None
     data: list[ParentSaasPromotion | PromotionChannelData] | None = None
 
 
@@ -264,6 +256,17 @@ class Promotion(BaseModel):
     saas_file_paths: list[str] | None = None
     target_paths: list[str] | None = None
 
+    @model_validator(mode="before")
+    @classmethod
+    def handle_gql_classes(cls, data: Any) -> Any:
+        if data.get("promotion_data"):
+            data["promotion_data"] = [
+                # convert a GQL class to a dict
+                item.model_dump() if hasattr(item, "model_dump") else item
+                for item in data["promotion_data"]
+            ]
+        return data
+
 
 @dataclass
 class ImageAuth:

reconcile/utils/saasherder/saasherder.py

@@ -16,7 +16,7 @@ from collections.abc import (
     Sequence,
 )
 from contextlib import suppress
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta
 from types import TracebackType
 from typing import Any
 
@@ -37,6 +37,7 @@ from sretoolbox.utils import (
 from reconcile.github_org import get_default_config
 from reconcile.status import RunningState
 from reconcile.utils import helm
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi, JobBuildState
@@ -91,8 +92,7 @@ from reconcile.utils.state import State
 from reconcile.utils.vcs import VCS
 
 TARGET_CONFIG_HASH = "target_config_hash"
-
-
+TEMPLATE_API_VERSION = "template.openshift.io/v1"
 UNIQUE_SAAS_FILE_ENV_COMBO_LEN = 56
 REQUEST_TIMEOUT = 60
 
@@ -764,7 +764,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 content = self.gitlab.get_raw_file(
                     project=project,
                     path=path,
@@ -800,7 +801,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 dir_contents = self.gitlab.get_directory_contents(
                     project,
                     ref=commit_sha,
@@ -825,7 +827,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:
@@ -870,10 +873,23 @@ class SaasHerder:
         """
         if parameter_name in consolidated_parameters:
             return False
-        for template_parameter in template.get("parameters", {}):
-            if template_parameter["name"] == parameter_name:
-                return True
-        return False
+        return any(
+            template_parameter["name"] == parameter_name
+            for template_parameter in template.get("parameters") or []
+        )
+
+    @staticmethod
+    def _pre_process_template(template: dict[str, Any]) -> dict[str, Any]:
+        """
+        The only supported apiVersion for OpenShift Template is "template.openshift.io/v1".
+        There are examples of templates using "v1", it can't pass validation on 4.19+ oc versions.
+
+        Args:
+            template (dict): The OpenShift template dictionary.
+        Returns:
+            dict: The OpenShift template dictionary with the correct apiVersion.
+        """
+        return template | {"apiVersion": TEMPLATE_API_VERSION}
 
     def _process_template(
         self, spec: TargetSpec
@@ -963,7 +979,8 @@ class SaasHerder:
             oc = OCLocal("cluster", None, None, local=True)
             try:
                 resources: Iterable[Mapping[str, Any]] = oc.process(
-                    template, consolidated_parameters
+                    template=self._pre_process_template(template),
+                    parameters=consolidated_parameters,
                 )
             except StatusCodeError as e:
                 logging.error(f"{error_prefix} error processing template: {e!s}")
@@ -1178,13 +1195,13 @@ class SaasHerder:
         images_list = threaded.run(
             self._collect_images, resources, self.available_thread_pool_size
         )
-        images = set(itertools.chain.from_iterable(images_list))
-        self.images.update(images)
-        if not images:
+        images_set = set(itertools.chain.from_iterable(images_list))
+        self.images.update(images_set)
+        if not images_set:
             return False  # no errors
         images = threaded.run(
             self._get_image,
-            images,
+            images_set,
             self.available_thread_pool_size,
             image_patterns=spec.image_patterns,
             image_auth=spec.image_auth,
@@ -1249,7 +1266,9 @@ class SaasHerder:
             self.saas_files,
             self.thread_pool_size,
         )
-        desired_state_specs = list(itertools.chain.from_iterable(results))
+        desired_state_specs: list[TargetSpec] = list(
+            itertools.chain.from_iterable(results)
+        )
         promotions = threaded.run(
             self.populate_desired_state_saas_file,
             desired_state_specs,
@@ -1897,21 +1916,23 @@ class SaasHerder:
             name=target.name,
             ref=target.ref,
             promotion=(
-                target.promotion.dict(by_alias=True) if target.promotion else None
+                target.promotion.model_dump(by_alias=True) if target.promotion else None
             ),
             secretParameters=(
-                [p.dict(by_alias=True) for p in target.secret_parameters]
+                [p.model_dump(by_alias=True) for p in target.secret_parameters]
                 if target.secret_parameters
                 else None
             ),
             slos=(
-                [slo.dict(by_alias=True) for slo in target.slos]
+                [slo.model_dump(by_alias=True) for slo in target.slos]
                 if target.slos
                 else None
             ),
-            upstream=(target.upstream.dict(by_alias=True) if target.upstream else None),
+            upstream=(
+                target.upstream.model_dump(by_alias=True) if target.upstream else None
+            ),
             images=(
-                [i.dict(by_alias=True) for i in target.images]
+                [i.model_dump(by_alias=True) for i in target.images]
                 if target.images
                 else None
             ),
@@ -1947,16 +1968,16 @@ class SaasHerder:
         )
         if saas_file.managed_resource_names:
             state_content["saas_file_managed_resource_names"] = [
-                m.dict() for m in saas_file.managed_resource_names
+                m.model_dump() for m in saas_file.managed_resource_names
             ]
         # include secret parameters from resource template and saas file
         if resource_template.secret_parameters:
             state_content["rt_secretparameters"] = [
-                p.dict() for p in resource_template.secret_parameters
+                p.model_dump() for p in resource_template.secret_parameters
             ]
         if saas_file.secret_parameters:
             state_content["saas_file_secretparameters"] = [
-                p.dict() for p in saas_file.secret_parameters
+                p.model_dump() for p in saas_file.secret_parameters
             ]
         return state_content
 
@@ -2025,7 +2046,7 @@ class SaasHerder:
         if promotion.commit_sha in self.hotfix_versions.get(promotion.url, set()):
             return True
 
-        now = datetime.now(UTC)
+        now = utc_now()
         passed_soak_days = timedelta(days=0)
 
         for channel in promotion.subscribe:
@@ -2127,7 +2148,7 @@ class SaasHerder:
         if not (self.state and self._promotion_state):
             raise Exception("state is not initialized")
 
-        now = datetime.now(UTC)
+        now = utc_now()
         for promotion in self.promotions:
             if promotion is None:
                 continue
@@ -2239,7 +2260,9 @@ class SaasHerder:
             for rt in saas_file.resource_templates:
                 for target in rt.targets:
                     template_vars = {
-                        "resource": {"namespace": target.namespace.dict(by_alias=True)}
+                        "resource": {
+                            "namespace": target.namespace.model_dump(by_alias=True)
+                        }
                     }
                     if target.parameters:
                         for param in target.parameters:

reconcile/utils/slack_api.py

@@ -71,14 +71,14 @@ class HasClientGlobalConfig(Protocol):
     max_retries: int | None
     timeout: int | None
 
-    def dict(self) -> dict[str, int | None]: ...
+    def model_dump(self) -> dict[str, int | None]: ...
 
 
 class HasClientMethodConfig(Protocol):
     name: str
     args: Any
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class HasClientConfig(Protocol):

reconcile/utils/sloth.py

@@ -1,5 +1,9 @@
+import subprocess
+import tempfile
 from io import StringIO
-from typing import NotRequired, TypedDict
+from typing import Any, NotRequired, TypedDict
+
+import yaml
 
 from reconcile.utils.ruamel import create_ruamel_instance
 
@@ -21,6 +25,44 @@ class PrometheusRuleSpec(TypedDict):
     groups: list[PrometheusRuleGroup]
 
 
+class SLOParametersDict(TypedDict):
+    window: str
+
+
+class SLO(TypedDict):
+    name: str
+    SLIType: str
+    SLISpecification: str
+    SLOTarget: float
+    SLOTargetUnit: str
+    SLOParameters: SLOParametersDict
+    SLODetails: str
+    dashboard: str
+    expr: str
+    SLIErrorQuery: NotRequired[str]
+    SLITotalQuery: NotRequired[str]
+
+
+class App(TypedDict):
+    name: str
+
+
+class SLODocument(TypedDict):
+    name: str
+    app: App
+    slos: NotRequired[list[SLO]]
+
+
+class SlothGenerateError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("sloth generate failed: " + str(msg))
+
+
+class SlothInputError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("sloth input validation failed: " + str(msg))
+
+
 def process_sloth_output(output_file_path: str) -> str:
     ruamel_instance = create_ruamel_instance()
     with open(output_file_path, encoding="utf-8") as f:
@@ -49,7 +91,134 @@ def process_sloth_output(output_file_path: str) -> str:
                 rule["annotations"] = annotations
             else:
                 rule.pop("annotations", None)
-
     with StringIO() as s:
         ruamel_instance.dump(data, s)
         return s.getvalue()
+
+
+def run_sloth(spec: dict[str, Any]) -> str:
+    with (
+        tempfile.NamedTemporaryFile(
+            encoding="utf-8", mode="w", suffix=".yml"
+        ) as input_file,
+        tempfile.NamedTemporaryFile(
+            encoding="utf-8", mode="w", suffix=".yml"
+        ) as output_file,
+    ):
+        yaml.dump(spec, input_file, allow_unicode=True)
+        cmd = ["sloth", "generate", "-i", input_file.name, "-o", output_file.name]
+        try:
+            subprocess.run(cmd, capture_output=True, check=True, text=True)
+        except subprocess.CalledProcessError as e:
+            error_msg = f"{e}"
+            if e.stdout:
+                error_msg += f"\nstdout: {e.stdout}"
+            if e.stderr:
+                error_msg += f"\nstderr: {e.stderr}"
+            raise SlothGenerateError(error_msg) from e
+        return process_sloth_output(output_file.name)
+
+
+def get_slo_target(slo: SLO) -> float:
+    """
+    Ensure SLO target unit aligns with format expected by sloth for 'Objective' attribute
+    https://pkg.go.dev/github.com/slok/sloth/pkg/prometheus/api/v1#section-readme
+    """
+    val = float(slo["SLOTarget"])
+    return val * (100.0 if slo.get("SLOTargetUnit") == "percent_0_1" else 1.0)
+
+
+def generate_sloth_rules(
+    slo_document: SLODocument,
+    version: str = "prometheus/v1",
+) -> str:
+    """Generate Prometheus rules for an slo_document_v1 using sloth
+
+    Args:
+        slo_document query:
+            {
+                slo_docs: slo_document_v1(filter: {name: "foo"}) {
+                    name
+                    app {
+                        name
+                    }
+                    slos {
+                        name
+                        SLIType
+                        SLOTargetUnit
+                        SLOParameters {
+                            window
+                        }
+                        expr
+                        SLOTarget
+                        SLIErrorQuery
+                        SLITotalQuery
+                        SLODetails
+                        dashboard
+                    }
+                }
+            }
+        version: Spec version (default: "prometheus/v1")
+
+    Returns:
+        Generated Prometheus rules as YAML string
+    """
+    if not slo_document.get("slos"):
+        raise SlothInputError("SLO document has no SLOs defined")
+
+    service = slo_document["app"]["name"]
+    # only process SLOs that have both error and total queries defined
+    slo_input = [
+        {
+            "name": slo["name"],
+            "objective": get_slo_target(slo),
+            "description": f"{slo['name']} SLO for {service}",
+            "sli": {
+                "events": {
+                    "error_query": slo["SLIErrorQuery"].replace(
+                        "{{window}}", "{{.window}}"
+                    ),
+                    "total_query": slo["SLITotalQuery"].replace(
+                        "{{window}}", "{{.window}}"
+                    ),
+                }
+            },
+            "alerting": {
+                "name": f"{service.title()}{slo['name'].title()}",
+                "annotations": {
+                    "summary": f"High error rate on {service} {slo['name']}",
+                    "message": f"High error rate on {service} {slo['name']}",
+                    "runbook": slo["SLODetails"],
+                    "dashboard": slo["dashboard"],
+                },
+                "page_alert": {
+                    "labels": {
+                        "severity": "critical",
+                        "service": service,
+                        "slo": slo["name"],
+                    }
+                },
+                "ticket_alert": {
+                    "labels": {
+                        "severity": "high",
+                        "service": service,
+                        "slo": slo["name"],
+                    }
+                },
+            },
+        }
+        for slo in slo_document["slos"]
+        if slo.get("SLIErrorQuery") and slo.get("SLITotalQuery")
+    ]
+
+    if not slo_input:
+        raise SlothInputError(
+            "No SLOs found with both SLIErrorQuery and SLITotalQuery defined"
+        )
+
+    spec = {
+        "version": version,
+        "service": service,
+        "slos": slo_input,
+    }
+    return run_sloth(spec)

reconcile/utils/structs.py

@@ -1,4 +1,4 @@
-from pydantic.dataclasses import dataclass
+from dataclasses import dataclass
 
 
 @dataclass

reconcile/utils/terraform_client.py

@@ -36,6 +36,7 @@ from reconcile.typed_queries.app_interface_custom_messages import (
 )
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.aws_helper import get_region_from_availability_zone
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
     ExternalResourceSpecInventory,
@@ -222,7 +223,7 @@ class TerraformClient:
         if disable_deletions_detected:
             raise RuntimeError("Terraform plan has disabled deletions detected")
 
-    @retry(no_retry_exceptions=RdsUpgradeValidationError)
+    @retry(no_retry_exceptions=(RdsUpgradeValidationError,))
     def terraform_plan(
         self, spec: TerraformSpec, enable_deletion: bool
     ) -> tuple[bool, list[AccountUser], bool]:
@@ -419,11 +420,11 @@ class TerraformClient:
         deletion_approvals = account.get("deletionApprovals")
         if not deletion_approvals:
             return False
-        now = datetime.utcnow()
+        now = utc_now()
         for da in deletion_approvals:
             try:
-                expiration = datetime.strptime(
-                    da["expiration"], DATE_FORMAT
+                expiration = ensure_utc(
+                    datetime.strptime(da["expiration"], DATE_FORMAT)  # noqa: DTZ007
                 ) + timedelta(days=1)
             except ValueError:
                 raise DeletionApprovalExpirationValueError(