qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/saasherder/saasherder.py

@@ -16,7 +16,7 @@ from collections.abc import (
     Sequence,
 )
 from contextlib import suppress
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta
 from types import TracebackType
 from typing import Any
 
@@ -37,10 +37,12 @@ from sretoolbox.utils import (
 from reconcile.github_org import get_default_config
 from reconcile.status import RunningState
 from reconcile.utils import helm
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi, JobBuildState
 from reconcile.utils.jjb_client import JJB
+from reconcile.utils.json import json_dumps
 from reconcile.utils.oc import (
     OCLocal,
     StatusCodeError,
@@ -90,8 +92,7 @@ from reconcile.utils.state import State
 from reconcile.utils.vcs import VCS
 
 TARGET_CONFIG_HASH = "target_config_hash"
-
-
+TEMPLATE_API_VERSION = "template.openshift.io/v1"
 UNIQUE_SAAS_FILE_ENV_COMBO_LEN = 56
 REQUEST_TIMEOUT = 60
 
@@ -763,7 +764,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 content = self.gitlab.get_raw_file(
                     project=project,
                     path=path,
@@ -799,7 +801,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 dir_contents = self.gitlab.get_directory_contents(
                     project,
                     ref=commit_sha,
@@ -824,7 +827,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:
@@ -869,12 +873,27 @@ class SaasHerder:
         """
         if parameter_name in consolidated_parameters:
             return False
-        for template_parameter in template.get("parameters", {}):
-            if template_parameter["name"] == parameter_name:
-                return True
-        return False
+        return any(
+            template_parameter["name"] == parameter_name
+            for template_parameter in template.get("parameters") or []
+        )
+
+    @staticmethod
+    def _pre_process_template(template: dict[str, Any]) -> dict[str, Any]:
+        """
+        The only supported apiVersion for OpenShift Template is "template.openshift.io/v1".
+        There are examples of templates using "v1", it can't pass validation on 4.19+ oc versions.
 
-    def _process_template(self, spec: TargetSpec) -> tuple[list[Any], Promotion | None]:
+        Args:
+            template (dict): The OpenShift template dictionary.
+        Returns:
+            dict: The OpenShift template dictionary with the correct apiVersion.
+        """
+        return template | {"apiVersion": TEMPLATE_API_VERSION}
+
+    def _process_template(
+        self, spec: TargetSpec
+    ) -> tuple[Iterable[Any], Promotion | None]:
         saas_file_name = spec.saas_file_name
         resource_template_name = spec.resource_template_name
         url = spec.url
@@ -959,7 +978,10 @@ class SaasHerder:
 
             oc = OCLocal("cluster", None, None, local=True)
             try:
-                resources = oc.process(template, consolidated_parameters)
+                resources: Iterable[Mapping[str, Any]] = oc.process(
+                    template=self._pre_process_template(template),
+                    parameters=consolidated_parameters,
+                )
             except StatusCodeError as e:
                 logging.error(f"{error_prefix} error processing template: {e!s}")
 
@@ -1173,13 +1195,13 @@ class SaasHerder:
         images_list = threaded.run(
             self._collect_images, resources, self.available_thread_pool_size
         )
-        images = set(itertools.chain.from_iterable(images_list))
-        self.images.update(images)
-        if not images:
+        images_set = set(itertools.chain.from_iterable(images_list))
+        self.images.update(images_set)
+        if not images_set:
             return False  # no errors
         images = threaded.run(
             self._get_image,
-            images,
+            images_set,
             self.available_thread_pool_size,
             image_patterns=spec.image_patterns,
             image_auth=spec.image_auth,
@@ -1244,7 +1266,9 @@ class SaasHerder:
             self.saas_files,
             self.thread_pool_size,
         )
-        desired_state_specs = list(itertools.chain.from_iterable(results))
+        desired_state_specs: list[TargetSpec] = list(
+            itertools.chain.from_iterable(results)
+        )
         promotions = threaded.run(
             self.populate_desired_state_saas_file,
             desired_state_specs,
@@ -1861,7 +1885,7 @@ class SaasHerder:
     @staticmethod
     def get_target_config_hash(target_config: Any) -> str:
         m = hashlib.sha256()
-        m.update(json.dumps(target_config, sort_keys=True).encode("utf-8"))
+        m.update(json_dumps(target_config).encode("utf-8"))
         digest = m.hexdigest()[:16]
         return digest
 
@@ -1892,21 +1916,23 @@ class SaasHerder:
             name=target.name,
             ref=target.ref,
             promotion=(
-                target.promotion.dict(by_alias=True) if target.promotion else None
+                target.promotion.model_dump(by_alias=True) if target.promotion else None
             ),
             secretParameters=(
-                [p.dict(by_alias=True) for p in target.secret_parameters]
+                [p.model_dump(by_alias=True) for p in target.secret_parameters]
                 if target.secret_parameters
                 else None
             ),
             slos=(
-                [slo.dict(by_alias=True) for slo in target.slos]
+                [slo.model_dump(by_alias=True) for slo in target.slos]
                 if target.slos
                 else None
             ),
-            upstream=(target.upstream.dict(by_alias=True) if target.upstream else None),
+            upstream=(
+                target.upstream.model_dump(by_alias=True) if target.upstream else None
+            ),
             images=(
-                [i.dict(by_alias=True) for i in target.images]
+                [i.model_dump(by_alias=True) for i in target.images]
                 if target.images
                 else None
             ),
@@ -1917,14 +1943,14 @@ class SaasHerder:
             # before the GQL classes are introduced, the parameters attribute
             # was a json string. Keep it that way to be backwards compatible.
             saas_file_parameters=(
-                json.dumps(saas_file.parameters, separators=(",", ":"))
+                json_dumps(saas_file.parameters, compact=True)
                 if saas_file.parameters is not None
                 else None
             ),
             # before the GQL classes are introduced, the parameters attribute
             # was a json string. Keep it that way to be backwards compatible.
             parameters=(
-                json.dumps(target.parameters, separators=(",", ":"))
+                json_dumps(target.parameters, compact=True)
                 if target.parameters is not None
                 else None
             ),
@@ -1935,23 +1961,23 @@ class SaasHerder:
             # before the GQL classes are introduced, the parameters attribute
             # was a json string. Keep it that way to be backwards compatible.
             rt_parameters=(
-                json.dumps(resource_template.parameters, separators=(",", ":"))
+                json_dumps(resource_template.parameters, compact=True)
                 if resource_template.parameters is not None
                 else None
             ),
         )
         if saas_file.managed_resource_names:
             state_content["saas_file_managed_resource_names"] = [
-                m.dict() for m in saas_file.managed_resource_names
+                m.model_dump() for m in saas_file.managed_resource_names
             ]
         # include secret parameters from resource template and saas file
         if resource_template.secret_parameters:
             state_content["rt_secretparameters"] = [
-                p.dict() for p in resource_template.secret_parameters
+                p.model_dump() for p in resource_template.secret_parameters
             ]
         if saas_file.secret_parameters:
             state_content["saas_file_secretparameters"] = [
-                p.dict() for p in saas_file.secret_parameters
+                p.model_dump() for p in saas_file.secret_parameters
             ]
         return state_content
 
@@ -2020,7 +2046,7 @@ class SaasHerder:
         if promotion.commit_sha in self.hotfix_versions.get(promotion.url, set()):
             return True
 
-        now = datetime.now(UTC)
+        now = utc_now()
         passed_soak_days = timedelta(days=0)
 
         for channel in promotion.subscribe:
@@ -2122,7 +2148,7 @@ class SaasHerder:
         if not (self.state and self._promotion_state):
             raise Exception("state is not initialized")
 
-        now = datetime.now(UTC)
+        now = utc_now()
         for promotion in self.promotions:
             if promotion is None:
                 continue
@@ -2234,7 +2260,9 @@ class SaasHerder:
             for rt in saas_file.resource_templates:
                 for target in rt.targets:
                     template_vars = {
-                        "resource": {"namespace": target.namespace.dict(by_alias=True)}
+                        "resource": {
+                            "namespace": target.namespace.model_dump(by_alias=True)
+                        }
                     }
                     if target.parameters:
                         for param in target.parameters:

reconcile/utils/secret_reader.py

@@ -146,14 +146,14 @@ class VaultSecretReader(SecretReaderBase):
     @property
     def vault_client(self) -> VaultClient:
         if self._vault_client is None:
-            self._vault_client = VaultClient()
+            self._vault_client = VaultClient.get_instance()
         return self._vault_client
 
     def _read_all(
         self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         try:
-            data = self.vault_client.read_all(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+            data = self.vault_client.read_all(
                 self._parameters_to_dict(
                     path=path,
                     field=field,
@@ -173,7 +173,7 @@ class VaultSecretReader(SecretReaderBase):
         self, path: str, field: str, format: str | None, version: int | None
     ) -> str:
         try:
-            data = self.vault_client.read(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+            data = self.vault_client.read(
                 self._parameters_to_dict(
                     path=path,
                     field=field,
@@ -251,7 +251,7 @@ class SecretReader(SecretReaderBase):
     @property
     def vault_client(self) -> VaultClient:
         if self._vault_client is None:
-            self._vault_client = VaultClient()
+            self._vault_client = VaultClient.get_instance()
         return self._vault_client
 
     def _read(
@@ -278,7 +278,7 @@ class SecretReader(SecretReaderBase):
 
         if self.settings and self.settings.get("vault"):
             try:
-                data = self.vault_client.read(params)  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+                data = self.vault_client.read(params)
             except vault.SecretNotFoundError as e:
                 raise SecretNotFoundError(*e.args) from e
         else:
@@ -312,7 +312,7 @@ class SecretReader(SecretReaderBase):
 
         if self.settings and self.settings.get("vault"):
             try:
-                data = self.vault_client.read_all(params)  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+                data = self.vault_client.read_all(params)
             except Forbidden:
                 raise VaultForbiddenError(
                     f"permission denied reading vault secret at {path}"

reconcile/utils/sharding.py

@@ -8,7 +8,7 @@ SHARDS = int(os.environ.get("SHARDS", "1"))
 SHARD_ID = int(os.environ.get("SHARD_ID", "0"))
 
 
-def is_in_shard(value):
+def is_in_shard(value: str) -> bool:
     if SHARDS == 1:
         return True
 

reconcile/utils/slack_api.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -26,6 +27,23 @@ if TYPE_CHECKING:
 MAX_RETRIES = 5
 TIMEOUT = 30
 
+# Slack API base URLs for different workspace types
+SLACK_API_BASE_URL = "https://slack.com/api/"
+SLACK_GOV_API_BASE_URL = "https://slack-gov.com/api/"
+
+
+def is_gov_slack_workspace() -> bool:
+    """
+    Determine if a workspace is a government Slack workspace.
+
+    :return: True if it's a gov-slack workspace, False otherwise
+    """
+    # Check GOV_SLACK environment variable from OpenShift YAML configuration
+    # If not set, defaults to False (regular Slack)
+    gov_slack_env = os.getenv("GOV_SLACK", "false")
+
+    return gov_slack_env.lower() == "true"
+
 
 class UserNotFoundError(Exception):
     pass
@@ -53,14 +71,14 @@ class HasClientGlobalConfig(Protocol):
     max_retries: int | None
     timeout: int | None
 
-    def dict(self) -> dict[str, int | None]: ...
+    def model_dump(self) -> dict[str, int | None]: ...
 
 
 class HasClientMethodConfig(Protocol):
     name: str
     args: Any
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class HasClientConfig(Protocol):
@@ -165,7 +183,6 @@ class SlackApi:
         api_config: SlackApiConfig | None = None,
         init_usergroups: bool = True,
         channel: str | None = None,
-        slack_url: str | None = None,
         **chat_kwargs: Any,
     ) -> None:
         """
@@ -187,10 +204,15 @@ class SlackApi:
         else:
             self.config = SlackApiConfig()
 
+        # Determine the appropriate Slack API base URL based on GOV_SLACK environment variable
+        base_url = (
+            SLACK_GOV_API_BASE_URL if is_gov_slack_workspace() else SLACK_API_BASE_URL
+        )
+
         self._sc = WebClient(
             token=token,
             timeout=self.config.timeout,
-            base_url=slack_url or WebClient.BASE_URL,
+            base_url=base_url,
         )
         self._configure_client_retry()
 

reconcile/utils/sloth.py

@@ -0,0 +1,224 @@
+import subprocess
+import tempfile
+from io import StringIO
+from typing import Any, NotRequired, TypedDict
+
+import yaml
+
+from reconcile.utils.ruamel import create_ruamel_instance
+
+
+class PrometheusRule(TypedDict):
+    record: NotRequired[str]
+    alert: NotRequired[str]
+    expr: str
+    labels: NotRequired[dict[str, str]]
+    annotations: NotRequired[dict[str, str]]
+
+
+class PrometheusRuleGroup(TypedDict):
+    name: str
+    rules: list[PrometheusRule]
+
+
+class PrometheusRuleSpec(TypedDict):
+    groups: list[PrometheusRuleGroup]
+
+
+class SLOParametersDict(TypedDict):
+    window: str
+
+
+class SLO(TypedDict):
+    name: str
+    SLIType: str
+    SLISpecification: str
+    SLOTarget: float
+    SLOTargetUnit: str
+    SLOParameters: SLOParametersDict
+    SLODetails: str
+    dashboard: str
+    expr: str
+    SLIErrorQuery: NotRequired[str]
+    SLITotalQuery: NotRequired[str]
+
+
+class App(TypedDict):
+    name: str
+
+
+class SLODocument(TypedDict):
+    name: str
+    app: App
+    slos: NotRequired[list[SLO]]
+
+
+class SlothGenerateError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("sloth generate failed: " + str(msg))
+
+
+class SlothInputError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("sloth input validation failed: " + str(msg))
+
+
+def process_sloth_output(output_file_path: str) -> str:
+    ruamel_instance = create_ruamel_instance()
+    with open(output_file_path, encoding="utf-8") as f:
+        data: PrometheusRuleSpec = ruamel_instance.load(f)
+    for group in data.get("groups", []):
+        for rule in group.get("rules", []):
+            labels = (
+                # sloth adds several sloth_* labels to alerting rules that are not compliant with prometheus-rule-1 schema
+                # see https://sloth.dev/examples/default/getting-started/#__tabbed_1_2
+                {k: v for k, v in rule["labels"].items() if not k.startswith("sloth")}
+                if rule.get("alert")
+                else rule["labels"]  # retain all labels on record rules
+            )
+            annotations = (
+                # sloth adds a `title` key within annotations for alert rules: https://sloth.dev/examples/default/getting-started/#__tabbed_1_2
+                # this is not compliant with schema and is discarded
+                {k: v for k, v in rule["annotations"].items() if k != "title"}
+                if rule.get("alert")
+                else {}  # record rules do not support annotations
+            )
+            if labels:
+                rule["labels"] = labels
+            else:
+                rule.pop("labels", None)
+            if annotations:
+                rule["annotations"] = annotations
+            else:
+                rule.pop("annotations", None)
+    with StringIO() as s:
+        ruamel_instance.dump(data, s)
+        return s.getvalue()
+
+
+def run_sloth(spec: dict[str, Any]) -> str:
+    with (
+        tempfile.NamedTemporaryFile(
+            encoding="utf-8", mode="w", suffix=".yml"
+        ) as input_file,
+        tempfile.NamedTemporaryFile(
+            encoding="utf-8", mode="w", suffix=".yml"
+        ) as output_file,
+    ):
+        yaml.dump(spec, input_file, allow_unicode=True)
+        cmd = ["sloth", "generate", "-i", input_file.name, "-o", output_file.name]
+        try:
+            subprocess.run(cmd, capture_output=True, check=True, text=True)
+        except subprocess.CalledProcessError as e:
+            error_msg = f"{e}"
+            if e.stdout:
+                error_msg += f"\nstdout: {e.stdout}"
+            if e.stderr:
+                error_msg += f"\nstderr: {e.stderr}"
+            raise SlothGenerateError(error_msg) from e
+        return process_sloth_output(output_file.name)
+
+
+def get_slo_target(slo: SLO) -> float:
+    """
+    Ensure SLO target unit aligns with format expected by sloth for 'Objective' attribute
+    https://pkg.go.dev/github.com/slok/sloth/pkg/prometheus/api/v1#section-readme
+    """
+    val = float(slo["SLOTarget"])
+    return val * (100.0 if slo.get("SLOTargetUnit") == "percent_0_1" else 1.0)
+
+
+def generate_sloth_rules(
+    slo_document: SLODocument,
+    version: str = "prometheus/v1",
+) -> str:
+    """Generate Prometheus rules for an slo_document_v1 using sloth
+
+    Args:
+        slo_document query:
+            {
+                slo_docs: slo_document_v1(filter: {name: "foo"}) {
+                    name
+                    app {
+                        name
+                    }
+                    slos {
+                        name
+                        SLIType
+                        SLOTargetUnit
+                        SLOParameters {
+                            window
+                        }
+                        expr
+                        SLOTarget
+                        SLIErrorQuery
+                        SLITotalQuery
+                        SLODetails
+                        dashboard
+                    }
+                }
+            }
+        version: Spec version (default: "prometheus/v1")
+
+    Returns:
+        Generated Prometheus rules as YAML string
+    """
+    if not slo_document.get("slos"):
+        raise SlothInputError("SLO document has no SLOs defined")
+
+    service = slo_document["app"]["name"]
+    # only process SLOs that have both error and total queries defined
+    slo_input = [
+        {
+            "name": slo["name"],
+            "objective": get_slo_target(slo),
+            "description": f"{slo['name']} SLO for {service}",
+            "sli": {
+                "events": {
+                    "error_query": slo["SLIErrorQuery"].replace(
+                        "{{window}}", "{{.window}}"
+                    ),
+                    "total_query": slo["SLITotalQuery"].replace(
+                        "{{window}}", "{{.window}}"
+                    ),
+                }
+            },
+            "alerting": {
+                "name": f"{service.title()}{slo['name'].title()}",
+                "annotations": {
+                    "summary": f"High error rate on {service} {slo['name']}",
+                    "message": f"High error rate on {service} {slo['name']}",
+                    "runbook": slo["SLODetails"],
+                    "dashboard": slo["dashboard"],
+                },
+                "page_alert": {
+                    "labels": {
+                        "severity": "critical",
+                        "service": service,
+                        "slo": slo["name"],
+                    }
+                },
+                "ticket_alert": {
+                    "labels": {
+                        "severity": "high",
+                        "service": service,
+                        "slo": slo["name"],
+                    }
+                },
+            },
+        }
+        for slo in slo_document["slos"]
+        if slo.get("SLIErrorQuery") and slo.get("SLITotalQuery")
+    ]
+
+    if not slo_input:
+        raise SlothInputError(
+            "No SLOs found with both SLIErrorQuery and SLITotalQuery defined"
+        )
+
+    spec = {
+        "version": version,
+        "service": service,
+        "slos": slo_input,
+    }
+    return run_sloth(spec)

reconcile/utils/sqs_gateway.py

@@ -1,7 +1,10 @@
 import json
 import os
+from collections.abc import Iterable, Mapping
+from typing import Any, Self
 
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.secret_reader import SecretReader
 
 
@@ -12,7 +15,9 @@ class SQSGatewayInitError(Exception):
 class SQSGateway:
     """Wrapper around SQS AWS SDK"""
 
-    def __init__(self, accounts, secret_reader: SecretReader):
+    def __init__(
+        self, accounts: Iterable[Mapping[str, Any]], secret_reader: SecretReader
+    ) -> None:
         queue_url = os.environ.get("gitlab_pr_submitter_queue_url")  # noqa: SIM112
         if not queue_url:
             raise SQSGatewayInitError(
@@ -30,17 +35,17 @@ class SQSGateway:
         self.sqs = self._aws_api.get_session_client(session, "sqs")
         self.queue_url = queue_url
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, *ext):
+    def __exit__(self, *ext: Any) -> None:
         self.cleanup()
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         self._aws_api.cleanup()
 
     @staticmethod
-    def get_queue_account(accounts, queue_url):
+    def get_queue_account(accounts: Iterable[Mapping[str, Any]], queue_url: str) -> str:
         queue_account_uid = queue_url.split("/")[3]
         queue_account_name = [
             a["name"] for a in accounts if a["uid"] == queue_account_uid
@@ -49,14 +54,14 @@ class SQSGateway:
             raise SQSGatewayInitError(f"account uid not found: {queue_account_uid}")
         return queue_account_name[0]
 
-    def send_message(self, body):
-        self.sqs.send_message(QueueUrl=self.queue_url, MessageBody=json.dumps(body))
+    def send_message(self, body: Mapping[str, Any]) -> None:
+        self.sqs.send_message(QueueUrl=self.queue_url, MessageBody=json_dumps(body))
 
     def receive_messages(
         self,
-        visibility_timeout=30,
-        wait_time_seconds=20,
-    ):
+        visibility_timeout: int = 30,
+        wait_time_seconds: int = 20,
+    ) -> list[tuple[str, dict[str, Any]]]:
         messages = self.sqs.receive_message(
             QueueUrl=self.queue_url,
             VisibilityTimeout=visibility_timeout,
@@ -64,5 +69,5 @@ class SQSGateway:
         ).get("Messages", [])
         return [(m["ReceiptHandle"], json.loads(m["Body"])) for m in messages]
 
-    def delete_message(self, receipt_handle):
+    def delete_message(self, receipt_handle: str) -> None:
         self.sqs.delete_message(QueueUrl=self.queue_url, ReceiptHandle=receipt_handle)

reconcile/utils/state.py

@@ -28,6 +28,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.typed_queries.get_state_aws_account import get_state_aws_account
 from reconcile.utils.aws_api import aws_config_file_path
+from reconcile.utils.json import json_dumps
 from reconcile.utils.secret_reader import (
     SecretReaderBase,
     create_secret_reader,
@@ -355,7 +356,7 @@ class State:
         self.client.put_object(
             Bucket=self.bucket,
             Key=f"{self.state_path}/{key}",
-            Body=json.dumps(value),
+            Body=json_dumps(value),
             Metadata=metadata or {},
         )