qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/aws_api_typed/support.py

@@ -0,0 +1,79 @@
+from enum import Enum
+from typing import TYPE_CHECKING, Literal
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from mypy_boto3_support import SupportClient
+else:
+    SupportClient = object
+
+
+class AWSCase(BaseModel):
+    case_id: str = Field(..., alias="caseId")
+    subject: str
+    status: str
+
+
+class SUPPORT_PLAN(Enum):
+    BASIC = "basic"
+    DEVELOPER = "developer"
+    BUSINESS = "business"
+    ENTERPRISE = "enterprise"
+
+
+class AWSApiSupport:
+    def __init__(self, client: SupportClient) -> None:
+        self.client = client
+
+    def create_case(
+        self,
+        subject: str,
+        message: str,
+        category: str = "other-account-issues",
+        service: str = "customer-account",
+        issue_type: Literal["customer-service", "technical"] = "customer-service",
+        language: Literal["en", "zh", "ja", "ko"] = "en",
+        severity: str = "high",
+    ) -> str:
+        """Create a support case and return the case id."""
+        case = self.client.create_case(
+            subject=subject,
+            communicationBody=message,
+            categoryCode=category,
+            serviceCode=service,
+            issueType=issue_type,
+            language=language,
+            severityCode=severity,
+        )
+        return case["caseId"]
+
+    def describe_case(self, case_id: str) -> AWSCase:
+        """Return the status of a support case."""
+        case = self.client.describe_cases(caseIdList=[case_id])["cases"][0]
+        return AWSCase(**case)
+
+    def get_support_level(self) -> SUPPORT_PLAN:
+        """Return the support level of the account."""
+
+        try:
+            response = self.client.describe_severity_levels(language="en")
+        except self.client.exceptions.ClientError as err:
+            if err.response["Error"]["Code"] == "SubscriptionRequiredException":
+                return SUPPORT_PLAN.BASIC
+            raise err
+
+        severity_levels = {
+            level["code"].lower() for level in response["severityLevels"]
+        }
+        if "critical" in severity_levels:
+            return SUPPORT_PLAN.ENTERPRISE
+        if "urgent" in severity_levels:
+            return SUPPORT_PLAN.BUSINESS
+        if "high" in severity_levels:
+            return SUPPORT_PLAN.BUSINESS
+        if "normal" in severity_levels:
+            return SUPPORT_PLAN.DEVELOPER
+        if "low" in severity_levels:
+            return SUPPORT_PLAN.DEVELOPER
+        return SUPPORT_PLAN.BASIC

reconcile/utils/aws_helper.py

@@ -1,6 +1,10 @@
 from collections.abc import Iterable
-from typing import Any
+from typing import Any, Protocol
 
+from reconcile.utils.disabled_integrations import (
+    HasDisableIntegrations,
+    integration_is_enabled,
+)
 from reconcile.utils.secret_reader import SecretReader
 
 
@@ -11,8 +15,8 @@ class AccountNotFoundError(Exception):
 Account = dict[str, Any]
 
 
-def get_user_id_from_arn(arn):
-    # arn:aws:iam::12345:user/id --> id
+def get_id_from_arn(arn):
+    # arn:aws:iam::12345:<arntype>/id --> id
     return arn.split("/")[1]
 
 
@@ -21,6 +25,17 @@ def get_account_uid_from_arn(arn):
     return arn.split(":")[4]
 
 
+def get_role_name_from_arn(arn: str) -> str:
+    # arn:aws:iam::12345:role/role-1 --> role-1
+    return arn.split("/")[-1]
+
+
+def is_aws_managed_resource(arn: str) -> bool:
+    # arn:aws:iam::aws:role/role-1 --> True
+    # arn:aws:iam::12345:role/role-1 --> False
+    return get_account_uid_from_arn(arn) == "aws"
+
+
 def get_details_from_role_link(role_link):
     # https://signin.aws.amazon.com/switchrole?
     # account=<uid>&roleName=<role_name> -->
@@ -61,3 +76,27 @@ def get_account(accounts: Iterable[Account], account_name: str) -> Account:
 
 def get_region_from_availability_zone(availability_zone: str) -> str:
     return availability_zone[:-1]
+
+
+class AccountSSO(HasDisableIntegrations, Protocol):
+    name: str
+    uid: str
+    sso: bool | None
+
+
+def unique_sso_aws_accounts(
+    integration: str, accounts: Iterable[AccountSSO], account_name: str | None = None
+) -> list[AccountSSO]:
+    """Return a unique list of AWS accounts with SSO enabled."""
+    filtered_account = {}
+    for account in accounts:
+        if account_name and account.name != account_name:
+            continue
+        if not account.sso:
+            continue
+        if not integration_is_enabled(integration, account):
+            continue
+        if account.uid in filtered_account:
+            continue
+        filtered_account[account.uid] = account
+    return list(filtered_account.values())

reconcile/utils/batches.py

@@ -0,0 +1,11 @@
+from collections.abc import Generator, Iterable
+from itertools import islice
+from typing import Any
+
+
+def batched(iterable: Iterable[Any], size: int) -> Generator:
+    if size < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    while batch := tuple(islice(it, size)):
+        yield batch

reconcile/utils/binary.py

@@ -1,6 +1,6 @@
 import re
+import shutil
 import subprocess
-from distutils.spawn import find_executable  # pylint: disable=deprecated-module
 from functools import wraps
 
 
@@ -13,7 +13,7 @@ def binary(binaries=None):
         @wraps(f)
         def f_binary(*args, **kwargs):
             for b in binaries:
-                if not find_executable(b):
+                if not shutil.which(b):
                     raise Exception(
                         f"Aborting: Could not find binary: {b}. "
                         + f"Hint: https://command-not-found.com/{b}"
@@ -25,7 +25,7 @@ def binary(binaries=None):
     return deco_binary
 
 
-def binary_version(binary, version_args, search_regex, expected_version):
+def binary_version(binary, version_args, search_regex, expected_versions):
     """Check that a binary exists and is a desired version"""
 
     def deco_binary_version(f):
@@ -36,15 +36,13 @@ def binary_version(binary, version_args, search_regex, expected_version):
             cmd = [binary]
             cmd.extend(version_args)
             try:
-                result = subprocess.run(
-                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
-                )
+                result = subprocess.run(cmd, capture_output=True, check=True)
             except subprocess.CalledProcessError as e:
                 msg = (
                     f"Could not execute binary '{binary}' "
                     f"for binary version check: {e}"
                 )
-                raise Exception(msg)
+                raise Exception(msg) from e
 
             found = False
             match = None
@@ -62,10 +60,10 @@ def binary_version(binary, version_args, search_regex, expected_version):
                 )
 
             version = match.group(1)
-            if version != expected_version:
+            if version not in expected_versions:
                 raise Exception(
                     f"Binary version check for binary {binary} failed! "
-                    f"Expected: {expected_version}, found: {version}"
+                    f"Expected: {expected_versions}, found: {version}"
                 )
 
             f(*args, **kwargs)

reconcile/utils/cloud_resource_best_practice/aws_rds.py

@@ -0,0 +1,66 @@
+import operator
+from typing import Any
+
+from terrascript.resource import aws_db_instance
+
+delete_automated_backups = "delete_automated_backups"
+skip_final_snapshot = "skip_final_snapshot"
+backup_retention_period = "backup_retention_period"
+
+
+# Keeping the exception specific now, we can build a generic exception and refactor later
+# as more cases arise in the future.
+class RDSResourceComplianceError(Exception):
+    def __init__(
+        self, field_expected_actual: list[tuple[str, Any, Any]], db_identifier: str
+    ):
+        self.actual_vs_expected = field_expected_actual
+
+        self.message = (
+            f"AWS RDS instance {db_identifier} does not comply with best practices\n"
+        )
+
+        for field, actual, expected in field_expected_actual:
+            self.message += (
+                f'Expected field {field} with value "{expected}", found "{actual}" \n'
+            )
+        super().__init__(self.message)
+
+
+__loss_impact_levels = {
+    "high": [
+        (delete_automated_backups, False, operator.__eq__),
+        (skip_final_snapshot, False, operator.__eq__),
+        (backup_retention_period, 7, operator.__ge__),
+    ],
+    "medium": [
+        (skip_final_snapshot, False, operator.__eq__),
+        (backup_retention_period, 7, operator.__ge__),
+    ],
+    "low": [(backup_retention_period, 3, operator.__ge__)],
+    "none": [(backup_retention_period, 0, operator.__ge__)],
+}
+
+
+# Keeping this method here for now, we can re-evaluate if this needs to go to separate util once more
+# use-cases are built.
+def _check(aws_db_instance: aws_db_instance, checks: list) -> None:
+    rds_fields_not_complied = []
+    for field, expected_value, op in checks:
+        if not op(aws_db_instance.get(field), expected_value):
+            rds_fields_not_complied.append((
+                field,
+                aws_db_instance.get(field),
+                expected_value,
+            ))
+
+    if len(rds_fields_not_complied) > 0:
+        raise RDSResourceComplianceError(rds_fields_not_complied, aws_db_instance._name)
+
+
+def verify_rds_best_practices(
+    aws_db_instance: aws_db_instance, data_classification: dict
+) -> None:
+    if data_classification is None:
+        return
+    _check(aws_db_instance, __loss_impact_levels[data_classification["loss_impact"]])

reconcile/utils/clusterhealth/providerbase.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel
+
+
+class ClusterHealth(BaseModel):
+    source: str
+    errors: set[str] | None = None
+
+    def has_health_errors(self) -> bool:
+        return bool(self.errors)
+
+
+def build_assumed_cluster_health() -> ClusterHealth:
+    return ClusterHealth(source="assumption")
+
+
+class ClusterHealthProvider(ABC):
+    """
+    A base class for cluster health providers.
+    """
+
+    @abstractmethod
+    def cluster_health(self, cluster_external_id: str, org_id: str) -> ClusterHealth:
+        """
+        Provides health information for an individual cluster in an organization
+        """
+
+
+class EmptyClusterHealthProvider(ClusterHealthProvider):
+    """
+    A default implementation of a cluster health provider that returns no health
+    information. Not every environment has a healthprovider available and this
+    implementation helps avoiding to check for the existance of a health provider
+    in various code places for such environments.
+    """
+
+    def cluster_health(self, cluster_external_id: str, org_id: str) -> ClusterHealth:
+        return build_assumed_cluster_health()

reconcile/utils/clusterhealth/telemeter.py

@@ -0,0 +1,39 @@
+from functools import lru_cache
+
+from reconcile.utils.clusterhealth.providerbase import (
+    ClusterHealth,
+    ClusterHealthProvider,
+)
+from reconcile.utils.grouping import group_by
+from reconcile.utils.prometheus import PrometheusQuerier
+
+TELEMETER_SOURCE = "telemeter"
+
+
+class TelemeterClusterHealthProvider(ClusterHealthProvider):
+    def __init__(self, querier: PrometheusQuerier):
+        self.querier = querier
+        self.cluster_health_for_org = lru_cache(maxsize=None)(
+            self._cluster_health_for_org
+        )
+
+    def _cluster_health_for_org(self, org_id: str) -> dict[str, ClusterHealth]:
+        vectors_by_cluster = group_by(
+            self.querier.instant_vector_query(telemeter_alert_query(org_id)),
+            lambda v: v.mandatory_label("_id"),
+        )
+        return {
+            cluster_id: ClusterHealth(
+                errors={alert.mandatory_label("alertname") for alert in vectors},
+                source=TELEMETER_SOURCE,
+            )
+            for cluster_id, vectors in vectors_by_cluster.items()
+        }
+
+    def cluster_health(self, cluster_external_id: str, org_id: str) -> ClusterHealth:
+        health = self.cluster_health_for_org(org_id=org_id).get(cluster_external_id)
+        return health or ClusterHealth(source=TELEMETER_SOURCE)
+
+
+def telemeter_alert_query(organization_id: str) -> str:
+    return f'alerts{{alertstate="firing", severity="critical"}} * on (_id) group_left(organization) max(ocm_subscription{{organization="{organization_id}"}}) by (_id, organization)'

reconcile/utils/config.py

@@ -12,12 +12,11 @@ class SecretNotFound(Exception):
 
 
 def get_config():
-    global _config
     return _config
 
 
 def init(config):
-    global _config
+    global _config  # noqa: PLW0603
     _config = config
     return _config
 
@@ -36,7 +35,7 @@ def read(secret):
             config = config[t]
         return config[field]
     except Exception as e:
-        raise SecretNotFound(f"key not found in config file {path}: {str(e)}")
+        raise SecretNotFound(f"key not found in config file {path}: {e!s}") from None
 
 
 def read_all(secret):
@@ -48,4 +47,4 @@ def read_all(secret):
             config = config[t]
         return config
     except Exception as e:
-        raise SecretNotFound(f"secret {path} not found in config file: {str(e)}")
+        raise SecretNotFound(f"secret {path} not found in config file: {e!s}") from None

reconcile/utils/deadmanssnitch_api.py

@@ -0,0 +1,86 @@
+import logging
+from typing import (
+    Any,
+    Self,
+)
+
+import requests
+from pydantic import BaseModel
+
+BASE_URL = "https://api.deadmanssnitch.com/v1/snitches"
+REQUEST_TIMEOUT = 60
+
+
+class DeadManssnitchException(Exception):
+    pass
+
+
+class Snitch(BaseModel):
+    token: str
+    href: str
+    name: str
+    tags: list[str]
+    notes: str
+    status: str
+    check_in_url: str
+    interval: str
+    alert_type: str
+    alert_email: list[str]
+    vault_data: str | None
+
+    def needs_vault_update(self) -> bool:
+        return self.vault_data is not None and self.check_in_url != self.vault_data
+
+
+class DeadMansSnitchApi:
+    def __init__(
+        self, token: str, url: str = BASE_URL, timeout: int = REQUEST_TIMEOUT
+    ) -> None:
+        self.token = token
+        self.url = url
+        self.timeout = timeout
+        self.session = requests.Session()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
+
+    def get_snitches(self, tags: list[str]) -> list[Snitch]:
+        logging.debug("Getting snitches for tags:%s", tags)
+        response = self.session.get(
+            url=self.url,
+            params={"tags": ",".join(tags)},
+            auth=(self.token, ""),
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        snitches = [Snitch(**item) for item in response.json()]
+        return snitches
+
+    def create_snitch(self, payload: dict) -> Snitch:
+        if payload.get("name") is None or payload.get("interval") is None:
+            raise DeadManssnitchException(
+                "Invalid payload,name and interval are mandatory"
+            )
+        headers = {"Content-Type": "application/json"}
+        logging.debug("Creating new snitch with name:: %s ", payload["name"])
+        response = self.session.post(
+            url=self.url,
+            json=payload,
+            auth=(self.token, ""),
+            headers=headers,
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        response_json = response.json()
+        return Snitch(**response_json)
+
+    def delete_snitch(self, token: str) -> None:
+        delete_api_url = f"{self.url}/{token}"
+        response = self.session.delete(
+            url=delete_api_url, auth=(self.token, ""), timeout=self.timeout
+        )
+        response.raise_for_status()
+        logging.debug("Successfully deleted snich: %s", token)

reconcile/utils/differ.py

@@ -0,0 +1,205 @@
+from collections.abc import (
+    Callable,
+    Iterable,
+    Mapping,
+)
+from dataclasses import dataclass
+from typing import (
+    Any,
+    Generic,
+    TypeVar,
+)
+
+T = TypeVar("T")
+Current = TypeVar("Current")
+Desired = TypeVar("Desired")
+Key = TypeVar("Key")
+
+
+@dataclass(frozen=True, eq=True)
+class DiffPair(Generic[Current, Desired]):
+    current: Current
+    desired: Desired
+
+
+@dataclass(frozen=True, eq=True)
+class DiffResult(Generic[Current, Desired, Key]):
+    add: dict[Key, Desired]
+    delete: dict[Key, Current]
+    change: dict[Key, DiffPair[Current, Desired]]
+    identical: dict[Key, DiffPair[Current, Desired]]
+
+
+def _default_equal(current: Current, desired: Desired) -> bool:
+    return current == desired
+
+
+def _default_key(item: Any) -> Any:
+    return item
+
+
+def diff_mappings(
+    current: Mapping[Key, Current],
+    desired: Mapping[Key, Desired],
+    equal: Callable[[Current, Desired], bool] = _default_equal,
+) -> DiffResult[Current, Desired, Key]:
+    """
+    Compare two mappings and return a `DiffResult` instance containing the differences between them.
+
+    :param current: The current mapping to compare.
+    :type current: Mapping[Key, Current]
+    :param desired: The desired mapping to compare.
+    :type desired: Mapping[Key, Desired]
+    :param equal: A function that compares two elements in the mappings and returns True if they are equal.
+        The default behavior is to use the `==` operator.
+    :type equal: Callable[[Current, Desired], bool]
+    :return: A `DiffResult` instance containing the differences between the `current` and `desired` mappings,
+        including elements that were added, deleted, changed or identical.
+    :rtype: DiffResult[Current, Desired, Key]
+    :raises: None
+
+    Example:
+        >>> current = {"i": 1, "c": 2, "d": 3}
+        >>> desired = {"i": 1, "c": 20, "a": 30}
+        >>> result = diff_mappings(current, desired, equal=lambda c, d: c == d)
+        DiffResult(
+            add={'a': 30},
+            delete={'d': 3},
+            change={'c': DiffPair(current=2, desired=20),
+            identical={'i': DiffPair(current=1, desired=1)},
+        )
+    """
+    add = {k: desired[k] for k in desired.keys() - current.keys()}
+    delete = {k: current[k] for k in current.keys() - desired.keys()}
+    change, identical = {}, {}
+    for k in current.keys() & desired.keys():
+        diff_pair = DiffPair(current=current[k], desired=desired[k])
+        if equal(diff_pair.current, diff_pair.desired):
+            identical[k] = diff_pair
+        else:
+            change[k] = diff_pair
+    return DiffResult(
+        add=add,
+        delete=delete,
+        change=change,
+        identical=identical,
+    )
+
+
+def diff_any_iterables(
+    current: Iterable[Current],
+    desired: Iterable[Desired],
+    current_key: Callable[[Current], Key] = _default_key,
+    desired_key: Callable[[Desired], Key] = _default_key,
+    equal: Callable[[Current, Desired], bool] = _default_equal,
+) -> DiffResult[Current, Desired, Key]:
+    """
+    Compare two iterables and return a `DiffResult` instance containing the differences between them.
+
+    :param current: The current iterable to compare.
+    :type current: Iterable[Current]
+    :param desired: The desired iterable to compare.
+    :type desired: Iterable[Desired]
+    :param current_key: A function that returns the key for an element in the `current` iterable.
+        If not provided, the default behavior is to use the element itself as the key.
+    :type current_key: Callable[[Current], Key]
+    :param desired_key: A function that returns the key for an element in the `desired` iterable.
+        If not provided, the default behavior is to use the element itself as the key.
+    :type desired_key: Callable[[Desired], Key]
+    :param equal: A function that compares two elements in the iterables and returns True if they are equal.
+        If not provided, the default behavior is to use the `==` operator.
+    :type equal: Callable[[Current, Desired], bool]
+    :return: A `DiffResult` instance containing the differences between the `current` and `desired` iterables,
+        including elements that were added, deleted, changed or identical.
+    :rtype: DiffResult[Current, Desired, Key]
+    :raises: None
+
+    Example:
+        >>> current = [
+        ...     {"name": "i", "value": 1},
+        ...     {"name": "c", "value": 2},
+        ...     {"name": "d", "value": 3},
+        ... ]
+        >>> desired = [
+        ...     {"name": "i", "value": 1},
+        ...     {"name": "c", "value": 20},
+        ...     {"name": "a", "value": 30},
+        ... ]
+        >>> result = diff_any_iterables(
+        ...     current,
+        ...     desired,
+        ...     lambda c: c["name"],
+        ...     lambda d: d["name"],
+        ...     equal=lambda c, d: c["value"] == d["value"],
+        ... )
+        DiffResult(
+            add={'a': {'name': 'a', 'value': 30}},
+            delete={'d': {'name': 'd', 'value': 3}},
+            change={'c': DiffPair(current={'name': 'c', 'value': 2}, desired={'name': 'c', 'value': 20})),
+            identical={'i': DiffPair(current={'name': 'i', 'value': 1}, desired={'name': 'i', 'value': 1})},
+        )
+    """
+    current_dict = {current_key(c): c for c in current}
+    desired_dict = {desired_key(d): d for d in desired}
+    return diff_mappings(
+        current_dict,
+        desired_dict,
+        equal=equal,
+    )
+
+
+def diff_iterables(
+    current: Iterable[T],
+    desired: Iterable[T],
+    key: Callable[[T], Key] = _default_key,
+    equal: Callable[[T, T], bool] = _default_equal,
+) -> DiffResult[T, T, Key]:
+    """
+    Compare two iterables with same type and return a `DiffResult` instance containing the differences between them.
+
+    :param current: The current iterable to compare.
+    :type current: Iterable[T]
+    :param desired: The desired iterable to compare.
+    :type desired: Iterable[T]
+    :param key: A function that returns the key for an element in the `current` and `desired` iterable.
+        If not provided, the default behavior is to use the element itself as the key.
+    :type key: Callable[[Current], Key]
+    :param equal: A function that compares two elements in the iterables and returns True if they are equal.
+        If not provided, the default behavior is to use the `==` operator.
+    :type equal: Callable[[Current, Desired], bool]
+    :return: A `DiffResult` instance containing the differences between the `current` and `desired` iterables,
+        including elements that were added, deleted, changed or identical.
+    :rtype: DiffResult[T, T, Key]
+    :raises: None
+
+    Example:
+        >>> current = [
+        ...     {"name": "i", "value": 1},
+        ...     {"name": "c", "value": 2},
+        ...     {"name": "d", "value": 3},
+        ... ]
+        >>> desired = [
+        ...     {"name": "i", "value": 1},
+        ...     {"name": "c", "value": 20},
+        ...     {"name": "a", "value": 30},
+        ... ]
+        >>> result = diff_iterables(
+        ...     current,
+        ...     desired,
+        ...     lambda x: x["name"],
+        ...     equal=lambda c, d: c["value"] == d["value"],
+        ... )
+        DiffResult(
+            add={'a': {'name': 'a', 'value': 30}},
+            delete={'d': {'name': 'd', 'value': 3}},
+            change={'c': DiffPair(current={'name': 'c', 'value': 2}, desired={'name': 'c', 'value': 20})),
+            identical={'i': DiffPair(current={'name': 'i', 'value': 1}, desired={'name': 'i', 'value': 1})},
+        )
+    """
+    return diff_any_iterables(
+        current,
+        desired,
+        current_key=key,
+        desired_key=key,
+        equal=equal,
+    )