qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/openshift_resource.py

@@ -1,4 +1,6 @@
+# ruff: noqa: SIM114
 import base64
+import contextlib
 import copy
 import datetime
 import hashlib
@@ -6,12 +8,12 @@ import json
 import re
 from collections.abc import Mapping
 from threading import Lock
-from typing import (
-    Optional,
-    Union,
-)
 
 import semver
+from pydantic import BaseModel
+
+from reconcile.external_resources.meta import SECRET_UPDATED_AT
+from reconcile.utils.metrics import GaugeMetric
 
 SECRET_MAX_KEY_LENGTH = 253
 
@@ -20,6 +22,10 @@ class ResourceKeyExistsError(Exception):
     pass
 
 
+class ResourceNotManagedError(Exception):
+    pass
+
+
 class ConstructResourceError(Exception):
     def __init__(self, msg):
         super().__init__("error constructing openshift resource: " + str(msg))
@@ -40,7 +46,7 @@ IGNORABLE_DATA_FIELDS = ["service-ca.crt"]
 # these labels existance and/or value is determined by a controller running
 # on the cluster. we need to ignore their existance in the current state,
 # otherwise we will deal with constant reconciliation
-CONTROLLER_MANAGED_LABELS: dict[str, set[Union[str, re.Pattern]]] = {
+CONTROLLER_MANAGED_LABELS: dict[str, set[str | re.Pattern]] = {
     "ManagedCluster": {
         "clusterID",
         "managed-by",
@@ -49,7 +55,22 @@ CONTROLLER_MANAGED_LABELS: dict[str, set[Union[str, re.Pattern]]] = {
     }
 }
 
+QONTRACT_ANNOTATION_INTEGRATION = "qontract.integration"
+QONTRACT_ANNOTATION_INTEGRATION_VERSION = "qontract.integration_version"
+QONTRACT_ANNOTATION_SHA256SUM = "qontract.sha256sum"
+QONTRACT_ANNOTATION_UPDATE = "qontract.update"
+QONTRACT_ANNOTATION_CALLER_NAME = "qontract.caller_name"
+
+QONTRACT_ANNOTATIONS = {
+    QONTRACT_ANNOTATION_INTEGRATION,
+    QONTRACT_ANNOTATION_INTEGRATION_VERSION,
+    QONTRACT_ANNOTATION_SHA256SUM,
+    QONTRACT_ANNOTATION_UPDATE,
+    QONTRACT_ANNOTATION_CALLER_NAME,
+}
+
 
+# pylint: disable=R0904
 class OpenshiftResource:
     def __init__(
         self,
@@ -81,7 +102,7 @@ class OpenshiftResource:
             for obj1_k, obj1_v in obj1.items():
                 obj2_v = obj2.get(obj1_k, None)
                 if obj2_v is None:
-                    if obj1_v not in [None, ""]:
+                    if obj1_v:
                         return False
                 if self.ignorable_field(obj1_k):
                     pass
@@ -100,7 +121,7 @@ class OpenshiftResource:
                     ]
                     if diff or not self.obj_intersect_equal(obj1_v, obj2_v, depth + 1):
                         return False
-                elif obj1_k in ["data", "matchLabels"]:
+                elif obj1_k in {"data", "matchLabels"}:
                     diff = [
                         k
                         for k in obj2_v
@@ -153,28 +174,22 @@ class OpenshiftResource:
             "uid",
             "fieldRef",
         ]
-        if val in ignorable_fields:
-            return True
-        return False
+        return val in ignorable_fields
 
     @staticmethod
     def ignorable_key_value_pair(key, val):
         ignorable_key_value_pair = {"annotations": None, "divisor": "0"}
-        if key in ignorable_key_value_pair and ignorable_key_value_pair[key] == val:
-            return True
-        return False
+        return bool(
+            key in ignorable_key_value_pair and ignorable_key_value_pair[key] == val
+        )
 
     @staticmethod
     def cpu_equal(val1, val2):
         # normalize both to string
-        try:
+        with contextlib.suppress(Exception):
             val1 = f"{int(float(val1) * 1000)}m"
-        except Exception:
-            pass
-        try:
+        with contextlib.suppress(Exception):
             val2 = f"{int(float(val2) * 1000)}m"
-        except Exception:
-            pass
         return val1 == val2
 
     @staticmethod
@@ -191,12 +206,20 @@ class OpenshiftResource:
 
     @property
     def name(self):
-        return self.body["metadata"]["name"]
+        # PipelineRun name can be empty when creating
+        if self.kind == "PipelineRun" and "name" not in self.body["metadata"]:
+            return self.body["metadata"]["generateName"][:-1]
+        else:
+            return self.body["metadata"]["name"]
 
     @property
     def kind(self):
         return self.body["kind"]
 
+    @property
+    def annotations(self):
+        return self.body["metadata"].get("annotations", {})
+
     @property
     def kind_and_group(self):
         return fully_qualified_kind(self.kind, self.body["apiVersion"])
@@ -213,20 +236,18 @@ class OpenshiftResource:
 
     def verify_valid_k8s_object(self):
         try:
-            self.name  # pylint: disable=pointless-statement
-            self.kind  # pylint: disable=pointless-statement
+            assert self.name
+            assert self.kind
         except (KeyError, TypeError) as e:
-            msg = "resource invalid data ({}). details: {}".format(
-                e.__class__.__name__, self.error_details
-            )
-            raise ConstructResourceError(msg)
+            msg = f"resource invalid data ({e.__class__.__name__}). details: {self.error_details}"
+            raise ConstructResourceError(msg) from None
 
-        if self.kind not in [
+        if self.kind not in {
             "Role",
             "RoleBinding",
             "ClusterRole",
             "ClusterRoleBinding",
-        ] and (
+        } and (
             not DNS_SUBDOMAIN_RE.match(self.name)
             or not len(self.name) <= DNS_SUBDOMAIN_MAX_LENGTH
         ):
@@ -283,15 +304,15 @@ class OpenshiftResource:
         try:
             annotations = self.body["metadata"]["annotations"]
 
-            assert annotations["qontract.integration"] == self.integration
+            assert annotations[QONTRACT_ANNOTATION_INTEGRATION] == self.integration
 
-            integration_version = annotations["qontract.integration_version"]
+            integration_version = annotations[QONTRACT_ANNOTATION_INTEGRATION_VERSION]
             assert (
                 semver.VersionInfo.parse(integration_version).major
                 == semver.VersionInfo.parse(self.integration_version).major
             )
 
-            assert annotations["qontract.sha256sum"] is not None
+            assert annotations[QONTRACT_ANNOTATION_SHA256SUM] is not None
         except KeyError:
             return False
         except AssertionError:
@@ -314,7 +335,7 @@ class OpenshiftResource:
         except KeyError:
             return False
 
-    def annotate(self):
+    def annotate(self, canonicalize=True):
         """
         Creates a OpenshiftResource with the qontract annotations, and removes
         unneeded Openshift fields.
@@ -323,10 +344,9 @@ class OpenshiftResource:
             openshift_resource: new OpenshiftResource object with
                 annotations.
         """
+        body = self.canonicalize(self.body) if canonicalize else self.body
 
-        # calculate sha256sum of canonical body
-        canonical_body = self.canonicalize(self.body)
-        sha256sum = self.calculate_sha256sum(self.serialize(canonical_body))
+        sha256sum = self.calculate_sha256sum(self.serialize(body))
 
         # create new body object
         body = copy.deepcopy(self.body)
@@ -339,13 +359,13 @@ class OpenshiftResource:
         annotations = body["metadata"]["annotations"]
 
         # add qontract annotations
-        annotations["qontract.integration"] = self.integration
-        annotations["qontract.integration_version"] = self.integration_version
-        annotations["qontract.sha256sum"] = sha256sum
+        annotations[QONTRACT_ANNOTATION_INTEGRATION] = self.integration
+        annotations[QONTRACT_ANNOTATION_INTEGRATION_VERSION] = self.integration_version
+        annotations[QONTRACT_ANNOTATION_SHA256SUM] = sha256sum
         now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
-        annotations["qontract.update"] = now
+        annotations[QONTRACT_ANNOTATION_UPDATE] = now
         if self.caller_name:
-            annotations["qontract.caller_name"] = self.caller_name
+            annotations[QONTRACT_ANNOTATION_CALLER_NAME] = self.caller_name
 
         return OpenshiftResource(body, self.integration, self.integration_version)
 
@@ -389,7 +409,7 @@ class OpenshiftResource:
 
         # Default fields for specific resource types
         # ConfigMaps and Secrets are by default Opaque
-        if body["kind"] in ("ConfigMap", "Secret") and body.get("type") == "Opaque":
+        if body["kind"] in {"ConfigMap", "Secret"} and body.get("type") == "Opaque":
             body.pop("type")
 
         if body["kind"] == "Secret":
@@ -397,7 +417,7 @@ class OpenshiftResource:
             if string_data:
                 body.setdefault("data", {})
                 for k, v in string_data.items():
-                    v = base64.b64encode(str(v).encode()).decode("utf-8")
+                    v = base64_encode_secret_field_value(str(v))
                     body["data"][k] = v
 
         if body["kind"] == "Deployment":
@@ -419,7 +439,7 @@ class OpenshiftResource:
                     tls.pop("key", None)
                     tls.pop("certificate", None)
             subdomain = body["spec"].get("subdomain", None)
-            if subdomain == "":
+            if not subdomain:
                 body["spec"].pop("subdomain", None)
 
         if body["kind"] == "ServiceAccount":
@@ -448,9 +468,6 @@ class OpenshiftResource:
                     and not rule["attributeRestrictions"]
                 ):
                     rule.pop("attributeRestrictions")
-            # TODO: remove this once we have no 3.11 clusters
-            if body["apiVersion"] == "authorization.openshift.io/v1":
-                body["apiVersion"] = "rbac.authorization.k8s.io/v1"
 
         if body["kind"] == "OperatorGroup":
             annotations.pop("olm.providedAPIs", None)
@@ -472,18 +489,11 @@ class OpenshiftResource:
                 if "namespace" in subject:
                     subject.pop("namespace")
                 if "apiGroup" in subject and (
-                    subject["apiGroup"] == ""
-                    or subject["apiGroup"] in body["apiVersion"]
+                    not subject["apiGroup"] or subject["apiGroup"] in body["apiVersion"]
                 ):
                     subject.pop("apiGroup")
-            # TODO: remove this once we have no 3.11 clusters
-            if body["apiVersion"] == "rbac.authorization.k8s.io/v1":
-                body["apiVersion"] = "authorization.openshift.io/v1"
 
         if body["kind"] == "ClusterRoleBinding":
-            # TODO: remove this once we have no 3.11 clusters
-            if body["apiVersion"] == "authorization.openshift.io/v1":
-                body["apiVersion"] = "rbac.authorization.k8s.io/v1"
             if "userNames" in body:
                 body.pop("userNames")
             if "roleRef" in body:
@@ -502,12 +512,11 @@ class OpenshiftResource:
                 spec.pop("clusterIP", None)
 
         # remove qontract specific params
-        annotations.pop("qontract.integration", None)
-        annotations.pop("qontract.integration_version", None)
-        annotations.pop("qontract.sha256sum", None)
-        annotations.pop("qontract.update", None)
-        annotations.pop("qontract.caller_name", None)
+        for a in QONTRACT_ANNOTATIONS:
+            annotations.pop(a, None)
 
+        # Remove external resources annotation used for optimistic locking
+        annotations.pop(SECRET_UPDATED_AT, None)
         return body
 
     @staticmethod
@@ -528,6 +537,25 @@ def fully_qualified_kind(kind: str, api_version: str) -> str:
     return kind
 
 
+class OpenshiftResourceBaseMetric(BaseModel):
+    "Base class Openshift Resource metrics"
+
+    integration: str
+
+
+class OpenshiftResourceInventoryGauge(OpenshiftResourceBaseMetric, GaugeMetric):
+    "Inventory Gauge"
+
+    cluster: str
+    namespace: str
+    kind: str
+    state: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_openshift_resource_inventory"
+
+
 class ResourceInventory:
     def __init__(self):
         self._clusters = {}
@@ -535,11 +563,23 @@ class ResourceInventory:
         self._error_registered_clusters = {}
         self._lock = Lock()
 
-    def initialize_resource_type(self, cluster, namespace, resource_type):
+    def initialize_resource_type(
+        self,
+        cluster,
+        namespace,
+        resource_type,
+        managed_names: list[str] | None = None,
+    ):
         self._clusters.setdefault(cluster, {})
         self._clusters[cluster].setdefault(namespace, {})
         self._clusters[cluster][namespace].setdefault(
-            resource_type, {"current": {}, "desired": {}, "use_admin_token": {}}
+            resource_type,
+            {
+                "current": {},
+                "desired": {},
+                "use_admin_token": {},
+                "managed_names": managed_names,
+            },
         )
 
     def is_cluster_present(self, cluster: str) -> bool:
@@ -575,6 +615,13 @@ class ResourceInventory:
         # mismatch between schema and implementation for now, it will enable
         # us to implement per-resource configuration in the future
         with self._lock:
+            # fail if the name of the resource is not within the managed names if they are defined
+            managed_names = self._clusters[cluster][namespace][resource_type][
+                "managed_names"
+            ]
+            if managed_names is not None and name not in managed_names:
+                raise ResourceNotManagedError(name)
+
             desired = self._clusters[cluster][namespace][resource_type]["desired"]
             if name in desired:
                 raise ResourceKeyExistsError(name)
@@ -630,10 +677,9 @@ def build_secret(
     integration_version: str,
     unencoded_data: Mapping[str, str],
     error_details: str = "",
-    caller_name: Optional[str] = None,
-    annotations: Optional[Mapping[str, str]] = None,
+    caller_name: str | None = None,
+    annotations: Mapping[str, str] | None = None,
 ) -> OpenshiftResource:
-
     encoded_data = {
         k: base64_encode_secret_field_value(v) for k, v in unencoded_data.items()
     }
@@ -655,7 +701,7 @@ def build_secret(
     )
 
 
-def base64_encode_secret_field_value(value: str) -> Optional[str]:
-    if value == "":
-        return None
+def base64_encode_secret_field_value(value: str) -> str:
+    if not value:
+        return ""
     return base64.b64encode(str(value).encode()).decode("utf-8")

reconcile/utils/output.py

@@ -1,19 +1,19 @@
 import json
+import re
 from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import Union
 
 import yaml
 from tabulate import tabulate
 
 
 def print_output(
-    options: Mapping[str, Union[str, bool]],
+    options: Mapping[str, str | bool],
     content: list[dict],
     columns: Iterable[str] = (),
-):
+) -> str | None:
     if options["sort"]:
         content.sort(key=lambda c: tuple(c.values()))
     if options.get("to_string"):
@@ -23,17 +23,26 @@ def print_output(
 
     output = options["output"]
 
+    formatted_content = None
     if output == "table":
-        print_table(content, columns)
+        formatted_content = format_table(content, columns)
+        print(formatted_content)
     elif output == "md":
-        print_table(content, columns, table_format="github")
+        formatted_content = re.sub(
+            r" +", " ", format_table(content, columns, table_format="github")
+        )
+        print(formatted_content)
     elif output == "json":
-        print(json.dumps(content))
+        formatted_content = json.dumps(content)
+        print(formatted_content)
     elif output == "yaml":
-        print(yaml.dump(content))
+        formatted_content = yaml.dump(content)
+        print(formatted_content)
     else:
         pass  # error
 
+    return formatted_content
+
 
 def format_table(content, columns, table_format="simple") -> str:
     headers = [column.upper() for column in columns]
@@ -53,10 +62,8 @@ def format_table(content, columns, table_format="simple") -> str:
                     cell = "<br />".join(cell)
                 else:
                     cell = "\n".join(cell)
+            if table_format == "github" and isinstance(cell, str):
+                cell = cell.replace("|", "&#124;")
             row_data.append(cell)
         table_data.append(row_data)
     return tabulate(table_data, headers=headers, tablefmt=table_format)
-
-
-def print_table(content, columns, table_format="simple"):
-    print(format_table(content, columns, table_format))

reconcile/utils/pagerduty_api.py

@@ -4,8 +4,8 @@ from collections.abc import (
     Iterable,
 )
 from datetime import datetime as dt
+from datetime import timedelta
 from typing import (
-    Optional,
     Protocol,
 )
 
@@ -44,7 +44,7 @@ class PagerDutyUser(Protocol):
     which must be implemented by a class to be compatible."""
 
     org_username: str
-    pagerduty_username: Optional[str]
+    pagerduty_username: str | None
 
 
 class PagerDutyTarget(Protocol):
@@ -53,8 +53,8 @@ class PagerDutyTarget(Protocol):
 
     name: str
     instance: PagerDutyInstance
-    escalation_policy_id: Optional[str]
-    schedule_id: Optional[str]
+    escalation_policy_id: str | None
+    schedule_id: str | None
 
 
 class PagerDutyConfig(BaseModel):
@@ -75,7 +75,7 @@ class PagerDutyApi:
             self.users: list[pypd.User] = []
 
     def init_users(self) -> None:
-        self.users = pypd.User.find()
+        self.users = pypd.User.find(limit=100)
 
     def get_pagerduty_users(
         self, resource_type: str, resource_id: str
@@ -104,7 +104,8 @@ class PagerDutyApi:
         return user.email.split("@")[0]
 
     def get_schedule_users(self, schedule_id: str, now: dt) -> list[pypd.User]:
-        s = pypd.Schedule.fetch(id=schedule_id, since=now, until=now, time_zone="UTC")
+        until = now + timedelta(seconds=60)
+        s = pypd.Schedule.fetch(id=schedule_id, since=now, until=until, time_zone="UTC")
         entries = s["final_schedule"]["rendered_schedule_entries"]
 
         return [
@@ -164,16 +165,21 @@ class PagerDutyMap:
 
 def get_pagerduty_map(
     secret_reader: SecretReader,
-    pagerduty_instances: Iterable[PagerDutyInstance],
+    pagerduty_instances: Iterable[PagerDutyInstance] | None,
     init_users: bool = True,
     pager_duty_api_class: type[PagerDutyApi] = PagerDutyApi,
 ) -> PagerDutyMap:
     """Initiate a PagerDutyMap for given PagerDuty instances."""
-    return PagerDutyMap(
-        instances=[
+    instances = (
+        [
             PagerDutyConfig(name=i.name, token=secret_reader.read_secret(i.token))
             for i in pagerduty_instances
-        ],
+        ]
+        if pagerduty_instances
+        else []
+    )
+    return PagerDutyMap(
+        instances=instances,
         init_users=init_users,
         pager_duty_api_class=pager_duty_api_class,
     )

reconcile/utils/parse_dhms_duration.py

@@ -31,6 +31,18 @@ HANDLE_UNIT_MAP = {
 }
 
 
+def seconds_to_hms(seconds: int) -> str:
+    minutes, s = divmod(seconds, 60)
+    if minutes == 0:
+        return f"{s}s"
+
+    h, m = divmod(minutes, 60)
+    if h == 0:
+        return f"{m}m{s}s"
+
+    return f"{h}h{m}m{s}s"
+
+
 def dhms_to_seconds(time_str: str) -> int:
     """Parses durations and returns seconds. The format is a subset of Go's
     ParseDuration format, only allowing from days to seconds in resolution,
@@ -46,7 +58,7 @@ def dhms_to_seconds(time_str: str) -> int:
         if s.isnumeric():
             previous_number += s
         else:
-            if previous_number == "":
+            if not previous_number:
                 raise BadHDMSDurationError(f"Invalid time duration {time_str}")
 
             if s in HANDLE_UNIT_MAP:

reconcile/utils/prometheus.py

@@ -0,0 +1,123 @@
+from typing import NamedTuple, Protocol
+
+import requests
+from pydantic import BaseModel, Field
+
+from reconcile.gql_definitions.fragments.prometheus_instance import (
+    PrometheusInstance,
+    PrometheusInstanceBearerAuthV1,
+    PrometheusInstanceOidcAuthV1,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+INSTANT_VECTOR_RESULT_TYPE = "vector"
+
+
+class PrometheusQueryError(Exception):
+    pass
+
+
+class PrometheusValue(NamedTuple):
+    timestamp: float
+    value: float
+
+
+class PrometheusVector(BaseModel):
+    metric: dict[str, str]
+    raw_value: PrometheusValue = Field(..., alias="value")
+
+    @property
+    def name(self) -> str:
+        return self.metric["__name__"]
+
+    @property
+    def timestamp(self) -> float:
+        return self.raw_value[0]
+
+    @property
+    def value(self) -> float:
+        return self.raw_value[1]
+
+    def mandatory_label(self, label_name: str) -> str:
+        return self.metric[label_name]
+
+    def label(self, label_name: str, default: str | None = None) -> str | None:
+        return self.metric.get(label_name, default)
+
+
+class PrometheusQuerier(Protocol):
+    """
+    A protocol for a querier of Prometheus.
+    """
+
+    def instant_vector_query(self, query: str) -> list[PrometheusVector]:
+        """
+        Query for instant vectors.
+        """
+
+
+class PrometheusHttpQuerier(BaseModel):
+    query_url: str
+    auth_token: str
+
+    def instant_vector_query(self, query: str) -> list[PrometheusVector]:
+        response = requests.get(
+            self.query_url,
+            params={"query": query},
+            headers={
+                "Authorization": f"Bearer {self.auth_token}",
+            },
+        )
+
+        response.raise_for_status()
+
+        # Parse the response JSON
+        try:
+            parsed = response.json()
+        except Exception as e:
+            raise PrometheusQueryError(
+                f"Query failed with invalid JSON response: {response.text[:100]}..."
+            ) from e
+        if parsed.get("status") != "success":
+            raise PrometheusQueryError(
+                f"Query response status was not `success`but {parsed.get('status')}"
+            )
+        if "data" not in parsed:
+            raise PrometheusQueryError(
+                f"Query response does not contain `data`: {response.text[:100]}..."
+            )
+        result_type = parsed.get("data", {}).get("resultType")
+        if result_type != INSTANT_VECTOR_RESULT_TYPE:
+            raise PrometheusQueryError(
+                f"Query failed with unexpected result type. Expected {INSTANT_VECTOR_RESULT_TYPE} got {result_type}"
+            )
+        return [PrometheusVector(**m) for m in parsed["data"]["result"]]
+
+
+def init_prometheus_http_querier_from_prometheus_instance(
+    prometheus: PrometheusInstance, secret_reader: SecretReaderBase
+) -> PrometheusHttpQuerier:
+    match prometheus.auth:
+        case PrometheusInstanceBearerAuthV1():
+            auth_token = secret_reader.read_secret(prometheus.auth.token)
+        case PrometheusInstanceOidcAuthV1():
+            client_secret = secret_reader.read_secret(
+                prometheus.auth.access_token_client_secret
+            )
+            data = {
+                "grant_type": "client_credentials",
+                "client_id": prometheus.auth.access_token_client_id,
+                "client_secret": client_secret,
+            }
+            response = requests.post(
+                prometheus.auth.access_token_url, data=data, timeout=15
+            )
+            response.raise_for_status()
+            auth_token = response.json().get("access_token")
+        case _:
+            raise Exception(f"Unsupported auth type: {prometheus.auth.provider}")
+
+    return PrometheusHttpQuerier(
+        query_url=f"{prometheus.base_url}/{prometheus.query_path}",
+        auth_token=auth_token,
+    )