qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/ocm/search_filters.py

@@ -5,14 +5,13 @@ from abc import (
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import (
+    UTC,
     datetime,
-    timezone,
 )
 from enum import Enum
 from typing import (
     Any,
     Optional,
-    Union,
 )
 
 import dateparser
@@ -57,7 +56,7 @@ class ListCondition(ABC, FilterCondition):
             raise InvalidFilterError(f"Cannot merge {self} with {other}")
         return type(self)(
             key=self.key,
-            values=sorted(list(set(self.values + other.values))),
+            values=sorted(set(self.values + other.values)),
         )
 
 
@@ -73,9 +72,7 @@ class EqCondition(ListCondition):
     def render(self) -> str:
         if len(self.values) == 1:
             return f"{self.key}='{self._escape_value(self.values[0])}'"
-        quoted_values = ",".join(
-            map(lambda x: f"'{self._escape_value(x)}'", self.values)
-        )
+        quoted_values = ",".join(f"'{self._escape_value(x)}'" for x in self.values)
         return f"{self.key} in ({quoted_values})"
 
     def _escape_value(self, value: Any) -> str:
@@ -94,9 +91,7 @@ class LikeCondition(ListCondition):
     def render(self) -> str:
         if len(self.values) == 1:
             return f"{self.key} like '{self.values[0]}'"
-        quoted_values = " or ".join(
-            map(lambda x: f"{self.key} like '{x}'", self.values)
-        )
+        quoted_values = " or ".join(f"{self.key} like '{x}'" for x in self.values)
         if len(self.values) > 1:
             return f"({quoted_values})"
         return quoted_values
@@ -128,8 +123,8 @@ class DateRangeCondition(FilterCondition):
     by the dateparser library is supported, e.g. "now", "today", "5 days ago"
     """
 
-    start: Optional[Union[datetime, str]]
-    end: Optional[Union[datetime, str]]
+    start: datetime | str | None
+    end: datetime | str | None
 
     def copy_and_merge(self, other: "FilterCondition") -> "FilterCondition":
         raise InvalidFilterError("daterange merge not supported")
@@ -150,7 +145,7 @@ class DateRangeCondition(FilterCondition):
             conditions.append(f"{self.key} <= '{resolved_end.isoformat()}'")
         return " and ".join(conditions)
 
-    def resolve_start(self) -> Optional[datetime]:
+    def resolve_start(self) -> datetime | None:
         """
         Resolves the start bound of the date range. If the start bound is a
         string, it is parsed by the dateparser library. If it is already a
@@ -158,7 +153,7 @@ class DateRangeCondition(FilterCondition):
         """
         return DateRangeCondition._resolve_date(self.start) if self.start else None
 
-    def resolve_end(self) -> Optional[datetime]:
+    def resolve_end(self) -> datetime | None:
         """
         Resolves the end bound of the date range. If the end bound is a
         string, it is parsed by the dateparser library. If it is already a
@@ -166,7 +161,7 @@ class DateRangeCondition(FilterCondition):
         return DateRangeCondition._resolve_date(self.end) if self.end else None
 
     @staticmethod
-    def _resolve_date(date: Union[datetime, str]) -> datetime:
+    def _resolve_date(date: datetime | str) -> datetime:
         if isinstance(date, datetime):
             return date
         parsed = dateparser.parse(
@@ -180,7 +175,7 @@ class DateRangeCondition(FilterCondition):
 
     @staticmethod
     def now() -> datetime:
-        return datetime.now(tz=timezone.utc)
+        return datetime.now(tz=UTC)
 
 
 class InvalidFilterError(Exception):
@@ -210,13 +205,13 @@ class Filter:
 
     def __init__(
         self,
-        conditions: Optional[list[FilterCondition]] = None,
+        conditions: list[FilterCondition] | None = None,
         mode: FilterMode = FilterMode.AND,
     ):
         self.conditions: list[FilterCondition] = conditions or []
         self.mode = mode
 
-    def condition_by_key(self, key: str) -> Optional[FilterCondition]:
+    def condition_by_key(self, key: str) -> FilterCondition | None:
         """
         Returns the condition with the given key, or None if it does not exist.
         """
@@ -258,7 +253,7 @@ class Filter:
         merged_condition = existing_condition.copy_and_merge(condition)
         return self.copy_and_override(merged_condition)
 
-    def eq(self, key: str, value: Optional[str]) -> "Filter":
+    def eq(self, key: str, value: str | None) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires
         the given key to be equal to the given value. If the value is None,
@@ -268,7 +263,7 @@ class Filter:
             return self.is_in(key, [value])
         return self
 
-    def like(self, key: str, value: Optional[str]) -> "Filter":
+    def like(self, key: str, value: str | None) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires
         the given key to be similar to the given value based on the % wildcard.
@@ -278,22 +273,19 @@ class Filter:
             return self.add_condition(LikeCondition(key, [value]))
         return self
 
-    def is_in(self, key: str, values: Optional[Iterable[Any]]) -> "Filter":
+    def is_in(self, key: str, values: Iterable[Any] | None) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires
         the given key to be equal to one of the given values. If the values
         are None or empty, the condition is not added.
         """
         if values:
-            if isinstance(values, list):
-                value_list = values
-            else:
-                value_list = list(values)
+            value_list = values if isinstance(values, list) else list(values)
             value_list.sort()
             return self.add_condition(EqCondition(key, value_list))
         return self
 
-    def before(self, key: str, date: Optional[Union[datetime, str]]) -> "Filter":
+    def before(self, key: str, date: datetime | str | None) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires
         the given key to be before the given date. If the date is None,
@@ -303,7 +295,7 @@ class Filter:
             return self.add_condition(DateRangeCondition(key, None, date))
         return self
 
-    def after(self, key: str, date: Optional[Union[datetime, str]]) -> "Filter":
+    def after(self, key: str, date: datetime | str | None) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires
         the given key to be after the given date. If the date is None,
@@ -316,8 +308,8 @@ class Filter:
     def between(
         self,
         key: str,
-        start: Optional[Union[datetime, str]],
-        end: Optional[Union[datetime, str]],
+        start: datetime | str | None,
+        end: datetime | str | None,
     ) -> "Filter":
         """
         Copies the filter and adds a condition to the copy, that requires

reconcile/utils/ocm/service_log.py

@@ -1,82 +1,32 @@
+from collections.abc import Generator
 from datetime import (
     datetime,
     timedelta,
 )
-from enum import Enum
-from typing import (
-    Generator,
-    Optional,
-)
-
-from pydantic import BaseModel
 
+from reconcile.utils.ocm.base import (
+    OCMClusterServiceLog,
+    OCMClusterServiceLogCreateModel,
+)
 from reconcile.utils.ocm.search_filters import Filter
 from reconcile.utils.ocm_base_client import OCMBaseClient
 
+CLUSTER_SERVICE_LOGS_LIST_ENDPOINT = "/api/service_logs/v1/clusters/cluster_logs"
+CLUSTER_SERVICE_LOGS_CREATE_ENDPOINT = "/api/service_logs/v1/cluster_logs"
 
-class OCMServiceLogSeverity(Enum):
-    """
-    Represents the severity of a service log.
-    """
-
-    Debug = "Debug"
-    Info = "Info"
-    Warning = "Warning"
-    Error = "Error"
-    Fatal = "Fatal"
-
-
-class OCMClusterServiceLogCreateModel(BaseModel):
-    cluster_uuid: str
-    """
-    The cluster UUID is the same as external ID on the OCM cluster_mgmt API
-    """
-
-    service_name: str
-    """
-    The name of the service a service log entry belongs to
-    """
-
-    summary: str
-    """
-    Short summary of the log entry.
-    """
 
-    description: str
-    """
-    Detailed description of the log entry.
-    """
-
-    severity: OCMServiceLogSeverity
-
-
-class OCMClusterServiceLog(OCMClusterServiceLogCreateModel):
-    """
-    Represents a service log entry for a cluster.
-    """
-
-    id: str
-    href: str
-    event_stream_id: str
-    username: str
-
-    cluster_id: str
-
-    timestamp: datetime
-    """
-    The time at which the log entry was created.
-    """
-
-
-def get_service_logs(
-    ocm_api: OCMBaseClient, filter: Filter
+def get_service_logs_for_cluster_uuid(
+    ocm_api: OCMBaseClient, cluster_uuid: str, filter: Filter | None = None
 ) -> Generator[OCMClusterServiceLog, None, None]:
     """
-    Returns a list of service logs matching the given filter.
+    Returns a list of service logs for a cluster, matching the optional filter.
     """
+    params = {"cluster_uuid": cluster_uuid}
+    if filter:
+        params["search"] = filter.render()
     for log_dict in ocm_api.get_paginated(
-        api_path="/api/service_logs/v1/cluster_logs",
-        params={"search": filter.render()},
+        api_path=CLUSTER_SERVICE_LOGS_LIST_ENDPOINT,
+        params=params,
         max_page_size=100,
     ):
         yield OCMClusterServiceLog(**log_dict)
@@ -85,25 +35,28 @@ def get_service_logs(
 def create_service_log(
     ocm_api: OCMBaseClient,
     service_log: OCMClusterServiceLogCreateModel,
-    dedup_interval: Optional[timedelta] = None,
+    dedup_interval: timedelta | None = None,
 ) -> OCMClusterServiceLog:
     if dedup_interval:
-        for previous_log in get_service_logs(
-            ocm_api=ocm_api,
-            filter=Filter()
-            .eq("cluster_uuid", service_log.cluster_uuid)
-            .eq("service_name", service_log.service_name)
-            .eq("severity", service_log.severity.value)
-            .eq("summary", service_log.summary)
-            .eq("description", service_log.description)
-            .after("created_at", datetime.utcnow() - dedup_interval),
-        ):
+        previous_log = next(
+            get_service_logs_for_cluster_uuid(
+                ocm_api=ocm_api,
+                cluster_uuid=service_log.cluster_uuid,
+                filter=Filter()
+                .eq("service_name", service_log.service_name)
+                .eq("severity", service_log.severity.value)
+                .eq("summary", service_log.summary)
+                .eq("description", service_log.description)
+                .after("created_at", datetime.utcnow() - dedup_interval),
+            ),
+            None,
+        )
+        if previous_log:
             return previous_log
 
-    # no similar service log record found... create a new one
     return OCMClusterServiceLog(
         **ocm_api.post(
-            api_path="/api/service_logs/v1/cluster_logs",
-            data=service_log.json(by_alias=True),
+            api_path=CLUSTER_SERVICE_LOGS_CREATE_ENDPOINT,
+            data=service_log.dict(by_alias=True),
         )
     )

reconcile/utils/ocm/sre_capability_labels.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+from pydantic import Field
+from pydantic.fields import ModelField
+
+from reconcile.utils.ocm.base import (
+    LabelContainer,
+    LabelSetTypeVar,
+)
+from reconcile.utils.ocm.labels import build_container_for_prefix
+
+
+def sre_capability_label_key(
+    sre_capability: str, config_atom: str | None = None
+) -> str:
+    """
+    Generates label keys compliant with the naming schema defined in
+    https://service.pages.redhat.com/dev-guidelines/docs/sre-capabilities/framework/ocm-labels/
+    """
+    if config_atom is None:
+        return f"sre-capabilities.{sre_capability}"
+    return f"sre-capabilities.{sre_capability}.{config_atom}"
+
+
+def labelset_groupfield(group_prefix: str) -> Any:
+    """
+    Helper function to build the FieldMeta for a labelset field that groups labels.
+    """
+    return Field(group_by_prefix=group_prefix)
+
+
+def build_labelset(
+    labels: LabelContainer, dataclass: type[LabelSetTypeVar]
+) -> LabelSetTypeVar:
+    """
+    Instantiates a dataclass from a set of labels.
+    """
+    raw_data = {
+        field.alias: _labelset_field_value(labels, field)
+        for field in dataclass.__fields__.values()
+    }
+    return dataclass(**raw_data)
+
+
+def _labelset_field_value(labels: LabelContainer, field: ModelField) -> Any | None:
+    key_prefix = field.field_info.extra.get("group_by_prefix")
+    if key_prefix:
+        return build_container_for_prefix(
+            labels, key_prefix, strip_key_prefix=True
+        ).get_values_dict()
+    return labels.get_label_value(field.alias)

reconcile/utils/ocm/status_board.py

@@ -0,0 +1,66 @@
+from typing import Any
+
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+APPLICATION_DESIRED_KEYS = {"id", "name", "fullname", "metadata"}
+PRODUCTS_DESIRED_KEYS = {"id", "name", "fullname", "metadata"}
+
+
+METADATA_MANAGED_BY_KEY = "managedBy"
+METADATA_MANAGED_BY_VALUE = "qontract-reconcile"
+
+
+def get_product_applications(
+    ocm_api: OCMBaseClient, product_id: str
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for application in ocm_api.get_paginated(
+        f"/api/status-board/v1/products/{product_id}/applications"
+    ):
+        if (
+            application.get("metadata", {}).get(METADATA_MANAGED_BY_KEY, "")
+            == METADATA_MANAGED_BY_VALUE
+        ):
+            results.append({
+                k: v for k, v in application.items() if k in APPLICATION_DESIRED_KEYS
+            })
+
+    return results
+
+
+def get_managed_products(ocm_api: OCMBaseClient) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for product in ocm_api.get_paginated("/api/status-board/v1/products/"):
+        if (
+            product.get("metadata", {}).get(METADATA_MANAGED_BY_KEY, "")
+            == METADATA_MANAGED_BY_VALUE
+        ):
+            results.append({
+                k: v for k, v in product.items() if k in PRODUCTS_DESIRED_KEYS
+            })
+    return results
+
+
+def create_product(ocm_api: OCMBaseClient, spec: dict[str, Any]) -> str:
+    if "metadata" not in spec or spec["metadata"] is None:
+        spec["metadata"] = {}
+    spec["metadata"][METADATA_MANAGED_BY_KEY] = METADATA_MANAGED_BY_VALUE
+    resp = ocm_api.post("/api/status-board/v1/products/", data=spec)
+    return resp["id"]
+
+
+def create_application(ocm_api: OCMBaseClient, spec: dict[str, Any]) -> str:
+    if "metadata" not in spec or spec["metadata"] is None:
+        spec["metadata"] = {}
+    spec["metadata"][METADATA_MANAGED_BY_KEY] = METADATA_MANAGED_BY_VALUE
+    resp = ocm_api.post("/api/status-board/v1/applications/", data=spec)
+    return resp["id"]
+
+
+def delete_product(ocm_api: OCMBaseClient, product_id: str) -> None:
+    ocm_api.delete(f"/api/status-board/v1/products/{product_id}")
+
+
+def delete_application(ocm_api: OCMBaseClient, application_id: str) -> None:
+    ocm_api.delete(f"/api/status-board/v1/applications/{application_id}")

reconcile/utils/ocm/subscriptions.py

@@ -1,60 +1,13 @@
-from datetime import datetime
-from enum import Enum
-from typing import Optional
+from pydantic import ValidationError
 
-from pydantic import BaseModel
-
-from reconcile.utils.ocm.labels import OCMSubscriptionLabel
+from reconcile.utils.ocm.base import (
+    OCMOrganization,
+    OCMSubscription,
+)
 from reconcile.utils.ocm.search_filters import Filter
 from reconcile.utils.ocm_base_client import OCMBaseClient
 
 
-class OCMCapability(BaseModel):
-    """
-    Represents a capability (feature/feature flag) of a subscription, e.g. becoming cluster admin
-    """
-
-    name: str
-    value: str
-
-
-class OCMSubscriptionStatus(Enum):
-
-    Active = "Active"
-    Deprovisioned = "Deprovisioned"
-    Stale = "Stale"
-    Archived = "Archived"
-    Reserved = "Reserved"
-    Disconnected = "Disconnected"
-
-
-class OCMSubscription(BaseModel):
-    """
-    Represents a subscription in OCM.
-    """
-
-    id: str
-    href: str
-    display_name: str
-    created_at: datetime
-    cluster_id: str
-
-    organization_id: str
-    managed: bool
-    """
-    A managed subscription is one that belongs to a cluster managed by OCM,
-    e.g. ROSA, OSD, etc.
-    """
-
-    status: OCMSubscriptionStatus
-
-    labels: Optional[list[OCMSubscriptionLabel]] = None
-    capabilities: Optional[list[OCMCapability]] = None
-    """
-    Capabilities are a list of features/features flags that are enabled for a subscription.
-    """
-
-
 def get_subscriptions(
     ocm_api: OCMBaseClient, filter: Filter
 ) -> dict[str, OCMSubscription]:
@@ -65,21 +18,58 @@ def get_subscriptions(
     subscriptions = {}
     chunk_size = 100
     for filter_chunk in filter.chunk_by("id", chunk_size, ignore_missing=True):
+        # Note, that pagination is currently broken.
+        # Each call will return a random order, meaning pages are not consistent.
+        # ALWAYS by default use "orderBy: id", as id has an index in the db.
         for subscription_dict in ocm_api.get_paginated(
             api_path="/api/accounts_mgmt/v1/subscriptions?fetchCapabilities=true&fetchLabels=true",
-            params={"search": filter_chunk.render()},
+            params={"search": filter_chunk.render(), "orderBy": "id"},
             max_page_size=chunk_size,
         ):
-            sub = OCMSubscription(
-                **subscription_dict,
-            )
-            subscriptions[sub.id] = sub
+            try:
+                sub = OCMSubscription(
+                    **subscription_dict,
+                )
+                subscriptions[sub.id] = sub
+            except ValidationError:
+                pass
+                # ignore subscriptions that fail to validate
+                # against the OCMSubscription, since the lack important fields
     return subscriptions
 
 
-def build_subscription_filter(state: str = "Active", managed: bool = True) -> Filter:
+def build_subscription_filter(
+    states: set[str] | None = None, managed: bool = True
+) -> Filter:
     """
     Helper function to create a subscription search filer for two very common
     fields: status and managed.
     """
-    return Filter().eq("status", state).eq("managed", str(managed).lower())
+    return Filter().is_in("status", states).eq("managed", str(managed).lower())
+
+
+def get_organizations(
+    ocm_api: OCMBaseClient, filter: Filter
+) -> dict[str, OCMOrganization]:
+    """
+    Returns a dictionary of organizations based on the provided filter.
+    The result is keyed by organization ID.
+    """
+    organizations = {}
+    chunk_size = 100
+    for filter_chunk in filter.chunk_by("id", chunk_size, ignore_missing=True):
+        for organization_dict in ocm_api.get_paginated(
+            api_path="/api/accounts_mgmt/v1/organizations?fetchCapabilities=true&fetchLabels=true",
+            params={"search": filter_chunk.render()},
+            max_page_size=chunk_size,
+        ):
+            try:
+                org = OCMOrganization(
+                    **organization_dict,
+                )
+                organizations[org.id] = org
+            except ValidationError:
+                pass
+                # ignore subscriptions that fail to validate
+                # against the OCMSubscription, since the lack important fields
+    return organizations

reconcile/utils/ocm/syncsets.py

@@ -0,0 +1,39 @@
+from collections.abc import Generator, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+
+@dataclass
+class SyncSet:
+    def __init__(self, cluster_id: str):
+        self.href = f"/api/clusters_mgmt/v1/clusters/{cluster_id}/external_configuration/syncsets"
+
+    href: str
+
+
+def get_syncsets(
+    ocm_client: OCMBaseClient, cluster_id: str
+) -> Generator[dict[str, Any], None, None]:
+    syncset = SyncSet(cluster_id)
+    return ocm_client.get_paginated(api_path=syncset.href)
+
+
+def get_syncset(ocm_client: OCMBaseClient, cluster_id: str, syncset_id: str) -> Any:
+    syncset = SyncSet(cluster_id)
+    return ocm_client.get(api_path=syncset.href + "/" + syncset_id)
+
+
+def create_syncset(
+    ocm_client: OCMBaseClient, cluster_id: str, syncset_map: Mapping
+) -> None:
+    syncset = SyncSet(cluster_id)
+    ocm_client.post(api_path=syncset.href, data=syncset_map)
+
+
+def patch_syncset(
+    ocm_client: OCMBaseClient, cluster_id: str, syncset_id: str, syncset_map: Mapping
+) -> None:
+    syncset = SyncSet(cluster_id)
+    ocm_client.patch(api_path=syncset.href + "/" + syncset_id, data=syncset_map)