qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/three_way_diff_strategy.py

@@ -0,0 +1,157 @@
+import base64
+import logging
+import re
+from collections.abc import Mapping
+from typing import Any
+
+import jsonpatch  # type: ignore
+from jsonpointer import resolve_pointer  # type: ignore
+
+from reconcile.utils.openshift_resource import QONTRACT_ANNOTATIONS
+from reconcile.utils.openshift_resource import OpenshiftResource as OR
+
+NORMALIZE_COMPARE_EXCLUDED_ATTRS = {
+    "creationTimestamp",
+    "resourceVersion",
+    "generation",
+    "selfLink",
+    "uid",
+    "fieldRef",
+    "managedFields",
+    "namespace",
+}
+K8S_ANNOTATION_LAC = "kubectl.kubernetes.io/last-applied-configuration"
+NORMALIZE_IGNORE_ANNOTATIONS = QONTRACT_ANNOTATIONS | {K8S_ANNOTATION_LAC}
+
+
+def _normalize_secret(secret: OR) -> None:
+    body = secret.body
+    string_data = body.get("stringData")
+
+    if string_data:
+        data = body.get("data") or {}
+        body["data"] = data | {
+            k: base64.b64encode(str(v).encode()).decode("utf-8")
+            for k, v in string_data.items()
+        }
+        secret.body = {k: v for k, v in body.items() if k != "stringData"}
+
+
+def _normalize_noop(item: OR) -> None:
+    pass
+
+
+NORMALIZERS = {"Secret": _normalize_secret}
+
+
+def normalize_object(item: OR) -> OR:
+    # Remove K8s managed attributes not needed to compare objects
+    metadata = {
+        k: v
+        for k, v in item.body["metadata"].items()
+        if k not in NORMALIZE_COMPARE_EXCLUDED_ATTRS
+    }
+
+    n = OR(
+        body=item.body | {"metadata": metadata},
+        integration=item.integration,
+        integration_version=item.integration_version,
+        error_details=item.error_details,
+        caller_name=item.caller_name,
+        validate_k8s_object=False,
+    )
+
+    annotations = n.body.get("metadata").get("annotations") or {}
+    metadata["annotations"] = {
+        k: v for k, v in annotations.items() if k not in NORMALIZE_IGNORE_ANNOTATIONS
+    }
+
+    # Run normalizers on Kinds with special needs
+    NORMALIZERS.get(n.body["kind"], _normalize_noop)(n)
+    return n
+
+
+CPU_REGEX = re.compile(r"/.*/(requests|limits)/cpu$")
+EMPTY_ENV_VALUE = re.compile(r"/.*/env/[0-9]+/value$")
+
+
+def is_cpu_mutation(current: OR, desired: OR, patch: Mapping[str, Any]) -> bool:
+    pointer = patch["path"]
+    if re.match(CPU_REGEX, pointer):
+        current_value = resolve_pointer(current.body, pointer)
+        desired_value = patch["value"]
+        return OR.cpu_equal(current_value, desired_value)
+
+    return False
+
+
+def is_empty_env_value(current: OR, desired: OR, patch: Mapping[str, Any]) -> bool:
+    """Check if the patch is an empty env value. Empty values are removed in the
+    current object
+
+    :param current: Current object
+    :param desired: Desired object
+    :param patch: JSON patch
+    :return: True if the change is not needed, False otherwise
+    """
+    pointer = patch["path"]
+    return bool(
+        patch["op"] == "add"
+        and not patch["value"]
+        and re.match(EMPTY_ENV_VALUE, pointer)
+    )
+
+
+def is_valid_change(current: OR, desired: OR, patch: Mapping[str, Any]) -> bool:
+    # Only consider added or replaced values on the Desired object
+    if patch["op"] not in {"add", "replace"}:
+        return False
+
+    # Check known mutations. Replaced values can happen if values have been
+    # mutated by the API server
+    if is_cpu_mutation(current, desired, patch):
+        return False
+
+    # Other cases
+    return not is_empty_env_value(current, desired, patch)
+
+
+def three_way_diff_using_hash(c_item: OR, d_item: OR) -> bool:
+    c_item_sha256 = ""
+    try:
+        annotations = c_item.body["metadata"]["annotations"]
+        c_item_sha256 = annotations["qontract.sha256sum"]
+    except KeyError:
+        logging.debug("Current object QR hash is missing -> Apply")
+        return False
+
+    if (
+        c_item_integration := annotations["qontract.integration"]
+    ) != d_item.integration:
+        logging.info(
+            f"resource switching integration from {c_item_integration} to {d_item.integration}"
+        )
+        return False
+
+    # Original object does not match Desired -> Apply
+    # Current object hash is not recalculated!
+    if c_item_sha256 != d_item.sha256sum():
+        logging.debug("Original and Desired objects hash differs -> Apply")
+        return False
+
+    # The patch only detects changes with attributes defined in the desired state.
+    # Values in the current state added by operators or other actors are not taken
+    # into account
+
+    current = normalize_object(c_item)
+    desired = normalize_object(d_item)
+
+    patch = jsonpatch.JsonPatch.from_diff(current.body, desired.body)
+    valid_changes = [
+        item for item in patch.patch if is_valid_change(current, desired, item)
+    ]
+    if len(valid_changes) > 0:
+        logging.debug("Desired and Current objects differ -> Apply: %s", valid_changes)
+        return False
+
+    return True

reconcile/utils/unleash/__init__.py

@@ -0,0 +1,11 @@
+from .client import (
+    get_feature_toggle_default,
+    get_feature_toggle_state,
+    get_feature_toggles,
+)
+
+__all__ = [
+    "get_feature_toggle_default",
+    "get_feature_toggle_state",
+    "get_feature_toggles",
+]

reconcile/utils/{unleash.py → unleash/client.py}

@@ -2,10 +2,7 @@ import logging
 import os
 import threading
 from collections.abc import Mapping
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from UnleashClient import (
     BaseCache,
@@ -13,84 +10,93 @@ from UnleashClient import (
 )
 from UnleashClient.strategies import Strategy
 
-client: Optional[UnleashClient] = None
+client: UnleashClient | None = None
 client_lock = threading.Lock()
 
 
 class CacheDict(BaseCache):
-    def __init__(self):
-        self.cache = {}
+    def __init__(self) -> None:
+        self.cache: dict = {}
 
-    def set(self, key: str, value: Any):
+    def set(self, key: str, value: Any) -> None:
         self.cache[key] = value
 
-    def mset(self, data: dict):
+    def mset(self, data: dict) -> None:
         self.cache.update(data)
 
-    def get(self, key: str, default: Optional[Any] = None):
+    def get(self, key: str, default: Any | None = None) -> Any:
         return self.cache.get(key, default)
 
-    def exists(self, key: str):
+    def exists(self, key: str) -> bool:
         return key in self.cache
 
-    def destroy(self):
+    def destroy(self) -> None:
         self.cache = {}
 
 
-class DisableClusterStrategy(Strategy):
+class ClusterStrategy(Strategy):
     def load_provisioning(self) -> list:
         return [x.strip() for x in self.parameters["cluster_name"].split(",")]
 
-    def apply(self, context: Optional[dict] = None) -> bool:
+
+class DisableClusterStrategy(ClusterStrategy):
+    def apply(self, context: dict | None = None) -> bool:
         enable = True
 
-        if context and "cluster_name" in context.keys():
+        if context and "cluster_name" in context:
             # if cluster in context is in clusters sent from server, disable
             enable = context["cluster_name"] not in self.parsed_provisioning
 
         return enable
 
 
+class EnableClusterStrategy(ClusterStrategy):
+    def apply(self, context: dict | None = None) -> bool:
+        enable = False
+
+        if context and "cluster_name" in context:
+            # if cluster in context is in clusters sent from server, enable
+            enable = context["cluster_name"] in self.parsed_provisioning
+
+        return enable
+
+
 def _get_unleash_api_client(api_url: str, auth_head: str) -> UnleashClient:
-    global client
+    global client  # noqa: PLW0603
     with client_lock:
         if client is None:
             logging.getLogger("apscheduler").setLevel(logging.ERROR)
             logging.getLogger("UnleashClient").setLevel(logging.ERROR)
-            headers = {"Authorization": f"Bearer {auth_head}"}
+            headers = {"Authorization": auth_head}
             client = UnleashClient(
                 url=api_url,
                 app_name="qontract-reconcile",
                 custom_headers=headers,
                 cache=CacheDict(),
-                custom_strategies={"disableCluster": DisableClusterStrategy},
+                custom_strategies={
+                    "enableCluster": EnableClusterStrategy,
+                    "disableCluster": DisableClusterStrategy,
+                },
             )
             client.initialize_client()
     return client
 
 
-def _shutdown_client():
-    # Intended for test usage only
-    with client_lock:
-        if client:
-            client.destroy()
-
-
-def get_feature_toggle_default(feature_name, context):
+def get_feature_toggle_default(feature_name: str, context: dict) -> bool:
     return True
 
 
-def get_feature_toggle_default_false(feature_name, context):
+def get_feature_toggle_default_false(feature_name: str, context: dict) -> bool:
     return False
 
 
 def get_feature_toggle_state(
-    integration_name: str, context: Optional[dict] = None, default: bool = True
+    integration_name: str, context: dict | None = None, default: bool = True
 ) -> bool:
     api_url = os.environ.get("UNLEASH_API_URL")
     client_access_token = os.environ.get("UNLEASH_CLIENT_ACCESS_TOKEN")
     if not (api_url and client_access_token):
-        return get_feature_toggle_default(None, None)
+        return get_feature_toggle_default("", {})
 
     c = _get_unleash_api_client(
         api_url,

reconcile/utils/unleash/server.py

@@ -0,0 +1,145 @@
+from enum import StrEnum
+
+import requests
+from pydantic import BaseModel, Field
+
+from reconcile.utils.rest_api_base import ApiBase, BearerTokenAuth
+
+
+class FeatureToggleType(StrEnum):
+    experiment = "experiment"
+    kill_switch = "kill-switch"
+    release = "release"
+    operational = "operational"
+    permission = "permission"
+
+
+class Environment(BaseModel):
+    name: str
+    enabled: bool
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Environment):
+            return self.name == other.name
+        return self.name == other
+
+
+class FeatureToggle(BaseModel):
+    name: str
+    type: FeatureToggleType = FeatureToggleType.release
+    description: str | None = None
+    impression_data: bool = Field(False, alias="impressionData")
+    environments: list[Environment]
+
+    class Config:
+        allow_population_by_field_name = True
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, FeatureToggle):
+            return self.name == other.name
+        return self.name == other
+
+
+class Project(BaseModel):
+    pk: str = Field(alias="id")
+    name: str
+    feature_toggles: list[FeatureToggle] = []
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class TokenAuth(BearerTokenAuth):
+    def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
+        r.headers["Authorization"] = self.token
+        return r
+
+
+class UnleashServer(ApiBase):
+    """Unleash server API client."""
+
+    def projects(self, include_feature_toggles: bool = False) -> list[Project]:
+        """List all projects."""
+        projects = []
+        for project_data in self._list("/api/admin/projects", attribute="projects"):
+            project = Project(**project_data)
+            if include_feature_toggles:
+                project.feature_toggles = self.feature_toggles(project.pk)
+            projects.append(project)
+        return projects
+
+    def feature_toggles(self, project_id: str) -> list[FeatureToggle]:
+        """List all feature toggles for a project."""
+        return [
+            FeatureToggle(**i)
+            for i in self._list(
+                f"/api/admin/projects/{project_id}/features", attribute="features"
+            )
+        ]
+
+    def environments(self, project_id: str) -> list[Environment]:
+        """Gets the environments that are available for this project. An environment is available for a project if enabled in the project configuration."""
+        return [
+            Environment(**i)
+            for i in self._list(
+                f"/api/admin/environments/project/{project_id}",
+                attribute="environments",
+            )
+        ]
+
+    def create_feature_toggle(
+        self,
+        project_id: str,
+        name: str,
+        description: str,
+        type: FeatureToggleType,
+        impression_data: bool,
+    ) -> None:
+        """Create a feature toggle."""
+        self._post(
+            f"/api/admin/projects/{project_id}/features",
+            data={
+                "name": name,
+                "description": description,
+                "type": type.value,
+                "impressionData": impression_data,
+            },
+        )
+
+    def update_feature_toggle(
+        self,
+        project_id: str,
+        name: str,
+        description: str,
+        type: FeatureToggleType,
+        impression_data: bool,
+    ) -> None:
+        """Create a feature toggle."""
+        self._put(
+            f"/api/admin/projects/{project_id}/features/{name}",
+            data={
+                "description": description,
+                "type": type.value,
+                "impressionData": impression_data,
+            },
+        )
+
+    def delete_feature_toggle(self, project_id: str, name: str) -> None:
+        """Delete a feature toggle."""
+        # First archive the feature toggle
+        self._delete(f"/api/admin/projects/{project_id}/features/{name}")
+        # Then delete it
+        self._post(
+            f"/api/admin/projects/{project_id}/delete",
+            data={"features": [name]},
+        )
+
+    def set_feature_toggle_state(
+        self, project_id: str, name: str, environment: str, enabled: bool
+    ) -> None:
+        """Set the state of a feature toggle."""
+        base_url = f"/api/admin/projects/{project_id}/features/{name}/environments/{environment}"
+        if enabled:
+            self._post(f"{base_url}/on")
+        else:
+            self._post(f"{base_url}/off")

reconcile/utils/vault.py

@@ -5,7 +5,6 @@ import threading
 import time
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import Optional
 
 import hvac
 import requests
@@ -60,11 +59,11 @@ class _VaultClient:
 
     def __init__(
         self,
-        server: Optional[str] = None,
-        role_id: Optional[str] = None,
-        secret_id: Optional[str] = None,
-        kube_auth_role: Optional[str] = None,
-        kube_auth_mount: Optional[str] = None,
+        server: str | None = None,
+        role_id: str | None = None,
+        secret_id: str | None = None,
+        kube_auth_role: str | None = None,
+        kube_auth_mount: str | None = None,
         auto_refresh: bool = True,
     ):
         config = get_config()
@@ -96,12 +95,9 @@ class _VaultClient:
         self._get_mount_version = lru_cache(maxsize=128)(self.__get_mount_version)
         self._read_all_v2 = lru_cache(maxsize=2048)(self.__read_all_v2)
 
-        # This is a threaded world. Let's define a big
-        # connections pool to live in that world
-        # (this avoids the warning "Connection pool is
-        # full, discarding connection: vault.devshift.net")
         session = requests.Session()
-        adapter = HTTPAdapter(pool_connections=100, pool_maxsize=100)
+        # There are at most 10 working threads in reconcile, plus 1 daemon thread for auto refresh
+        adapter = HTTPAdapter(pool_maxsize=11)
         session.mount("https://", adapter)
         self._client = hvac.Client(url=server, session=session)
         self._close_lock = threading.Lock()
@@ -113,7 +109,10 @@ class _VaultClient:
                 self._refresh_client_auth()
                 authenticated = self._client.is_authenticated()
                 break
-            except requests.exceptions.ConnectionError:
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.TooManyRedirects,
+            ):
                 time.sleep(1)
 
         if not authenticated:
@@ -154,7 +153,7 @@ class _VaultClient:
     def _refresh_client_auth(self):
         if self.kube_auth_enabled:
             # must read each time to account for sa token refresh
-            with open(self.kube_sa_token_path) as f:
+            with open(self.kube_sa_token_path, encoding="locale") as f:
                 try:
                     self._client.auth_kubernetes(
                         role=self.kube_auth_role,
@@ -167,7 +166,7 @@ class _VaultClient:
             self._client.auth_approle(self.role_id, self.secret_id)
 
     @retry()
-    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, Optional[str]]:
+    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, str | None]:
         """Returns a dictionary of keys and values in a Vault secret and the
         version of the secret, for V1 secrets, version will be None.
 
@@ -194,7 +193,6 @@ class _VaultClient:
 
         return data, version
 
-    @retry()
     def read_all(self, secret: Mapping) -> dict:
         """Returns a dictionary of keys and values in a Vault secret.
 
@@ -219,9 +217,7 @@ class _VaultClient:
 
         return version
 
-    def __read_all_v2(
-        self, path: str, version: Optional[str]
-    ) -> tuple[dict, Optional[str]]:
+    def __read_all_v2(self, path: str, version: str | None) -> tuple[dict, str | None]:
         path_split = path.split("/")
         mount_point = path_split[0]
         read_path = "/".join(path_split[1:])
@@ -241,10 +237,10 @@ class _VaultClient:
             )
         except InvalidPath:
             msg = f"version '{version}' not found " f"for secret with path '{path}'."
-            raise SecretVersionNotFound(msg)
+            raise SecretVersionNotFound(msg) from None
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing secret '{path}'"
-            raise SecretAccessForbidden(msg)
+            raise SecretAccessForbidden(msg) from None
         if secret is None or "data" not in secret or "data" not in secret["data"]:
             raise SecretNotFound(path)
 
@@ -257,7 +253,7 @@ class _VaultClient:
             secret = self._client.read(path)
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing secret '{path}'"
-            raise SecretAccessForbidden(msg)
+            raise SecretAccessForbidden(msg) from None
 
         if secret is None or "data" not in secret:
             raise SecretNotFound(path)
@@ -302,7 +298,7 @@ class _VaultClient:
         try:
             secret_field = data[field]
         except KeyError:
-            raise SecretFieldNotFound(f"{path}/{field} ({version})")
+            raise SecretFieldNotFound(f"{path}/{field} ({version})") from None
         return secret_field
 
     def _read_v1(self, path, field):
@@ -310,7 +306,7 @@ class _VaultClient:
         try:
             secret_field = data[field]
         except KeyError:
-            raise SecretFieldNotFound("{}/{}".format(path, field))
+            raise SecretFieldNotFound(f"{path}/{field}") from None
         return secret_field
 
     @retry()
@@ -362,14 +358,14 @@ class _VaultClient:
             self._read_all_v2.cache_clear()
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing secret '{path}'"
-            raise SecretAccessForbidden(msg)
+            raise SecretAccessForbidden(msg) from None
 
     def _write_v1(self, path, data):
         try:
             self._client.write(path, **data)
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing secret '{path}'"
-            raise SecretAccessForbidden(msg)
+            raise SecretAccessForbidden(msg) from None
 
     def _list_kv2(self, path: str) -> dict:
         try:
@@ -380,23 +376,23 @@ class _VaultClient:
             return response
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing path '{path}'"
-            raise PathAccessForbidden(msg)
+            raise PathAccessForbidden(msg) from None
 
     def _list(self, path: str) -> dict:
         try:
             return self._client.list(path)
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing path '{path}'"
-            raise PathAccessForbidden(msg)
+            raise PathAccessForbidden(msg) from None
 
     def list(self, path: str) -> list[str]:
         """Returns a list of secrets in a given path."""
         kv_version = self._get_mount_version_by_secret_path(path)
-        if kv_version == 2:
-            path_list = self._list_kv2(path)
-        else:
-            path_list = self._list(path)
+        path_list = self._list_kv2(path) if kv_version == 2 else self._list(path)
 
+        if not path_list:
+            # path list can be None if the path does not exist
+            return []
         return path_list["data"]["keys"] or []
 
     def list_all(self, path):
@@ -411,9 +407,23 @@ class _VaultClient:
                 secrets.append(secret_path)
         return secrets
 
+    @retry()
+    def delete(self, path: str) -> None:
+        """Deletes a V1 secret from Vault."""
+        kv_version = self._get_mount_version_by_secret_path(path)
+        if kv_version == 2:
+            raise ValueError("deleting V2 secrets is not supported yet")
+        self._delete_v1(path)
+
+    def _delete_v1(self, path):
+        try:
+            self._client.delete(path)
+        except hvac.exceptions.Forbidden:
+            msg = f"permission denied accessing secret '{path}'"
+            raise SecretAccessForbidden(msg) from None
 
-class VaultClient:
 
+class VaultClient:
     _instance_lock = threading.Lock()
     _instance = None
 

reconcile/utils/vaultsecretref.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass
 from typing import (
-    Optional,
     cast,
 )
 
@@ -12,13 +11,12 @@ from reconcile.utils.vault import (
 
 @dataclass
 class VaultSecretRef:
-
     _ALL_FIELDS = "all"
 
     path: str
     field: str
-    format: Optional[str] = None
-    version: Optional[int] = None
+    format: str | None = None
+    version: int | None = None
 
     def get(self, field=None, default=None):
         secret_content = self._resolve_secret()