qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/disabled_integrations.py

@@ -1,26 +1,24 @@
 from collections.abc import Mapping
 from typing import (
     Any,
-    Optional,
     Protocol,
-    Union,
     runtime_checkable,
 )
 
 
 class HasIntegrations(Protocol):
-    integrations: Optional[list[str]]
+    integrations: list[str] | None
 
 
 @runtime_checkable
 class HasDisableIntegrations(Protocol):
     @property
-    def disable(self) -> Optional[HasIntegrations]:
+    def disable(self) -> HasIntegrations | None:
         pass
 
 
 def disabled_integrations(
-    disable_obj: Optional[Union[Mapping[str, Any], HasDisableIntegrations]]
+    disable_obj: Mapping[str, Any] | HasDisableIntegrations | None,
 ) -> list[str]:
     """Returns all disabled integrations"""
     if not disable_obj:
@@ -38,7 +36,7 @@ def disabled_integrations(
 
 def integration_is_enabled(
     integration: str,
-    disable_obj: Optional[Union[Mapping[str, Any], HasDisableIntegrations]],
+    disable_obj: Mapping[str, Any] | HasDisableIntegrations | None,
 ) -> bool:
     """A convenient method to check whether an integration is enabled or not."""
     return integration not in disabled_integrations(disable_obj)

reconcile/utils/dynatrace/client.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from dynatrace import Dynatrace
+from dynatrace.environment_v2.tokens_api import ApiTokenUpdate
+from pydantic import BaseModel
+
+
+class DynatraceTokenCreationError(Exception):
+    pass
+
+
+class DynatraceTokenUpdateError(Exception):
+    pass
+
+
+class DynatraceTokenRetrievalError(Exception):
+    pass
+
+
+class DynatraceAPITokenCreated(BaseModel):
+    """
+    Wrap dynatrace.ApiTokenCreated for decoupling
+    """
+
+    token: str
+    id: str
+
+
+class DynatraceAPIToken(BaseModel):
+    id: str
+    scopes: list[str]
+
+
+class DynatraceClient:
+    def __init__(self, environment_url: str, api: Dynatrace) -> None:
+        self._environment_url = environment_url
+        self._api = api
+
+    def create_api_token(
+        self, name: str, scopes: Iterable[str]
+    ) -> DynatraceAPITokenCreated:
+        try:
+            token = self._api.tokens.create(name=name, scopes=scopes)
+        except Exception as e:
+            raise DynatraceTokenCreationError(
+                f"{self._environment_url=} Failed to create token for {name=}", e
+            ) from e
+        return DynatraceAPITokenCreated(token=token.token, id=token.id)
+
+    def get_token_ids_map_for_name_prefix(self, prefix: str) -> dict[str, str]:
+        try:
+            dt_tokens = self._api.tokens.list()
+        except Exception as e:
+            raise DynatraceTokenRetrievalError(
+                f"{self._environment_url=} Failed to retrieve tokens for {prefix=}", e
+            ) from e
+        return {
+            token.id: token.name for token in dt_tokens if token.name.startswith(prefix)
+        }
+
+    def get_token_by_id(self, token_id: str) -> DynatraceAPIToken:
+        try:
+            token = self._api.tokens.get(token_id=token_id)
+        except Exception as e:
+            raise DynatraceTokenRetrievalError(
+                f"{self._environment_url=} Failed to retrieve token for {token_id=}", e
+            ) from e
+        return DynatraceAPIToken(id=token.id, scopes=token.scopes)
+
+    def update_token(self, token_id: str, name: str, scopes: list[str]) -> None:
+        try:
+            self._api.tokens.put(
+                token_id=token_id,
+                api_token=ApiTokenUpdate(
+                    name=name,
+                    scopes=scopes,
+                ),
+            )
+        except Exception as e:
+            raise DynatraceTokenUpdateError(
+                f"{self._environment_url=} Failed to update token scopes for {token_id=}",
+                e,
+            ) from e
+
+    @staticmethod
+    def create(
+        environment_url: str, token: str | None, api: Dynatrace | None
+    ) -> DynatraceClient:
+        if not api:
+            api = Dynatrace(base_url=environment_url, token=token)
+        return DynatraceClient(environment_url=environment_url, api=api)

reconcile/utils/early_exit_cache.py

@@ -0,0 +1,289 @@
+from collections.abc import Mapping
+from datetime import UTC, datetime, timedelta
+from enum import Enum
+from functools import cached_property
+from typing import Any, Self
+
+from deepdiff import DeepHash
+from pydantic import BaseModel
+
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.state import State, init_state
+
+STATE_INTEGRATION = "early-exit-cache"
+EXPIRE_AT_METADATA_KEY = "expire-at"
+CACHE_SOURCE_DIGEST_METADATA_KEY = "cache-source-digest"
+LATEST_CACHE_SOURCE_DIGEST_METADATA_KEY = "latest-cache-source-digest"
+
+
+class CacheKeyWithDigest(BaseModel):
+    integration: str
+    integration_version: str
+    dry_run: bool
+    cache_source_digest: str
+    shard: str
+
+    def __str__(self) -> str:
+        return self.dry_run_path() if self.dry_run else self.no_dry_run_path()
+
+    def dry_run_path(self) -> str:
+        """
+        /<integration>/<integration_version>/dry-run(/<shard>)/<cache_source_digest>
+        """
+        return "/".join(
+            [
+                self.integration,
+                self.integration_version,
+                "dry-run",
+            ]
+            + ([self.shard] if self.shard else [])
+            + [
+                self.cache_source_digest,
+            ]
+        )
+
+    def no_dry_run_path(self) -> str:
+        """
+        /<integration>/<integration_version>/no-dry-run(/<shard>)/latest
+        """
+        return "/".join(
+            [
+                self.integration,
+                self.integration_version,
+                "no-dry-run",
+            ]
+            + ([self.shard] if self.shard else [])
+            + [
+                "latest",
+            ]
+        )
+
+    def build_cli_delete_args(self) -> str:
+        args = [
+            f"--integration {self.integration}",
+            f"--integration-version {self.integration_version}",
+            "--dry-run" if self.dry_run else "--no-dry-run",
+            f"--cache-source-digest {self.cache_source_digest}",
+        ]
+        if self.shard:
+            args.append(f"--shard {self.shard}")
+        return " ".join(args)
+
+    class Config:
+        frozen = True
+
+
+class CacheKey(BaseModel):
+    integration: str
+    integration_version: str
+    dry_run: bool
+    cache_source: object
+    shard: str
+
+    def __str__(self) -> str:
+        return str(self.cache_key_with_digest)
+
+    @cached_property
+    def cache_source_digest(self) -> str:
+        """
+        Calculate a consistent hash of the cache source, use @cached_property to avoid recalculating
+
+        :return: hash of the cache source
+        """
+        return DeepHash(self.cache_source)[self.cache_source]
+
+    @cached_property
+    def cache_key_with_digest(self) -> CacheKeyWithDigest:
+        return CacheKeyWithDigest(
+            integration=self.integration,
+            integration_version=self.integration_version,
+            dry_run=self.dry_run,
+            cache_source_digest=self.cache_source_digest,
+            shard=self.shard,
+        )
+
+    def dry_run_path(self) -> str:
+        """
+        /<integration>/<integration_version>/dry-run(/<shard>)/<cache_source_digest>
+        """
+        return self.cache_key_with_digest.dry_run_path()
+
+    def no_dry_run_path(self) -> str:
+        """
+        /<integration>/<integration_version>/no-dry-run(/<shard>)/latest
+        """
+        return self.cache_key_with_digest.no_dry_run_path()
+
+    def build_cli_delete_args(self) -> str:
+        """
+        Generate delete command arguments.
+        :return: delete command arguments
+        """
+        return self.cache_key_with_digest.build_cli_delete_args()
+
+    class Config:
+        frozen = True
+        keep_untouched = (cached_property,)
+
+
+class CacheValue(BaseModel):
+    payload: object
+    log_output: str
+    applied_count: int
+
+
+class CacheStatus(Enum):
+    MISS = "MISS"
+    HIT = "HIT"
+    EXPIRED = "EXPIRED"  # TTL expired
+    STALE = "STALE"  # latest cache source digest is different, which means there are new applied changes
+
+
+class CacheHeadResult(BaseModel):
+    status: CacheStatus
+    latest_cache_source_digest: str
+
+
+class EarlyExitCache:
+    def __init__(self, state: State):
+        self.state = state
+
+    @classmethod
+    def build(
+        cls,
+        secret_reader: SecretReaderBase | None = None,
+    ) -> Self:
+        state = init_state(STATE_INTEGRATION, secret_reader)
+        return cls(state)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        self.state.cleanup()
+
+    def get(self, key: CacheKey) -> CacheValue:
+        value = self.state.get(str(key))
+        return CacheValue.parse_obj(value)
+
+    def set(
+        self,
+        key: CacheKey,
+        value: CacheValue,
+        ttl_seconds: int,
+        latest_cache_source_digest: str,
+    ) -> None:
+        """
+        Set the cache value for the given key.
+
+        :param key: cache key
+        :param value: cache value
+        :param ttl_seconds: time to live in seconds
+        :param latest_cache_source_digest: latest cache source digest, used to check stale for dry run cache
+        :return: None
+        """
+        expire_at = datetime.now(tz=UTC) + timedelta(seconds=ttl_seconds)
+        metadata = {
+            EXPIRE_AT_METADATA_KEY: str(int(expire_at.timestamp())),
+            CACHE_SOURCE_DIGEST_METADATA_KEY: key.cache_source_digest,
+            LATEST_CACHE_SOURCE_DIGEST_METADATA_KEY: latest_cache_source_digest,
+        }
+        self.state.add(
+            str(key),
+            value.dict(),
+            metadata=metadata,
+            force=True,
+        )
+
+    def head(self, key: CacheKey) -> CacheHeadResult:
+        """
+        Get the cache status and latest cache source digest for the given key.
+        Additionally, for dry run key, it will use latest cache source digest to check stale.
+
+        :param key: CacheKey
+        :return: CacheHeadResult
+        """
+        return self._head_dry_run(key) if key.dry_run else self._head_no_dry_run(key)
+
+    def delete(self, key: CacheKeyWithDigest) -> None:
+        """
+        Delete the cache value for the given key.
+
+        :param key: CacheKey
+        :return: None
+        """
+        self.state.rm(str(key))
+
+    @staticmethod
+    def _is_stale(
+        metadata: Mapping[str, str],
+        latest_cache_source_digest: str,
+    ) -> bool:
+        cached_latest_cache_source_digest = (
+            metadata.get(LATEST_CACHE_SOURCE_DIGEST_METADATA_KEY) or ""
+        )
+        return cached_latest_cache_source_digest != latest_cache_source_digest
+
+    @staticmethod
+    def _is_expired(metadata: Mapping[str, str]) -> bool:
+        expire_at = datetime.fromtimestamp(
+            int(metadata[EXPIRE_AT_METADATA_KEY]),
+            tz=UTC,
+        )
+        now = datetime.now(UTC)
+        return now >= expire_at
+
+    def _head_dry_run_status(
+        self,
+        key: CacheKey,
+        latest_cache_source_digest: str,
+    ) -> CacheStatus:
+        exists, metadata = self.state.head(key.dry_run_path())
+        if not exists:
+            return CacheStatus.MISS
+        if self._is_expired(metadata):
+            return CacheStatus.EXPIRED
+        if self._is_stale(metadata, latest_cache_source_digest):
+            return CacheStatus.STALE
+        return CacheStatus.HIT
+
+    def _head_dry_run(self, key: CacheKey) -> CacheHeadResult:
+        _, latest_metadata = self.state.head(key.no_dry_run_path())
+        latest_cache_source_digest = (
+            latest_metadata.get(CACHE_SOURCE_DIGEST_METADATA_KEY) or ""
+        )
+        return CacheHeadResult(
+            status=self._head_dry_run_status(key, latest_cache_source_digest),
+            latest_cache_source_digest=latest_cache_source_digest,
+        )
+
+    def _head_no_dry_run_status(
+        self,
+        key: CacheKey,
+        exists: bool,
+        metadata: Mapping[str, str],
+        latest_cache_source_digest: str,
+    ) -> CacheStatus:
+        if not exists or key.cache_source_digest != latest_cache_source_digest:
+            return CacheStatus.MISS
+        if self._is_expired(metadata):
+            return CacheStatus.EXPIRED
+        return CacheStatus.HIT
+
+    def _head_no_dry_run(self, key: CacheKey) -> CacheHeadResult:
+        exists, metadata = self.state.head(key.no_dry_run_path())
+        latest_cache_source_digest = (
+            metadata.get(CACHE_SOURCE_DIGEST_METADATA_KEY) or ""
+        )
+        return CacheHeadResult(
+            status=self._head_no_dry_run_status(
+                key,
+                exists,
+                metadata,
+                latest_cache_source_digest,
+            ),
+            latest_cache_source_digest=latest_cache_source_digest,
+        )

reconcile/utils/elasticsearch_exceptions.py

@@ -11,3 +11,8 @@ class ElasticSearchResourceMissingSubnetIdError(Exception):
 class ElasticSearchResourceZoneAwareSubnetInvalidError(Exception):
     def __init__(self, msg):
         super().__init__(str(msg))
+
+
+class ElasticSearchResourceColdStorageError(Exception):
+    def __init__(self, msg):
+        super().__init__(str(msg))

reconcile/utils/environ.py

@@ -11,8 +11,8 @@ def environ(variables=None):
         @wraps(f)
         def f_environ(*args, **kwargs):
             for e in variables:
-                if e not in os.environ or os.environ[e] == "":
-                    raise KeyError("Could not find environment variable: {}".format(e))
+                if not os.environ.get(e):
+                    raise KeyError(f"Could not find environment variable: {e}")
             f(*args, **kwargs)
 
         return f_environ

reconcile/utils/exceptions.py

@@ -16,6 +16,10 @@ class AppInterfaceSmtpSettingsError(AppInterfaceSettingsError):
     pass
 
 
+class AppInterfaceLdapGroupsSettingsError(AppInterfaceSettingsError):
+    pass
+
+
 class SecretIncompleteError(Exception):
     pass
 

reconcile/utils/expiration.py

@@ -1,10 +1,8 @@
 import datetime
 from collections.abc import Iterable
 from typing import (
-    Optional,
     Protocol,
     TypeVar,
-    Union,
     cast,
 )
 
@@ -12,12 +10,10 @@ DATE_FORMAT = "%Y-%m-%d"
 
 
 class FilterableRole(Protocol):
-    expiration_date: Optional[str]
+    expiration_date: str | None
 
 
-DictsOrRoles = TypeVar(
-    "DictsOrRoles", bound=Union[Iterable[FilterableRole], Iterable[dict]]
-)
+DictsOrRoles = TypeVar("DictsOrRoles", bound=Iterable[FilterableRole] | Iterable[dict])
 
 
 def date_expired(date: str) -> bool:
@@ -26,7 +22,7 @@ def date_expired(date: str) -> bool:
     return current_date >= exp_date
 
 
-def filter(roles: Optional[DictsOrRoles]) -> DictsOrRoles:
+def filter(roles: DictsOrRoles | None) -> DictsOrRoles:
     """Filters roles and returns the ones which are not yet expired."""
     filtered = []
     for r in roles or []:
@@ -43,6 +39,6 @@ def filter(roles: Optional[DictsOrRoles]) -> DictsOrRoles:
         except ValueError:
             raise ValueError(
                 f"{key} field is not formatted as YYYY-MM-DD, currently set as {expiration_date}"
-            )
+            ) from None
 
     return cast(DictsOrRoles, filtered)