qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/extended_early_exit.py

@@ -0,0 +1,210 @@
+import logging
+from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from io import StringIO
+from logging import Logger
+
+from pydantic import BaseModel
+
+from reconcile.utils.early_exit_cache import (
+    CacheKey,
+    CacheStatus,
+    CacheValue,
+    EarlyExitCache,
+)
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+    inc_counter,
+    normalize_integration_name,
+    set_gauge,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class ExtendedEarlyExitRunnerResult(BaseModel):
+    payload: object
+    applied_count: int
+
+
+class ExtendedEarlyExitBaseMetric(BaseModel):
+    integration: str
+    integration_version: str
+    dry_run: bool
+    cache_status: str
+    shard: str
+
+
+class ExtendedEarlyExitCounter(ExtendedEarlyExitBaseMetric, CounterMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit"
+
+
+class ExtendedEarlyExitAppliedCountGauge(ExtendedEarlyExitBaseMetric, GaugeMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit_applied_count"
+
+
+def _publish_metrics(
+    cache_key: CacheKey,
+    cache_status: CacheStatus,
+    applied_count: int,
+) -> None:
+    inc_counter(
+        ExtendedEarlyExitCounter(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+            shard=cache_key.shard,
+        ),
+    )
+    set_gauge(
+        ExtendedEarlyExitAppliedCountGauge(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+            shard=cache_key.shard,
+        ),
+        applied_count,
+    )
+
+
+def _ttl_seconds(
+    applied_count: int,
+    ttl_seconds: int,
+) -> int:
+    """
+    Pick the ttl based on the applied count.
+    If the applied count is greater than 0, then we want to set ttl to 0 so that the next run will not hit the cache,
+    this will allow us to easy debug reconcile loops, as we will be able to see the logs of the next run,
+    and check cached value for more details.
+
+    :param applied_count: The number of resources that were applied
+    :param ttl_seconds: A ttl in seconds
+    :return: The ttl in seconds
+    """
+    return 0 if applied_count > 0 else ttl_seconds
+
+
+@contextmanager
+def log_stream_handler(
+    logger: Logger,
+) -> Generator[StringIO, None, None]:
+    """
+    Add a stream handler to the logger, and return the stream generator, automatically remove the handler when done.
+
+    :param logger: A logger
+    :return: A stream generator
+    """
+    with StringIO() as log_stream:
+        log_handler = logging.StreamHandler(log_stream)
+        logger.addHandler(log_handler)
+        try:
+            yield log_stream
+        finally:
+            logger.removeHandler(log_handler)
+
+
+def _log_cached_log_output(
+    cache: EarlyExitCache,
+    key: CacheKey,
+    logger: Logger,
+) -> None:
+    log_output = cache.get(key).log_output
+    logger.info(
+        "logging cached log output, to delete cache, use cli command "
+        "`qontract-cli --config config.toml early-exit-cache delete $ARGS` or "
+        "comment on merge request `/cache delete $ARGS`, "
+        "replace `$ARGS` with `%s`",
+        key.build_cli_delete_args(),
+    )
+    for line in log_output.splitlines():
+        logger.info(line)
+
+
+def extended_early_exit_run(
+    integration: str,
+    integration_version: str,
+    dry_run: bool,
+    cache_source: object,
+    shard: str,
+    ttl_seconds: int,
+    logger: Logger,
+    runner: Callable[..., ExtendedEarlyExitRunnerResult],
+    runner_params: Mapping | None = None,
+    secret_reader: SecretReaderBase | None = None,
+    log_cached_log_output: bool = False,
+) -> None:
+    """
+    Run the runner based on the cache status. Early exit when cache hit.
+    Runner log output will be extracted and stored in cache value,
+    and will be logged when hit if log_cached_log_output is True,
+    this is mainly used to show all log output from different integrations in one place (CI).
+    When runner returns no applies (applied_count is 0), the ttl will be set to ttl_seconds,
+    otherwise it will be set to 0.
+
+    :param integration: The integration name
+    :param integration_version: The integration version
+    :param dry_run: True if the run is in dry run mode, False otherwise
+    :param cache_source: The cache source, usually the static desired state
+    :param shard: The shard name
+    :param ttl_seconds: A ttl in seconds
+    :param logger: A Logger
+    :param runner: A runner can return ExtendedEarlyExitRunnerResult when called
+    :param runner_params: Runner params, will be spread into kwargs when calling runner
+    :param secret_reader: A secret reader
+    :param log_cached_log_output: Whether to log the cached log output when there is a cache hit
+    :return: None
+    """
+    with EarlyExitCache.build(secret_reader) as cache:
+        key = CacheKey(
+            integration=normalize_integration_name(integration),
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=cache_source,
+            shard=shard,
+        )
+        cache_result = cache.head(key)
+        logger.debug(
+            "Early exit cache head result for key=%s: status: %s, latest cache source digest: %s",
+            key,
+            cache_result.status,
+            cache_result.latest_cache_source_digest,
+        )
+
+        if cache_result.status == CacheStatus.HIT:
+            if log_cached_log_output:
+                _log_cached_log_output(cache, key, logger)
+            _publish_metrics(
+                cache_key=key,
+                cache_status=cache_result.status,
+                applied_count=0,
+            )
+            return
+
+        with log_stream_handler(logger) as log_stream:
+            result = runner(**(runner_params or {}))
+            log_output = log_stream.getvalue()
+
+        value = CacheValue(
+            payload=result.payload,
+            log_output=log_output,
+            applied_count=result.applied_count,
+        )
+        ttl = _ttl_seconds(result.applied_count, ttl_seconds)
+        logger.debug(
+            "Set early exit cache for key=%s with ttl=%d and latest_cache_source_digest=%s",
+            key,
+            ttl,
+            cache_result.latest_cache_source_digest,
+        )
+        cache.set(key, value, ttl, cache_result.latest_cache_source_digest)
+        _publish_metrics(
+            cache_key=key,
+            cache_status=cache_result.status,
+            applied_count=result.applied_count,
+        )

reconcile/utils/external_resource_spec.py

@@ -7,14 +7,15 @@ from collections.abc import (
 from dataclasses import field
 from typing import (
     Any,
-    Optional,
     cast,
 )
 
 import yaml
+from pydantic import BaseModel
 from pydantic.dataclasses import dataclass
 
-from reconcile import openshift_resources_base
+from reconcile.utils.jinja2.utils import process_jinja2_template
+from reconcile.utils.metrics import GaugeMetric
 from reconcile.utils.openshift_resource import (
     SECRET_MAX_KEY_LENGTH,
     OpenshiftResource,
@@ -50,16 +51,13 @@ class OutputFormatProcessor:
 
 @dataclass
 class GenericSecretOutputFormatConfig(OutputFormatProcessor):
-
-    data: Optional[str] = None
+    data: str | None = None
 
     def render(self, vars: Mapping[str, str]) -> dict[str, str]:
         if self.data:
             # the jinja2 rendering has the capabilitiy to change the passed
             # vars dict - make a copy to protect against it
-            rendered_data = openshift_resources_base.process_jinja2_template(
-                self.data, dict(vars)
-            )
+            rendered_data = process_jinja2_template(self.data, dict(vars))
             parsed_data = yaml.safe_load(rendered_data)
             self.validate_k8s_secret_data(parsed_data)
             return cast(dict[str, str], parsed_data)
@@ -68,9 +66,8 @@ class GenericSecretOutputFormatConfig(OutputFormatProcessor):
 
 @dataclass
 class OutputFormat:
-
     provider: str
-    data: Optional[str] = None
+    data: str | None = None
 
     @property
     def _formatter(self) -> OutputFormatProcessor:
@@ -84,12 +81,20 @@ class OutputFormat:
 
 @dataclass
 class ExternalResourceSpec:
-
     provision_provider: str
     provisioner: Mapping[str, Any]
     resource: MutableMapping[str, Any]
     namespace: Mapping[str, Any]
     secret: Mapping[str, str] = field(init=False, default_factory=lambda: {})
+    # Metadata is used for processing data that shuold not be included in the secret data
+    # e.g: ERV2 adds a updated_at attribute that acts as optimistic lock.
+    metadata: MutableMapping[str, Any] = field(
+        init=False, compare=False, repr=False, hash=False, default_factory=lambda: {}
+    )
+
+    @property
+    def marked_to_delete(self) -> bool:
+        return self.metadata.get("delete") or False
 
     @property
     def provider(self) -> str:
@@ -142,7 +147,7 @@ class ExternalResourceSpec:
             "app": self.namespace["app"]["name"],
         }
 
-    def get_secret_field(self, field: str) -> Optional[str]:
+    def get_secret_field(self, field: str) -> str | None:
         return self.secret.get(field)
 
     def id_object(self) -> "ExternalResourceUniqueKey":
@@ -172,7 +177,6 @@ class ExternalResourceSpec:
 
 @dataclass(frozen=True)
 class ExternalResourceUniqueKey:
-
     provision_provider: str
     provisioner_name: str
     identifier: str
@@ -192,6 +196,24 @@ class ExternalResourceUniqueKey:
         )
 
 
+class ExternalResourceBaseMetric(BaseModel):
+    "Base class External Resource metrics"
+
+    integration: str
+
+
+class ExternalResourceInventoryGauge(ExternalResourceBaseMetric, GaugeMetric):
+    "Inventory Gauge"
+
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_external_resource_inventory"
+
+
 ExternalResourceSpecInventory = MutableMapping[
     ExternalResourceUniqueKey, ExternalResourceSpec
 ]

reconcile/utils/external_resources.py

@@ -1,25 +1,29 @@
 import json
-from collections.abc import (
-    Mapping,
-    MutableMapping,
-)
-from typing import (
-    Any,
-    Optional,
-)
+from collections import Counter
+from collections.abc import Mapping, MutableMapping
+from typing import Any
 
 import anymarkup
 
-from reconcile.utils import gql
+from reconcile.external_resources.model import ExternalResourcesInventory
+from reconcile.utils import (
+    gql,
+    metrics,
+)
 from reconcile.utils.exceptions import FetchResourceError
-from reconcile.utils.external_resource_spec import ExternalResourceSpec
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceInventoryGauge,
+    ExternalResourceSpec,
+    ExternalResourceSpecInventory,
+)
 
 PROVIDER_AWS = "aws"
 PROVIDER_CLOUDFLARE = "cloudflare"
 
 
 def get_external_resource_specs(
-    namespace_info: Mapping[str, Any], provision_provider: Optional[str] = None
+    namespace_info: Mapping[str, Any],
+    provision_provider: str | None = None,
 ) -> list[ExternalResourceSpec]:
     specs: list[ExternalResourceSpec] = []
     if not managed_external_resources(namespace_info):
@@ -55,10 +59,34 @@ def get_provision_providers(namespace_info: Mapping[str, Any]) -> set[str]:
 
 
 def managed_external_resources(namespace_info: Mapping[str, Any]) -> bool:
-    if namespace_info.get("managedExternalResources"):
-        return True
-
-    return False
+    return bool(namespace_info.get("managedExternalResources"))
+
+
+def get_inventory_count_combinations(
+    inventory: ExternalResourceSpecInventory | ExternalResourcesInventory,
+) -> Counter[tuple]:
+    return Counter(
+        (k.provision_provider, k.provisioner_name, k.provider) for k in inventory
+    )
+
+
+def publish_metrics(
+    inventory: ExternalResourceSpecInventory | ExternalResourcesInventory,
+    integration: str,
+) -> None:
+    count_combinations = get_inventory_count_combinations(inventory)
+    integration_name = metrics.normalize_integration_name(integration)
+    for combination, count in count_combinations.items():
+        provision_provider, provisioner_name, provider = combination
+        metrics.set_gauge(
+            ExternalResourceInventoryGauge(
+                integration=integration_name,
+                provision_provider=provision_provider,
+                provisioner_name=provisioner_name,
+                provider=provider,
+            ),
+            count,
+        )
 
 
 class ResourceValueResolver:
@@ -87,7 +115,7 @@ class ResourceValueResolver:
     def __init__(
         self,
         spec: ExternalResourceSpec,
-        integration_tag: Optional[str] = None,
+        integration_tag: str | None = None,
         identifier_as_value: bool = False,
     ):
         """
@@ -110,7 +138,7 @@ class ResourceValueResolver:
         resource = self._spec.resource
 
         keys_to_add = [
-            key for key in self._spec.resource.keys() if key not in self._IGNORE_KEYS
+            key for key in self._spec.resource if key not in self._IGNORE_KEYS
         ]
 
         defaults_path = resource.get("defaults", None)
@@ -145,7 +173,7 @@ class ResourceValueResolver:
             values.pop("$schema", None)
         except anymarkup.AnyMarkupError:
             e_msg = "Could not parse data. Skipping resource: {}"
-            raise FetchResourceError(e_msg.format(path))
+            raise FetchResourceError(e_msg.format(path)) from None
         return values
 
     @staticmethod
@@ -154,7 +182,7 @@ class ResourceValueResolver:
         try:
             raw_values = gqlapi.get_resource(path)
         except gql.GqlGetResourceError as e:
-            raise FetchResourceError(str(e))
+            raise FetchResourceError(str(e)) from e
         return raw_values
 
     @staticmethod
@@ -173,7 +201,7 @@ class ResourceValueResolver:
 
     @staticmethod
     def _override_values(
-        values: MutableMapping[str, Any], overrides: Optional[str]
+        values: MutableMapping[str, Any], overrides: str | None
     ) -> None:
         if overrides is None:
             return

reconcile/utils/filtering.py

@@ -0,0 +1,16 @@
+from typing import (
+    TypeVar,
+)
+
+KeyType = TypeVar("KeyType")
+ValueType = TypeVar("ValueType")
+
+
+def remove_none_values_from_dict(
+    a_dict: dict[KeyType, ValueType | None],
+) -> dict[KeyType, ValueType]:
+    """
+    Creates a new dictionary based on the input dictionary but skips items
+    with None values.
+    """
+    return {key: value for key, value in a_dict.items() if value is not None}

reconcile/utils/git.py

@@ -6,30 +6,65 @@ class GitError(Exception):
     pass
 
 
-def clone(repo_url, wd):
-    # pylint: disable=subprocess-run-check
-    cmd = ["git", "clone", repo_url, wd]
-    result = subprocess.run(cmd, cwd=wd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+def clone(repo_url, wd, depth=None, verify=True):
+    cmd = ["git"]
+    if not verify:
+        cmd += ["-c", "http.sslVerify=false"]
+    cmd += ["clone"]
+    if depth:
+        cmd += ["--depth", str(depth)]
+    cmd += [repo_url, wd]
+    result = subprocess.run(cmd, cwd=wd, capture_output=True, check=False)
     if result.returncode != 0:
         raise GitError(f"git clone failed: {repo_url}")
 
 
-def checkout(commit, wd):
-    # pylint: disable=subprocess-run-check
-    cmd = ["git", "checkout", commit]
-    result = subprocess.run(cmd, cwd=wd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+def rev_parse(ref: str, wd: str) -> str:
+    cmd = ["git", "rev-parse", ref]
+    result = subprocess.run(cmd, cwd=wd, capture_output=True, text=True, check=True)
+    return result.stdout.strip()
+
+
+def is_current_ref(ref: str, wd: str) -> bool:
+    return rev_parse("HEAD", wd) == rev_parse(ref, wd)
+
+
+def fetch(
+    ref: str,
+    wd: str,
+    remote: str = "origin",
+    depth: int | None = None,
+    verify: bool = True,
+):
+    cmd = ["git"]
+    if not verify:
+        cmd += ["-c", "http.sslVerify=false"]
+    cmd += ["fetch", remote, ref]
+    if depth:
+        cmd += ["--depth", str(depth)]
+    result = subprocess.run(cmd, cwd=wd, capture_output=True, text=True, check=False)
     if result.returncode != 0:
-        raise GitError(f"git checkout failed: {commit}")
+        raise GitError(f"git fetch failed for {ref}: {result.stderr}")
+
+
+def checkout(
+    ref: str,
+    wd: str,
+    verify: bool = True,
+):
+    if not is_current_ref(ref, wd):
+        fetch(ref, wd, depth=1, verify=verify)
+    cmd = ["git", "checkout", ref]
+    result = subprocess.run(cmd, cwd=wd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise GitError(f"git checkout failed for {ref}: {result.stderr}")
 
 
 def is_file_in_git_repo(file_path):
     real_path = os.path.realpath(file_path)
     dir_path = os.path.dirname(real_path)
-    # pylint: disable=subprocess-run-check
     cmd = ["git", "rev-parse", "--is-inside-work-tree"]
-    result = subprocess.run(
-        cmd, cwd=dir_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-    )
+    result = subprocess.run(cmd, cwd=dir_path, capture_output=True, check=False)
     return result.returncode == 0
 
 
@@ -38,9 +73,7 @@ def has_uncommited_changes() -> bool:
     result = subprocess.run(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True
     )
-    if result.stdout:
-        return True
-    return False
+    return bool(result.stdout)
 
 
 def show_uncommited_changes() -> str:

reconcile/utils/github_api.py

@@ -1,13 +1,10 @@
 import os
 from pathlib import Path
 from types import TracebackType
-from typing import (
-    Optional,
-    Type,
-)
 from urllib.parse import urlparse
 
 from github import (
+    Commit,
     Github,
     UnknownObjectException,
 )
@@ -32,7 +29,7 @@ class GithubRepositoryApi:
         repo_url: str,
         token: str,
         timeout: int = 30,
-        github: Optional[Github] = None,
+        github: Github | None = None,
     ):
         parsed_repo_url = urlparse(repo_url)
         repo = parsed_repo_url.path.strip("/")
@@ -47,9 +44,9 @@ class GithubRepositoryApi:
 
     def __exit__(
         self,
-        exc_type: Optional[Type[BaseException]],
-        exc_value: Optional[BaseException],
-        traceback: Optional[TracebackType],
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
     ) -> None:
         self.cleanup()
 
@@ -66,7 +63,7 @@ class GithubRepositoryApi:
         return tree_items
 
     @retry()
-    def get_file(self, path: str, ref: str = "master") -> Optional[bytes]:
+    def get_file(self, path: str, ref: str = "master") -> bytes | None:
         try:
             content = self._repo.get_contents(path=path, ref=ref)
             if isinstance(content, list):
@@ -81,3 +78,7 @@ class GithubRepositoryApi:
     @retry()
     def get_commit_sha(self, ref: str) -> str:
         return self._repo.get_commit(sha=ref).sha
+
+    @retry()
+    def compare(self, commit_from: str, commit_to: str) -> list[Commit.Commit]:
+        return self._repo.compare(commit_from, commit_to).commits