qontract-reconcile 0.10.1rc537__py3-none-any.whl → 0.10.1rc539__py3-none-any.whl

reconcile/test/test_terraform_resources.py

@@ -4,6 +4,7 @@ from collections.abc import (
     Mapping,
 )
 from typing import Any
+from unittest.mock import MagicMock, create_autospec
 
 import pytest
 from pytest_mock import MockerFixture
@@ -12,6 +13,7 @@ import reconcile.terraform_resources as integ
 from reconcile.gql_definitions.terraform_resources.terraform_resources_namespaces import (
     NamespaceV1,
 )
+from reconcile.utils.secret_reader import SecretReaderBase
 
 
 def test_cannot_use_exclude_accounts_if_not_dry_run():
@@ -71,7 +73,7 @@ def test_cannot_pass_two_aws_account_if_not_dry_run():
 def test_filter_accounts_by_name():
     accounts = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
 
-    filtered = integ.filter_accounts_by_name(accounts, filter=("a", "b"))
+    filtered = integ.filter_accounts_by_name(accounts, names=("a", "b"))
 
     assert filtered == [{"name": "a"}, {"name": "b"}]
 
@@ -79,7 +81,7 @@ def test_filter_accounts_by_name():
 def test_exclude_accounts_by_name():
     accounts = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
 
-    filtered = integ.exclude_accounts_by_name(accounts, filter=("a", "b"))
+    filtered = integ.exclude_accounts_by_name(accounts, names=("a", "b"))
 
     assert filtered == [{"name": "c"}]
 
@@ -258,27 +260,255 @@ def test_filter_tf_namespaces_namespace_deleted(gql_class_factory: Callable):
     assert filtered == [ns2]
 
 
-def test_empty_run(mocker: MockerFixture) -> None:
+def setup_mocks(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+    aws_accounts: list[dict[str, Any]],
+    tf_namespaces: list[NamespaceV1],
+    feature_toggle_state: bool = True,
+) -> dict[str, Any]:
     mocked_queries = mocker.patch("reconcile.terraform_resources.queries")
-    mocked_queries.get_aws_accounts.return_value = [{"name": "a"}]
+    mocked_queries.get_aws_accounts.return_value = aws_accounts
     mocked_queries.get_app_interface_settings.return_value = []
 
-    mocker.patch("reconcile.terraform_resources.get_namespaces").return_value = []
+    mocker.patch(
+        "reconcile.terraform_resources.get_namespaces"
+    ).return_value = tf_namespaces
 
-    mocked_ts = mocker.patch("reconcile.terraform_resources.Terrascript", autospec=True)
-    mocked_ts.return_value.resource_spec_inventory = {}
+    mocked_ts = mocker.patch(
+        "reconcile.terraform_resources.Terrascript", autospec=True
+    ).return_value
+    mocked_ts.resource_spec_inventory = {}
 
-    mocked_tf = mocker.patch("reconcile.terraform_resources.Terraform", autospec=True)
-    mocked_tf.return_value.plan.return_value = (False, None)
-    mocked_tf.return_value.should_apply = False
+    mocked_tf = mocker.patch(
+        "reconcile.terraform_resources.Terraform", autospec=True
+    ).return_value
+    mocked_tf.plan.return_value = (False, None)
+    mocked_tf.should_apply.return_value = False
 
     mocker.patch("reconcile.terraform_resources.AWSApi", autospec=True)
-    mocker.patch("reconcile.terraform_resources.sys")
 
     mocked_logging = mocker.patch("reconcile.terraform_resources.logging")
 
+    mocker.patch("reconcile.terraform_resources.get_app_interface_vault_settings")
+
+    mocker.patch(
+        "reconcile.terraform_resources.create_secret_reader",
+        return_value=secret_reader,
+    )
+
+    mock_extended_early_exit_run = mocker.patch(
+        "reconcile.terraform_resources.extended_early_exit_run"
+    )
+
+    get_feature_toggle_state = mocker.patch(
+        "reconcile.terraform_resources.get_feature_toggle_state",
+        return_value=feature_toggle_state,
+    )
+
+    return {
+        "queries": mocked_queries,
+        "ts": mocked_ts,
+        "tf": mocked_tf,
+        "logging": mocked_logging,
+        "extended_early_exit_run": mock_extended_early_exit_run,
+        "get_feature_toggle_state": get_feature_toggle_state,
+    }
+
+
+def test_empty_run(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+
     integ.run(True, account_name="a")
 
-    mocked_logging.warning.assert_called_once_with(
+    mocks["logging"].warning.assert_called_once_with(
         "No terraform namespaces found, consider disabling this integration, account names: a"
     )
+
+
+def test_run_with_extended_early_exit_run_enabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+    defer = MagicMock()
+    expected_runner_params = integ.RunnerParams(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=mocks["tf"],
+        ts=mocks["ts"],
+        secret_reader=secret_reader,
+        dry_run=True,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    integ.run.__wrapped__(
+        True,
+        account_name="a",
+        enable_extended_early_exit=True,
+        extended_early_exit_cache_ttl_seconds=60,
+        log_cached_log_output=True,
+        defer=defer,
+    )
+
+    mocks["extended_early_exit_run"].assert_called_once_with(
+        integration=integ.QONTRACT_INTEGRATION,
+        integration_version=integ.QONTRACT_INTEGRATION_VERSION,
+        dry_run=True,
+        cache_source=mocks["ts"].terraform_configurations.return_value,
+        ttl_seconds=60,
+        logger=mocks["logging"].getLogger.return_value,
+        runner=integ.runner,
+        runner_params=expected_runner_params,
+        secret_reader=secret_reader,
+        log_cached_log_output=True,
+    )
+
+
+def test_run_with_extended_early_exit_run_disabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+
+    integ.run(
+        True,
+        account_name="a",
+        enable_extended_early_exit=False,
+    )
+
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["tf"].plan.assert_called_once_with(False)
+
+
+def test_run_with_extended_early_exit_run_feature_disabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+        feature_toggle_state=False,
+    )
+
+    integ.run(
+        True,
+        account_name="a",
+        enable_extended_early_exit=True,
+    )
+
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["tf"].plan.assert_called_once_with(False)
+    mocks["get_feature_toggle_state"].assert_called_once_with(
+        "terraform-resources-extended-early-exit",
+        default=False,
+    )
+
+
+def test_terraform_resources_runner_dry_run(
+    secret_reader: SecretReaderBase,
+) -> None:
+    tf = create_autospec(integ.Terraform)
+    tf.plan.return_value = (False, None)
+
+    ts = create_autospec(integ.Terrascript)
+    terraform_configurations = {"a": "b"}
+    ts.terraform_configurations.return_value = terraform_configurations
+
+    defer = MagicMock()
+
+    runner_params = dict(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=tf,
+        ts=ts,
+        secret_reader=secret_reader,
+        dry_run=True,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    result = integ.runner(**runner_params)
+
+    assert result == integ.ExtendedEarlyExitRunnerResult(
+        payload=terraform_configurations,
+        applied_count=0,
+    )
+
+
+def test_terraform_resources_runner_no_dry_run(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    tf = create_autospec(integ.Terraform)
+    tf.plan.return_value = (False, None)
+    tf.apply_count = 1
+    tf.should_apply.return_value = True
+    tf.apply.return_value = False
+
+    ts = create_autospec(integ.Terrascript)
+    terraform_configurations = {"a": "b"}
+    ts.terraform_configurations.return_value = terraform_configurations
+    ts.resource_spec_inventory = {}
+
+    defer = MagicMock()
+
+    mocked_ob = mocker.patch("reconcile.terraform_resources.ob")
+    mocked_ob.realize_data.return_value = [{"action": "applied"}]
+
+    runner_params = dict(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=tf,
+        ts=ts,
+        secret_reader=secret_reader,
+        dry_run=False,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    result = integ.runner(**runner_params)
+
+    assert result == integ.ExtendedEarlyExitRunnerResult(
+        payload=terraform_configurations,
+        applied_count=2,
+    )

reconcile/utils/early_exit_cache.py

@@ -1,6 +1,6 @@
 from datetime import UTC, datetime, timedelta
 from enum import Enum
-from typing import Any, Optional, Self
+from typing import Any, Self
 
 from deepdiff import DeepHash
 from pydantic import BaseModel
@@ -16,19 +16,19 @@ class CacheKey(BaseModel):
     integration: str
     integration_version: str
     dry_run: bool
-    cache_desired_state: object
+    cache_source: object
 
     def __str__(self) -> str:
         return "/".join([
             self.integration,
             self.integration_version,
             "dry-run" if self.dry_run else "no-dry-run",
-            DeepHash(self.cache_desired_state)[self.cache_desired_state],
+            DeepHash(self.cache_source)[self.cache_source],
         ])
 
 
 class CacheValue(BaseModel):
-    desired_state: object
+    payload: object
     log_output: str
     applied_count: int
 
@@ -46,7 +46,7 @@ class EarlyExitCache:
     @classmethod
     def build(
         cls,
-        secret_reader: Optional[SecretReaderBase] = None,
+        secret_reader: SecretReaderBase | None = None,
     ) -> Self:
         state = init_state(STATE_INTEGRATION, secret_reader)
         return cls(state)

reconcile/utils/extended_early_exit.py

@@ -0,0 +1,177 @@
+import logging
+from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from io import StringIO
+from logging import Logger
+
+from pydantic import BaseModel
+
+from reconcile.utils.early_exit_cache import (
+    CacheKey,
+    CacheStatus,
+    CacheValue,
+    EarlyExitCache,
+)
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+    inc_counter,
+    normalize_integration_name,
+    set_gauge,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class ExtendedEarlyExitRunnerResult(BaseModel):
+    payload: object
+    applied_count: int
+
+
+class ExtendedEarlyExitBaseMetric(BaseModel):
+    integration: str
+    integration_version: str
+    dry_run: bool
+    cache_status: str
+
+
+class ExtendedEarlyExitCounter(ExtendedEarlyExitBaseMetric, CounterMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit"
+
+
+class ExtendedEarlyExitAppliedCountGauge(ExtendedEarlyExitBaseMetric, GaugeMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit_applied_count"
+
+
+def _publish_metrics(
+    cache_key: CacheKey,
+    cache_status: CacheStatus,
+    applied_count: int,
+) -> None:
+    inc_counter(
+        ExtendedEarlyExitCounter(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+        ),
+    )
+    set_gauge(
+        ExtendedEarlyExitAppliedCountGauge(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+        ),
+        applied_count,
+    )
+
+
+def _ttl_seconds(
+    applied_count: int,
+    ttl_seconds: int,
+) -> int:
+    """
+    Pick the ttl based on the applied count.
+    If the applied count is greater than 0, then we want to set ttl to 0 so that the next run will not hit the cache,
+    this will allow us to easy debug reconcile loops, as we will be able to see the logs of the next run,
+    and check cached value for more details.
+
+    :param applied_count: The number of resources that were applied
+    :param ttl_seconds: A ttl in seconds
+    :return: The ttl in seconds
+    """
+    return 0 if applied_count > 0 else ttl_seconds
+
+
+@contextmanager
+def log_stream_handler(
+    logger: Logger,
+) -> Generator[StringIO, None, None]:
+    """
+    Add a stream handler to the logger, and return the stream generator, automatically remove the handler when done.
+
+    :param logger: A logger
+    :return: A stream generator
+    """
+    log_stream = StringIO()
+    log_handler = logging.StreamHandler(log_stream)
+    logger.addHandler(log_handler)
+    try:
+        yield log_stream
+    finally:
+        logger.removeHandler(log_handler)
+
+
+def extended_early_exit_run(
+    integration: str,
+    integration_version: str,
+    dry_run: bool,
+    cache_source: object,
+    ttl_seconds: int,
+    logger: Logger,
+    runner: Callable[..., ExtendedEarlyExitRunnerResult],
+    runner_params: Mapping | None = None,
+    secret_reader: SecretReaderBase | None = None,
+    log_cached_log_output: bool = False,
+) -> None:
+    """
+    Run the runner based on the cache status. Early exit when cache hit.
+    Runner log output will be extracted and stored in cache value,
+    and will be logged when hit if log_cached_log_output is True,
+    this is mainly used to show all log output from different integrations in one place (CI).
+    When runner returns no applies (applied_count is 0), the ttl will be set to ttl_seconds,
+    otherwise it will be set to 0.
+
+    :param integration: The integration name
+    :param integration_version: The integration version
+    :param dry_run: True if the run is in dry run mode, False otherwise
+    :param cache_source: The cache source, usually the static desired state
+    :param ttl_seconds: A ttl in seconds
+    :param logger: A Logger
+    :param runner: A runner can return ExtendedEarlyExitRunnerResult when called
+    :param runner_params: Runner params, will be spread into kwargs when calling runner
+    :param secret_reader: A secret reader
+    :param log_cached_log_output: Whether to log the cached log output when there is a cache hit
+    :return: None
+    """
+    with EarlyExitCache.build(secret_reader) as cache:
+        key = CacheKey(
+            integration=normalize_integration_name(integration),
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=cache_source,
+        )
+        cache_status = cache.head(key)
+        logger.debug("Early exit cache status for key=%s: %s", key, cache_status)
+
+        if cache_status == CacheStatus.HIT:
+            if log_cached_log_output:
+                logger.info(cache.get(key).log_output)
+            _publish_metrics(
+                cache_key=key,
+                cache_status=cache_status,
+                applied_count=0,
+            )
+            return
+
+        with log_stream_handler(logger) as log_stream:
+            result = runner(**(runner_params or {}))
+            log_output = log_stream.getvalue()
+
+        value = CacheValue(
+            payload=result.payload,
+            log_output=log_output,
+            applied_count=result.applied_count,
+        )
+        ttl = _ttl_seconds(result.applied_count, ttl_seconds)
+        logger.debug("Set early exit cache for key=%s with ttl=%d", key, ttl)
+        cache.set(key, value, ttl)
+        _publish_metrics(
+            cache_key=key,
+            cache_status=cache_status,
+            applied_count=result.applied_count,
+        )

reconcile/utils/external_resources.py

@@ -79,11 +79,12 @@ def get_inventory_count_combinations(
 
 def publish_metrics(inventory: ExternalResourceSpecInventory, integration: str) -> None:
     count_combinations = get_inventory_count_combinations(inventory)
+    integration_name = metrics.normalize_integration_name(integration)
     for combination, count in count_combinations.items():
         provision_provider, provisioner_name, provider = combination
         metrics.set_gauge(
             ExternalResourceInventoryGauge(
-                integration=integration.replace("_", "-"),
+                integration=integration_name,
                 provision_provider=provision_provider,
                 provisioner_name=provisioner_name,
                 provider=provider,

reconcile/utils/glitchtip/models.py

@@ -154,6 +154,10 @@ class Project(BaseModel):
     platform: Optional[str]
     teams: list[Team] = []
     alerts: list[ProjectAlert] = []
+    event_throttle_rate: int = Field(0, alias="eventThrottleRate")
+
+    class Config:
+        allow_population_by_field_name = True
 
     @root_validator
     def slugify(  # pylint: disable=no-self-argument
@@ -174,7 +178,11 @@ class Project(BaseModel):
         return self.slug == other.slug
 
     def diff(self, other: Project) -> bool:
-        return self.name != other.name or self.platform != other.platform
+        return (
+            self.name != other.name
+            or self.platform != other.platform
+            or self.event_throttle_rate != other.event_throttle_rate
+        )
 
     def __hash__(self) -> int:
         return hash(self.slug)

reconcile/utils/metrics.py

@@ -563,3 +563,10 @@ class ErrorRateMetricSet:
         if exc_value:
             self.fail(exc_value)
         inc_counter(self._error_counter, by=(1 if self._errors else 0))
+
+
+def normalize_integration_name(integration: str) -> str:
+    """
+    Normalize the integration name to be used in prometheus.
+    """
+    return integration.replace("_", "-")

reconcile/utils/terraform_client.py

@@ -90,7 +90,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         self.thread_pool_size = thread_pool_size
         self._aws_api = aws_api
         self._log_lock = Lock()
-        self.should_apply = False
+        self.apply_count = 0
 
         self.specs: list[TerraformSpec] = []
         self.init_specs()
@@ -111,6 +111,12 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             for account, output in self.outputs.items()
         }
 
+    def increment_apply_count(self):
+        self.apply_count += 1
+
+    def should_apply(self) -> bool:
+        return self.apply_count > 0
+
     def get_new_users(self):
         new_users = []
         self.init_outputs()  # get updated output
@@ -282,7 +288,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             after = output_change.get("after")
             if before != after:
                 logging.info(["update", name, "output", output_name])
-                self.should_apply = True
+                self.increment_apply_count()
 
         # A way to detect deleted outputs is by comparing
         # the prior state with the output changes.
@@ -295,7 +301,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         deleted_outputs = [po for po in prior_outputs if po not in output_changes]
         for output_name in deleted_outputs:
             logging.info(["delete", name, "output", output_name])
-            self.should_apply = True
+            self.increment_apply_count()
 
         resource_changes = output.get("resource_changes")
         if resource_changes is None:
@@ -339,7 +345,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                         resource_name,
                         self._resource_diff_changed_fields(action, resource_change),
                     ])
-                    self.should_apply = True
+                    self.increment_apply_count()
                 if action == "create":
                     if resource_type == "aws_iam_user_login_profile":
                         created_users.append(AccountUser(name, resource_name))

reconcile/utils/terrascript_aws_client.py

@@ -3890,22 +3890,30 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             if os.path.isfile(print_to_file):
                 os.remove(print_to_file)
 
-        for name, ts in self.tss.items():
+        for name, ts in self.terraform_configurations().items():
             if print_to_file:
                 with open(print_to_file, "a", encoding="locale") as f:
                     f.write(f"##### {name} #####\n")
-                    f.write(str(ts))
+                    f.write(ts)
                     f.write("\n")
             if existing_dirs is None:
                 wd = tempfile.mkdtemp(prefix=TMP_DIR_PREFIX)
             else:
                 wd = working_dirs[name]
             with open(wd + "/config.tf.json", "w", encoding="locale") as f:
-                f.write(str(ts))
+                f.write(ts)
             working_dirs[name] = wd
 
         return working_dirs
 
+    def terraform_configurations(self) -> dict[str, str]:
+        """
+        Return the Terraform configurations (in JSON format) for each AWS account.
+
+        :return: key is AWS account name and value is terraform configuration
+        """
+        return {name: str(ts) for name, ts in self.tss.items()}
+
     def init_values(self, spec: ExternalResourceSpec, init_tags: bool = True) -> dict:
         """
         Initialize the values of the terraform resource and merge the defaults and