qontract-reconcile 0.10.1rc536__py3-none-any.whl → 0.10.1rc538__py3-none-any.whl

reconcile/test/test_acs_policies.py

@@ -20,6 +20,10 @@ from reconcile.gql_definitions.acs.acs_policies import (
 )
 from reconcile.utils.acs.policies import AcsPolicyApi, Policy, PolicyCondition, Scope
 
+CLUSTER_NAME_ONE = "app-sre-stage"
+CLUSTER_ID_ONE = "5211d395-5cf7-4185-a1fb-d88f41bc7542"
+CLUSTER_NAME_TWO = "app-sre-prod"
+CLUSTER_ID_TWO = "a217cca7-d85a-4be1-9703-f58866fdbe2d"
 CUSTOM_POLICY_ONE_NAME = "app-sre-clusters-fixable-cve-7-fixable"
 CUSTOM_POLICY_ONE_ID = "365d4e71-3241-4448-9f3d-eb0eed1c1820"
 CUSTOM_POLICY_TWO_NAME = "app-sre-namespaces-severity-critical"
@@ -41,8 +45,8 @@ def query_data_desired_state() -> AcsPolicyQueryData:
                 scope=AcsPolicyScopeClusterV1(
                     level="cluster",
                     clusters=[
-                        ClusterV1(name="app-sre-stage"),
-                        ClusterV1(name="app-sre-prod"),
+                        ClusterV1(name=CLUSTER_NAME_ONE),
+                        ClusterV1(name=CLUSTER_NAME_TWO),
                     ],
                 ),
                 conditions=[
@@ -91,8 +95,8 @@ def modeled_acs_policies() -> list[Policy]:
             notifiers=[JIRA_NOTIFIER_ID],
             categories=["Vulnerability Management"],
             scope=[
-                Scope(cluster="app-sre-prod", namespace=""),
-                Scope(cluster="app-sre-stage", namespace=""),
+                Scope(cluster=CLUSTER_ID_ONE, namespace=""),
+                Scope(cluster=CLUSTER_ID_TWO, namespace=""),
             ],
             conditions=[
                 PolicyCondition(field_name="CVSS", values=[">=7"], negate=False),
@@ -106,8 +110,8 @@ def modeled_acs_policies() -> list[Policy]:
             notifiers=[],
             categories=["DevOps Best Practices", "Vulnerability Management"],
             scope=[
-                Scope(cluster="app-sre-prod", namespace="app-interface-production"),
-                Scope(cluster="app-sre-stage", namespace="app-interface-stage"),
+                Scope(cluster=CLUSTER_ID_ONE, namespace="app-interface-stage"),
+                Scope(cluster=CLUSTER_ID_TWO, namespace="app-interface-production"),
             ],
             conditions=[
                 PolicyCondition(
@@ -175,8 +179,8 @@ def api_response_policies_specific() -> list[Any]:
             "eventSource": "NOT_APPLICABLE",
             "exclusions": [],
             "scope": [
-                {"cluster": "app-sre-stage", "namespace": "", "label": None},
-                {"cluster": "app-sre-prod", "namespace": "", "label": None},
+                {"cluster": CLUSTER_ID_ONE, "namespace": "", "label": None},
+                {"cluster": CLUSTER_ID_TWO, "namespace": "", "label": None},
             ],
             "severity": "HIGH_SEVERITY",
             "enforcementActions": [],
@@ -216,12 +220,12 @@ def api_response_policies_specific() -> list[Any]:
             "exclusions": [],
             "scope": [
                 {
-                    "cluster": "app-sre-stage",
+                    "cluster": CLUSTER_ID_ONE,
                     "namespace": "app-interface-stage",
                     "label": None,
                 },
                 {
-                    "cluster": "app-sre-prod",
+                    "cluster": CLUSTER_ID_TWO,
                     "namespace": "app-interface-production",
                     "label": None,
                 },
@@ -257,11 +261,20 @@ def api_response_list_notifiers() -> list[AcsPolicyApi.NotifierIdentifiers]:
     ]
 
 
+@pytest.fixture
+def api_response_list_clusters() -> list[AcsPolicyApi.ClusterIdentifiers]:
+    return [
+        AcsPolicyApi.ClusterIdentifiers(id=CLUSTER_ID_ONE, name=CLUSTER_NAME_ONE),
+        AcsPolicyApi.ClusterIdentifiers(id=CLUSTER_ID_TWO, name=CLUSTER_NAME_TWO),
+    ]
+
+
 def test_get_desired_state(
     mocker: MockerFixture,
     query_data_desired_state: AcsPolicyQueryData,
     modeled_acs_policies: list[Policy],
     api_response_list_notifiers: list[AcsPolicyApi.NotifierIdentifiers],
+    api_response_list_clusters: list[AcsPolicyApi.ClusterIdentifiers],
 ) -> None:
     query_func = mocker.patch(
         "reconcile.gql_definitions.acs.acs_policies.query", autospec=True
@@ -270,7 +283,9 @@ def test_get_desired_state(
 
     integration = AcsPoliciesIntegration()
     result = integration.get_desired_state(
-        query_func=query_func, notifiers=api_response_list_notifiers
+        query_func=query_func,
+        notifiers=api_response_list_notifiers,
+        clusters=api_response_list_clusters,
     )
     assert result == modeled_acs_policies
 

reconcile/test/test_terraform_resources.py

@@ -4,6 +4,7 @@ from collections.abc import (
     Mapping,
 )
 from typing import Any
+from unittest.mock import MagicMock, create_autospec
 
 import pytest
 from pytest_mock import MockerFixture
@@ -12,6 +13,7 @@ import reconcile.terraform_resources as integ
 from reconcile.gql_definitions.terraform_resources.terraform_resources_namespaces import (
     NamespaceV1,
 )
+from reconcile.utils.secret_reader import SecretReaderBase
 
 
 def test_cannot_use_exclude_accounts_if_not_dry_run():
@@ -71,7 +73,7 @@ def test_cannot_pass_two_aws_account_if_not_dry_run():
 def test_filter_accounts_by_name():
     accounts = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
 
-    filtered = integ.filter_accounts_by_name(accounts, filter=("a", "b"))
+    filtered = integ.filter_accounts_by_name(accounts, names=("a", "b"))
 
     assert filtered == [{"name": "a"}, {"name": "b"}]
 
@@ -79,7 +81,7 @@ def test_filter_accounts_by_name():
 def test_exclude_accounts_by_name():
     accounts = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
 
-    filtered = integ.exclude_accounts_by_name(accounts, filter=("a", "b"))
+    filtered = integ.exclude_accounts_by_name(accounts, names=("a", "b"))
 
     assert filtered == [{"name": "c"}]
 
@@ -258,27 +260,255 @@ def test_filter_tf_namespaces_namespace_deleted(gql_class_factory: Callable):
     assert filtered == [ns2]
 
 
-def test_empty_run(mocker: MockerFixture) -> None:
+def setup_mocks(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+    aws_accounts: list[dict[str, Any]],
+    tf_namespaces: list[NamespaceV1],
+    feature_toggle_state: bool = True,
+) -> dict[str, Any]:
     mocked_queries = mocker.patch("reconcile.terraform_resources.queries")
-    mocked_queries.get_aws_accounts.return_value = [{"name": "a"}]
+    mocked_queries.get_aws_accounts.return_value = aws_accounts
     mocked_queries.get_app_interface_settings.return_value = []
 
-    mocker.patch("reconcile.terraform_resources.get_namespaces").return_value = []
+    mocker.patch(
+        "reconcile.terraform_resources.get_namespaces"
+    ).return_value = tf_namespaces
 
-    mocked_ts = mocker.patch("reconcile.terraform_resources.Terrascript", autospec=True)
-    mocked_ts.return_value.resource_spec_inventory = {}
+    mocked_ts = mocker.patch(
+        "reconcile.terraform_resources.Terrascript", autospec=True
+    ).return_value
+    mocked_ts.resource_spec_inventory = {}
 
-    mocked_tf = mocker.patch("reconcile.terraform_resources.Terraform", autospec=True)
-    mocked_tf.return_value.plan.return_value = (False, None)
-    mocked_tf.return_value.should_apply = False
+    mocked_tf = mocker.patch(
+        "reconcile.terraform_resources.Terraform", autospec=True
+    ).return_value
+    mocked_tf.plan.return_value = (False, None)
+    mocked_tf.should_apply.return_value = False
 
     mocker.patch("reconcile.terraform_resources.AWSApi", autospec=True)
-    mocker.patch("reconcile.terraform_resources.sys")
 
     mocked_logging = mocker.patch("reconcile.terraform_resources.logging")
 
+    mocker.patch("reconcile.terraform_resources.get_app_interface_vault_settings")
+
+    mocker.patch(
+        "reconcile.terraform_resources.create_secret_reader",
+        return_value=secret_reader,
+    )
+
+    mock_extended_early_exit_run = mocker.patch(
+        "reconcile.terraform_resources.extended_early_exit_run"
+    )
+
+    get_feature_toggle_state = mocker.patch(
+        "reconcile.terraform_resources.get_feature_toggle_state",
+        return_value=feature_toggle_state,
+    )
+
+    return {
+        "queries": mocked_queries,
+        "ts": mocked_ts,
+        "tf": mocked_tf,
+        "logging": mocked_logging,
+        "extended_early_exit_run": mock_extended_early_exit_run,
+        "get_feature_toggle_state": get_feature_toggle_state,
+    }
+
+
+def test_empty_run(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+
     integ.run(True, account_name="a")
 
-    mocked_logging.warning.assert_called_once_with(
+    mocks["logging"].warning.assert_called_once_with(
         "No terraform namespaces found, consider disabling this integration, account names: a"
     )
+
+
+def test_run_with_extended_early_exit_run_enabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+    defer = MagicMock()
+    expected_runner_params = integ.RunnerParams(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=mocks["tf"],
+        ts=mocks["ts"],
+        secret_reader=secret_reader,
+        dry_run=True,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    integ.run.__wrapped__(
+        True,
+        account_name="a",
+        enable_extended_early_exit=True,
+        extended_early_exit_cache_ttl_seconds=60,
+        log_cached_log_output=True,
+        defer=defer,
+    )
+
+    mocks["extended_early_exit_run"].assert_called_once_with(
+        integration=integ.QONTRACT_INTEGRATION,
+        integration_version=integ.QONTRACT_INTEGRATION_VERSION,
+        dry_run=True,
+        cache_source=mocks["ts"].terraform_configurations.return_value,
+        ttl_seconds=60,
+        logger=mocks["logging"].getLogger.return_value,
+        runner=integ.runner,
+        runner_params=expected_runner_params,
+        secret_reader=secret_reader,
+        log_cached_log_output=True,
+    )
+
+
+def test_run_with_extended_early_exit_run_disabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+    )
+
+    integ.run(
+        True,
+        account_name="a",
+        enable_extended_early_exit=False,
+    )
+
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["tf"].plan.assert_called_once_with(False)
+
+
+def test_run_with_extended_early_exit_run_feature_disabled(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    mocks = setup_mocks(
+        mocker,
+        secret_reader,
+        aws_accounts=[{"name": "a"}],
+        tf_namespaces=[],
+        feature_toggle_state=False,
+    )
+
+    integ.run(
+        True,
+        account_name="a",
+        enable_extended_early_exit=True,
+    )
+
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["tf"].plan.assert_called_once_with(False)
+    mocks["get_feature_toggle_state"].assert_called_once_with(
+        "terraform-resources-extended-early-exit",
+        default=False,
+    )
+
+
+def test_terraform_resources_runner_dry_run(
+    secret_reader: SecretReaderBase,
+) -> None:
+    tf = create_autospec(integ.Terraform)
+    tf.plan.return_value = (False, None)
+
+    ts = create_autospec(integ.Terrascript)
+    terraform_configurations = {"a": "b"}
+    ts.terraform_configurations.return_value = terraform_configurations
+
+    defer = MagicMock()
+
+    runner_params = dict(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=tf,
+        ts=ts,
+        secret_reader=secret_reader,
+        dry_run=True,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    result = integ.runner(**runner_params)
+
+    assert result == integ.ExtendedEarlyExitRunnerResult(
+        payload=terraform_configurations,
+        applied_count=0,
+    )
+
+
+def test_terraform_resources_runner_no_dry_run(
+    mocker: MockerFixture,
+    secret_reader: SecretReaderBase,
+) -> None:
+    tf = create_autospec(integ.Terraform)
+    tf.plan.return_value = (False, None)
+    tf.apply_count = 1
+    tf.should_apply.return_value = True
+    tf.apply.return_value = False
+
+    ts = create_autospec(integ.Terrascript)
+    terraform_configurations = {"a": "b"}
+    ts.terraform_configurations.return_value = terraform_configurations
+    ts.resource_spec_inventory = {}
+
+    defer = MagicMock()
+
+    mocked_ob = mocker.patch("reconcile.terraform_resources.ob")
+    mocked_ob.realize_data.return_value = [{"action": "applied"}]
+
+    runner_params = dict(
+        accounts=[{"name": "a"}],
+        account_names={"a"},
+        tf_namespaces=[],
+        tf=tf,
+        ts=ts,
+        secret_reader=secret_reader,
+        dry_run=False,
+        enable_deletion=False,
+        thread_pool_size=10,
+        internal=None,
+        use_jump_host=True,
+        light=False,
+        vault_output_path="",
+        defer=defer,
+    )
+
+    result = integ.runner(**runner_params)
+
+    assert result == integ.ExtendedEarlyExitRunnerResult(
+        payload=terraform_configurations,
+        applied_count=2,
+    )

reconcile/utils/acs/policies.py

@@ -151,3 +151,13 @@ class AcsPolicyApi(AcsBaseApi):
             self.NotifierIdentifiers(id=c["id"], name=c["name"])
             for c in self.generic_request("/v1/notifiers", "GET").json()["notifiers"]
         ]
+
+    class ClusterIdentifiers(BaseModel):
+        id: str
+        name: str
+
+    def list_clusters(self) -> list[ClusterIdentifiers]:
+        return [
+            self.ClusterIdentifiers(id=c["id"], name=c["name"])
+            for c in self.generic_request("/v1/clusters", "GET").json()["clusters"]
+        ]

reconcile/utils/early_exit_cache.py

@@ -1,6 +1,6 @@
 from datetime import UTC, datetime, timedelta
 from enum import Enum
-from typing import Any, Optional, Self
+from typing import Any, Self
 
 from deepdiff import DeepHash
 from pydantic import BaseModel
@@ -16,19 +16,19 @@ class CacheKey(BaseModel):
     integration: str
     integration_version: str
     dry_run: bool
-    cache_desired_state: object
+    cache_source: object
 
     def __str__(self) -> str:
         return "/".join([
             self.integration,
             self.integration_version,
             "dry-run" if self.dry_run else "no-dry-run",
-            DeepHash(self.cache_desired_state)[self.cache_desired_state],
+            DeepHash(self.cache_source)[self.cache_source],
         ])
 
 
 class CacheValue(BaseModel):
-    desired_state: object
+    payload: object
     log_output: str
     applied_count: int
 
@@ -46,7 +46,7 @@ class EarlyExitCache:
     @classmethod
     def build(
         cls,
-        secret_reader: Optional[SecretReaderBase] = None,
+        secret_reader: SecretReaderBase | None = None,
     ) -> Self:
         state = init_state(STATE_INTEGRATION, secret_reader)
         return cls(state)

reconcile/utils/extended_early_exit.py

@@ -0,0 +1,177 @@
+import logging
+from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from io import StringIO
+from logging import Logger
+
+from pydantic import BaseModel
+
+from reconcile.utils.early_exit_cache import (
+    CacheKey,
+    CacheStatus,
+    CacheValue,
+    EarlyExitCache,
+)
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+    inc_counter,
+    normalize_integration_name,
+    set_gauge,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class ExtendedEarlyExitRunnerResult(BaseModel):
+    payload: object
+    applied_count: int
+
+
+class ExtendedEarlyExitBaseMetric(BaseModel):
+    integration: str
+    integration_version: str
+    dry_run: bool
+    cache_status: str
+
+
+class ExtendedEarlyExitCounter(ExtendedEarlyExitBaseMetric, CounterMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit"
+
+
+class ExtendedEarlyExitAppliedCountGauge(ExtendedEarlyExitBaseMetric, GaugeMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_extended_early_exit_applied_count"
+
+
+def _publish_metrics(
+    cache_key: CacheKey,
+    cache_status: CacheStatus,
+    applied_count: int,
+) -> None:
+    inc_counter(
+        ExtendedEarlyExitCounter(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+        ),
+    )
+    set_gauge(
+        ExtendedEarlyExitAppliedCountGauge(
+            integration=cache_key.integration,
+            integration_version=cache_key.integration_version,
+            dry_run=cache_key.dry_run,
+            cache_status=cache_status.value,
+        ),
+        applied_count,
+    )
+
+
+def _ttl_seconds(
+    applied_count: int,
+    ttl_seconds: int,
+) -> int:
+    """
+    Pick the ttl based on the applied count.
+    If the applied count is greater than 0, then we want to set ttl to 0 so that the next run will not hit the cache,
+    this will allow us to easy debug reconcile loops, as we will be able to see the logs of the next run,
+    and check cached value for more details.
+
+    :param applied_count: The number of resources that were applied
+    :param ttl_seconds: A ttl in seconds
+    :return: The ttl in seconds
+    """
+    return 0 if applied_count > 0 else ttl_seconds
+
+
+@contextmanager
+def log_stream_handler(
+    logger: Logger,
+) -> Generator[StringIO, None, None]:
+    """
+    Add a stream handler to the logger, and return the stream generator, automatically remove the handler when done.
+
+    :param logger: A logger
+    :return: A stream generator
+    """
+    log_stream = StringIO()
+    log_handler = logging.StreamHandler(log_stream)
+    logger.addHandler(log_handler)
+    try:
+        yield log_stream
+    finally:
+        logger.removeHandler(log_handler)
+
+
+def extended_early_exit_run(
+    integration: str,
+    integration_version: str,
+    dry_run: bool,
+    cache_source: object,
+    ttl_seconds: int,
+    logger: Logger,
+    runner: Callable[..., ExtendedEarlyExitRunnerResult],
+    runner_params: Mapping | None = None,
+    secret_reader: SecretReaderBase | None = None,
+    log_cached_log_output: bool = False,
+) -> None:
+    """
+    Run the runner based on the cache status. Early exit when cache hit.
+    Runner log output will be extracted and stored in cache value,
+    and will be logged when hit if log_cached_log_output is True,
+    this is mainly used to show all log output from different integrations in one place (CI).
+    When runner returns no applies (applied_count is 0), the ttl will be set to ttl_seconds,
+    otherwise it will be set to 0.
+
+    :param integration: The integration name
+    :param integration_version: The integration version
+    :param dry_run: True if the run is in dry run mode, False otherwise
+    :param cache_source: The cache source, usually the static desired state
+    :param ttl_seconds: A ttl in seconds
+    :param logger: A Logger
+    :param runner: A runner can return ExtendedEarlyExitRunnerResult when called
+    :param runner_params: Runner params, will be spread into kwargs when calling runner
+    :param secret_reader: A secret reader
+    :param log_cached_log_output: Whether to log the cached log output when there is a cache hit
+    :return: None
+    """
+    with EarlyExitCache.build(secret_reader) as cache:
+        key = CacheKey(
+            integration=normalize_integration_name(integration),
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=cache_source,
+        )
+        cache_status = cache.head(key)
+        logger.debug("Early exit cache status for key=%s: %s", key, cache_status)
+
+        if cache_status == CacheStatus.HIT:
+            if log_cached_log_output:
+                logger.info(cache.get(key).log_output)
+            _publish_metrics(
+                cache_key=key,
+                cache_status=cache_status,
+                applied_count=0,
+            )
+            return
+
+        with log_stream_handler(logger) as log_stream:
+            result = runner(**(runner_params or {}))
+            log_output = log_stream.getvalue()
+
+        value = CacheValue(
+            payload=result.payload,
+            log_output=log_output,
+            applied_count=result.applied_count,
+        )
+        ttl = _ttl_seconds(result.applied_count, ttl_seconds)
+        logger.debug("Set early exit cache for key=%s with ttl=%d", key, ttl)
+        cache.set(key, value, ttl)
+        _publish_metrics(
+            cache_key=key,
+            cache_status=cache_status,
+            applied_count=result.applied_count,
+        )

reconcile/utils/external_resources.py

@@ -79,11 +79,12 @@ def get_inventory_count_combinations(
 
 def publish_metrics(inventory: ExternalResourceSpecInventory, integration: str) -> None:
     count_combinations = get_inventory_count_combinations(inventory)
+    integration_name = metrics.normalize_integration_name(integration)
     for combination, count in count_combinations.items():
         provision_provider, provisioner_name, provider = combination
         metrics.set_gauge(
             ExternalResourceInventoryGauge(
-                integration=integration.replace("_", "-"),
+                integration=integration_name,
                 provision_provider=provision_provider,
                 provisioner_name=provisioner_name,
                 provider=provider,

reconcile/utils/metrics.py

@@ -563,3 +563,10 @@ class ErrorRateMetricSet:
         if exc_value:
             self.fail(exc_value)
         inc_counter(self._error_counter, by=(1 if self._errors else 0))
+
+
+def normalize_integration_name(integration: str) -> str:
+    """
+    Normalize the integration name to be used in prometheus.
+    """
+    return integration.replace("_", "-")