qontract-reconcile 0.10.1rc879__py3-none-any.whl → 0.10.1rc894__py3-none-any.whl

tools/qontract_cli.py

@@ -9,16 +9,13 @@ import re
 import sys
 from collections import defaultdict
 from datetime import (
+    UTC,
     datetime,
     timedelta,
-    timezone,
 )
 from operator import itemgetter
 from statistics import median
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import boto3
 import click
@@ -59,6 +56,14 @@ from reconcile.cli import (
     config_file,
     use_jump_host,
 )
+from reconcile.external_resources.manager import (
+    FLAG_RESOURCE_MANAGED_BY_ERV2,
+    setup_factories,
+)
+from reconcile.external_resources.model import (
+    ExternalResourcesInventory,
+    load_module_inventory,
+)
 from reconcile.gql_definitions.advanced_upgrade_service.aus_clusters import (
     query as aus_clusters_query,
 )
@@ -75,6 +80,11 @@ from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.clusters import get_clusters
+from reconcile.typed_queries.external_resources import (
+    get_modules,
+    get_namespaces,
+    get_settings,
+)
 from reconcile.typed_queries.saas_files import get_saas_files
 from reconcile.typed_queries.slo_documents import get_slo_documents
 from reconcile.typed_queries.status_board import get_status_board
@@ -352,8 +362,8 @@ def get_upgrade_policies_data(
 
     def soaking_str(
         soaking: dict[str, Any],
-        upgrade_policy: Optional[AbstractUpgradePolicy],
-        upgradeable_version: Optional[str],
+        upgrade_policy: AbstractUpgradePolicy | None,
+        upgradeable_version: str | None,
     ) -> str:
         if upgrade_policy:
             upgrade_version = upgrade_policy.version
@@ -764,9 +774,9 @@ def ocm_addon_upgrade_policies(ctx: click.core.Context) -> None:
 @click.pass_context
 def sd_app_sre_alert_report(
     ctx: click.core.Context,
-    days: Optional[int],
-    from_timestamp: Optional[int],
-    to_timestamp: Optional[int],
+    days: int | None,
+    from_timestamp: int | None,
+    to_timestamp: int | None,
 ) -> None:
     import tools.sd_app_sre_alert_report as report
 
@@ -1407,9 +1417,7 @@ def rosa_create_cluster_command(ctx, cluster_name):
 @click.argument("jumphost_hostname", required=False)
 @click.argument("cluster_name", required=False)
 @click.pass_context
-def sshuttle_command(
-    ctx, jumphost_hostname: Optional[str], cluster_name: Optional[str]
-):
+def sshuttle_command(ctx, jumphost_hostname: str | None, cluster_name: str | None):
     jumphosts_query_data = queries.get_jumphosts(hostname=jumphost_hostname)
     jumphosts = jumphosts_query_data.jumphosts or []
     for jh in jumphosts:
@@ -2358,7 +2366,7 @@ def ec2_jenkins_workers(ctx, aws_access_key_id, aws_secret_access_key, aws_regio
     client = boto3.client("autoscaling")
     ec2 = boto3.resource("ec2")
     results = []
-    now = datetime.now(timezone.utc)
+    now = datetime.now(UTC)
     DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
     columns = [
         "type",
@@ -2511,7 +2519,7 @@ def alerts(ctx, file_path):
             case _:
                 return BIG_NUMBER
 
-    with open(file_path, "r", encoding="locale") as f:
+    with open(file_path, encoding="locale") as f:
         content = json.loads(f.read())
 
     columns = [
@@ -2605,7 +2613,7 @@ def osd_component_versions(ctx):
 @get.command()
 @click.pass_context
 def maintenances(ctx):
-    now = datetime.now(timezone.utc)
+    now = datetime.now(UTC)
     maintenances = maintenances_gql.query(gql.get_api().query).maintenances or []
     data = [
         {
@@ -3474,7 +3482,7 @@ def saas_dev(ctx, app_name=None, saas_file_name=None, env_name=None) -> None:
 @click.option("--app-name", default=None, help="app to act on.")
 @click.pass_context
 def saas_targets(
-    ctx, saas_file_name: Optional[str] = None, app_name: Optional[str] = None
+    ctx, saas_file_name: str | None = None, app_name: str | None = None
 ) -> None:
     """Resolve namespaceSelectors and print all resulting targets of a saas file."""
     console = Console()
@@ -3663,6 +3671,46 @@ def gpg_encrypt(
     ).execute()
 
 
+@root.command()
+@click.option("--channel", help="the channel that state is part of")
+@click.option("--sha", help="the commit sha we want state for")
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+def get_promotion_state(channel: str, sha: str):
+    from tools.saas_promotion_state.saas_promotion_state import (
+        SaasPromotionState,
+    )
+
+    promotion_state = SaasPromotionState.create(promotion_state=None, saas_files=None)
+    for publisher_id, state in promotion_state.get(channel=channel, sha=sha).items():
+        print()
+        if not state:
+            print(f"No state found for {publisher_id=}")
+        else:
+            print(f"State for {publisher_id=}:")
+            print(state)
+
+
+@root.command()
+@click.option("--channel", help="the channel that state is part of")
+@click.option("--sha", help="the commit sha we want state for")
+@click.option("--publisher-id", help="the publisher id we want state for")
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+def mark_promotion_state_successful(channel: str, sha: str, publisher_id: str):
+    from tools.saas_promotion_state.saas_promotion_state import (
+        SaasPromotionState,
+    )
+
+    promotion_state = SaasPromotionState.create(promotion_state=None, saas_files=None)
+    print(f"Current states for {publisher_id=}")
+    print(promotion_state.get(channel=channel, sha=sha).get(publisher_id, None))
+    print()
+    print("Pushing new state ...")
+    promotion_state.set_successful(channel=channel, sha=sha, publisher_uid=publisher_id)
+    print()
+    print(f"New state for {publisher_id=}")
+    print(promotion_state.get(channel=channel, sha=sha).get(publisher_id, None))
+
+
 @root.command()
 @click.option("--change-type-name")
 @click.option("--role-name")
@@ -3770,5 +3818,45 @@ def remove(ctx, sso_client_vault_secret_path: str):
     )
 
 
+@root.group()
+@click.pass_context
+def external_resources(ctx):
+    """External resources commands"""
+
+
+@external_resources.command()
+@click.argument("provision-provider", required=True)
+@click.argument("provisioner", required=True)
+@click.argument("provider", required=True)
+@click.argument("identifier", required=True)
+@click.pass_context
+def get_input(
+    ctx, provision_provider: str, provisioner: str, provider: str, identifier: str
+):
+    """Gets the input data for an external resource asset. Input data is what is used
+    in the Reconciliation Job to manage the resource.
+
+    e.g: qontract-reconcile --config=<config> external-resources get-input aws app-sre-stage rds dashdotdb-stage
+    """
+    namespaces = [ns for ns in get_namespaces() if ns.external_resources]
+    er_inventory = ExternalResourcesInventory(namespaces)
+
+    spec = er_inventory.get_inventory_spec(
+        provision_provider=provision_provider,
+        provisioner=provisioner,
+        provider=provider,
+        identifier=identifier,
+    )
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    er_settings = get_settings()[0]
+    m_inventory = load_module_inventory(get_modules())
+    factories = setup_factories(er_settings, m_inventory, er_inventory, secret_reader)
+    f = factories.get_factory(spec.provision_provider)
+    resource = f.create_external_resource(spec)
+    f.validate_external_resource(resource)
+    print(resource.json(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}}))
+
+
 if __name__ == "__main__":
     root()  # pylint: disable=no-value-for-parameter

tools/saas_promotion_state/saas_promotion_state.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from reconcile.openshift_saas_deploy import (
+    QONTRACT_INTEGRATION as OPENSHIFT_SAAS_DEPLOY,
+)
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.saas_files import SaasFile, get_saas_files
+from reconcile.utils.promotion_state import PromotionData, PromotionState
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.state import init_state
+
+
+class SaasPromotionStateException(Exception):
+    pass
+
+
+class SaasPromotionStateMissingException(Exception):
+    pass
+
+
+class SaasPromotionState:
+    def __init__(
+        self, promotion_state: PromotionState, saas_files: Iterable[SaasFile]
+    ) -> None:
+        self._promotion_state = promotion_state
+        self._saas_files = saas_files
+
+    def _publisher_ids_for_channel(
+        self, channel: str, saas_files: Iterable[SaasFile]
+    ) -> list[str]:
+        publisher_uids: list[str] = []
+        for saas_file in saas_files:
+            for resource_template in saas_file.resource_templates:
+                for target in resource_template.targets:
+                    if not target.promotion:
+                        continue
+                    for publish_channel in target.promotion.publish or []:
+                        if publish_channel == channel:
+                            publisher_uids.append(
+                                target.uid(
+                                    parent_saas_file_name=saas_file.name,
+                                    parent_resource_template_name=resource_template.name,
+                                )
+                            )
+        return publisher_uids
+
+    def get(self, channel: str, sha: str) -> dict[str, PromotionData | None]:
+        return {
+            publisher_id: self._promotion_state.get_promotion_data(
+                sha=sha,
+                channel=channel,
+                use_cache=False,
+                target_uid=publisher_id,
+                pre_check_sha_exists=False,
+            )
+            for publisher_id in self._publisher_ids_for_channel(
+                channel=channel, saas_files=self._saas_files
+            )
+        }
+
+    def set_successful(self, channel: str, sha: str, publisher_uid: str) -> None:
+        current_data = self._promotion_state.get_promotion_data(
+            sha=sha,
+            channel=channel,
+            target_uid=publisher_uid,
+            use_cache=False,
+            pre_check_sha_exists=False,
+        )
+
+        if not current_data:
+            raise SaasPromotionStateMissingException(
+                f"No promotion state in S3 for given {publisher_uid=} {sha=} {channel=}"
+            )
+
+        if current_data.success:
+            raise SaasPromotionStateException(
+                f"The current promotion state is already marked successful for given {publisher_uid=} {sha=} {channel=}",
+                current_data,
+            )
+
+        current_data.success = True
+        self._promotion_state.publish_promotion_data(
+            data=current_data, sha=sha, channel=channel, target_uid=publisher_uid
+        )
+
+    @staticmethod
+    def create(
+        promotion_state: PromotionState | None, saas_files: Iterable[SaasFile] | None
+    ) -> SaasPromotionState:
+        if not promotion_state:
+            vault_settings = get_app_interface_vault_settings()
+            secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+            saas_deploy_state = init_state(
+                integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
+            )
+            promotion_state = PromotionState(state=saas_deploy_state)
+        if not saas_files:
+            saas_files = get_saas_files()
+        return SaasPromotionState(
+            promotion_state=promotion_state, saas_files=saas_files
+        )

tools/template_validation.py

@@ -27,7 +27,7 @@ def load_clean_yaml(path: str) -> dict:
 
 def load_yaml(to_load: str) -> dict:
     ruamel_instance = create_ruamel_instance()
-    with open(to_load, "r", encoding="utf-8") as file:
+    with open(to_load, encoding="utf-8") as file:
         return ruamel_instance.load(file)
 
 

tools/test/conftest.py

@@ -1,16 +1,17 @@
 from collections.abc import (
     Callable,
+    Iterable,
+    Mapping,
     MutableMapping,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from pathlib import Path
+from typing import Any
 
 import pytest
 from pydantic import BaseModel
 from pydantic.error_wrappers import ValidationError
 
+from reconcile.typed_queries.saas_files import SaasFile
 from reconcile.utils.models import data_default_none
 
 
@@ -18,17 +19,55 @@ class GQLClassFactoryError(Exception):
     pass
 
 
+@pytest.fixture
+def saas_files_builder(
+    gql_class_factory: Callable[[type[SaasFile], Mapping], SaasFile],
+) -> Callable[[Iterable[MutableMapping]], list[SaasFile]]:
+    def builder(data: Iterable[MutableMapping]) -> list[SaasFile]:
+        for d in data:
+            if "app" not in d:
+                d["app"] = {}
+            if "pipelinesProvider" not in d:
+                d["pipelinesProvider"] = {}
+            if "managedResourceTypes" not in d:
+                d["managedResourceTypes"] = []
+            if "imagePatterns" not in d:
+                d["imagePatterns"] = []
+            for rt in d.get("resourceTemplates", []):
+                for t in rt.get("targets", []):
+                    ns = t["namespace"]
+                    if "name" not in ns:
+                        ns["name"] = "some_name"
+                    if "environment" not in ns:
+                        ns["environment"] = {}
+                    if "app" not in ns:
+                        ns["app"] = {}
+                    if "cluster" not in ns:
+                        ns["cluster"] = {}
+        return [gql_class_factory(SaasFile, d) for d in data]
+
+    return builder
+
+
+@pytest.fixture
+def fx() -> Callable:
+    def _fx(name: str) -> str:
+        return (Path(__file__).parent / "fixtures" / name).read_text()
+
+    return _fx
+
+
 @pytest.fixture
 def gql_class_factory() -> (
     Callable[
-        [type[BaseModel], Optional[MutableMapping[str, Any]]],
+        [type[BaseModel], MutableMapping[str, Any] | None],
         BaseModel,
     ]
 ):
     """Create a GQL class from a fixture and set default values to None."""
 
     def _gql_class_factory(
-        klass: type[BaseModel], data: Optional[MutableMapping[str, Any]] = None
+        klass: type[BaseModel], data: MutableMapping[str, Any] | None = None
     ) -> BaseModel:
         try:
             return klass(**data_default_none(klass, data or {}))

tools/test/test_saas_promotion_state.py

@@ -0,0 +1,187 @@
+from collections.abc import (
+    Callable,
+    Iterable,
+    Mapping,
+)
+from unittest.mock import (
+    create_autospec,
+)
+
+from pytest import raises
+
+from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.utils.promotion_state import PromotionData, PromotionState
+from tools.saas_promotion_state.saas_promotion_state import (
+    SaasPromotionState,
+    SaasPromotionStateException,
+    SaasPromotionStateMissingException,
+)
+
+
+def test_get_saas_promotion_state(
+    saas_files_builder: Callable[[Iterable[Mapping]], list[SaasFile]],
+) -> None:
+    saas_files = saas_files_builder([
+        {
+            "path": "/saas1.yml",
+            "name": "saas_1",
+            "resourceTemplates": [
+                {
+                    "name": "template_1",
+                    "url": "repo1/url",
+                    "targets": [
+                        {
+                            "ref": "main",
+                            "namespace": {"path": "/namespace1.yml"},
+                            "promotion": {
+                                "publish": ["channel-a"],
+                            },
+                        }
+                    ],
+                }
+            ],
+        },
+        {
+            "path": "/saas2.yml",
+            "name": "saas_2",
+            "resourceTemplates": [
+                {
+                    "name": "template_2",
+                    "url": "repo2/url",
+                    "targets": [
+                        {
+                            "ref": "main",
+                            "namespace": {"path": "/namespace2.yml"},
+                            "promotion": {
+                                "publish": ["channel-b"],
+                                "subscribe": ["channel-a"],
+                            },
+                        },
+                        {
+                            "ref": "main",
+                            "namespace": {"path": "/namespace3.yml"},
+                        },
+                    ],
+                }
+            ],
+        },
+    ])
+
+    expected = PromotionData(
+        check_in="test1",
+        saas_file="test2",
+        success=True,
+        target_config_hash="test3",
+    )
+    promotion_state = create_autospec(spec=PromotionState)
+    promotion_state.get_promotion_data.return_value = expected
+    saas_promotion_state = SaasPromotionState.create(
+        promotion_state=promotion_state, saas_files=saas_files
+    )
+    result = saas_promotion_state.get(channel="channel-a", sha="main")
+
+    assert result == {"616af45d7fad7f4eea8d52b8b5e8a058cef82ab0": expected}
+    promotion_state.get_promotion_data.assert_called_once_with(
+        sha="main",
+        channel="channel-a",
+        use_cache=False,
+        target_uid="616af45d7fad7f4eea8d52b8b5e8a058cef82ab0",
+        pre_check_sha_exists=False,
+    )
+
+
+def test_set_saas_promotion_state_success(
+    saas_files_builder: Callable[[Iterable[Mapping]], list[SaasFile]],
+) -> None:
+    saas_files = saas_files_builder([{"resourceTemplates": []}])
+
+    current_data = PromotionData(
+        check_in="test1",
+        saas_file="test2",
+        success=False,
+        target_config_hash="test3",
+    )
+    promotion_state = create_autospec(spec=PromotionState)
+    promotion_state.get_promotion_data.return_value = current_data
+    saas_promotion_state = SaasPromotionState.create(
+        promotion_state=promotion_state, saas_files=saas_files
+    )
+    saas_promotion_state.set_successful(
+        channel="test-channel", sha="test-sha", publisher_uid="test-uid"
+    )
+
+    promotion_state.get_promotion_data.assert_called_once_with(
+        sha="test-sha",
+        channel="test-channel",
+        use_cache=False,
+        target_uid="test-uid",
+        pre_check_sha_exists=False,
+    )
+    promotion_state.publish_promotion_data.assert_called_once_with(
+        data=PromotionData(
+            check_in="test1",
+            saas_file="test2",
+            success=True,
+            target_config_hash="test3",
+        ),
+        channel="test-channel",
+        sha="test-sha",
+        target_uid="test-uid",
+    )
+
+
+def test_set_saas_promotion_state_missing(
+    saas_files_builder: Callable[[Iterable[Mapping]], list[SaasFile]],
+) -> None:
+    saas_files = saas_files_builder([{"resourceTemplates": []}])
+    promotion_state = create_autospec(spec=PromotionState)
+    promotion_state.get_promotion_data.return_value = None
+    saas_promotion_state = SaasPromotionState.create(
+        promotion_state=promotion_state, saas_files=saas_files
+    )
+
+    with raises(SaasPromotionStateMissingException):
+        saas_promotion_state.set_successful(
+            channel="test-channel", sha="test-sha", publisher_uid="test-uid"
+        )
+
+    promotion_state.get_promotion_data.assert_called_once_with(
+        sha="test-sha",
+        channel="test-channel",
+        use_cache=False,
+        target_uid="test-uid",
+        pre_check_sha_exists=False,
+    )
+    promotion_state.publish_promotion_data.assert_not_called()
+
+
+def test_set_saas_promotion_state_already_successful(
+    saas_files_builder: Callable[[Iterable[Mapping]], list[SaasFile]],
+) -> None:
+    saas_files = saas_files_builder([{"resourceTemplates": []}])
+
+    current_data = PromotionData(
+        check_in="test1",
+        saas_file="test2",
+        success=True,
+        target_config_hash="test3",
+    )
+    promotion_state = create_autospec(spec=PromotionState)
+    promotion_state.get_promotion_data.return_value = current_data
+    saas_promotion_state = SaasPromotionState.create(
+        promotion_state=promotion_state, saas_files=saas_files
+    )
+
+    with raises(SaasPromotionStateException):
+        saas_promotion_state.set_successful(
+            channel="test-channel", sha="test-sha", publisher_uid="test-uid"
+        )
+
+    promotion_state.get_promotion_data.assert_called_once_with(
+        sha="test-sha",
+        channel="test-channel",
+        use_cache=False,
+        target_uid="test-uid",
+        pre_check_sha_exists=False,
+    )
+    promotion_state.publish_promotion_data.assert_not_called()