qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/test/test_wrong_region.py

@@ -1,78 +0,0 @@
-import pytest
-
-from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.terrascript_aws_client import TerrascriptClient
-
-
-@pytest.fixture
-def default_region():
-    return "default-region"
-
-
-@pytest.fixture
-def tf_bucket_region():
-    return "tf-bucket-region"
-
-
-@pytest.fixture
-def accounts(default_region):
-    return [
-        {
-            "name": "some-account",
-            "automationToken": {
-                "path": "path",
-            },
-            "resourcesDefaultRegion": default_region,
-            "supportedDeploymentRegions": [default_region],
-            "providerVersion": "1.2.3",
-            "uid": "123",
-            "terraformState": {},
-        }
-    ]
-
-
-@pytest.fixture
-def secret(tf_bucket_region):
-    return {
-        "aws_access_key_id": "key_id",
-        "aws_secret_access_key": "access_key",
-        "region": tf_bucket_region,
-        "bucket": "tf-bucket-name",
-        "_key": "tf_key.tfstate",
-    }
-
-
-@pytest.fixture
-def aws_api(accounts, secret, mocker):
-    mock_secret_reader = mocker.patch(
-        "reconcile.utils.aws_api.SecretReader", autospec=True
-    )
-    mock_secret_reader.return_value.read_all.return_value = secret
-    return AWSApi(1, accounts, init_users=False)
-
-
-@pytest.fixture
-def terrascript(accounts, secret, mocker):
-    mock_secret_reader = mocker.patch(
-        "reconcile.utils.terrascript_aws_client.SecretReader", autospec=True
-    )
-    mock_secret_reader.return_value.read_all.return_value = secret
-    return TerrascriptClient("", "", 1, accounts)
-
-
-def test_wrong_region_aws_api(aws_api, accounts, default_region):
-    for a in accounts:
-        assert aws_api.sessions[a["name"]].region_name == default_region
-
-
-def test_wrong_region_terrascript(terrascript, accounts, tf_bucket_region):
-    for a in accounts:
-        assert terrascript.configs[a["name"]]["region"] == tf_bucket_region
-
-
-def test_wrong_region_both(
-    aws_api, terrascript, accounts, default_region, tf_bucket_region
-):
-    for a in accounts:
-        assert aws_api.sessions[a["name"]].region_name == default_region
-        assert terrascript.configs[a["name"]]["region"] == tf_bucket_region

reconcile/typed_queries/glitchtip_settings.py

@@ -1,18 +0,0 @@
-from reconcile.gql_definitions.glitchtip.glitchtip_settings import query
-from reconcile.utils import gql
-
-
-def get_glitchtip_settings(
-    read_timeout: int = 30, max_retries: int = 3, mail_domain: str = "redhat.com"
-) -> tuple[int, int, str]:
-    """Returns Glitchtip Settings."""
-    gqlapi = gql.get_api()
-    if _s := query(query_func=gqlapi.query).settings:
-        if _gs := _s[0].glitchtip:
-            if _gs.read_timeout is not None:
-                read_timeout = _gs.read_timeout
-            if _gs.max_retries is not None:
-                max_retries = _gs.max_retries
-            if _gs.mail_domain is not None:
-                mail_domain = _gs.mail_domain
-    return read_timeout, max_retries, mail_domain

reconcile/typed_queries/ocp_release_mirror.py

@@ -1,11 +0,0 @@
-from reconcile.gql_definitions.ocp_release_mirror.ocp_release_mirror import (
-    OcpReleaseMirrorV1,
-    query,
-)
-from reconcile.utils import gql
-
-
-def get_ocp_release_mirrors() -> list[OcpReleaseMirrorV1]:
-    gqlapi = gql.get_api()
-    data = query(gqlapi.query)
-    return list(data.ocp_release_mirror or [])

reconcile/unleash_watcher.py

@@ -1,120 +0,0 @@
-import logging
-
-from reconcile import queries
-from reconcile.slack_base import slackapi_from_slack_workspace
-from reconcile.utils.defer import defer
-from reconcile.utils.secret_reader import SecretReader
-from reconcile.utils.slack_api import SlackApi
-from reconcile.utils.state import init_state
-from reconcile.utils.unleash import get_feature_toggles
-
-QONTRACT_INTEGRATION = "unleash-watcher"
-
-
-def fetch_current_state(unleash_instance, secret_reader: SecretReader):
-    api_url = f"{unleash_instance['url']}/api"
-    admin_access_token = secret_reader.read(unleash_instance["token"])
-    return get_feature_toggles(api_url, admin_access_token)
-
-
-def fetch_previous_state(state, instance_name):
-    return state.get_all(instance_name)
-
-
-def format_message(url, key, event, previous_state=None, current_state=None):
-    info = (
-        ": {} -> {}".format(previous_state, current_state)
-        if previous_state and current_state
-        else ""
-    )
-    return "{} {} {}{}".format(url, key, event, info)
-
-
-def calculate_diff(current_state, previous_state):
-    diffs = []
-
-    for toggle, current_value in current_state.items():
-        # new toggles
-        if toggle not in previous_state:
-            diff = {"event": "created", "toggle": toggle, "to": current_value}
-            diffs.append(diff)
-        # updated toggles
-        else:
-            previous_value = previous_state[toggle]
-            if current_value != previous_value:
-                diff = {
-                    "event": "updated",
-                    "toggle": toggle,
-                    "from": previous_value,
-                    "to": current_value,
-                }
-                diffs.append(diff)
-
-    # deleted toggles
-    for toggle in previous_state:
-        if toggle not in current_state:
-            diff = {"event": "deleted", "toggle": toggle}
-            diffs.append(diff)
-
-    return diffs
-
-
-def init_slack_map(
-    unleash_instance, secret_reader: SecretReader
-) -> dict[str, SlackApi]:
-    return {
-        slack_info["channel"]: slackapi_from_slack_workspace(
-            slack_info, secret_reader, QONTRACT_INTEGRATION, init_usergroups=False
-        )
-        for slack_info in unleash_instance["notifications"]["slack"]
-    }
-
-
-def act(dry_run, state, unleash_instance, diffs, secret_reader: SecretReader):
-    if not dry_run and diffs:
-        slack_notifications = unleash_instance.get(
-            "notifications"
-        ) and unleash_instance["notifications"].get("slack")
-        if not slack_notifications:
-            return
-        slack_map = init_slack_map(unleash_instance, secret_reader)
-
-    for diff in reversed(diffs):
-        event = diff["event"]
-        toggle = diff["toggle"]
-
-        msg = f"Feature toggle {toggle} {event}"
-        if event == "updated":
-            msg += f": {diff['from']} -> {diff['to']}"
-        logging.info(msg)
-        if not dry_run:
-            for slack in slack_map.values():
-                slack.chat_post_message(msg)
-            key = f"{unleash_instance['name']}/{toggle}"
-            if event == "created":
-                state.add(key, diff["to"])
-            elif event == "deleted":
-                state.rm(key)
-            elif event == "updated":
-                state.add(key, diff["to"], force=True)
-
-
-@defer
-def run(dry_run, defer=None):
-    secret_reader = SecretReader(settings=queries.get_secret_reader_settings())
-    unleash_instances = queries.get_unleash_instances()
-    state = init_state(QONTRACT_INTEGRATION, secret_reader)
-    defer(state.cleanup)
-    for unleash_instance in unleash_instances:
-        instance_name = unleash_instance["name"]
-        current_state = fetch_current_state(unleash_instance, secret_reader)
-        if not current_state:
-            logging.warning(
-                "not acting on empty Unleash instances. "
-                + "please create a feature toggle to get started."
-            )
-            continue
-        previous_state = fetch_previous_state(state, instance_name)
-        diffs = calculate_diff(current_state, previous_state)
-        if diffs:
-            act(dry_run, state, unleash_instance, diffs, secret_reader)

reconcile/utils/git_secrets.py

@@ -1,63 +0,0 @@
-import logging
-import os
-import subprocess
-import tempfile
-
-import requests
-from sretoolbox.utils import retry
-
-from reconcile.utils import git
-
-
-@retry()
-def scan_history(repo_url, existing_keys):
-    logging.info("scanning {}".format(repo_url))
-    if requests.get(repo_url, timeout=60).status_code == 404:
-        logging.info("not found {}".format(repo_url))
-        return []
-
-    with tempfile.TemporaryDirectory() as wd:
-        git.clone(repo_url, wd)
-        subprocess.run(["git", "secrets", "--install"], check=False, cwd=wd)
-        result = subprocess.run(
-            ["git", "secrets", "--scan-history"],
-            capture_output=True,
-            check=False,
-            cwd=wd,
-        )
-        if result.returncode == 0:
-            return []
-        logging.info("found suspects in {}".format(repo_url))
-        suspected_files = get_suspected_files(result.stderr.decode("utf-8"))
-        leaked_keys = get_leaked_keys(wd, suspected_files, existing_keys)
-        if leaked_keys:
-            logging.info("found suspected leaked keys: {}".format(leaked_keys))
-        return leaked_keys
-
-
-def get_suspected_files(error):
-    suspects = []
-    for e in error.split("\n"):
-        if e == "":
-            break
-        if e.startswith("warning"):
-            continue
-        commit_path_split = e.split(" ")[0].split(":")
-        commit, path = commit_path_split[0], commit_path_split[1]
-
-        suspects.append((commit, path))
-    return set(suspects)
-
-
-def get_leaked_keys(repo_wd, suspected_files, existing_keys):
-    all_leaked_keys = []
-    for s in suspected_files:
-        commit, file_relative_path = s[0], s[1]
-        git.checkout(commit, repo_wd)
-        file_path = os.path.join(repo_wd, file_relative_path)
-        with open(file_path, "r") as f:
-            content = f.read()
-        leaked_keys = [key for key in existing_keys if key in content]
-        all_leaked_keys.extend(leaked_keys)
-
-    return all_leaked_keys

reconcile/utils/mr/auto_promoter.py

@@ -1,218 +0,0 @@
-import hashlib
-import json
-import logging
-from collections.abc import (
-    Iterable,
-    MutableMapping,
-)
-from dataclasses import (
-    asdict,
-    dataclass,
-)
-from typing import (
-    Any,
-    Sequence,
-    Union,
-)
-
-from ruamel import yaml
-
-from reconcile.utils.gitlab_api import GitLabApi
-from reconcile.utils.mr.base import MergeRequestBase
-from reconcile.utils.mr.labels import AUTO_MERGE
-from reconcile.utils.saasherder.interfaces import SaasPromotion
-from reconcile.utils.saasherder.models import Promotion
-
-LOG = logging.getLogger(__name__)
-
-
-@dataclass
-class ParentSaasConfigPromotion:
-    TYPE = "parent_saas_config"
-    parent_saas: str
-    target_config_hash: str
-    type: str = TYPE
-
-
-class AutoPromoter(MergeRequestBase):
-    name = "auto_promoter"
-
-    def __init__(
-        self, promotions: Union[Sequence[SaasPromotion], Sequence[dict[str, Any]]]
-    ):
-        # !!! Attention !!!
-        # AutoPromoter is also initialized with promitions as dict by 'gitlab_mr_sqs_consumer'
-        # loaded from SQS message body, therefore self.promotions must be json serializable
-        self.promotions = [
-            p.dict(by_alias=True) if isinstance(p, SaasPromotion) else p
-            for p in promotions
-        ]
-
-        # the parent class stores self.promotions (the json serializable one) in self.sqs_msg_data
-        super().__init__()
-        # create an internal list with Promotion objects out of self.promotions
-        self._promotions = [Promotion(**p) for p in self.promotions]
-        self.labels = [AUTO_MERGE]
-
-    @property
-    def title(self) -> str:
-        """
-        to make the MR title unique, add a sha256sum of the promotions to it
-        TODO: while adding a digest ensures uniqueness, this title is
-              still not very descriptive
-        """
-        m = hashlib.sha256()
-        m.update(json.dumps(self.promotions, sort_keys=True).encode("utf-8"))
-        digest = m.hexdigest()[:6]
-        return f"[{self.name}] openshift-saas-deploy automated " f"promotion {digest}"
-
-    @property
-    def description(self) -> str:
-        return "openshift-saas-deploy automated promotion"
-
-    @staticmethod
-    def init_promotion_data(channel: str, promotion: SaasPromotion) -> dict[str, Any]:
-        psc = ParentSaasConfigPromotion(
-            parent_saas=promotion.saas_file,
-            target_config_hash=promotion.target_config_hash,
-        )
-        return {"channel": channel, "data": [asdict(psc)]}
-
-    @staticmethod
-    def process_promotion(
-        promotion: SaasPromotion,
-        target_promotion: MutableMapping[str, Any],
-        target_channels: Iterable[str],
-    ) -> bool:
-        # Existent subscribe data channel data
-        promotion_data = {
-            v["channel"]: v["data"]
-            for v in target_promotion.get("promotion_data", [])
-            if v["channel"] in target_channels
-        }
-
-        if not promotion_data:
-            target_promotion["promotion_data"] = []
-
-        modified = False
-        for channel in target_channels:
-            channel_data = promotion_data.get(channel)
-            if channel_data is None:
-                channel_data = AutoPromoter.init_promotion_data(channel, promotion)
-                target_promotion["promotion_data"].append(channel_data)
-                modified = True
-            else:
-                for i, item in enumerate(channel_data):
-                    if item["type"] == ParentSaasConfigPromotion.TYPE:
-                        target_psc = ParentSaasConfigPromotion(**item)
-                        promotion_psc = ParentSaasConfigPromotion(
-                            parent_saas=promotion.saas_file,
-                            target_config_hash=promotion.target_config_hash,
-                        )
-                        if target_psc != promotion_psc:
-                            channel_data[i] = asdict(promotion_psc)
-                            modified = True
-
-        return modified
-
-    def process_target(
-        self, target: MutableMapping[str, Any], promotion: SaasPromotion
-    ) -> bool:
-        target_updated = False
-        target_promotion = target.get("promotion")
-        if not target_promotion:
-            return target_updated
-        target_auto = target_promotion.get("auto")
-        if not target_auto:
-            return target_updated
-        subscribe = target_promotion.get("subscribe")
-        if not subscribe:
-            return target_updated
-        if not promotion.publish:
-            return target_updated
-
-        channels = [c for c in subscribe if c in promotion.publish]
-        if channels:
-            # Update REF on target if differs.
-            if target["ref"] != promotion.commit_sha:
-                target["ref"] = promotion.commit_sha
-                target_updated = True
-
-            # Update Promotion data
-            modified = self.process_promotion(promotion, target_promotion, channels)
-
-            if modified:
-                target_updated = True
-
-        return target_updated
-
-    def process(self, gitlab_cli: GitLabApi) -> None:
-        for promotion in self._promotions:
-            if not promotion.publish:
-                continue
-            if not promotion.commit_sha:
-                continue
-            for saas_file_path in promotion.saas_file_paths or []:
-                saas_file_updated = False
-                try:
-                    # This will only work with gitlab cli, not with SQS
-                    # this method is only triggered by gitlab_sqs_consumer
-                    # not by openshift_saas_deploy
-                    raw_file = gitlab_cli.project.files.get(
-                        file_path=saas_file_path, ref=self.branch
-                    )
-                except Exception as e:
-                    logging.error(e)
-
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
-
-                for rt in content["resourceTemplates"]:
-                    for target in rt["targets"]:
-                        if self.process_target(target, promotion):
-                            saas_file_updated = True
-
-                if saas_file_updated:
-                    new_content = "---\n"
-                    new_content += yaml.dump(content, Dumper=yaml.RoundTripDumper) or ""
-                    msg = f"auto promote {promotion.commit_sha} in {saas_file_path}"
-                    gitlab_cli.update_file(
-                        branch_name=self.branch,
-                        file_path=saas_file_path,
-                        commit_message=msg,
-                        content=new_content,
-                    )
-                else:
-                    LOG.info(
-                        f"commit sha {promotion.commit_sha} has already been "
-                        f"promoted to all targets in {content['name']} "
-                        f"subscribing to {','.join(promotion.publish)}"
-                    )
-
-            for target_path in promotion.target_paths or []:
-                try:
-                    # This will only work with gitlab cli, not with SQS
-                    # this method is only triggered by gitlab_sqs_consumer
-                    # not by openshift_saas_deploy
-                    raw_file = gitlab_cli.project.files.get(
-                        file_path=target_path, ref=self.branch
-                    )
-                except Exception as e:
-                    logging.error(e)
-
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
-                if self.process_target(content, promotion):
-                    new_content = "---\n"
-                    new_content += yaml.dump(content, Dumper=yaml.RoundTripDumper)  # type: ignore
-                    msg = f"auto promote {promotion.commit_sha} in {target_path}"
-                    gitlab_cli.update_file(
-                        branch_name=self.branch,
-                        file_path=target_path,
-                        commit_message=msg,
-                        content=new_content,
-                    )
-                else:
-                    LOG.info(
-                        f"commit sha {promotion.commit_sha} has already been "
-                        f"promoted to all targets in {content['name']} "
-                        f"subscribing to {','.join(promotion.publish)}"
-                    )