qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/vcs.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+
+from gitlab.v4.objects import ProjectMergeRequest
+
+from reconcile.typed_queries.github_orgs import GithubOrgV1
+from reconcile.typed_queries.gitlab_instances import GitlabInstanceV1
+from reconcile.utils.github_api import GithubRepositoryApi
+from reconcile.utils.gitlab_api import (
+    GitLabApi,
+    MRState,
+)
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.secret_reader import (
+    HasSecret,
+    SecretReaderBase,
+)
+
+
+class MRCheckStatus(Enum):
+    NONE = 0
+    SUCCESS = 1
+    FAILED = 2
+    RUNNING = 3
+
+
+@dataclass(order=True)
+class Commit:
+    repo: str
+    sha: str
+    date: datetime
+
+
+class VCSMissingSourceBranchError(Exception):
+    pass
+
+
+class VCS:
+    """
+    Abstraction layer for aggregating different Version Control Systems.
+    The idea is to abstract away the differences between
+    Gitlab and Github for fetching publisher state.
+    Further this acts as a wrapper around our gitlab client
+    for interactions with the app-interface repo. That makes
+    setting up tests easier.
+    """
+
+    def __init__(
+        self,
+        secret_reader: SecretReaderBase,
+        github_orgs: Iterable[GithubOrgV1],
+        gitlab_instances: Iterable[GitlabInstanceV1],
+        app_interface_repo_url: str,
+        dry_run: bool,
+        allow_deleting_mrs: bool = False,
+        allow_opening_mrs: bool = False,
+        gitlab_instance: GitLabApi | None = None,
+        default_gh_token: str | None = None,
+        app_interface_api: GitLabApi | None = None,
+        github_api_per_repo_url: dict[str, GithubRepositoryApi] | None = None,
+    ):
+        self._dry_run = dry_run
+        self._allow_deleting_mrs = allow_deleting_mrs
+        self._allow_opening_mrs = allow_opening_mrs
+        self._secret_reader = secret_reader
+        self._gh_per_repo_url: dict[str, GithubRepositoryApi] = (
+            github_api_per_repo_url if github_api_per_repo_url else {}
+        )
+        self._default_gh_token = (
+            default_gh_token
+            if default_gh_token
+            else self._get_default_gh_token(github_orgs=github_orgs)
+        )
+        self._gitlab_instance = (
+            gitlab_instance
+            if gitlab_instance
+            else self._gitlab_api(gitlab_instances=gitlab_instances)
+        )
+        self._app_interface_api = (
+            app_interface_api
+            if app_interface_api
+            else self._init_app_interface_api(
+                gitlab_instances=gitlab_instances,
+                app_interface_repo_url=app_interface_repo_url,
+            )
+        )
+        self._is_commit_sha_regex = re.compile(r"^[0-9a-f]{40}$")
+
+    def _get_default_gh_token(
+        self,
+        github_orgs: Iterable[GithubOrgV1],
+    ) -> str:
+        defaults: list[str] = []
+        for org in github_orgs:
+            if not org.default:
+                continue
+            token = self._secret_reader.read_secret(org.token)
+            defaults.append(token)
+        if len(defaults) == 0:
+            raise RuntimeError("No default GitHub token found.")
+        if len(defaults) > 1:
+            raise RuntimeError("More than 1 default token for GitHub found.")
+        return defaults[0]
+
+    def _init_github(
+        self, repo_url: str, auth_code: HasSecret | None
+    ) -> GithubRepositoryApi:
+        if repo_url not in self._gh_per_repo_url:
+            if auth_code:
+                token = self._secret_reader.read_secret(auth_code)
+            else:
+                token = self._default_gh_token
+            self._gh_per_repo_url[repo_url] = self._github_api(
+                token=token, repo_url=repo_url
+            )
+        return self._gh_per_repo_url[repo_url]
+
+    def _github_api(self, token: str, repo_url: str) -> GithubRepositoryApi:
+        return GithubRepositoryApi(repo_url=repo_url, token=token)
+
+    def _gitlab_api(
+        self,
+        gitlab_instances: Iterable[GitlabInstanceV1],
+    ) -> GitLabApi:
+        return GitLabApi(
+            next(iter(gitlab_instances)).dict(by_alias=True),
+            secret_reader=self._secret_reader,
+        )
+
+    def _init_app_interface_api(
+        self,
+        gitlab_instances: Iterable[GitlabInstanceV1],
+        app_interface_repo_url: str,
+    ) -> GitLabApi:
+        return GitLabApi(
+            next(iter(gitlab_instances)).dict(by_alias=True),
+            secret_reader=self._secret_reader,
+            project_url=app_interface_repo_url,
+        )
+
+    def get_gitlab_mr_check_status(self, mr: ProjectMergeRequest) -> MRCheckStatus:
+        pipelines = self._gitlab_instance.get_merge_request_pipelines(mr)
+        if not pipelines:
+            return MRCheckStatus.NONE
+        # available status codes https://docs.gitlab.com/ee/api/pipelines.html
+        last_pipeline_result = pipelines[0]["status"]
+        match last_pipeline_result:
+            case "success":
+                return MRCheckStatus.SUCCESS
+            case "running":
+                return MRCheckStatus.RUNNING
+            case "failed":
+                return MRCheckStatus.FAILED
+            case _:
+                # Lets assume all other states as non-present
+                return MRCheckStatus.NONE
+
+    def get_commit_sha(
+        self, repo_url: str, ref: str, auth_code: HasSecret | None
+    ) -> str:
+        if bool(self._is_commit_sha_regex.search(ref)):
+            return ref
+        if repo_url.startswith("https://github.com/"):
+            github = self._init_github(repo_url=repo_url, auth_code=auth_code)
+            return github.get_commit_sha(ref=ref)
+        # assume gitlab by default
+        return self._gitlab_instance.get_commit_sha(ref=ref, repo_url=repo_url)
+
+    def get_commits_between(
+        self,
+        repo_url: str,
+        commit_from: str,
+        commit_to: str,
+        auth_code: HasSecret | None,
+    ) -> list[Commit]:
+        """
+        Return a list of commits between two commits.
+        Note, that the commit_to is included in the result list, whereas commit_from is not included.
+        """
+        if repo_url.startswith("https://github.com/"):
+            github = self._init_github(repo_url=repo_url, auth_code=auth_code)
+            data = github.compare(commit_from=commit_from, commit_to=commit_to)
+            return [
+                Commit(
+                    repo=repo_url,
+                    sha=gh_commit.sha,
+                    date=gh_commit.commit.committer.date,
+                )
+                for gh_commit in data
+            ]
+        # assume gitlab by default
+        else:
+            data = self._gitlab_instance.repository_compare(
+                repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
+            )
+            return [
+                Commit(
+                    repo=repo_url,
+                    sha=gl_commit["id"],
+                    date=datetime.fromisoformat(gl_commit["committed_date"]),
+                )
+                for gl_commit in data
+            ]
+
+    def close_app_interface_mr(self, mr: ProjectMergeRequest, comment: str) -> None:
+        if not self._allow_deleting_mrs:
+            logging.info("Deleting MRs is disabled. Skipping.")
+        if not self._dry_run and self._allow_deleting_mrs:
+            self._app_interface_api.add_comment_to_merge_request(
+                merge_request=mr,
+                body=comment,
+            )
+            source_branch = mr.attributes.get("source_branch")
+            if not source_branch:
+                raise VCSMissingSourceBranchError(
+                    f"Source branch is missing for MR {mr.attributes.get('iid')}"
+                )
+            self._app_interface_api.close(mr)
+            self._app_interface_api.delete_branch(source_branch)
+
+    def get_file_content_from_app_interface_master(self, file_path: str) -> str:
+        file_path = (
+            f"data/{file_path.lstrip('/')}"
+            if not file_path.startswith("data")
+            else file_path
+        )
+        return self._app_interface_api.project.files.get(
+            file_path=file_path, ref="master"
+        ).decode()
+
+    def get_open_app_interface_merge_requests(self) -> list[ProjectMergeRequest]:
+        return self._app_interface_api.get_merge_requests(state=MRState.OPENED)
+
+    def open_app_interface_merge_request(self, mr: MergeRequestBase) -> None:
+        if not self._allow_opening_mrs:
+            logging.info("Creating MRs is disabled. Skipping.")
+        if not self._dry_run and self._allow_opening_mrs:
+            mr.submit_to_gitlab(gitlab_cli=self._app_interface_api)
+
+    def cleanup(self) -> None:
+        for gh_client in self._gh_per_repo_url.values():
+            gh_client.cleanup()
+        self._gitlab_instance.cleanup()
+        self._app_interface_api.cleanup()

reconcile/vault_replication.py

@@ -2,8 +2,6 @@ import logging
 import re
 from collections.abc import Iterable
 from typing import (
-    Optional,
-    Union,
     cast,
 )
 
@@ -36,6 +34,7 @@ from reconcile.utils.vault import (
 )
 
 QONTRACT_INTEGRATION = "vault-replication"
+SECRET_PATH_PATTERN = re.compile(r"^[\w/-]+?(?P<folder>/\*?)?$")
 
 
 class VaultInvalidPaths(Exception):
@@ -100,9 +99,12 @@ def write_dummy_versions(
     that a secret engine stores"""
 
     write_dict = {"path": path, "data": {"dummy": "data"}}
-    logging.info(
-        ["replicate_vault_secret", "generate_dummy_data", secret_version, path]
-    )
+    logging.info([
+        "replicate_vault_secret",
+        "generate_dummy_data",
+        secret_version,
+        path,
+    ])
     if not dry_run:
         # Using force=True to write the dummy data to force the vault client
         # to write the version even if the data is the same as the previous version.
@@ -120,7 +122,7 @@ def copy_vault_secret(
     except SecretAccessForbidden:
         # Raise exception if we can't read the secret from the source vault.
         # This is likely to be related to the approle permissions.
-        raise SecretAccessForbidden("Cannot read secret from source vault")
+        raise SecretAccessForbidden("Cannot read secret from source vault") from None
     except SecretNotFound:
         # If the secret is present in vault, but there are no versions of it
         # we want to be aware of it, but not cause a failure of the complete
@@ -176,7 +178,7 @@ def copy_vault_secret(
 
 def check_invalid_paths(
     path_list: Iterable[str],
-    policy_paths: Optional[Iterable[str]],
+    policy_paths: Iterable[str] | None,
 ) -> None:
     """Checks if the paths to be replicated are present in the policy used to limit the secrets
     that are going to be replicated."""
@@ -231,14 +233,21 @@ def get_policy_secret_list(
     vault_instance: _VaultClient, policy_paths: Iterable[str]
 ) -> list[str]:
     """Returns a list of secrets to be copied from the given policy"""
-    secret_list = []
+    secrets = set()
     for path in policy_paths:
-        # Remove the * at the end of the path because list method expects
-        # a folder path without any secret or wilcard
-        path = path[:-1] if path.endswith("*") else path
-        secret_list.extend(vault_instance.list_all(path))
+        match = SECRET_PATH_PATTERN.match(path)
+        if not match:
+            logging.error(["get_policy_secret_list", "Invalid path to replicate", path])
+            raise VaultInvalidPaths
 
-    return secret_list
+        if match.group("folder"):
+            # Remove the * at the end of the path because list method expects
+            # a folder path without any secret or wilcard
+            secrets.update(vault_instance.list_all(path.rstrip("*")))
+        else:
+            secrets.add(path)
+
+    return list(secrets)
 
 
 def get_jenkins_secret_list(
@@ -280,12 +289,10 @@ def get_jenkins_secret_list(
 
 
 def get_vault_credentials(
-    vault_auth: Union[
-        VaultReplicationConfigV1_VaultInstanceAuthV1,
-        VaultInstanceV1_VaultReplicationConfigV1_VaultInstanceAuthV1,
-    ],
+    vault_auth: VaultReplicationConfigV1_VaultInstanceAuthV1
+    | VaultInstanceV1_VaultReplicationConfigV1_VaultInstanceAuthV1,
     vault_address: str,
-) -> dict[str, Optional[str]]:
+) -> dict[str, str | None]:
     """Returns a dictionary with the credentials used to authenticate with Vault,
     retrieved from the values present on AppInterface and comming from Vault itself."""
     vault_creds = {}
@@ -330,7 +337,6 @@ def replicate_paths(
         return
 
     for path in replications.paths:
-
         if isinstance(path, VaultReplicationJenkinsV1):
             if path.policy is not None:
                 vault_query_data = vault_policies.query(query_func=gql.get_api().query)
@@ -369,12 +375,11 @@ def replicate_paths(
 
 
 def _get_start_end_secret(path: str) -> tuple[str, str]:
-
     start = path[0 : path.index("{")]
     if start[-1] != "/":
         start = start.rsplit("/", 1)[0] + "/"
     try:
-        end = path[path[path.rindex("}") : :].index("/") + path.rindex("}") : :]
+        end = path[path[path.rindex("}") :].index("/") + path.rindex("}") :]
     except ValueError:
         end = ""
 
@@ -419,7 +424,6 @@ def get_secrets_from_templated_path(path: str, vault_list: Iterable[str]) -> lis
 
 
 def run(dry_run: bool) -> None:
-
     query_data = vault_instances.query(query_func=gql.get_api().query)
 
     if query_data.vault_instances:
@@ -436,15 +440,18 @@ def run(dry_run: bool) -> None:
                     # Private class _VaultClient is used because the public class is
                     # defined as a singleton, and we need to create multiple instances
                     # as the source vault is different than the replication.
-                    with _VaultClient(
-                        server=source_creds["server"],
-                        role_id=source_creds["role_id"],
-                        secret_id=source_creds["secret_id"],
-                    ) as source_vault, _VaultClient(
-                        server=dest_creds["server"],
-                        role_id=dest_creds["role_id"],
-                        secret_id=dest_creds["secret_id"],
-                    ) as dest_vault:
+                    with (
+                        _VaultClient(
+                            server=source_creds["server"],
+                            role_id=source_creds["role_id"],
+                            secret_id=source_creds["secret_id"],
+                        ) as source_vault,
+                        _VaultClient(
+                            server=dest_creds["server"],
+                            role_id=dest_creds["role_id"],
+                            secret_id=dest_creds["secret_id"],
+                        ) as dest_vault,
+                    ):
                         replicate_paths(
                             dry_run=dry_run,
                             source_vault=source_vault,

reconcile/vpc_peerings_validator.py

@@ -1,10 +1,11 @@
+import ipaddress
 import logging
 import sys
 from typing import (
-    Union,
     cast,
 )
 
+from reconcile import queries
 from reconcile.gql_definitions.vpc_peerings_validator import vpc_peerings_validator
 from reconcile.gql_definitions.vpc_peerings_validator.vpc_peerings_validator import (
     ClusterPeeringConnectionClusterAccepterV1,
@@ -14,10 +15,80 @@ from reconcile.gql_definitions.vpc_peerings_validator.vpc_peerings_validator imp
 )
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
+from reconcile.utils.aws_api import AWSApi
 
 QONTRACT_INTEGRATION = "vpc-peerings-validator"
 
 
+def validate_no_cidr_overlap(
+    query_data: VpcPeeringsValidatorQueryData,
+) -> bool:
+    clusters: list[ClusterV1] = query_data.clusters or []
+
+    for cluster in clusters:
+        if cluster.peering:
+            peerings_entries = [
+                {
+                    "provider": "cluster-self-vpc",
+                    "vpc_name": cluster.name,
+                    "cidr_block": cluster.network.vpc,  # type: ignore[union-attr]
+                },
+            ]
+            for peering in cluster.peering.connections:
+                if peering.provider == "account-vpc-mesh":
+                    aws_account_uid = peering.account.uid  # type: ignore[union-attr]
+                    settings = queries.get_secret_reader_settings()
+                    accounts = queries.get_aws_accounts(uid=aws_account_uid)
+                    awsapi = AWSApi(1, accounts, settings=settings, init_users=False)
+                    tags = peering.tags or "{}"  # type: ignore[union-attr]
+                    mesh_results = awsapi.get_vpcs_details(accounts[0], tags)
+                    for mesh_result in mesh_results:
+                        vpc_peering_info = {
+                            "provider": peering.provider,
+                            "vpc_name": mesh_result["vpc_id"],
+                            "cidr_block": mesh_result["cidr_block"],
+                        }
+                        peerings_entries.append(vpc_peering_info)
+                if peering.provider == "account-vpc":
+                    cidr_block = str(peering.vpc.cidr_block)  # type: ignore[union-attr]
+                    vpc_peering_info = {
+                        "provider": peering.provider,
+                        "vpc_name": peering.vpc.name,  # type: ignore[union-attr]
+                        "cidr_block": cidr_block,
+                    }
+                    peerings_entries.append(vpc_peering_info)
+                if peering.provider in {
+                    "cluster-vpc-requester",
+                    "cluster-vpc-accepter",
+                }:
+                    vpc_peering_info = {
+                        "provider": peering.provider,
+                        "vpc_name": peering.cluster.name,  # type: ignore[union-attr]
+                        "cidr_block": peering.cluster.network.vpc,  # type: ignore[union-attr]
+                    }
+                    peerings_entries.append(vpc_peering_info)
+            find_overlap = find_cidr_overlap(cluster.name, peerings_entries)
+            if find_overlap:
+                return False
+    return True
+
+
+def find_cidr_overlap(cluster_name: str, input_list: list):
+    for i in range(len(input_list)):  # pylint: disable=consider-using-enumerate
+        compared_vpc = input_list[i]
+        for j in range(i + 1, len(input_list)):
+            comparing_vpc = input_list[j]
+            if ipaddress.ip_network(compared_vpc["cidr_block"]).overlaps(
+                ipaddress.ip_network(comparing_vpc["cidr_block"])
+            ):
+                logging.error(f"VPC peering error in cluster {cluster_name}")
+                logging.error(
+                    f"vpc [{compared_vpc['vpc_name']}] with cidr block [{compared_vpc['cidr_block']}] provided by [{compared_vpc['provider']}] overlaps with vpc [{comparing_vpc['vpc_name']}] with cidr block [{comparing_vpc['cidr_block']}] provided by [{comparing_vpc['provider']}]"
+                )
+                return True
+    return False
+
+
 def validate_no_internal_to_public_peerings(
     query_data: VpcPeeringsValidatorQueryData,
 ) -> bool:
@@ -29,16 +100,14 @@ def validate_no_internal_to_public_peerings(
         if not cluster.internal or not cluster.peering:
             continue
         for connection in cluster.peering.connections:
-            if connection.provider not in [
+            if connection.provider not in {
                 "cluster-vpc-accepter",
                 "cluster-vpc-requester",
-            ]:
+            }:
                 continue
             connection = cast(
-                Union[
-                    ClusterPeeringConnectionClusterAccepterV1,
-                    ClusterPeeringConnectionClusterRequesterV1,
-                ],
+                ClusterPeeringConnectionClusterAccepterV1
+                | ClusterPeeringConnectionClusterRequesterV1,
                 connection,
             )
             peer = connection.cluster
@@ -72,16 +141,14 @@ def validate_no_public_to_public_peerings(
         ):
             continue
         for connection in cluster.peering.connections:
-            if connection.provider not in [
+            if connection.provider not in {
                 "cluster-vpc-accepter",
                 "cluster-vpc-requester",
-            ]:
+            }:
                 continue
             connection = cast(
-                Union[
-                    ClusterPeeringConnectionClusterAccepterV1,
-                    ClusterPeeringConnectionClusterRequesterV1,
-                ],
+                ClusterPeeringConnectionClusterAccepterV1
+                | ClusterPeeringConnectionClusterRequesterV1,
                 connection,
             )
             peer = connection.cluster
@@ -108,6 +175,8 @@ def run(dry_run: bool):
         valid = False
     if not validate_no_public_to_public_peerings(query_data):
         valid = False
+    if not validate_no_cidr_overlap(query_data):
+        valid = False
 
     if not valid:
         sys.exit(ExitCodes.ERROR)

tools/app_interface_metrics_exporter.py

@@ -0,0 +1,70 @@
+import logging
+from collections.abc import Mapping
+
+import click
+from pydantic import BaseModel
+
+from reconcile.cli import (
+    config_file,
+    dry_run,
+    log_level,
+)
+from reconcile.typed_queries.app_interface_metrics_exporter.onboarding_status import (
+    get_onboarding_status,
+)
+from reconcile.utils import (
+    gql,
+    metrics,
+)
+from reconcile.utils.metrics import GaugeMetric
+from reconcile.utils.runtime.environment import init_env
+
+INTEGRATION = "app-interface-metrics-exporter"
+
+
+class OverviewBaseMetric(BaseModel):
+    """Base class for overview metrics"""
+
+    integration: str
+
+
+class OverviewOnboardingStatus(OverviewBaseMetric, GaugeMetric):
+    """Overview of onboarding status"""
+
+    status: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "qontract_reconcile_onboarding_status"
+
+
+def publish_onboarding_status_metrics(
+    onboarding_status: Mapping[str, int],
+) -> None:
+    logging.debug("Publishing onboarding status metrics: %s", onboarding_status)
+    for status, count in onboarding_status.items():
+        metrics.set_gauge(
+            OverviewOnboardingStatus(
+                integration=INTEGRATION,
+                status=status,
+            ),
+            count,
+        )
+
+
+@click.command()
+@config_file
+@dry_run
+@log_level
+def main(
+    configfile: str,
+    dry_run: bool,
+    log_level: str,
+) -> None:
+    init_env(log_level=log_level, config_file=configfile)
+    onboarding_status = get_onboarding_status(gql.get_api())
+    publish_onboarding_status_metrics(onboarding_status)
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter