qontract-reconcile 0.10.1rc1020__py3-none-any.whl → 0.10.1rc1021__py3-none-any.whl

qontract_reconcile-0.10.1rc1021.dist-info/METADATA

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: qontract-reconcile
+Version: 0.10.1rc1021
+Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
+Home-page: https://github.com/app-sre/qontract-reconcile
+Author: Red Hat App-SRE Team
+Author-email: sd-app-sre@redhat.com
+License: Apache License 2.0
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Requires-Dist: sretoolbox~=2.5
+Requires-Dist: Click<9.0,>=7.0
+Requires-Dist: gql==3.1.0
+Requires-Dist: toml<0.11.0,>=0.10.0
+Requires-Dist: jsonpath-rw<1.5.0,>=1.4.0
+Requires-Dist: PyGithub<1.59,>=1.58
+Requires-Dist: hvac<0.8.0,>=0.7.0
+Requires-Dist: ldap3<2.10.0,>=2.9.1
+Requires-Dist: anymarkup<0.9.0,>=0.7.0
+Requires-Dist: python-gitlab~=4.6
+Requires-Dist: semver~=3.0
+Requires-Dist: boto3==1.34.94
+Requires-Dist: botocore==1.34.94
+Requires-Dist: urllib3~=2.2
+Requires-Dist: slack-sdk<4.0,>=3.10
+Requires-Dist: pypd<1.2.0,>=1.1.0
+Requires-Dist: Jinja2<3.2.0,>=2.10.1
+Requires-Dist: jira~=3.1
+Requires-Dist: pyOpenSSL~=23.0
+Requires-Dist: ruamel.yaml<0.18.0,>=0.17.22
+Requires-Dist: terrascript==0.9.0
+Requires-Dist: tabulate<0.9.0,>=0.8.6
+Requires-Dist: UnleashClient~=5.11
+Requires-Dist: prometheus-client~=0.8
+Requires-Dist: sentry-sdk~=2.0
+Requires-Dist: jenkins-job-builder~=4.3.0
+Requires-Dist: parse==1.18.0
+Requires-Dist: sendgrid<6.5.0,>=6.4.8
+Requires-Dist: dnspython~=2.1
+Requires-Dist: requests~=2.32
+Requires-Dist: kubernetes~=24.0
+Requires-Dist: websocket-client<0.55.0,>=0.35
+Requires-Dist: sshtunnel>=0.4.0
+Requires-Dist: croniter<1.1.0,>=1.0.15
+Requires-Dist: pydantic~=1.10.6
+Requires-Dist: MarkupSafe==2.1.1
+Requires-Dist: filetype~=1.2.0
+Requires-Dist: psycopg2~=2.9
+Requires-Dist: packaging~=23.1
+Requires-Dist: deepdiff==6.7.1
+Requires-Dist: jsonpath-ng==1.5.3
+Requires-Dist: networkx~=2.8
+Requires-Dist: rich<14.0.0,>=13.3.0
+Requires-Dist: dateparser~=1.1.7
+Requires-Dist: pyjwt~=2.7
+Requires-Dist: requests-oauthlib~=1.3
+Requires-Dist: dt==1.1.61
+Requires-Dist: jsonpatch~=1.33
+Requires-Dist: jsonpointer~=2.4
+Requires-Dist: yamllint==1.34.0
+

{qontract_reconcile-0.10.1rc1020.dist-info → qontract_reconcile-0.10.1rc1021.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=q96mwr2KeEQ5bpNniGlgIMZTxiuLSodcYfX-t
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
 reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
-reconcile/cli.py,sha256=lLVw-FxEUR8zU6UAKZzJk7XwRbjsXPaGUAQqXuBYSaU,105825
+reconcile/cli.py,sha256=rfT0jrWjDjvbUE0JWVvSladV2vTKqBBdNrthlH2Ekyc,106979
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=rLh16BOlBOxTmJ8Si3wWyyEpmMlhh4Znx1Gc36qsmOc,4865
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=l34QDu1G96_Ctnh7ZXdxXgSeCE93GQMdLAkWxmN6vDA,4775
@@ -185,6 +185,10 @@ reconcile/dynatrace_token_provider/integration.py,sha256=QY-k5vsbBOm80yW_RB6G2JZ
 reconcile/dynatrace_token_provider/metrics.py,sha256=xiKkl8fTEBQaXJelGCPNTZhHAWdO1M3pCXNr_Tei63c,1285
 reconcile/dynatrace_token_provider/model.py,sha256=gkpqo5rRRueBXnIMjp4EEHqBUBuU65TRI8zpdb8GJ0A,241
 reconcile/dynatrace_token_provider/ocm.py,sha256=iHMsgbsLs-dlrB9UXmWNDF7E4UDe49JOsLa9rnowKfo,4282
+reconcile/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/endpoints_discovery/integration.py,sha256=OFdSno8C5VVOoXALI5BFxHA3hfIhTZ55lJaMa0jLt0s,12210
+reconcile/endpoints_discovery/merge_request.py,sha256=NktpwMUoQ9TvBzOeHSPC09OqxWmgZBZVy-Eqmsm_vrA,2909
+reconcile/endpoints_discovery/merge_request_manager.py,sha256=oY3lovy5DcUgHNEDzWrbOPabzZtaGMns6YLl5ipZJ0c,6346
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
@@ -280,6 +284,8 @@ reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py,sha256=zUa-CmpOwi
 reconcile/gql_definitions/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py,sha256=5gTuAnR2rnx2k6Rn7FMEAzw6GCZ6F5HZbqkmJ9-3NI4,2244
 reconcile/gql_definitions/dynatrace_token_provider/token_specs.py,sha256=XGsMuB8gowRpqJjkD_KRomx-1OswzyWbF4qjVdhionk,2555
+reconcile/gql_definitions/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/endpoints_discovery/namespaces.py,sha256=FqJ0H7NdsIm5BgVnuJV9wLcj7i667VhCN559tWJ-WsA,3054
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
 reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
@@ -859,8 +865,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1020.dist-info/METADATA,sha256=nlUFKwjsUyVZVDAbzvIF_4FhV-g1XlrJk91Z7lLRsjI,2263
-qontract_reconcile-0.10.1rc1020.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc1020.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1020.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1020.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1021.dist-info/METADATA,sha256=Ob87vLcMc2mnkX4nPY_xWLWfzwuDB_Hgp2zMZVknZds,2213
+qontract_reconcile-0.10.1rc1021.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1021.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1021.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1021.dist-info/RECORD,,

{qontract_reconcile-0.10.1rc1020.dist-info → qontract_reconcile-0.10.1rc1021.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

reconcile/cli.py

@@ -1802,6 +1802,49 @@ def openshift_prometheus_rules(
     )
 
 
+@integration.command(short_help="Discover routes and update endpoints")
+@threaded()
+@binary(["oc"])
+@binary_version("oc", ["version", "--client"], OC_VERSION_REGEX, OC_VERSIONS)
+@internal()
+@use_jump_host()
+@cluster_name
+@namespace_name
+@click.option(
+    "--endpoint-tmpl-resource",
+    help="Resource name of the endpoint template in the app-interface.",
+    required=False,
+)
+@click.pass_context
+def endpoints_discovery(
+    ctx,
+    thread_pool_size,
+    internal,
+    use_jump_host,
+    cluster_name,
+    namespace_name,
+    endpoint_tmpl_resource,
+):
+    from reconcile.endpoints_discovery.integration import (
+        EndpointsDiscoveryIntegration,
+        EndpointsDiscoveryIntegrationParams,
+    )
+
+    params = EndpointsDiscoveryIntegrationParams(
+        thread_pool_size=thread_pool_size,
+        internal=internal,
+        use_jump_host=use_jump_host,
+        cluster_name=cluster_name,
+        namespace_name=namespace_name,
+    )
+    if endpoint_tmpl_resource:
+        params.endpoint_tmpl_resource = endpoint_tmpl_resource
+    run_class_integration(
+        integration=EndpointsDiscoveryIntegration(params),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Configures the teams and members in Quay.")
 @click.pass_context
 def quay_membership(ctx):

reconcile/endpoints_discovery/integration.py

@@ -0,0 +1,336 @@
+import logging
+from collections.abc import Callable, Iterable
+from typing import TypedDict
+
+import jinja2
+from pydantic import BaseModel
+
+from reconcile.endpoints_discovery.merge_request import Renderer, create_parser
+from reconcile.endpoints_discovery.merge_request_manager import (
+    App,
+    Endpoint,
+    EndpointsToAdd,
+    EndpointsToChange,
+    EndpointsToDelete,
+    MergeRequestManager,
+)
+from reconcile.gql_definitions.endpoints_discovery.namespaces import (
+    AppEndPointsV1,
+    NamespaceV1,
+)
+from reconcile.gql_definitions.endpoints_discovery.namespaces import (
+    query as namespaces_query,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils import gql
+from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_any_iterables
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
+from reconcile.utils.oc_map import OCMap, init_oc_map_from_namespaces
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.unleash import get_feature_toggle_state
+from reconcile.utils.vcs import VCS
+
+QONTRACT_INTEGRATION = "endpoints-discovery"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+
+
+class EndpointsDiscoveryIntegrationParams(PydanticRunParams):
+    thread_pool_size: int = 10
+    internal: bool | None = None
+    use_jump_host: bool = True
+    cluster_name: set[str] | None = None
+    namespace_name: str | None = None
+    endpoint_tmpl_resource: str = "/endpoints-discovery/endpoint-template.yml"
+    # extended early exit parameters
+    enable_extended_early_exit: bool = False
+    extended_early_exit_cache_ttl_seconds: int = 7200  # run every 2 hours
+    log_cached_log_output: bool = False
+
+
+class Route(BaseModel):
+    name: str
+    host: str
+    tls: bool
+
+    @property
+    def url(self) -> str:
+        return f"{self.host}:{443 if self.tls else 80}"
+
+
+def endpoint_prefix(namespace: NamespaceV1) -> str:
+    return f"{QONTRACT_INTEGRATION}-{namespace.cluster.name}-{namespace.name}"
+
+
+def compile_endpoint_name(endpoint_prefix: str, route: Route) -> str:
+    return f"{endpoint_prefix}-{route.name}"
+
+
+def render_template(template: str, endpoint_name: str, route: Route) -> dict:
+    yml = create_ruamel_instance()
+    return yml.load(
+        jinja2.Template(
+            template,
+            undefined=jinja2.StrictUndefined,
+            trim_blocks=True,
+            lstrip_blocks=True,
+            keep_trailing_newline=True,
+        ).render({"endpoint_name": endpoint_name, "route": route})
+    )
+
+
+class RunnerParams(TypedDict):
+    oc_map: OCMap
+    merge_request_manager: MergeRequestManager
+    endpoint_template: str
+    namespaces: Iterable[NamespaceV1]
+
+
+class EndpointsDiscoveryIntegration(
+    QontractReconcileIntegration[EndpointsDiscoveryIntegrationParams]
+):
+    """Discover routes from all OpenShift clusters and update endPoints in app-interface."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_desired_state_shard_config(self) -> None:
+        """Sharding (per cluster) is not supported for this integration.
+
+        An application can have endpoints in multiple clusters and this may cause merge conflicts."""
+        return None
+
+    def get_namespaces(
+        self,
+        query_func: Callable,
+        cluster_names: Iterable[str] | None = None,
+        namespace_name: str | None = None,
+    ) -> list[NamespaceV1]:
+        """Return namespaces to consider for the integration."""
+        return [
+            ns
+            for ns in namespaces_query(query_func).namespaces or []
+            if integration_is_enabled(self.name, ns.cluster)
+            and (not cluster_names or ns.cluster.name in cluster_names)
+            and (not namespace_name or ns.name == namespace_name)
+            and not ns.delete
+        ]
+
+    def get_routes(self, oc_map: OCMap, namespace: NamespaceV1) -> list[Route]:
+        """Return the routes for the given namespace."""
+        oc = oc_map.get_cluster(namespace.cluster.name)
+        if not oc.project_exists(namespace.name):
+            logging.info(
+                f"{namespace.cluster.name}/{namespace.name}: Namespace does not exist (yet). Skipping for now!"
+            )
+            return []
+
+        return [
+            Route(
+                name=item["metadata"]["name"],
+                host=item["spec"]["host"],
+                tls=bool(item["spec"].get("tls")),
+            )
+            for item in oc.get_items(kind="Route", namespace=namespace.name)
+        ]
+
+    def get_endpoint_changes(
+        self,
+        app: str,
+        endpoint_prefix: str,
+        endpoint_template: str,
+        endpoints: Iterable[AppEndPointsV1],
+        routes: Iterable[Route],
+    ) -> tuple[EndpointsToAdd, EndpointsToChange, EndpointsToDelete]:
+        """Get all new/changed/deleted endpoints for the given namespace."""
+        if not routes and not endpoints:
+            # nothing to do
+            return [], [], []
+
+        diff = diff_any_iterables(
+            # exclude manual endpoints
+            current=[
+                endpoint
+                for endpoint in endpoints
+                if endpoint.name.startswith(endpoint_prefix)
+            ]
+            or [],
+            desired=routes,
+            # names are unique, so we can use them as keys
+            current_key=lambda endpoint: endpoint.name,
+            desired_key=lambda route: compile_endpoint_name(endpoint_prefix, route),
+            # compare the endpoint and route by url.
+            # we can't use other endpoint attributes because we don't want to query them.
+            # there is a note about that behavior in the template.
+            equal=lambda endpoint, route: endpoint.url == route.url,
+        )
+
+        endpoints_to_add = []
+        endpoints_to_change = []
+        endpoints_to_delete = []
+
+        for add in diff.add.values():
+            logging.info(f"{app}: Adding endpoint for route {add.name}")
+            endpoints_to_add.append(
+                Endpoint(
+                    name=compile_endpoint_name(endpoint_prefix, add),
+                    data=render_template(
+                        endpoint_template,
+                        endpoint_name=compile_endpoint_name(endpoint_prefix, add),
+                        route=add,
+                    ),
+                )
+            )
+
+        for pair in diff.change.values():
+            logging.info(
+                f"{app}: Changing endpoint {pair.current.name} for route {pair.desired.name}"
+            )
+            endpoints_to_change.append(
+                Endpoint(
+                    name=pair.current.name,
+                    data=render_template(
+                        endpoint_template,
+                        endpoint_name=compile_endpoint_name(
+                            endpoint_prefix, pair.desired
+                        ),
+                        route=pair.desired,
+                    ),
+                )
+            )
+        for delete in diff.delete.values():
+            logging.info(f"{app}: Deleting endpoint for route {delete.name}")
+            endpoints_to_delete.append(Endpoint(name=delete.name))
+        return endpoints_to_add, endpoints_to_change, endpoints_to_delete
+
+    def get_apps(
+        self, oc_map: OCMap, endpoint_template: str, namespaces: Iterable[NamespaceV1]
+    ) -> list[App]:
+        """Compile a list of apps with their endpoints to add, change and delete."""
+        apps: dict[str, App] = {}
+        for namespace in namespaces:
+            routes = self.get_routes(oc_map, namespace)
+            endpoints_to_add, endpoints_to_change, endpoints_to_delete = (
+                self.get_endpoint_changes(
+                    app=namespace.app.name,
+                    endpoint_prefix=endpoint_prefix(namespace),
+                    endpoint_template=endpoint_template,
+                    endpoints=namespace.app.end_points or [],
+                    routes=routes,
+                )
+            )
+            # update the app with the endpoints per namespace
+            app = apps.setdefault(
+                namespace.app.path,
+                App(name=namespace.app.name, path=namespace.app.path),
+            )
+            app.endpoints_to_add += endpoints_to_add
+            app.endpoints_to_change += endpoints_to_change
+            app.endpoints_to_delete += endpoints_to_delete
+
+        # return only apps endpoint changes
+        return [
+            app
+            for app in apps.values()
+            if app.endpoints_to_add
+            or app.endpoints_to_change
+            or app.endpoints_to_delete
+        ]
+
+    def runner(
+        self,
+        oc_map: OCMap,
+        merge_request_manager: MergeRequestManager,
+        endpoint_template: str,
+        namespaces: Iterable[NamespaceV1],
+    ) -> ExtendedEarlyExitRunnerResult:
+        """Reconcile the endpoints for all namespaces."""
+        apps = self.get_apps(oc_map, endpoint_template, namespaces)
+        merge_request_manager.create_merge_request(apps=apps)
+        return ExtendedEarlyExitRunnerResult(payload={}, applied_count=len(apps))
+
+    @defer
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
+        """Run the integration."""
+        gql_api = gql.get_api()
+        namespaces = self.get_namespaces(
+            gql_api.query,
+            cluster_names=self.params.cluster_name,
+            namespace_name=self.params.namespace_name,
+        )
+        if not namespaces:
+            # nothing to do
+            return
+
+        oc_map = init_oc_map_from_namespaces(
+            namespaces=namespaces,
+            secret_reader=self.secret_reader,
+            integration=QONTRACT_INTEGRATION,
+            use_jump_host=self.params.use_jump_host,
+            thread_pool_size=self.params.thread_pool_size,
+            internal=self.params.internal,
+            init_projects=True,
+        )
+
+        if defer:
+            defer(oc_map.cleanup)
+
+        vcs = VCS(
+            secret_reader=self.secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=True,
+            allow_opening_mrs=True,
+        )
+        if defer:
+            defer(vcs.cleanup)
+        merge_request_manager = MergeRequestManager(
+            vcs=vcs,
+            renderer=Renderer(),
+            parser=create_parser(),
+            auto_merge_enabled=get_feature_toggle_state(
+                integration_name=f"{self.name}-allow-auto-merge-mrs", default=False
+            ),
+        )
+        endpoint_template = gql_api.get_resource(
+            path=self.params.endpoint_tmpl_resource
+        )["content"]
+
+        runner_params: RunnerParams = {
+            "oc_map": oc_map,
+            "merge_request_manager": merge_request_manager,
+            "endpoint_template": endpoint_template,
+            "namespaces": namespaces,
+        }
+
+        if self.params.enable_extended_early_exit and get_feature_toggle_state(
+            f"{QONTRACT_INTEGRATION}-extended-early-exit", default=True
+        ):
+            extended_early_exit_run(
+                integration=QONTRACT_INTEGRATION,
+                integration_version=QONTRACT_INTEGRATION_VERSION,
+                dry_run=dry_run,
+                cache_source=self.get_early_exit_desired_state(),
+                shard="",
+                ttl_seconds=self.params.extended_early_exit_cache_ttl_seconds,
+                logger=logging.getLogger(),
+                runner=self.runner,
+                runner_params=runner_params,
+                log_cached_log_output=self.params.log_cached_log_output,
+            )
+        else:
+            self.runner(**runner_params)

reconcile/endpoints_discovery/merge_request.py

@@ -0,0 +1,92 @@
+import re
+import string
+
+from pydantic import BaseModel
+from ruamel.yaml.compat import StringIO
+
+from reconcile.utils.merge_request_manager.parser import Parser
+from reconcile.utils.ruamel import create_ruamel_instance
+
+VERSION = "1.0.0"
+INTEGRATION = "endpoints-discovery"
+LABEL = "ENDPOINTS-DISCOVERY"
+
+INTEGRATION_REF = "integration"
+VERSION_REF = "endpoints-discover-version"
+HASH_REF = "hash"
+COMPILED_REGEXES = {
+    i: re.compile(rf".*{i}: (.*)$", re.MULTILINE)
+    for i in [INTEGRATION_REF, VERSION_REF, HASH_REF]
+}
+
+PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
+DESC = string.Template(
+    f"""
+This MR is triggered by app-interface's [endpoints-discovery](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/endpoints_discovery).
+
+Please **do not remove the `{LABEL}` label** from this MR!
+
+Parts of this description are used to manage the MR.
+
+{PROMOTION_DATA_SEPARATOR}
+
+* {INTEGRATION_REF}: {INTEGRATION}
+* {VERSION_REF}: {VERSION}
+* {HASH_REF}: $hash
+"""
+)
+
+
+class EPDInfo(BaseModel):
+    integration: str = INTEGRATION
+    hash: str
+
+
+def create_parser() -> Parser:
+    return Parser[EPDInfo](
+        klass=EPDInfo,
+        compiled_regexes=COMPILED_REGEXES,
+        version_ref=VERSION_REF,
+        expected_version=VERSION,
+        data_separator=PROMOTION_DATA_SEPARATOR,
+    )
+
+
+class Renderer:
+    """
+    This class is only concerned with rendering text for MRs.
+    Most logic evolves around ruamel yaml modification.
+    This class makes testing for MergeRequestManager easier.
+
+    Note, that this class is very susceptible to schema changes
+    as it mainly works on raw dicts.
+    """
+
+    def render_merge_request_content(
+        self,
+        current_content: str,
+        endpoints_to_add: list[dict],
+        endpoints_to_change: dict[str, dict],
+        endpoints_to_delete: list[str],
+    ) -> str:
+        """Update the app-interface app file for a merge request."""
+        yml = create_ruamel_instance(explicit_start=True)
+        content = yml.load(current_content)
+        app_endpoints = content.setdefault("endPoints", [])
+        for i, app_endpoint in enumerate(app_endpoints):
+            if app_endpoint["name"] in endpoints_to_delete:
+                app_endpoints.remove(app_endpoint)
+            if app_endpoint["name"] in endpoints_to_change:
+                app_endpoints[i] = endpoints_to_change[app_endpoint["name"]]
+        app_endpoints.extend(endpoints_to_add)
+        with StringIO() as stream:
+            yml.dump(content, stream)
+            return stream.getvalue()
+
+    def render_description(self, hash: str) -> str:
+        """Render the description for a merge request."""
+        return DESC.safe_substitute(EPDInfo(hash=hash).dict())
+
+    def render_title(self) -> str:
+        """Render the title for a merge request."""
+        return f"[auto] {INTEGRATION}: update application endpoints"

reconcile/endpoints_discovery/merge_request_manager.py

@@ -0,0 +1,175 @@
+import hashlib
+import json
+import logging
+from collections.abc import Sequence
+from typing import Any, TypeAlias
+
+from gitlab.exceptions import GitlabGetError
+from pydantic import BaseModel
+
+from reconcile.endpoints_discovery.merge_request import (
+    INTEGRATION,
+    INTEGRATION_REF,
+    LABEL,
+    EPDInfo,
+    Parser,
+    Renderer,
+)
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+
+class EndpointsDiscoveryMR(MergeRequestBase):
+    name = "endpoints-discovery"
+
+    def __init__(self, title: str, description: str, labels: list[str]):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self.labels = labels
+        self._commits: list[tuple[str, str, str]] = []
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def add_commit(self, path: str, content: str, msg: str) -> None:
+        self._commits.append((path, content, msg))
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        for path, content, msg in self._commits:
+            gitlab_cli.update_file(
+                branch_name=self.branch,
+                file_path=path,
+                commit_message=msg,
+                content=content,
+            )
+
+
+class Endpoint(BaseModel):
+    # the current endpoint name to change or delete. It doesn't matter for new endpoints
+    name: str
+    # the endpoint data will be generated and rendered from the endpoint template resource
+    # see EndpointsDiscoveryIntegrationParams.endpoint_tmpl_resource
+    data: dict[str, Any] = {}
+
+    @property
+    def hash(self) -> str:
+        return hashlib.sha256(
+            json.dumps(self.dict(), sort_keys=True).encode()
+        ).hexdigest()
+
+
+EndpointsToAdd: TypeAlias = list[Endpoint]
+EndpointsToChange: TypeAlias = list[Endpoint]
+EndpointsToDelete: TypeAlias = list[Endpoint]
+
+
+class App(BaseModel):
+    name: str
+    path: str
+    endpoints_to_add: EndpointsToAdd = EndpointsToAdd()
+    endpoints_to_change: EndpointsToChange = EndpointsToChange()
+    endpoints_to_delete: EndpointsToDelete = EndpointsToDelete()
+
+    @property
+    def hash(self) -> str:
+        return hashlib.sha256(
+            f"""
+                {self.path}
+                {[i.hash for i in sorted(self.endpoints_to_add, key=lambda i: i.name)]}
+                {[i.hash for i in sorted(self.endpoints_to_change, key=lambda i: i.name)]}
+                {[i.hash for i in sorted(self.endpoints_to_delete, key=lambda i: i.name)]}
+            """.encode()
+        ).hexdigest()
+
+
+def hash_apps(apps: Sequence[App]) -> str:
+    return hashlib.sha256(
+        ",".join(app.hash for app in sorted(apps, key=lambda i: i.name)).encode()
+    ).hexdigest()
+
+
+class MergeRequestManager(MergeRequestManagerBase[EPDInfo]):
+    """
+    Manager for AVS merge requests. This class
+    is responsible for housekeeping (closing old/bad MRs) and
+    opening new MRs for external resources that have new versions.
+
+    For each external resource, there are exist just one MR to update
+    the version number in the App-Interface. Old or obsolete MRs are
+    closed automatically.
+    """
+
+    def __init__(
+        self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
+    ):
+        super().__init__(vcs, parser, LABEL)
+        self._renderer = renderer
+        self._auto_merge_enabled = auto_merge_enabled
+
+    def create_merge_request(self, apps: Sequence[App]) -> None:
+        """Open new MR (if not already present) for apps and close any outdated before."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        apps_hash = hash_apps(apps)
+        # we support only one MR at a time for all apps
+        if mr := self._merge_request_already_exists({INTEGRATION_REF: INTEGRATION}):
+            if mr.mr_info.hash == apps_hash:
+                logging.info(
+                    f"Found an open MR for {INTEGRATION} and it's up-to-date - doing nothing."
+                )
+                return None
+            logging.info(f"Found an outdated MR for {INTEGRATION} - closing it.")
+            self._vcs.close_app_interface_mr(
+                mr.raw, "Closing this MR because it's outdated."
+            )
+            # don't open a new MR right now, because the deletion of the old MRs could be
+            # disabled. In this case, we would end up with multiple open MRs for the
+            # same external resource.
+            return None
+
+        endpoints_discovery_mr = EndpointsDiscoveryMR(
+            title=self._renderer.render_title(),
+            description=self._renderer.render_description(hash=apps_hash),
+            labels=[LABEL] + ([AUTO_MERGE] if self._auto_merge_enabled else []),
+        )
+        for app in apps:
+            try:
+                content = self._vcs.get_file_content_from_app_interface_master(
+                    file_path=app.path
+                )
+            except GitlabGetError as e:
+                if e.response_code == 404:
+                    logging.error(
+                        "The file %s does not exist anylonger. Most likely qontract-server data not in synch. This should resolve soon on its own.",
+                        app.path,
+                    )
+                    return None
+                raise e
+            content = self._renderer.render_merge_request_content(
+                current_content=content,
+                endpoints_to_add=[item.data for item in app.endpoints_to_add],
+                endpoints_to_change={
+                    item.name: item.data for item in app.endpoints_to_change
+                },
+                endpoints_to_delete=[item.name for item in app.endpoints_to_delete],
+            )
+            endpoints_discovery_mr.add_commit(
+                path=f"data/{app.path.lstrip('/')}",
+                content=content,
+                msg=f"endpoints-discovery: update application endpoints for {app.name}",
+            )
+
+        logging.info("Open MR for %d app(s)", len(apps))
+        self._vcs.open_app_interface_merge_request(mr=endpoints_discovery_mr)

reconcile/gql_definitions/endpoints_discovery/namespaces.py

@@ -0,0 +1,127 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.oc_connection_cluster import OcConnectionCluster
+
+
+DEFINITION = """
+fragment CommonJumphostFields on ClusterJumpHost_v1 {
+  hostname
+  knownHosts
+  user
+  port
+  remotePort
+  identity {
+    ... VaultSecret
+  }
+}
+
+fragment OcConnectionCluster on Cluster_v1 {
+  name
+  serverUrl
+  internal
+  insecureSkipTLSVerify
+  jumpHost {
+    ...CommonJumphostFields
+  }
+  automationToken {
+    ...VaultSecret
+  }
+  clusterAdminAutomationToken {
+    ...VaultSecret
+  }
+  disable {
+    integrations
+  }
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query EndPointsDiscoveryNamespaces {
+  namespaces: namespaces_v1 {
+    name
+    delete
+    clusterAdmin
+    cluster {
+      ... OcConnectionCluster
+    }
+    app {
+      path
+      name
+      endPoints {
+        name
+        url
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AppEndPointsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    url: str = Field(..., alias="url")
+
+
+class AppV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    name: str = Field(..., alias="name")
+    end_points: Optional[list[AppEndPointsV1]] = Field(..., alias="endPoints")
+
+
+class NamespaceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    delete: Optional[bool] = Field(..., alias="delete")
+    cluster_admin: Optional[bool] = Field(..., alias="clusterAdmin")
+    cluster: OcConnectionCluster = Field(..., alias="cluster")
+    app: AppV1 = Field(..., alias="app")
+
+
+class EndPointsDiscoveryNamespacesQueryData(ConfiguredBaseModel):
+    namespaces: Optional[list[NamespaceV1]] = Field(..., alias="namespaces")
+
+
+def query(query_func: Callable, **kwargs: Any) -> EndPointsDiscoveryNamespacesQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        EndPointsDiscoveryNamespacesQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return EndPointsDiscoveryNamespacesQueryData(**raw_data)

qontract_reconcile-0.10.1rc1020.dist-info/METADATA

@@ -1,64 +0,0 @@
-Metadata-Version: 2.1
-Name: qontract-reconcile
-Version: 0.10.1rc1020
-Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
-Home-page: https://github.com/app-sre/qontract-reconcile
-Author: Red Hat App-SRE Team
-Author-email: sd-app-sre@redhat.com
-License: Apache License 2.0
-Classifier: Development Status :: 2 - Pre-Alpha
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.11
-Requires-Dist: sretoolbox ~=2.5
-Requires-Dist: Click <9.0,>=7.0
-Requires-Dist: gql ==3.1.0
-Requires-Dist: toml <0.11.0,>=0.10.0
-Requires-Dist: jsonpath-rw <1.5.0,>=1.4.0
-Requires-Dist: PyGithub <1.59,>=1.58
-Requires-Dist: hvac <0.8.0,>=0.7.0
-Requires-Dist: ldap3 <2.10.0,>=2.9.1
-Requires-Dist: anymarkup <0.9.0,>=0.7.0
-Requires-Dist: python-gitlab ~=4.6
-Requires-Dist: semver ~=3.0
-Requires-Dist: boto3 ==1.34.94
-Requires-Dist: botocore ==1.34.94
-Requires-Dist: urllib3 ~=2.2
-Requires-Dist: slack-sdk <4.0,>=3.10
-Requires-Dist: pypd <1.2.0,>=1.1.0
-Requires-Dist: Jinja2 <3.2.0,>=2.10.1
-Requires-Dist: jira ~=3.1
-Requires-Dist: pyOpenSSL ~=23.0
-Requires-Dist: ruamel.yaml <0.18.0,>=0.17.22
-Requires-Dist: terrascript ==0.9.0
-Requires-Dist: tabulate <0.9.0,>=0.8.6
-Requires-Dist: UnleashClient ~=5.11
-Requires-Dist: prometheus-client ~=0.8
-Requires-Dist: sentry-sdk ~=2.0
-Requires-Dist: jenkins-job-builder ~=4.3.0
-Requires-Dist: parse ==1.18.0
-Requires-Dist: sendgrid <6.5.0,>=6.4.8
-Requires-Dist: dnspython ~=2.1
-Requires-Dist: requests ~=2.32
-Requires-Dist: kubernetes ~=24.0
-Requires-Dist: websocket-client <0.55.0,>=0.35
-Requires-Dist: sshtunnel >=0.4.0
-Requires-Dist: croniter <1.1.0,>=1.0.15
-Requires-Dist: pydantic ~=1.10.6
-Requires-Dist: MarkupSafe ==2.1.1
-Requires-Dist: filetype ~=1.2.0
-Requires-Dist: psycopg2 ~=2.9
-Requires-Dist: packaging ~=23.1
-Requires-Dist: deepdiff ==6.7.1
-Requires-Dist: jsonpath-ng ==1.5.3
-Requires-Dist: networkx ~=2.8
-Requires-Dist: rich <14.0.0,>=13.3.0
-Requires-Dist: dateparser ~=1.1.7
-Requires-Dist: pyjwt ~=2.7
-Requires-Dist: requests-oauthlib ~=1.3
-Requires-Dist: dt ==1.1.61
-Requires-Dist: jsonpatch ~=1.33
-Requires-Dist: jsonpointer ~=2.4
-Requires-Dist: yamllint ==1.34.0
-