qontract-reconcile 0.10.2.dev14__py3-none-any.whl → 0.10.2.dev16__py3-none-any.whl

reconcile/email_sender.py

@@ -53,8 +53,7 @@ def collect_to(to):
             if not service_owners:
                 continue
 
-            for service_owner in service_owners:
-                audience.add(service_owner["email"])
+            audience.update(service_owner["email"] for service_owner in service_owners)
 
     # TODO: implement clusters and namespaces
 
@@ -65,8 +64,7 @@ def collect_to(to):
             if not account_owners:
                 continue
 
-            for account_owner in account_owners:
-                audience.add(account_owner["email"])
+            audience.update(account_owner["email"] for account_owner in account_owners)
 
     roles = to.get("roles")
     if roles:
@@ -75,13 +73,11 @@ def collect_to(to):
             if not users:
                 continue
 
-            for user in users:
-                audience.add(user["org_username"])
+            audience.update(user["org_username"] for user in users)
 
     users = to.get("users")
     if users:
-        for user in users:
-            audience.add(user["org_username"])
+        audience.update(user["org_username"] for user in users)
 
     return audience
 

reconcile/endpoints_discovery/integration.py

@@ -15,12 +15,13 @@ from reconcile.endpoints_discovery.merge_request_manager import (
     EndpointsToDelete,
     MergeRequestManager,
 )
-from reconcile.gql_definitions.endpoints_discovery.namespaces import (
+from reconcile.gql_definitions.endpoints_discovery.apps import (
     AppEndPointsV1,
+    AppV1,
     NamespaceV1,
 )
-from reconcile.gql_definitions.endpoints_discovery.namespaces import (
-    query as namespaces_query,
+from reconcile.gql_definitions.endpoints_discovery.apps import (
+    query as apps_query,
 )
 from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
 from reconcile.typed_queries.github_orgs import get_github_orgs
@@ -44,7 +45,7 @@ from reconcile.utils.unleash import get_feature_toggle_state
 from reconcile.utils.vcs import VCS
 
 QONTRACT_INTEGRATION = "endpoints-discovery"
-QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 1)
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 1, 0)
 
 
 class EndpointsDiscoveryIntegrationParams(PydanticRunParams):
@@ -52,7 +53,7 @@ class EndpointsDiscoveryIntegrationParams(PydanticRunParams):
     internal: bool | None = None
     use_jump_host: bool = True
     cluster_name: set[str] | None = None
-    namespace_name: str | None = None
+    app_name: str | None = None
     endpoint_tmpl_resource: str = "/endpoints-discovery/endpoint-template.yml"
     # extended early exit parameters
     enable_extended_early_exit: bool = False
@@ -71,14 +72,25 @@ class Route(BaseModel):
 
 
 def endpoint_prefix(namespace: NamespaceV1) -> str:
+    """Return the prefix for the endpoint name."""
     return f"{QONTRACT_INTEGRATION}/{namespace.cluster.name}/{namespace.name}/"
 
 
+def parse_endpoint_name(endpoint_name: str) -> tuple[str, str, list[str]]:
+    """Parse the endpoint name into its components."""
+    integration_name, cluster, namespace, route_names = endpoint_name.split("/")
+    if integration_name != QONTRACT_INTEGRATION:
+        raise ValueError("Invalid integration name")
+    return cluster, namespace, route_names.split("|")
+
+
 def compile_endpoint_name(endpoint_prefix: str, route: Route) -> str:
+    """Compile the endpoint name from the prefix and route."""
     return f"{endpoint_prefix}{route.name}"
 
 
 def render_template(template: str, endpoint_name: str, route: Route) -> dict:
+    """Render the endpoint yaml template used in the merge request."""
     yml = create_ruamel_instance()
     return yml.load(
         jinja2.Template(
@@ -95,7 +107,7 @@ class RunnerParams(TypedDict):
     oc_map: OCMap
     merge_request_manager: MergeRequestManager
     endpoint_template: str
-    namespaces: Iterable[NamespaceV1]
+    apps: Iterable[AppV1]
 
 
 class EndpointsDiscoveryIntegration(
@@ -113,20 +125,16 @@ class EndpointsDiscoveryIntegration(
         An application can have endpoints in multiple clusters and this may cause merge conflicts."""
         return None
 
-    def get_namespaces(
+    def get_apps(
         self,
         query_func: Callable,
-        cluster_names: Iterable[str] | None = None,
-        namespace_name: str | None = None,
-    ) -> list[NamespaceV1]:
-        """Return namespaces to consider for the integration."""
+        app_name: str | None = None,
+    ) -> list[AppV1]:
+        """Return all applications to consider for the integration."""
         return [
-            ns
-            for ns in namespaces_query(query_func).namespaces or []
-            if integration_is_enabled(self.name, ns.cluster)
-            and (not cluster_names or ns.cluster.name in cluster_names)
-            and (not namespace_name or ns.name == namespace_name)
-            and not ns.delete
+            app
+            for app in apps_query(query_func).apps or []
+            if (not app_name or app.name == app_name)
         ]
 
     def get_routes(self, oc_map: OCMap, namespace: NamespaceV1) -> list[Route]:
@@ -155,9 +163,8 @@ class EndpointsDiscoveryIntegration(
             for (host, tls), names in routes.items()
         ]
 
-    def get_endpoint_changes(
+    def get_namespace_endpoint_changes(
         self,
-        app: str,
         endpoint_prefix: str,
         endpoint_template: str,
         endpoints: Iterable[AppEndPointsV1],
@@ -186,108 +193,140 @@ class EndpointsDiscoveryIntegration(
             equal=lambda endpoint, route: endpoint.url == route.url,
         )
 
-        endpoints_to_add = []
-        endpoints_to_change = []
-        endpoints_to_delete = []
-
-        for add in diff.add.values():
-            logging.info(f"{app}: Adding endpoint for route {add.name}")
-            endpoints_to_add.append(
-                Endpoint(
-                    name=compile_endpoint_name(endpoint_prefix, add),
-                    data=render_template(
-                        endpoint_template,
-                        endpoint_name=compile_endpoint_name(endpoint_prefix, add),
-                        route=add,
-                    ),
-                )
-            )
-
-        for pair in diff.change.values():
-            logging.info(
-                f"{app}: Changing endpoint {pair.current.name} for route {pair.desired.name}"
+        endpoints_to_add = [
+            Endpoint(
+                name=compile_endpoint_name(endpoint_prefix, add),
+                data=render_template(
+                    endpoint_template,
+                    endpoint_name=compile_endpoint_name(endpoint_prefix, add),
+                    route=add,
+                ),
             )
-            endpoints_to_change.append(
-                Endpoint(
-                    name=pair.current.name,
-                    data=render_template(
-                        endpoint_template,
-                        endpoint_name=compile_endpoint_name(
-                            endpoint_prefix, pair.desired
-                        ),
-                        route=pair.desired,
-                    ),
-                )
+            for add in diff.add.values()
+        ]
+        endpoints_to_change = [
+            Endpoint(
+                name=pair.current.name,
+                data=render_template(
+                    endpoint_template,
+                    endpoint_name=compile_endpoint_name(endpoint_prefix, pair.desired),
+                    route=pair.desired,
+                ),
             )
-        for delete in diff.delete.values():
-            logging.info(f"{app}: Deleting endpoint for route {delete.name}")
-            endpoints_to_delete.append(Endpoint(name=delete.name))
+            for pair in diff.change.values()
+        ]
+        endpoints_to_delete = [
+            Endpoint(name=delete.name) for delete in diff.delete.values()
+        ]
         return endpoints_to_add, endpoints_to_change, endpoints_to_delete
 
-    def get_apps(
-        self, oc_map: OCMap, endpoint_template: str, namespaces: Iterable[NamespaceV1]
+    def process(
+        self,
+        oc_map: OCMap,
+        endpoint_template: str,
+        apps: Iterable[AppV1],
+        cluster_names: Iterable[str] | None = None,
     ) -> list[App]:
         """Compile a list of apps with their endpoints to add, change and delete."""
-        apps: dict[str, App] = {}
-        for namespace in namespaces:
-            logging.debug(
-                f"Processing namespace {namespace.cluster.name}/{namespace.name}"
-            )
-            routes = self.get_routes(oc_map, namespace)
-            endpoints_to_add, endpoints_to_change, endpoints_to_delete = (
-                self.get_endpoint_changes(
-                    app=namespace.app.name,
-                    endpoint_prefix=endpoint_prefix(namespace),
-                    endpoint_template=endpoint_template,
-                    endpoints=namespace.app.end_points or [],
-                    routes=routes,
+        apps_with_changes: list[App] = []
+        for app in apps:
+            app_endpoints = App(name=app.name, path=app.path)
+            for namespace in app.namespaces or []:
+                if not self.is_enabled(namespace, cluster_names=cluster_names):
+                    continue
+
+                logging.debug(
+                    f"Processing namespace {namespace.cluster.name}/{namespace.name}"
                 )
-            )
-            # update the app with the endpoints per namespace
-            app = apps.setdefault(
-                namespace.app.path,
-                App(name=namespace.app.name, path=namespace.app.path),
-            )
-            app.endpoints_to_add += endpoints_to_add
-            app.endpoints_to_change += endpoints_to_change
-            app.endpoints_to_delete += endpoints_to_delete
-
-        # return only apps endpoint changes
-        return [
-            app
-            for app in apps.values()
-            if app.endpoints_to_add
-            or app.endpoints_to_change
-            or app.endpoints_to_delete
-        ]
+                routes = self.get_routes(oc_map, namespace)
+                endpoints_to_add, endpoints_to_change, endpoints_to_delete = (
+                    self.get_namespace_endpoint_changes(
+                        endpoint_prefix=endpoint_prefix(namespace),
+                        endpoint_template=endpoint_template,
+                        endpoints=app.end_points or [],
+                        routes=routes,
+                    )
+                )
+                # update the app with the endpoints per namespace
+                app_endpoints.endpoints_to_add += endpoints_to_add
+                app_endpoints.endpoints_to_change += endpoints_to_change
+                app_endpoints.endpoints_to_delete += endpoints_to_delete
+
+            # remove endpoints from deleted namespaces
+            namspace_names = {(ns.cluster.name, ns.name) for ns in app.namespaces or []}
+            for ep in app.end_points or []:
+                try:
+                    ep_cluster, ep_namespace, _ = parse_endpoint_name(ep.name)
+                except ValueError:
+                    continue
+                if (ep_cluster, ep_namespace) not in namspace_names:
+                    app_endpoints.endpoints_to_delete.append(Endpoint(name=ep.name))
+
+            # log the changes
+            for add in app_endpoints.endpoints_to_add:
+                logging.info(f"{app.name}: Adding endpoint for route {add.name}")
+
+            for change in app_endpoints.endpoints_to_change:
+                logging.info(f"{app.name}: Changing endpoint for route {change.name}")
+
+            for delete in app_endpoints.endpoints_to_delete:
+                logging.info(f"{app.name}: Deleting endpoint for route {delete.name}")
+
+            if (
+                app_endpoints.endpoints_to_add
+                or app_endpoints.endpoints_to_change
+                or app_endpoints.endpoints_to_delete
+            ):
+                # ignore apps without changes
+                apps_with_changes.append(app_endpoints)
+
+        return apps_with_changes
 
     def runner(
         self,
         oc_map: OCMap,
         merge_request_manager: MergeRequestManager,
         endpoint_template: str,
-        namespaces: Iterable[NamespaceV1],
+        apps: Iterable[AppV1],
     ) -> ExtendedEarlyExitRunnerResult:
         """Reconcile the endpoints for all namespaces."""
-        apps = self.get_apps(oc_map, endpoint_template, namespaces)
-        merge_request_manager.create_merge_request(apps=apps)
-        return ExtendedEarlyExitRunnerResult(payload={}, applied_count=len(apps))
+        apps_with_changes = self.process(
+            oc_map,
+            endpoint_template,
+            apps,
+            cluster_names=self.params.cluster_name,
+        )
+        merge_request_manager.create_merge_request(apps=apps_with_changes)
+        return ExtendedEarlyExitRunnerResult(
+            payload={}, applied_count=len(apps_with_changes)
+        )
+
+    def is_enabled(
+        self, namespace: NamespaceV1, cluster_names: Iterable[str] | None = None
+    ) -> bool:
+        """Check if the integration is enabled for the given namespace."""
+        return (
+            integration_is_enabled(self.name, namespace.cluster)
+            and (not cluster_names or namespace.cluster.name in cluster_names)
+            and not namespace.delete
+        )
 
     @defer
     def run(self, dry_run: bool, defer: Callable | None = None) -> None:
         """Run the integration."""
         gql_api = gql.get_api()
-        namespaces = self.get_namespaces(
-            gql_api.query,
-            cluster_names=self.params.cluster_name,
-            namespace_name=self.params.namespace_name,
-        )
-        if not namespaces:
+        apps = self.get_apps(gql_api.query, app_name=self.params.app_name)
+        if not apps:
             # nothing to do
             return
 
         oc_map = init_oc_map_from_namespaces(
-            namespaces=namespaces,
+            namespaces=[
+                ns
+                for app in apps
+                for ns in app.namespaces or []
+                if self.is_enabled(ns, self.params.cluster_name)
+            ],
             secret_reader=self.secret_reader,
             integration=QONTRACT_INTEGRATION,
             use_jump_host=self.params.use_jump_host,
@@ -326,7 +365,7 @@ class EndpointsDiscoveryIntegration(
             "oc_map": oc_map,
             "merge_request_manager": merge_request_manager,
             "endpoint_template": endpoint_template,
-            "namespaces": namespaces,
+            "apps": apps,
         }
 
         if self.params.enable_extended_early_exit and get_feature_toggle_state(

reconcile/external_resources/secrets_sync.py

@@ -120,8 +120,8 @@ class OutputSecretsFormatter:
     def _format_value(self, value: str) -> str:
         decoded_value = base64.b64decode(value).decode("utf-8")
         if decoded_value.startswith("__vault__:"):
-            _secret_ref = json.loads(decoded_value.replace("__vault__:", ""))
-            secret_ref = VaultSecret(**_secret_ref)
+            secret_ref_ = json.loads(decoded_value.replace("__vault__:", ""))
+            secret_ref = VaultSecret(**secret_ref_)
             return self.secret_reader.read_secret(secret_ref)
         else:
             return decoded_value

reconcile/external_resources/state.py

@@ -74,47 +74,47 @@ class DynamoDBStateAdapter:
     MODCONF_DRIFT_MINS = "drift_detection_minutes"
     MODCONF_TIMEOUT_MINS = "timeout_minutes"
 
-    def _get_value(self, item: Mapping[str, Any], key: str, _type: str = "S") -> Any:
-        return item[key][_type]
+    def _get_value(self, item: Mapping[str, Any], key: str, type: str = "S") -> Any:
+        return item[key][type]
 
     def deserialize(
         self,
         item: Mapping[str, Any],
         partial_data: bool = False,
     ) -> ExternalResourceState:
-        _key = self._get_value(item, self.ER_KEY, _type="M")
+        key_ = self._get_value(item, self.ER_KEY, type="M")
         key = ExternalResourceKey(
-            provision_provider=self._get_value(_key, self.ER_KEY_PROVISION_PROVIDER),
-            provisioner_name=self._get_value(_key, self.ER_KEY_PROVISIONER_NAME),
-            provider=self._get_value(_key, self.ER_KEY_PROVIDER),
-            identifier=self._get_value(_key, self.ER_KEY_IDENTIFIER),
+            provision_provider=self._get_value(key_, self.ER_KEY_PROVISION_PROVIDER),
+            provisioner_name=self._get_value(key_, self.ER_KEY_PROVISIONER_NAME),
+            provider=self._get_value(key_, self.ER_KEY_PROVIDER),
+            identifier=self._get_value(key_, self.ER_KEY_IDENTIFIER),
         )
-        _reconciliation = self._get_value(item, self.RECONC, _type="M")
+        reconciliation = self._get_value(item, self.RECONC, type="M")
 
         if partial_data:
             r = Reconciliation(
                 key=key,
                 resource_hash=self._get_value(
-                    _reconciliation, self.RECONC_RESOURCE_HASH
+                    reconciliation, self.RECONC_RESOURCE_HASH
                 ),
             )
         else:
-            _modconf = self._get_value(_reconciliation, self.MODCONF, _type="M")
+            modconf = self._get_value(reconciliation, self.MODCONF, type="M")
             r = Reconciliation(
                 key=key,
                 resource_hash=self._get_value(
-                    _reconciliation, self.RECONC_RESOURCE_HASH
+                    reconciliation, self.RECONC_RESOURCE_HASH
                 ),
-                input=self._get_value(_reconciliation, self.RECONC_INPUT),
-                action=self._get_value(_reconciliation, self.RECONC_ACTION),
+                input=self._get_value(reconciliation, self.RECONC_INPUT),
+                action=self._get_value(reconciliation, self.RECONC_ACTION),
                 module_configuration=ExternalResourceModuleConfiguration(
-                    image=self._get_value(_modconf, self.MODCONF_IMAGE),
-                    version=self._get_value(_modconf, self.MODCONF_VERSION),
+                    image=self._get_value(modconf, self.MODCONF_IMAGE),
+                    version=self._get_value(modconf, self.MODCONF_VERSION),
                     reconcile_drift_interval_minutes=self._get_value(
-                        _modconf, self.MODCONF_DRIFT_MINS, _type="N"
+                        modconf, self.MODCONF_DRIFT_MINS, type="N"
                     ),
                     reconcile_timeout_minutes=self._get_value(
-                        _modconf, self.MODCONF_TIMEOUT_MINS, _type="N"
+                        modconf, self.MODCONF_TIMEOUT_MINS, type="N"
                     ),
                 ),
             )

reconcile/gabi_authorized_users.py

@@ -67,7 +67,7 @@ def fetch_desired_state(
         expiration_date = datetime.strptime(g["expirationDate"], "%Y-%m-%d").date()
         if (expiration_date - date.today()).days > EXPIRATION_DAYS_MAX:
             raise RunnerException(
-                f'The maximum expiration date of {g["name"]} shall not '
+                f"The maximum expiration date of {g['name']} shall not "
                 f"exceed {EXPIRATION_DAYS_MAX} days from today"
             )
         for i in g["instances"]:
@@ -92,8 +92,8 @@ def fetch_desired_state(
                     break
             if not found:
                 raise RunnerException(
-                    f'[gabi:{g["name"]} (path: {g["path"]})] Could not find RDS identifier {identifier} '
-                    f'for account {account} in namespace {namespace["name"]}. '
+                    f"[gabi:{g['name']} (path: {g['path']})] Could not find RDS identifier {identifier} "
+                    f"for account {account} in namespace {namespace['name']}. "
                     "If this is a removed read only instance, consider updating the identifier to the source replica."
                 )
             users = get_usernames(g["users"], cluster)

reconcile/gcr_mirror.py

@@ -147,7 +147,7 @@ class QuayMirror:
         for org, data in summary.items():
             for item in data:
                 image = Image(
-                    f'{item["server_url"]}/{org}/{item["name"]}',
+                    f"{item['server_url']}/{org}/{item['name']}",
                     session=self.session,
                     timeout=REQUEST_TIMEOUT,
                 )
@@ -267,7 +267,7 @@ class QuayMirror:
             raw_data = self.secret_reader.read_all(push_secret)
             project = project_data["name"]
             token = base64.b64decode(raw_data["token"]).decode()
-            creds[project] = f'{raw_data["user"]}:{token}'
+            creds[project] = f"{raw_data['user']}:{token}"
         return creds
 
 

reconcile/github_org.py

@@ -120,7 +120,7 @@ def get_config(default=False):
             raise KeyError("default github org config not found")
         if len(found_defaults) > 1:
             raise KeyError(
-                "multiple default github org configs found: " f"{found_defaults}"
+                f"multiple default github org configs found: {found_defaults}"
             )
 
     return config
@@ -206,15 +206,11 @@ def fetch_desired_state(infer_clusters=True):
         if not permissions:
             continue
 
-        members = []
-
-        for user in role["users"]:
-            members.append(user["github_username"])
-
-        for bot in role["bots"]:
-            if "github_username" in bot:
-                members.append(bot["github_username"])
-        members = [m.lower() for m in members]
+        user_members = [user["github_username"] for user in role["users"]]
+        bot_members = [
+            bot["github_username"] for bot in role["bots"] if "github_username" in bot
+        ]
+        members = [m.lower() for m in user_members + bot_members]
 
         for permission in permissions:
             if permission["service"] == "github-org":
@@ -436,9 +432,9 @@ def run(dry_run):
     current_orgs = {item["params"]["org"] for item in current_state.dump()}
     desired_orgs = {item["params"]["org"] for item in desired_state.dump()}
 
-    assert (
-        current_orgs == desired_orgs
-    ), f"Current orgs ({current_orgs}) don't match desired orgs ({desired_orgs})"
+    assert current_orgs == desired_orgs, (
+        f"Current orgs ({current_orgs}) don't match desired orgs ({desired_orgs})"
+    )
 
     # Calculate diff
     diff = current_state.diff(desired_state)

reconcile/gitlab_housekeeping.py

@@ -174,7 +174,7 @@ def clean_pipelines(
                 gl_piplelines.get(p["id"]).cancel()
             except gitlab.exceptions.GitlabPipelineCancelError as err:
                 logging.error(
-                    f'unable to cancel {p["web_url"]} - '
+                    f"unable to cancel {p['web_url']} - "
                     f"error message {err.error_message}"
                 )
 

reconcile/gitlab_owners.py

@@ -198,7 +198,7 @@ class MRApproval:
         markdown_report = ""
 
         closest_approvers = []
-        for _, owners in report.items():
+        for owners in report.values():
             new_group = []
 
             if "closest_approvers" not in owners:
@@ -232,10 +232,10 @@ class MRApproval:
                 )
 
         for group in sorted(closest_approvers):
-            markdown_report += f'* {", ".join(group)}\n'
+            markdown_report += f"* {', '.join(group)}\n"
 
         approvers = set()
-        for _, owners in report.items():
+        for owners in report.values():
             if "approvers" not in owners:
                 continue
 
@@ -254,10 +254,10 @@ class MRApproval:
                 "\nIn case of emergency, the override approvers "
                 "(from parent directories) are:\n\n"
             )
-            markdown_report += f'* {", ".join(sorted(approvers))}\n'
+            markdown_report += f"* {', '.join(sorted(approvers))}\n"
 
         closest_reviewers = set()
-        for _, owners in report.items():
+        for owners in report.values():
             if "closest_reviewers" not in owners:
                 continue
 
@@ -274,11 +274,11 @@ class MRApproval:
                     closest_reviewers.add(closest_reviewer)
 
         if closest_reviewers:
-            markdown_report += "\nRelevant reviewers (with no " "merge rights) are:\n\n"
-            markdown_report += f'* {", ".join(sorted(closest_reviewers))}\n'
+            markdown_report += "\nRelevant reviewers (with no merge rights) are:\n\n"
+            markdown_report += f"* {', '.join(sorted(closest_reviewers))}\n"
 
         reviewers = set()
-        for _, owners in report.items():
+        for owners in report.values():
             if "reviewers" not in owners:
                 continue
 
@@ -303,7 +303,7 @@ class MRApproval:
                 "merge rights) from parent "
                 "directories are:\n\n"
             )
-            markdown_report += f'* {", ".join(sorted(reviewers))}\n'
+            markdown_report += f"* {', '.join(sorted(reviewers))}\n"
 
         return markdown_report.rstrip()
 
@@ -328,9 +328,7 @@ def act(repo, dry_run, instance, settings, defer=None):
 
         if mr_approval.top_commit_created_at is None:
             _LOG.info([
-                f"Project:{gitlab_cli.project.id} "
-                f"Merge Request:{mr.iid} "
-                f"- skipping"
+                f"Project:{gitlab_cli.project.id} Merge Request:{mr.iid} - skipping"
             ])
             continue
 

reconcile/gitlab_permissions.py

@@ -91,11 +91,12 @@ class GroupPermissionHandler:
         current_state: dict[str, GroupSpec],
     ) -> None:
         # gather list of app-interface managed repos
-        managed_repos: set[str] = set()
         instance = queries.get_gitlab_instance()
-        for project_request in instance.get("projectRequests", []):
-            for r in project_request.get("projects", []):
-                managed_repos.add(f"{instance['url']}/{project_request['group']}/{r}")
+        managed_repos = {
+            f"{instance['url']}/{project_request['group']}/{r}"
+            for project_request in instance.get("projectRequests", [])
+            for r in project_request.get("projects", [])
+        }
 
         # get the diff data
         diff_data = diff_mappings(