qontract-reconcile 0.10.2.dev186__tar.gz → 0.10.2.dev187__tar.gz

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev186
+Version: 0.10.2.dev187
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/pyproject.toml

@@ -263,7 +263,6 @@ module = [
     "reconcile.jenkins_webhooks_cleaner",
     "reconcile.jenkins_webhooks",
     "reconcile.jira_watcher",
-    "reconcile.ldap_users",
     "reconcile.mr_client_gateway",
     "reconcile.ocm_additional_routers",
     "reconcile.ocm_addons",
@@ -383,7 +382,6 @@ module = [
     "reconcile.test.test_integrations_manager",
     "reconcile.test.test_jenkins_worker_fleets",
     "reconcile.test.test_jump_host",
-    "reconcile.test.test_ldap_users",
     "reconcile.test.test_make",
     "reconcile.test.test_ocm_additional_routers",
     "reconcile.test.test_ocm_clusters_manifest_updates",

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/reconcile/github_users.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import re
+from collections import defaultdict
 from collections.abc import Callable
 
 from github import Github
@@ -17,9 +18,9 @@ from reconcile import (
     typed_queries,
 )
 from reconcile.github_org import get_default_config
-from reconcile.ldap_users import init_users as init_users_and_paths
 from reconcile.utils.defer import defer
 from reconcile.utils.mr import CreateDeleteUserAppInterface
+from reconcile.utils.mr.user_maintenance import PathSpec, PathTypes
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.smtp_client import (
     DEFAULT_SMTP_TIMEOUT,
@@ -39,6 +40,33 @@ def init_github() -> Github:
     return Github(token, base_url=GH_BASE_URL)
 
 
+def init_users_and_paths() -> list[dict[str, list]]:
+    app_int_users = queries.get_users(refs=True)
+
+    users = defaultdict(list)
+    for user in app_int_users:
+        u = user["org_username"]
+        item = PathSpec(type=PathTypes.USER, path=user["path"])
+        users[u].append(item)
+        for r in user.get("requests"):
+            item = PathSpec(type=PathTypes.REQUEST, path=r["path"])
+            users[u].append(item)
+        for q in user.get("queries"):
+            item = PathSpec(type=PathTypes.QUERY, path=q["path"])
+            users[u].append(item)
+        for g in user.get("gabi_instances"):
+            item = PathSpec(type=PathTypes.GABI, path=g["path"])
+            users[u].append(item)
+        for a in user.get("aws_accounts", []):
+            item = PathSpec(type=PathTypes.AWS_ACCOUNTS, path=a["path"])
+            users[u].append(item)
+        for s in user.get("schedules"):
+            item = PathSpec(type=PathTypes.SCHEDULE, path=s["path"])
+            users[u].append(item)
+
+    return [{"username": username, "paths": paths} for username, paths in users.items()]
+
+
 @retry(exceptions=(GithubException, ReadTimeout))
 def get_user_company(user: dict, github: Github) -> UserAndCompany:
     gh_user = github.get_user(login=user["github_username"])

qontract_reconcile-0.10.2.dev187/reconcile/gql_definitions/common/ldap_settings.py

@@ -0,0 +1,68 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query LdapSettings {
+  settings: app_interface_settings_v1 {
+    ldap {
+      serverUrl
+      baseDn
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class LdapSettingsV1(ConfiguredBaseModel):
+    server_url: str = Field(..., alias="serverUrl")
+    base_dn: str = Field(..., alias="baseDn")
+
+
+class AppInterfaceSettingsV1(ConfiguredBaseModel):
+    ldap: Optional[LdapSettingsV1] = Field(..., alias="ldap")
+
+
+class LdapSettingsQueryData(ConfiguredBaseModel):
+    settings: Optional[list[AppInterfaceSettingsV1]] = Field(..., alias="settings")
+
+
+def query(query_func: Callable, **kwargs: Any) -> LdapSettingsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        LdapSettingsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return LdapSettingsQueryData(**raw_data)

qontract_reconcile-0.10.2.dev187/reconcile/gql_definitions/common/users_with_paths.py

@@ -0,0 +1,102 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query UsersWithPaths {
+  users: users_v1 {
+    path
+    org_username
+    aws_accounts {
+      path
+    }
+    requests {
+      path
+    }
+    queries {
+      path
+    }
+    gabi_instances {
+      path
+    }
+    schedules {
+      path
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AWSAccountV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class CredentialsRequestV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class AppInterfaceSqlQueryV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class GabiInstanceV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class ScheduleV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class UserV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    org_username: str = Field(..., alias="org_username")
+    aws_accounts: Optional[list[AWSAccountV1]] = Field(..., alias="aws_accounts")
+    requests: Optional[list[CredentialsRequestV1]] = Field(..., alias="requests")
+    queries: Optional[list[AppInterfaceSqlQueryV1]] = Field(..., alias="queries")
+    gabi_instances: Optional[list[GabiInstanceV1]] = Field(..., alias="gabi_instances")
+    schedules: Optional[list[ScheduleV1]] = Field(..., alias="schedules")
+
+
+class UsersWithPathsQueryData(ConfiguredBaseModel):
+    users: Optional[list[UserV1]] = Field(..., alias="users")
+
+
+def query(query_func: Callable, **kwargs: Any) -> UsersWithPathsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        UsersWithPathsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return UsersWithPathsQueryData(**raw_data)

qontract_reconcile-0.10.2.dev187/reconcile/ldap_users.py

@@ -0,0 +1,126 @@
+import logging
+from collections.abc import Callable
+
+from pydantic import BaseModel, Field
+
+from reconcile import (
+    mr_client_gateway,
+)
+from reconcile.gql_definitions.common.ldap_settings import LdapSettingsV1
+from reconcile.gql_definitions.common.users_with_paths import UserV1
+from reconcile.typed_queries.ldap_settings import get_ldap_settings
+from reconcile.typed_queries.users_with_paths import get_users_with_paths
+from reconcile.utils.defer import defer
+from reconcile.utils.ldap_client import LdapClient
+from reconcile.utils.mr import (
+    CreateDeleteUserAppInterface,
+    CreateDeleteUserInfra,
+)
+from reconcile.utils.mr.user_maintenance import PathSpec, PathTypes
+
+QONTRACT_INTEGRATION = "ldap-users"
+
+
+class UserPaths(BaseModel):
+    username: str
+    paths: list[PathSpec] = Field(default_factory=list)
+
+
+def transform_users_with_paths(users_with_paths: list[UserV1]) -> list[UserPaths]:
+    """Converts UserV1 objects into UserPaths, consolidating all associated resource paths by type."""
+
+    users_paths = []
+    for user in users_with_paths:
+        up = UserPaths(username=user.org_username)
+        up.paths.append(PathSpec(type=PathTypes.USER, path=user.path))
+        for r in user.requests or []:
+            up.paths.append(PathSpec(type=PathTypes.REQUEST, path=r.path))
+        for q in user.queries or []:
+            up.paths.append(PathSpec(type=PathTypes.QUERY, path=q.path))
+        for g in user.gabi_instances or []:
+            up.paths.append(PathSpec(type=PathTypes.GABI, path=g.path))
+        for a in user.aws_accounts or []:
+            up.paths.append(PathSpec(type=PathTypes.AWS_ACCOUNTS, path=a.path))
+        for s in user.schedules or []:
+            up.paths.append(PathSpec(type=PathTypes.SCHEDULE, path=s.path))
+
+        users_paths.append(up)
+
+    return users_paths
+
+
+def get_usernames(users_paths: list[UserPaths]) -> list[str]:
+    """Extracts a list of usernames from a list of UserPaths objects."""
+    return [u.username for u in users_paths]
+
+
+def filter_users_not_exists(
+    users_paths: list[UserPaths], ldap_users: set[str]
+) -> list[UserPaths]:
+    """Filters out UserPaths objects whose usernames are not present in the given set of LDAP users."""
+
+    return [u for u in users_paths if u.username not in ldap_users]
+
+
+def get_ldap_users(ldap_settings: LdapSettingsV1, usernames: list[str]) -> set[str]:
+    """Retrieves existing usernames from LDAP based on the provided list and connection settings."""
+    with LdapClient.from_params(
+        server_url=ldap_settings.server_url,
+        user=None,
+        password=None,
+        base_dn=ldap_settings.base_dn,
+    ) as ldap_client:
+        return ldap_client.get_users(usernames)
+
+
+@defer
+def delete_user_from_app_interface(
+    dry_run: bool,
+    app_interface_project_id: str | int | None,
+    users: list[UserPaths],
+    defer: Callable | None = None,
+) -> None:
+    """Delete user data stored in the repository app-interface related with the given users."""
+    if not dry_run:
+        mr_cli_app_interface = mr_client_gateway.init(
+            gitlab_project_id=app_interface_project_id, sqs_or_gitlab="gitlab"
+        )
+        if defer:
+            defer(mr_cli_app_interface.cleanup)
+
+    for user in users:
+        logging.info(["delete_user", user.username])
+
+        if not dry_run:
+            mr = CreateDeleteUserAppInterface(user.username, user.paths)
+            mr.submit(cli=mr_cli_app_interface)
+
+
+def delete_user_from_infra(
+    dry_run: bool,
+    infra_project_id: str | int | None,
+    usernames: list[str],
+) -> None:
+    """Delete user data stored in the repository infra related with the given usernames."""
+    if not dry_run:
+        with mr_client_gateway.init(
+            gitlab_project_id=infra_project_id, sqs_or_gitlab="gitlab"
+        ) as mr_cli_infra:
+            mr_infra = CreateDeleteUserInfra(usernames)
+            mr_infra.submit(cli=mr_cli_infra)
+
+
+def run(dry_run: bool, app_interface_project_id: str, infra_project_id: str) -> None:
+    """
+    Synchronizes user data stored in the repositories app_interface and infra
+    associated with users that are no longer found in the LDAP.
+    """
+    users_paths = transform_users_with_paths(get_users_with_paths())
+
+    ldap_users = get_ldap_users(get_ldap_settings(), get_usernames(users_paths))
+
+    users_to_delete = filter_users_not_exists(users_paths, ldap_users)
+    usernames_to_delete = get_usernames(users_to_delete)
+
+    delete_user_from_app_interface(dry_run, app_interface_project_id, users_to_delete)
+    delete_user_from_infra(dry_run, infra_project_id, usernames_to_delete)

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/reconcile/oum/providers.py

@@ -3,7 +3,7 @@ from abc import (
     abstractmethod,
 )
 
-from reconcile.ldap_users import get_ldap_settings
+from reconcile.typed_queries.ldap_settings import get_ldap_settings
 from reconcile.utils.ldap_client import LdapClient
 
 
@@ -53,7 +53,10 @@ def init_ldap_group_member_provider(group_base_dn: str) -> LdapGroupMemberProvid
     settings = get_ldap_settings()
     return LdapGroupMemberProvider(
         LdapClient.from_params(
-            settings["ldap"]["serverUrl"], None, None, settings["ldap"]["baseDn"]
+            server_url=settings.server_url,
+            user=None,
+            password=None,
+            base_dn=settings.base_dn,
         ),
         group_base_dn,
     )

qontract_reconcile-0.10.2.dev187/reconcile/typed_queries/ldap_settings.py

@@ -0,0 +1,20 @@
+from collections.abc import Callable
+
+from reconcile.gql_definitions.common.ldap_settings import (
+    LdapSettingsV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.exceptions import AppInterfaceSettingsError
+
+
+def get_ldap_settings(
+    query_func: Callable | None = None,
+) -> LdapSettingsV1:
+    """Returns App Interface Settings and raises err if none are found"""
+    if not query_func:
+        query_func = gql.get_api().query
+    data = query(query_func)
+    if data.settings and len(data.settings) == 1 and data.settings[0].ldap:
+        return data.settings[0].ldap
+    raise AppInterfaceSettingsError("Ldap settings is not defined.")

qontract_reconcile-0.10.2.dev187/reconcile/typed_queries/users_with_paths.py

@@ -0,0 +1,10 @@
+from collections.abc import Callable
+
+from reconcile.gql_definitions.common.users_with_paths import UserV1, query
+from reconcile.utils import gql
+
+
+def get_users_with_paths(query_func: Callable | None = None) -> list[UserV1]:
+    if not query_func:
+        query_func = gql.get_api().query
+    return query(query_func=query_func).users or []

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/reconcile/utils/ldap_client.py

@@ -56,13 +56,6 @@ class LdapClient:
 
         return dict(groups_and_members)
 
-    @classmethod
-    def from_settings(cls, settings: dict) -> "LdapClient":
-        """Requires a nested dictionary with key 'ldap' in addition sub keys 'serverUrl' and 'baseDn'."""
-        return LdapClient.from_params(
-            settings["ldap"]["serverUrl"], None, None, settings["ldap"]["baseDn"]
-        )
-
     @classmethod
     def from_params(
         cls, server_url: str, user: str | None, password: str | None, base_dn: str

{qontract_reconcile-0.10.2.dev186 → qontract_reconcile-0.10.2.dev187}/reconcile/utils/mr/user_maintenance.py

@@ -1,5 +1,8 @@
+from collections.abc import Iterable
+from enum import Enum
 from pathlib import Path
 
+from pydantic import BaseModel, validator
 from ruamel import yaml
 
 from reconcile.utils.gitlab_api import GitLabApi
@@ -7,7 +10,7 @@ from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
 
 
-class PathTypes:
+class PathTypes(Enum):
     USER = 0
     REQUEST = 1
     QUERY = 2
@@ -16,10 +19,19 @@ class PathTypes:
     SCHEDULE = 5
 
 
+class PathSpec(BaseModel):
+    type: PathTypes
+    path: str
+
+    @validator("path")
+    def prepend_data_to_path(cls, v):
+        return "data" + v
+
+
 class CreateDeleteUserAppInterface(MergeRequestBase):
     name = "create_delete_user_mr"
 
-    def __init__(self, username, paths):
+    def __init__(self, username, paths: Iterable[PathSpec]):
         self.username = username
         self.paths = paths
 
@@ -37,8 +49,8 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
 
     def process(self, gitlab_cli: GitLabApi) -> None:
         for path_spec in self.paths:
-            path_type = path_spec["type"]
-            path = path_spec["path"]
+            path_type = path_spec.type
+            path = path_spec.path
             if path_type in {PathTypes.USER, PathTypes.REQUEST, PathTypes.QUERY}:
                 gitlab_cli.delete_file(
                     branch_name=self.branch, file_path=path, commit_message=self.title

qontract_reconcile-0.10.2.dev186/reconcile/ldap_users.py

@@ -1,99 +0,0 @@
-import logging
-from collections import defaultdict
-
-from reconcile import (
-    mr_client_gateway,
-    queries,
-)
-from reconcile.utils import gql
-from reconcile.utils.defer import defer
-from reconcile.utils.ldap_client import LdapClient
-from reconcile.utils.mr import (
-    CreateDeleteUserAppInterface,
-    CreateDeleteUserInfra,
-)
-from reconcile.utils.mr.user_maintenance import PathTypes
-
-QONTRACT_INTEGRATION = "ldap-users"
-
-
-def init_users() -> list[dict[str, list]]:
-    app_int_users = queries.get_users(refs=True)
-
-    users = defaultdict(list)
-    for user in app_int_users:
-        u = user["org_username"]
-        item = {"type": PathTypes.USER, "path": "data" + user["path"]}
-        users[u].append(item)
-        for r in user.get("requests"):
-            item = {"type": PathTypes.REQUEST, "path": "data" + r["path"]}
-            users[u].append(item)
-        for q in user.get("queries"):
-            item = {"type": PathTypes.QUERY, "path": "data" + q["path"]}
-            users[u].append(item)
-        for g in user.get("gabi_instances"):
-            item = {"type": PathTypes.GABI, "path": "data" + g["path"]}
-            users[u].append(item)
-        for a in user.get("aws_accounts", []):
-            item = {"type": PathTypes.AWS_ACCOUNTS, "path": "data" + a["path"]}
-            users[u].append(item)
-        for s in user.get("schedules"):
-            item = {"type": PathTypes.SCHEDULE, "path": "data" + s["path"]}
-            users[u].append(item)
-
-    return [{"username": username, "paths": paths} for username, paths in users.items()]
-
-
-LDAP_SETTINGS_QUERY = """
-{
-  settings: app_interface_settings_v1 {
-    ldap {
-      serverUrl
-      baseDn
-    }
-  }
-}
-"""
-
-
-def get_ldap_settings() -> dict:
-    """Returns LDAP settings"""
-    gqlapi = gql.get_api()
-    settings = gqlapi.query(LDAP_SETTINGS_QUERY)["settings"]
-    if settings:
-        # assuming a single settings file for now
-        return settings[0]
-    raise ValueError("no app-interface-settings settings found")
-
-
-@defer
-def run(dry_run, app_interface_project_id, infra_project_id, defer=None):
-    users = init_users()
-    with LdapClient.from_settings(get_ldap_settings()) as ldap_client:
-        ldap_users = ldap_client.get_users([u["username"] for u in users])
-
-    users_to_delete = [u for u in users if u["username"] not in ldap_users]
-
-    if not dry_run:
-        mr_cli_app_interface = mr_client_gateway.init(
-            gitlab_project_id=app_interface_project_id, sqs_or_gitlab="gitlab"
-        )
-        defer(mr_cli_app_interface.cleanup)
-        mr_cli_infra = mr_client_gateway.init(
-            gitlab_project_id=infra_project_id, sqs_or_gitlab="gitlab"
-        )
-        defer(mr_cli_infra.cleanup)
-
-    for u in users_to_delete:
-        username = u["username"]
-        paths = u["paths"]
-        logging.info(["delete_user", username])
-
-        if not dry_run:
-            mr = CreateDeleteUserAppInterface(username, paths)
-            mr.submit(cli=mr_cli_app_interface)
-
-    if not dry_run:
-        usernames = [u["username"] for u in users_to_delete]
-        mr_infra = CreateDeleteUserInfra(usernames)
-        mr_infra.submit(cli=mr_cli_infra)