qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/jump_host.py

@@ -4,7 +4,6 @@ import shutil
 import tempfile
 import threading
 from dataclasses import dataclass
-from typing import Optional
 
 from sshtunnel import SSHTunnelForwarder
 
@@ -27,9 +26,9 @@ class JumphostParameters:
     hostname: str
     known_hosts: str
     user: str
-    port: Optional[int]
-    remote_port: Optional[int]
-    local_port: Optional[int]
+    port: int | None
+    remote_port: int | None
+    local_port: int | None
     key: str
 
 
@@ -45,7 +44,7 @@ class JumpHostBase:
         self._identity_dir = tempfile.mkdtemp()
 
         identity_file = self._identity_dir + "/id"
-        with open(identity_file, "w") as f:
+        with open(identity_file, "w", encoding="locale") as f:
             f.write(self._identity)
         os.chmod(identity_file, 0o600)
         self._identity_file = identity_file
@@ -60,7 +59,7 @@ class JumpHostSSH(JumpHostBase):
     tunnel_lock = threading.Lock()
 
     def __init__(
-        self, parameters: JumphostParameters, gql_api: Optional[gql.GqlApi] = None
+        self, parameters: JumphostParameters, gql_api: gql.GqlApi | None = None
     ):
         JumpHostBase.__init__(self, parameters=parameters)
 
@@ -75,7 +74,7 @@ class JumpHostSSH(JumpHostBase):
         self._remote_port = parameters.remote_port
 
     @property
-    def local_port(self) -> Optional[int]:
+    def local_port(self) -> int | None:
         return self._local_port
 
     @staticmethod
@@ -91,18 +90,18 @@ class JumpHostSSH(JumpHostBase):
         try:
             known_hosts = self._gql_api.get_resource(known_hosts_path)
         except gql.GqlGetResourceError as e:
-            raise FetchResourceError(str(e))
+            raise FetchResourceError(str(e)) from None
         return known_hosts["content"]
 
     def _init_known_hosts_file(self) -> None:
         known_hosts_file = self._identity_dir + "/known_hosts"
-        with open(known_hosts_file, "w") as f:
+        with open(known_hosts_file, "w", encoding="locale") as f:
             f.write(self._known_hosts)
         os.chmod(known_hosts_file, 0o600)
         self.known_hosts_file = known_hosts_file
 
     def get_ssh_base_cmd(self) -> list[str]:
-        user_host = "{}@{}".format(self._user, self._hostname)
+        user_host = f"{self._user}@{self._hostname}"
 
         return [
             "ssh",
@@ -115,7 +114,7 @@ class JumpHostSSH(JumpHostBase):
             "-o",
             "StrictHostKeyChecking=yes",
             "-o",
-            "UserKnownHostsFile={}".format(self.known_hosts_file),
+            f"UserKnownHostsFile={self.known_hosts_file}",
             "-i",
             self._identity_file,
             "-p",
@@ -128,12 +127,14 @@ class JumpHostSSH(JumpHostBase):
             if self._local_port not in JumpHostSSH.bastion_tunnel:
                 # Hide connect messages from sshtunnel
                 with toggle_logger():
+                    # equivalent to: ssh -i identity_file -L localhost:local_port:localhost:remote_port user@bastion
+                    # localhost:local_port -> bastion:22 -> bastion_localhost:remote_port(tinyproxy) -> cluster_host:6443
                     tunnel = SSHTunnelForwarder(
                         ssh_address_or_host=self._hostname,
                         ssh_port=self._port,
                         ssh_username=self._user,
                         ssh_pkey=self._identity_file,
-                        remote_bind_address=(self._hostname, self._remote_port),
+                        remote_bind_address=("localhost", self._remote_port),
                         local_bind_address=("localhost", self._local_port),
                     )
                     tunnel.start()

reconcile/utils/keycloak.py

@@ -0,0 +1,106 @@
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+import requests
+from pydantic import BaseModel
+
+
+class SSOClient(BaseModel):
+    # these attributes come from the Keycloak API
+    client_id: str
+    client_id_issued_at: int
+    client_name: str
+    client_secret: str
+    client_secret_expires_at: int
+    grant_types: list[str]
+    redirect_uris: list[str]
+    registration_access_token: str
+    registration_client_uri: str
+    request_uris: list[str]
+    response_types: list[str]
+    subject_type: str
+    tls_client_certificate_bound_access_tokens: bool
+    token_endpoint_auth_method: str
+    # attribute added by the reconcile code and not part of the SSO client data
+    issuer: str
+
+
+class KeycloakInstance(BaseModel):
+    url: str
+    initial_access_token: str | None = None
+
+
+class KeycloakAPI:
+    def __init__(
+        self,
+        url: str | None = None,
+        initial_access_token: str | None = None,
+        timeout: int = 30,
+    ) -> None:
+        self.url = url
+        self.initial_access_token = initial_access_token
+        self.timeout = timeout
+        self._openid_configuration = self._init_openid_configuration()
+
+    def _init_openid_configuration(self) -> dict[str, Any] | None:
+        if not self.url:
+            return None
+        return requests.get(
+            f"{self.url}/.well-known/openid-configuration",
+            timeout=self.timeout,
+        ).json()
+
+    def register_client(
+        self,
+        client_name: str,
+        redirect_uris: Sequence[str],
+        initiate_login_uri: str,
+        request_uris: Sequence[str],
+        contacts: Sequence[str],
+    ) -> SSOClient:
+        """Create a new SSO client."""
+        if not self.initial_access_token:
+            raise ValueError("initial_access_token is required")
+        if not self._openid_configuration:
+            raise ValueError("openid_configuration is required")
+
+        response = requests.post(
+            self._openid_configuration["registration_endpoint"],
+            json={
+                "client_name": client_name,
+                "redirect_uris": redirect_uris,
+                "response_types": ["code"],
+                "grant_types": ["authorization_code"],
+                "application_type": "web",
+                "contacts": contacts,
+                "initiate_login_uri": initiate_login_uri,
+                "request_uris": request_uris,
+            },
+            headers={"Authorization": f"Bearer {self.initial_access_token}"},
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        return SSOClient(**response.json(), issuer=self.url)
+
+    def delete_client(
+        self, registration_client_uri: str, registration_access_token: str
+    ) -> None:
+        response = requests.delete(
+            registration_client_uri,
+            headers={"Authorization": f"Bearer {registration_access_token}"},
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+
+
+class KeycloakMap:
+    def __init__(self, keycloak_instances: Iterable[KeycloakInstance]) -> None:
+        self._map = {
+            instance.url: KeycloakAPI(
+                url=instance.url, initial_access_token=instance.initial_access_token
+            )
+            for instance in keycloak_instances
+        }
+
+    def get(self, keycloak_url: str) -> KeycloakAPI:
+        return self._map[keycloak_url]

reconcile/utils/ldap_client.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from collections.abc import Iterable
 
 from ldap3 import (
@@ -28,20 +29,48 @@ class LdapClient:
         self.connection.unbind()
 
     def get_users(self, uids: Iterable[str]) -> set[str]:
-        user_filter = "".join((f"(uid={u})" for u in uids))
+        user_filter = "".join(f"(uid={u})" for u in uids)
         _, _, results, _ = self.connection.search(
             self.base_dn, f"(&(objectclass=person)(|{user_filter}))", attributes=["uid"]
         )
-        return set(r["attributes"]["uid"][0] for r in results)
+        return {r["attributes"]["uid"][0] for r in results}
+
+    def get_group_members(self, groups_dns: set[str]) -> dict[str, set[str]]:
+        """
+        Returns a dictionary of group dns and their members.
+        """
+        if not groups_dns:
+            return {}
+        filter = f"(|{''.join([f'(memberOf={dn})' for dn in sorted(groups_dns)])})"
+
+        _, _, users, _ = self.connection.search(
+            self.base_dn,
+            filter,
+            attributes=["uid", "memberOf"],
+        )
+        groups_and_members: dict[str, set[str]] = defaultdict(set[str])
+        for u in users:
+            uid = u["attributes"]["uid"][0]
+            for group in set(u["attributes"]["memberOf"]).intersection(groups_dns):
+                groups_and_members[group].add(uid)
+
+        return dict(groups_and_members)
 
     @classmethod
     def from_settings(cls, settings: dict) -> "LdapClient":
         """Requires a nested dictionary with key 'ldap' in addition sub keys 'serverUrl' and 'baseDn'."""
+        return LdapClient.from_params(
+            settings["ldap"]["serverUrl"], None, None, settings["ldap"]["baseDn"]
+        )
+
+    @classmethod
+    def from_params(
+        cls, server_url: str, user: str | None, password: str | None, base_dn: str
+    ) -> "LdapClient":
         connection = Connection(
-            Server(settings["ldap"]["serverUrl"], get_info=ALL),
-            None,
-            None,
+            Server(server_url, get_info=ALL),
+            user,
+            password,
             client_strategy=SAFE_SYNC,
         )
-        base_dn = settings["ldap"]["baseDn"]
         return cls(base_dn, connection)

reconcile/utils/lean_terraform_client.py

@@ -1,6 +1,9 @@
 import json
 import logging
+import os
 import subprocess
+from collections.abc import Mapping
+from typing import Any
 
 
 def state_rm_access_key(working_dirs, account, user):
@@ -8,20 +11,126 @@ def state_rm_access_key(working_dirs, account, user):
     init_result = subprocess.run(["terraform", "init"], check=False, cwd=wd)
     if init_result.returncode != 0:
         return False
-    resource = "aws_iam_access_key.{}".format(user)
+    resource = f"aws_iam_access_key.{user}"
     result = subprocess.run(["terraform", "state", "rm", resource], check=False, cwd=wd)
     return result.returncode == 0
 
 
-def show_json(working_dir, out_file):
+def _compute_terraform_env(
+    env: Mapping[str, str] | None = None,
+) -> Mapping[str, str]:
+    default_env = os.environ.copy()
+    return default_env if env is None else {**default_env, **env}
+
+
+def _terraform_command(
+    args: list[str],
+    working_dir: str,
+    env: Mapping[str, str] | None = None,
+) -> tuple[int, str, str]:
     result = subprocess.run(
-        ["terraform", "show", "-no-color", "-json", out_file],
+        args,
         capture_output=True,
         check=False,
         cwd=working_dir,
+        env=_compute_terraform_env(env),
+    )
+    return_code = result.returncode
+    stdout = result.stdout.decode("utf-8")
+    stderr = result.stderr.decode("utf-8")
+    return return_code, stdout, stderr
+
+
+def show_json(working_dir: str, path: str) -> dict[str, Any]:
+    """
+    Run terraform show -no-color -json <path>.
+
+    :param working_dir: The directory where the terraform files are located
+    :param path: The path to the plan file
+    :return: Deserialized JSON from the terraform show command
+    """
+    return_code, stdout, stderr = _terraform_command(
+        args=["terraform", "show", "-no-color", "-json", path],
+        working_dir=working_dir,
     )
-    if result.returncode != 0:
-        msg = f"[{out_file}] terraform show failed: {result.stderr.decode('utf-8')}"
+    if return_code != 0:
+        msg = f"[{path}] terraform show failed: {stderr}"
         logging.warning(msg)
         raise Exception(msg)
-    return json.loads(result.stdout)
+    return json.loads(stdout)
+
+
+def init(
+    working_dir: str,
+    env: Mapping[str, str] | None = None,
+) -> tuple[int, str, str]:
+    """
+    Run terraform init -input=false -no-color.
+
+    :param working_dir: The directory where the terraform files are located
+    :param env: Environment variables to pass to the terraform command
+    :return: (return_code, stdout, stderr)
+    """
+    return _terraform_command(
+        args=["terraform", "init", "-input=false", "-no-color"],
+        working_dir=working_dir,
+        env=env,
+    )
+
+
+def output(
+    working_dir: str,
+    env: Mapping[str, str] | None = None,
+) -> tuple[int, str, str]:
+    """
+    Run terraform output -json.
+
+    :param working_dir: The directory where the terraform files are located
+    :param env: Environment variables to pass to the terraform command
+    :return: (return_code, stdout, stderr)
+    """
+    return _terraform_command(
+        args=["terraform", "output", "-json"],
+        working_dir=working_dir,
+        env=env,
+    )
+
+
+def plan(
+    working_dir: str,
+    out: str,
+    env: Mapping[str, str] | None = None,
+) -> tuple[int, str, str]:
+    """
+    Run terraform plan -out=<out> -input=false -no-color.
+
+    :param working_dir: The directory where the terraform files are located
+    :param out: The path to the plan file
+    :param env: Environment variables to pass to the terraform command
+    :return: (return_code, stdout, stderr)
+    """
+    return _terraform_command(
+        args=["terraform", "plan", f"-out={out}", "-input=false", "-no-color"],
+        working_dir=working_dir,
+        env=env,
+    )
+
+
+def apply(
+    working_dir: str,
+    dir_or_plan: str,
+    env: Mapping[str, str] | None = None,
+) -> tuple[int, str, str]:
+    """
+    Run terraform apply -input=false -no-color <dir_or_plan>.
+
+    :param working_dir: The directory where the terraform files are located
+    :param dir_or_plan: The path to the directory or plan file
+    :param env: Environment variables to pass to the terraform command
+    :return: (return_code, stdout, stderr)
+    """
+    return _terraform_command(
+        args=["terraform", "apply", "-input=false", "-no-color", dir_or_plan],
+        working_dir=working_dir,
+        env=env,
+    )

reconcile/utils/membershipsources/app_interface_resolver.py

@@ -0,0 +1,60 @@
+import base64
+from collections.abc import Generator
+from contextlib import contextmanager
+
+from reconcile import queries
+from reconcile.gql_definitions.fragments.membership_source import (
+    AppInterfaceMembershipProviderSourceV1,
+)
+from reconcile.gql_definitions.membershipsources.roles import RoleV1
+from reconcile.gql_definitions.membershipsources.roles import (
+    query as mebershipsource_query,
+)
+from reconcile.utils import gql
+from reconcile.utils.membershipsources.models import (
+    ProviderGroup,
+    RoleBot,
+    RoleMember,
+    RoleUser,
+)
+from reconcile.utils.secret_reader import SecretReader
+
+
+@contextmanager
+def gql_api_for_source(
+    source: AppInterfaceMembershipProviderSourceV1,
+) -> Generator[gql.GqlApi, None, None]:
+    settings = queries.get_secret_reader_settings()
+    secret_reader = SecretReader(settings=settings)
+    username = secret_reader.read_secret(source.username)
+    password = secret_reader.read_secret(source.password)
+    basic_auth_info = base64.b64encode(f"{username}:{password}".encode()).decode()
+    gql_api = gql.get_api_for_server(
+        source.url, f"Basic {basic_auth_info}", None, False
+    )
+    try:
+        yield gql_api
+    finally:
+        gql_api.close()
+
+
+def resolve_app_interface_membership_source(
+    provider_name: str,
+    source: AppInterfaceMembershipProviderSourceV1,
+    groups: set[str],
+) -> dict[ProviderGroup, list[RoleMember]]:
+    with gql_api_for_source(source) as gql_api:
+        roles = (
+            mebershipsource_query(
+                gql_api.query, variables={"filter": {"name": {"in": list(groups)}}}
+            ).roles
+            or []
+        )
+        return {(provider_name, r.name): build_member_list(r) for r in roles}
+
+
+def build_member_list(role: RoleV1) -> list[RoleMember]:
+    members: list[RoleMember] = []
+    members.extend([RoleUser(**u.dict()) for u in role.users or []])
+    members.extend([RoleBot(**b.dict()) for b in role.bots or [] if b.org_username])
+    return members

reconcile/utils/membershipsources/models.py

@@ -0,0 +1,91 @@
+from collections.abc import Callable, Sequence
+from typing import (
+    Any,
+    Protocol,
+    TypeVar,
+)
+
+from pydantic import (
+    BaseModel,
+    Extra,
+)
+
+from reconcile.gql_definitions.fragments.membership_source import (
+    MembershipProviderSourceV1,
+    RoleMembershipSource,
+)
+
+
+class User(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def org_username(self) -> str: ...
+
+    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+
+
+class Bot(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def org_username(self) -> str | None: ...
+
+    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+
+
+class RoleWithMemberships(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def users(self) -> Sequence[User]: ...
+
+    @property
+    def bots(self) -> Sequence[Bot]: ...
+
+    @property
+    def member_sources(self) -> Sequence[RoleMembershipSource] | None: ...
+
+
+class RoleUser(BaseModel):
+    name: str
+    org_username: str
+    github_username: str | None
+    quay_username: str | None
+    slack_username: str | None
+    pagerduty_username: str | None
+    aws_username: str | None
+    cloudflare_user: str | None
+    public_gpg_key: str | None
+    tag_on_cluster_updates: bool | None = False
+    tag_on_merge_requests: bool | None = False
+
+    class Config:
+        extra = Extra.ignore
+
+
+class RoleBot(BaseModel):
+    name: str
+    description: str | None
+    org_username: str | None
+    github_username: str | None
+    gitlab_username: str | None
+    openshift_serviceaccount: str | None
+    quay_username: str | None
+
+    class Config:
+        extra = Extra.ignore
+
+
+RoleMember = RoleUser | RoleBot
+
+ProviderGroup = tuple[str, str]
+
+ProviderSource = TypeVar("ProviderSource", bound=MembershipProviderSourceV1)
+
+ProviderResolver = Callable[
+    [str, ProviderSource, set[str]], dict[ProviderGroup, list[RoleMember]]
+]

reconcile/utils/membershipsources/resolver.py

@@ -0,0 +1,110 @@
+from collections.abc import (
+    Iterable,
+    Sequence,
+)
+from dataclasses import dataclass
+from itertools import chain
+
+from sretoolbox.utils import threaded
+
+from reconcile.gql_definitions.fragments.membership_source import (
+    AppInterfaceMembershipProviderSourceV1,
+    MembershipProviderSourceV1,
+    MembershipProviderV1,
+)
+from reconcile.utils.grouping import group_by
+from reconcile.utils.membershipsources.app_interface_resolver import (
+    resolve_app_interface_membership_source,
+)
+from reconcile.utils.membershipsources.models import (
+    ProviderResolver,
+    RoleBot,
+    RoleMember,
+    RoleUser,
+    RoleWithMemberships,
+)
+
+
+@dataclass
+class GroupResolverJob:
+    provider: MembershipProviderV1
+    groups: set[str]
+
+
+def build_resolver_jobs(
+    roles: Sequence[RoleWithMemberships],
+) -> list[GroupResolverJob]:
+    """
+    Bundles groups to be resolve by provider so that they can be resolved
+    in batches.
+    """
+    groups = group_by(
+        (ms for r in roles for ms in r.member_sources or []),
+        key=lambda ms: ms.provider.name,
+    )
+    return [
+        GroupResolverJob(
+            provider=g[0].provider,
+            groups={ms.group for ms in g},
+        )
+        for g in groups.values()
+    ]
+
+
+ProviderGroup = tuple[str, str]
+
+
+def get_resolver_for_provider_source(
+    source: MembershipProviderSourceV1,
+) -> ProviderResolver:
+    match source:
+        case AppInterfaceMembershipProviderSourceV1():
+            return resolve_app_interface_membership_source
+        case _:
+            raise ValueError(
+                "No resolver available for membership provider source",
+                type(source),
+            )
+
+
+def resolve_groups(job: GroupResolverJob) -> dict[ProviderGroup, list[RoleMember]]:
+    """
+    Resolves groups and returns a dict with group name as key and a list
+    of members as value.
+    """
+    resolver = get_resolver_for_provider_source(job.provider.source)
+    return resolver(job.provider.name, job.provider.source, job.groups)
+
+
+def resolve_role_members(
+    roles: Sequence[RoleWithMemberships], thread_pool: int = 5
+) -> dict[str, list[RoleMember]]:
+    """
+    Resolves members of roles, combining local members and the ones from
+    membership sources.
+    """
+    resolver_jobs = build_resolver_jobs(roles)
+    processed_jobs: Iterable[dict[ProviderGroup, list[RoleMember]]] = threaded.run(
+        func=resolve_groups,
+        iterable=resolver_jobs,
+        thread_pool_size=thread_pool,
+    )
+    resolved_groups: dict[ProviderGroup, list[RoleMember]] = dict(
+        chain.from_iterable(d.items() for d in processed_jobs)
+    )
+
+    members_by_group = {}
+    for r in roles:
+        members: list[RoleMember] = []
+
+        # bring in the local users and bots ...
+        members.extend(RoleUser(**u.dict()) for u in r.users or [])
+        members.extend(RoleBot(**b.dict()) for b in r.bots or [] if b.org_username)
+
+        # ... and enhance with the ones from member sources
+        for ms in r.member_sources or []:
+            members.extend(resolved_groups.get((ms.provider.name, ms.group), []))
+
+        members_by_group[r.name] = members
+
+    return members_by_group