qontract-reconcile 0.10.2.dev135__py3-none-any.whl → 0.10.2.dev136__py3-none-any.whl

reconcile/email_sender.py

@@ -1,13 +1,21 @@
 import logging
 import sys
+from collections.abc import Callable
 
-from reconcile import (
-    queries,
-    typed_queries,
+from reconcile import typed_queries
+from reconcile.gql_definitions.email_sender.apps import query as apps_query
+from reconcile.gql_definitions.email_sender.emails import (
+    AppInterfaceEmailAudienceV1,
+    AppInterfaceEmailV1,
 )
+from reconcile.gql_definitions.email_sender.emails import query as emails_query
+from reconcile.gql_definitions.email_sender.users import query as users_query
+from reconcile.gql_definitions.fragments.email_service import EmailServiceOwners
+from reconcile.gql_definitions.fragments.email_user import EmailUser
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
+from reconcile.utils import gql
 from reconcile.utils.defer import defer
 from reconcile.utils.secret_reader import create_secret_reader
 from reconcile.utils.smtp_client import (
@@ -20,11 +28,17 @@ from reconcile.utils.state import init_state
 QONTRACT_INTEGRATION = "email-sender"
 
 
-def collect_to(to):
+def collect_to(
+    to: AppInterfaceEmailAudienceV1,
+    all_users: list[EmailUser],
+    all_services: list[EmailServiceOwners],
+) -> set[str]:
     """Collect audience to send email to from to object
 
     Arguments:
-        to {dict} -- AppInterfaceEmailAudience_v1 object
+        to -- AppInterfaceEmailAudience_v1 object
+        all_users -- List of all app-interface users
+        all_services -- List of all app-interface apps/services with owners
 
     Raises:
         AttributeError: Unknown alias
@@ -32,63 +46,58 @@ def collect_to(to):
     Returns:
         set -- Audience to send email to
     """
-    audience = set()
-
-    aliases = to.get("aliases")
-    if aliases:
-        for alias in aliases:
-            if alias == "all-users":
-                users = queries.get_users()
-                to["users"] = users
-            elif alias == "all-service-owners":
-                services = queries.get_apps()
-                to["services"] = services
-            else:
+    audience: set[str] = set()
+
+    for alias in to.aliases or []:
+        match alias:
+            case "all-users":
+                to.users = all_users
+            case "all-service-owners":
+                to.services = all_services
+            case _:
                 raise AttributeError(f"unknown alias: {alias}")
 
-    services = to.get("services")
-    if services:
-        for service in services:
-            service_owners = service.get("serviceOwners")
-            if not service_owners:
-                continue
+    for service in to.services or []:
+        audience.update(
+            service_owner.email for service_owner in service.service_owners or []
+        )
 
-            audience.update(service_owner["email"] for service_owner in service_owners)
+    for account in to.aws_accounts or []:
+        audience.update(
+            account_owner.email for account_owner in account.account_owners or []
+        )
 
-    # TODO: implement clusters and namespaces
+    for role in to.roles or []:
+        audience.update(user.org_username for user in role.users or [])
 
-    aws_accounts = to.get("aws_accounts")
-    if aws_accounts:
-        for account in aws_accounts:
-            account_owners = account.get("accountOwners")
-            if not account_owners:
-                continue
+    # SmtpClient supports sending to org_username and email addresses
+    audience.update(user.org_username for user in to.users or [])
 
-            audience.update(account_owner["email"] for account_owner in account_owners)
-
-    roles = to.get("roles")
-    if roles:
-        for role in roles:
-            users = role.get("users")
-            if not users:
-                continue
-
-            audience.update(user["org_username"] for user in users)
+    if to.clusters or to.namespaces:
+        raise NotImplementedError("clusters and namespaces are not implemented yet")
+    return audience
 
-    users = to.get("users")
-    if users:
-        audience.update(user["org_username"] for user in users)
 
-    return audience
+def get_emails(query_func: Callable) -> list[AppInterfaceEmailV1]:
+    return emails_query(query_func).emails or []
 
 
 @defer
-def run(dry_run, defer=None):
+def run(dry_run: bool, defer: Callable | None = None) -> None:
+    gql_api = gql.get_api()
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
     state = init_state(integration=QONTRACT_INTEGRATION, secret_reader=secret_reader)
-    defer(state.cleanup)
-    emails = queries.get_app_interface_emails()
+    if defer:
+        defer(state.cleanup)
+
+    emails = get_emails(gql_api.query)
+    if not emails:
+        logging.info("no emails to send")
+        sys.exit(0)
+
+    all_users = users_query(gql_api.query).users or []
+    all_services = apps_query(gql_api.query).apps or []
     smtp_settings = typed_queries.smtp.settings()
     smtp_client = SmtpClient(
         server=get_smtp_server_connection(
@@ -99,18 +108,18 @@ def run(dry_run, defer=None):
         timeout=smtp_settings.timeout or DEFAULT_SMTP_TIMEOUT,
     )
     # validate no 2 emails have the same name
-    email_names = {e["name"] for e in emails}
+    email_names = {e.name for e in emails}
     if len(emails) != len(email_names):
         logging.error("email names must be unique.")
         sys.exit(1)
 
-    emails_to_send = [e for e in emails if not state.exists(e["name"])]
+    emails_to_send = [e for e in emails if not state.exists(e.name)]
     for email in emails_to_send:
-        logging.info(["send_email", email["name"], email["subject"]])
+        logging.info(["send_email", email.name, email.subject])
 
         if not dry_run:
-            names = collect_to(email["to"])
-            subject = email["subject"]
-            body = email["body"]
-            smtp_client.send_mail(names, subject, body)
-            state.add(email["name"])
+            names = collect_to(
+                email.q_to, all_users=all_users, all_services=all_services
+            )
+            smtp_client.send_mail(names, email.subject, email.body)
+            state.add(email.name)

reconcile/gcr_mirror.py

@@ -66,7 +66,7 @@ class QuayMirror:
     }
     """
 
-    def __init__(self, dry_run=False):
+    def __init__(self, dry_run: bool = False) -> None:
         self.dry_run = dry_run
         self.gqlapi = gql.get_api()
         settings = queries.get_app_interface_settings()
@@ -81,7 +81,7 @@ class QuayMirror:
     def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
         self.session.close()
 
-    def run(self):
+    def run(self) -> None:
         sync_tasks = self.process_sync_tasks()
         for org, data in sync_tasks.items():
             for item in data:
@@ -95,7 +95,7 @@ class QuayMirror:
                 except SkopeoCmdError as details:
                     _LOG.error("[%s]", details)
 
-    def process_repos_query(self):
+    def process_repos_query(self) -> dict[str, list[dict[str, Any]]]:
         result = self.gqlapi.query(self.GCR_REPOS_QUERY)
 
         summary = defaultdict(list)
@@ -122,7 +122,9 @@ class QuayMirror:
         return summary
 
     @staticmethod
-    def sync_tag(tags, tags_exclude, candidate):
+    def sync_tag(
+        tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
+    ) -> bool:
         if tags is not None:
             # When tags is defined, we don't look at tags_exclude
             return any(re.match(tag, candidate) for tag in tags)
@@ -137,7 +139,7 @@ class QuayMirror:
         # tag must be synced
         return True
 
-    def process_sync_tasks(self):
+    def process_sync_tasks(self) -> dict[str, list[dict[str, Any]]]:
         eight_hours = 28800  # 60 * 60 * 8
         is_deep_sync = self._is_deep_sync(interval=eight_hours)
 
@@ -233,7 +235,7 @@ class QuayMirror:
 
         return sync_tasks
 
-    def _is_deep_sync(self, interval):
+    def _is_deep_sync(self, interval: int) -> bool:
         control_file_name = "qontract-reconcile-gcr-mirror.timestamp"
         control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
         try:
@@ -251,11 +253,11 @@ class QuayMirror:
         return False
 
     @staticmethod
-    def _record_timestamp(path):
+    def _record_timestamp(path: str) -> None:
         with open(path, "w", encoding="locale") as file_object:
             file_object.write(str(time.time()))
 
-    def _get_push_creds(self):
+    def _get_push_creds(self) -> dict[str, str]:
         result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)
 
         creds = {}
@@ -271,6 +273,6 @@ class QuayMirror:
         return creds
 
 
-def run(dry_run):
+def run(dry_run: bool) -> None:
     with QuayMirror(dry_run) as gcr_mirror:
         gcr_mirror.run()

reconcile/github_org.py

@@ -1,9 +1,12 @@
 import logging
 import os
+from collections.abc import Callable, Iterable, KeysView
 from typing import Any
 
 from github import Github
 from github.GithubObject import NotSet  # type: ignore
+from github.Organization import Organization
+from github.Team import Team
 from sretoolbox.utils import retry
 
 from reconcile import (
@@ -95,15 +98,15 @@ CLUSTERS_QUERY = """
 QONTRACT_INTEGRATION = "github"
 
 
-def get_orgs():
+def get_orgs() -> list[dict[str, Any]]:
     gqlapi = gql.get_api()
     return gqlapi.query(ORGS_QUERY)["orgs"]
 
 
-def get_config(default=False):
+def get_config(default: bool = False) -> dict[str, Any]:
     orgs = get_orgs()
     secret_reader = SecretReader(queries.get_secret_reader_settings())
-    config = {"github": {}}
+    config: dict[str, Any] = {"github": {}}
     found_defaults = []
     for org in orgs:
         org_name = org["name"]
@@ -126,24 +129,52 @@ def get_config(default=False):
     return config
 
 
-def get_default_config():
+def get_default_config() -> dict[str, Any]:
     github_config = get_config(default=True)
     return next(iter(github_config["github"].values()))
 
 
 @retry()
-def get_org_and_teams(github, org_name):
+def get_org_and_teams(
+    github: Github, org_name: str
+) -> tuple[Organization, Iterable[Team]]:
     org = github.get_organization(org_name)
     teams = org.get_teams()
     return org, teams
 
 
 @retry()
-def get_members(unit):
+def get_members(unit: Organization) -> list[str]:
     return [member.login for member in unit.get_members()]
 
 
-def fetch_current_state(gh_api_store):
+class GHApiStore:
+    _orgs: dict[str, Any] = {}
+
+    def __init__(self, config: dict) -> None:
+        for org_name, org_config in config["github"].items():
+            token = org_config["token"]
+            managed_teams = org_config.get("managed_teams", None)
+            self._orgs[org_name] = (
+                Github(token, base_url=GH_BASE_URL),
+                RawGithubApi(token),
+                managed_teams,
+            )
+
+    def orgs(self) -> KeysView[str]:
+        return self._orgs.keys()
+
+    def github(self, org_name: str) -> Github:
+        return self._orgs[org_name][0]
+
+    def raw_github_api(self, org_name: str) -> RawGithubApi:
+        return self._orgs[org_name][1]
+
+    def managed_teams(self, org_name: str) -> list[str] | None:
+        return self._orgs[org_name][2]
+
+
+def fetch_current_state(gh_api_store: GHApiStore) -> AggregatedList:
     state = AggregatedList()
 
     for org_name in gh_api_store.orgs():
@@ -164,7 +195,7 @@ def fetch_current_state(gh_api_store):
 
         all_team_members = []
         for team in teams:
-            if not is_managed and team.name not in managed_teams:
+            if not is_managed and team.name not in (managed_teams or []):
                 continue
 
             members = get_members(team)
@@ -190,11 +221,11 @@ def fetch_current_state(gh_api_store):
     return state
 
 
-def fetch_desired_state(infer_clusters=True):
+def fetch_desired_state(infer_clusters: bool = True) -> AggregatedList:
     gqlapi = gql.get_api()
     state = AggregatedList()
 
-    roles = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"])
+    roles: list[dict[str, Any]] = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"])
     for role in roles:
         permissions = list(
             filter(
@@ -264,41 +295,15 @@ def fetch_desired_state(infer_clusters=True):
     return state
 
 
-class GHApiStore:
-    _orgs: dict[str, Any] = {}
-
-    def __init__(self, config):
-        for org_name, org_config in config["github"].items():
-            token = org_config["token"]
-            managed_teams = org_config.get("managed_teams", None)
-            self._orgs[org_name] = (
-                Github(token, base_url=GH_BASE_URL),
-                RawGithubApi(token),
-                managed_teams,
-            )
-
-    def orgs(self):
-        return self._orgs.keys()
-
-    def github(self, org_name):
-        return self._orgs[org_name][0]
-
-    def raw_github_api(self, org_name):
-        return self._orgs[org_name][1]
-
-    def managed_teams(self, org_name):
-        return self._orgs[org_name][2]
-
-
 class RunnerAction:
-    def __init__(self, dry_run, gh_api_store):
+    def __init__(self, dry_run: bool, gh_api_store: GHApiStore) -> None:
         self.dry_run = dry_run
         self.gh_api_store = gh_api_store
 
-    def add_to_team(self):
+    def add_to_team(self) -> Callable:
         label = "add_to_team"
 
-        def action(params, items):
+        def action(params: dict, items: dict) -> None:
             org = params["org"]
             team = params["team"]
 
@@ -318,10 +323,10 @@ class RunnerAction:
 
         return action
 
-    def del_from_team(self):
+    def del_from_team(self) -> Callable:
         label = "del_from_team"
 
-        def action(params, items):
+        def action(params: dict, items: dict) -> None:
             org = params["org"]
             team = params["team"]
 
@@ -346,10 +351,10 @@ class RunnerAction:
 
         return action
 
-    def create_team(self):
+    def create_team(self) -> Callable:
         label = "create_team"
 
-        def action(params, items):
+        def action(params: dict, items: dict) -> None:
             org = params["org"]
             team = params["team"]
 
@@ -367,10 +372,10 @@ class RunnerAction:
 
         return action
 
-    def add_to_org(self):
+    def add_to_org(self) -> Callable:
         label = "add_to_org"
 
-        def action(params, items):
+        def action(params: dict, items: dict) -> None:
             org = params["org"]
 
             if self.dry_run:
@@ -387,10 +392,10 @@ class RunnerAction:
 
         return action
 
-    def del_from_org(self):
+    def del_from_org(self) -> Callable:
         label = "del_from_org"
 
-        def action(params, items):
+        def action(params: dict, items: dict) -> None:
             org = params["org"]
 
             if self.dry_run:
@@ -410,18 +415,18 @@ class RunnerAction:
         return action
 
     @staticmethod
-    def raise_exception(msg):
-        def raiseException(params, items):
+    def raise_exception(msg: str) -> Callable:
+        def raiseException(params: dict, items: dict) -> None:
             raise Exception(msg)
 
         return raiseException
 
 
-def service_is(service):
+def service_is(service: str) -> Callable:
     return lambda params: params.get("service") == service
 
 
-def run(dry_run):
+def run(dry_run: bool) -> None:
     config = get_config()
     gh_api_store = GHApiStore(config)
 
@@ -507,7 +512,7 @@ def run(dry_run):
     runner.run()
 
 
-def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
+def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
     return {
         "github_orgs": get_orgs(),
         "github_org_members": fetch_desired_state().dump(),

reconcile/github_owners.py

@@ -1,6 +1,7 @@
 import logging
 import os
 
+import github
 from github import Github
 from sretoolbox.utils import retry
 
@@ -40,10 +41,10 @@ ROLES_QUERY = """
 QONTRACT_INTEGRATION = "github-owners"
 
 
-def fetch_desired_state():
-    desired_state = {}
+def fetch_desired_state() -> dict[str, list[str]]:
+    desired_state: dict[str, list[str]] = {}
     gqlapi = gql.get_api()
-    roles = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"])
+    roles: list[dict] = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"])
     for role in roles:
         permissions = [
             p
@@ -67,7 +68,9 @@ def fetch_desired_state():
 
 
 @retry()
-def get_current_github_usernames(github_org_name, github, raw_github):
+def get_current_github_usernames(
+    github_org_name: str, github: Github, raw_github: RawGithubApi
+) -> tuple[github.Organization.Organization, list[str]]:
     gh_org = github.get_organization(github_org_name)
     gh_org_members = gh_org.get_members(role="admin")
     current_github_usernames = [m.login for m in gh_org_members]
@@ -77,7 +80,7 @@ def get_current_github_usernames(github_org_name, github, raw_github):
     return gh_org, current_github_usernames
 
 
-def run(dry_run):
+def run(dry_run: bool) -> None:
     base_url = os.environ.get("GITHUB_API", "https://api.github.com")
     config = get_config()
     desired_state = fetch_desired_state()

reconcile/github_repo_invites.py

@@ -95,7 +95,7 @@ def get_settings() -> Mapping[str, Any]:
     raise ValueError("no app-interface-settings found")
 
 
-def run(dry_run):
+def run(dry_run: bool) -> set[str]:
     gqlapi = gql.get_api()
     settings = get_settings()
     secret_reader = SecretReader(settings=settings)

reconcile/github_repo_permissions_validator.py

@@ -16,7 +16,7 @@ QONTRACT_INTEGRATION = "github-repo-permissions-validator"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
 
 
-def get_jobs(jjb: JJB, instance_name: str):
+def get_jobs(jjb: JJB, instance_name: str) -> list[dict] | None:
     pr_check_jobs = jjb.get_all_jobs(
         job_types=["gh-pr-check"], instance_name=instance_name
     ).get(instance_name)
@@ -24,13 +24,13 @@ def get_jobs(jjb: JJB, instance_name: str):
     return pr_check_jobs
 
 
-def init_github():
+def init_github() -> Github:
     base_url = os.environ.get("GITHUB_API", "https://api.github.com")
     token = get_default_config()["token"]
     return Github(token, base_url=base_url)
 
 
-def run(dry_run, instance_name):
+def run(dry_run: bool, instance_name: str) -> None:
     secret_reader = SecretReader(queries.get_secret_reader_settings())
     jjb: JJB = init_jjb(secret_reader)
     pr_check_jobs = get_jobs(jjb, instance_name)

reconcile/github_users.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import re
+from collections.abc import Callable
 
 from github import Github
 from github.GithubException import GithubException
@@ -30,19 +31,21 @@ GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 
 QONTRACT_INTEGRATION = "github-users"
 
+UserAndCompany = tuple[str, str | None]
 
-def init_github():
+
+def init_github() -> Github:
     token = get_default_config()["token"]
     return Github(token, base_url=GH_BASE_URL)
 
 
 @retry(exceptions=(GithubException, ReadTimeout))
-def get_user_company(user, github):
+def get_user_company(user: dict, github: Github) -> UserAndCompany:
     gh_user = github.get_user(login=user["github_username"])
     return user["org_username"], gh_user.company
 
 
-def get_users_to_delete(results):
+def get_users_to_delete(results: list[UserAndCompany]) -> list[dict]:
     pattern = r"^.*[Rr]ed ?[Hh]at.*$"
     org_usernames_to_delete = [
         u for u, c in results if c is None or not re.search(pattern, c)
@@ -51,7 +54,7 @@ def get_users_to_delete(results):
     return [u for u in users_and_paths if u["username"] in org_usernames_to_delete]
 
 
-def send_email_notification(user, smtp_client: SmtpClient):
+def send_email_notification(user: dict, smtp_client: SmtpClient) -> None:
     msg_template = """
 Hello,
 
@@ -79,13 +82,13 @@ App-Interface repository: https://gitlab.cee.redhat.com/service/app-interface
 
 @defer
 def run(
-    dry_run,
-    gitlab_project_id=None,
-    thread_pool_size=10,
-    enable_deletion=False,
-    send_mails=False,
-    defer=None,
-):
+    dry_run: bool,
+    gitlab_project_id: str | None = None,
+    thread_pool_size: int = 10,
+    enable_deletion: bool = False,
+    send_mails: bool = False,
+    defer: Callable | None = None,
+) -> None:
     smtp_settings = typed_queries.smtp.settings()
     smtp_client = SmtpClient(
         server=get_smtp_server_connection(
@@ -104,7 +107,8 @@ def run(
 
     if not dry_run and enable_deletion:
         mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
-        defer(mr_cli.cleanup)
+        if defer:
+            defer(mr_cli.cleanup)
 
     for user in users_to_delete:
         username = user["username"]

reconcile/github_validator.py

@@ -10,7 +10,7 @@ from reconcile.utils.secret_reader import SecretReader
 QONTRACT_INTEGRATION = "github-validator"
 
 
-def run(dry_run):
+def run(dry_run: bool) -> None:
     base_url = os.environ.get("GITHUB_API", "https://api.github.com")
     orgs = queries.get_github_orgs()
     settings = queries.get_app_interface_settings()