qontract-reconcile 0.10.2.dev135__tar.gz → 0.10.2.dev137__tar.gz

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev135
+Version: 0.10.2.dev137
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/pyproject.toml

@@ -250,31 +250,8 @@ disallow_incomplete_defs = true
 [[tool.mypy.overrides]]
 # Untyped modules
 module = [
-    "reconcile.aws_ami_share",
-    "reconcile.aws_ecr_image_pull_secrets",
-    "reconcile.aws_garbage_collector",
-    "reconcile.aws_iam_keys",
-    "reconcile.aws_iam_password_reset",
-    "reconcile.aws_support_cases_sos",
     "reconcile.cli",
-    "reconcile.closedbox_endpoint_monitoring_base",
-    "reconcile.cna.client",
-    "reconcile.cna.integration",
-    "reconcile.cna.state",
-    "reconcile.email_sender",
-    "reconcile.gabi_authorized_users",
-    "reconcile.gcr_mirror",
-    "reconcile.github_org",
-    "reconcile.github_owners",
-    "reconcile.github_repo_invites",
-    "reconcile.github_repo_permissions_validator",
-    "reconcile.github_users",
-    "reconcile.github_validator",
-    "reconcile.gitlab_fork_compliance",
     "reconcile.gitlab_housekeeping",
-    "reconcile.gitlab_labeler",
-    "reconcile.gitlab_mr_sqs_consumer",
-    "reconcile.gitlab_owners",
     "reconcile.gitlab_permissions",
     "reconcile.gitlab_projects",
     "reconcile.integrations_manager",
@@ -365,12 +342,6 @@ module = [
     "reconcile.vpc_peerings_validator",
     "tools.sre_checkpoints.util",
     # untyped tests
-    "reconcile.test.cna.test_integration",
-    "reconcile.test.cna.test_state_assembly",
-    "reconcile.test.cna.test_state_diff",
-    "reconcile.test.cna.test_state_overrides",
-    "reconcile.test.conftest",
-    "reconcile.test.fixtures",
     "reconcile.test.oc.test_oc_connection_parameters",
     "reconcile.test.oc.test_oc_map",
     "reconcile.test.ocm_labels.test_ocm_labels_integration",
@@ -402,19 +373,12 @@ module = [
     "reconcile.test.statuspage.test_statuspage_state",
     "reconcile.test.statuspage.test_statuspage_status",
     "reconcile.test.test_acs_rbac",
-    "reconcile.test.test_aws_ami_share",
-    "reconcile.test.test_aws_iam_keys",
-    "reconcile.test.test_aws_iam_password_reset",
-    "reconcile.test.test_aws_support_cases_sos",
     "reconcile.test.test_checkpoint",
     "reconcile.test.test_cli",
     "reconcile.test.test_closedbox_endpoint_monitoring",
     "reconcile.test.test_dashdotdb_dora",
     "reconcile.test.test_database_access_manager",
-    "reconcile.test.test_github_org",
-    "reconcile.test.test_github_repo_invites",
     "reconcile.test.test_gitlab_housekeeping",
-    "reconcile.test.test_gitlab_labeler",
     "reconcile.test.test_gitlab_members",
     "reconcile.test.test_integrations_manager",
     "reconcile.test.test_jenkins_worker_fleets",

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_ami_share.py

@@ -1,5 +1,6 @@
 import logging
 from collections.abc import (
+    Callable,
     Iterable,
     Mapping,
 )
@@ -37,12 +38,13 @@ def get_region(
 
 
 @defer
-def run(dry_run, defer=None):
+def run(dry_run: bool, defer: Callable | None = None) -> None:
     accounts = queries.get_aws_accounts(sharing=True)
     sharing_accounts = filter_accounts(accounts)
     settings = queries.get_app_interface_settings()
     aws_api = AWSApi(1, sharing_accounts, settings=settings, init_users=False)
-    defer(aws_api.cleanup)
+    if defer:
+        defer(aws_api.cleanup)
 
     for src_account in sharing_accounts:
         sharing = src_account.get("sharing")

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_ecr_image_pull_secrets.py

@@ -1,6 +1,8 @@
 import base64
 import json
 import logging
+from collections.abc import Mapping
+from typing import Any
 
 from reconcile import queries
 from reconcile.utils.aws_api import AWSApi
@@ -9,15 +11,15 @@ from reconcile.utils.vault import VaultClient
 QONTRACT_INTEGRATION = "aws-ecr-image-pull-secrets"
 
 
-def enc_dec(data):
+def enc_dec(data: str) -> str:
     return base64.b64encode(data.encode("utf-8")).decode("utf-8")
 
 
-def get_password(token):
+def get_password(token: str) -> str:
     return base64.b64decode(token).decode("utf-8").split(":")[1]
 
 
-def construct_dockercfg_secret_data(data):
+def construct_dockercfg_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
     auth_data = data["authorizationData"][0]
     server = auth_data["proxyEndpoint"]
     token = auth_data["authorizationToken"]
@@ -36,7 +38,7 @@ def construct_dockercfg_secret_data(data):
     return {".dockerconfigjson": enc_dec(json.dumps(data))}
 
 
-def construct_basic_auth_secret_data(data):
+def construct_basic_auth_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
     auth_data = data["authorizationData"][0]
     token = auth_data["authorizationToken"]
     password = get_password(token)
@@ -44,17 +46,23 @@ def construct_basic_auth_secret_data(data):
     return {"user": enc_dec("AWS"), "token": enc_dec(password), "url": url}
 
 
-def write_output_to_vault(dry_run, vault_path, account, secret_data, name):
+def write_output_to_vault(
+    dry_run: bool,
+    vault_path: str,
+    account: str,
+    secret_data: Mapping[str, str],
+    name: str,
+) -> None:
     integration_name = QONTRACT_INTEGRATION
     secret_path = f"{vault_path}/{integration_name}/{account}/{name}"
     secret = {"path": secret_path, "data": secret_data}
     logging.info(["write_secret", secret_path])
     vault_client = VaultClient()
     if not dry_run:
-        vault_client.write(secret)
+        vault_client.write(secret)  # type: ignore
 
 
-def run(dry_run, vault_output_path=""):
+def run(dry_run: bool, vault_output_path: str = "") -> None:
     accounts = [a for a in queries.get_aws_accounts() if a.get("ecrs")]
     settings = queries.get_app_interface_settings()
     with AWSApi(1, accounts, settings=settings, init_ecr_auth_tokens=True) as aws:

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_garbage_collector.py

@@ -4,7 +4,7 @@ from reconcile.utils.aws_api import AWSApi
 QONTRACT_INTEGRATION = "aws-garbage-collector"
 
 
-def run(dry_run, thread_pool_size=10):
+def run(dry_run: bool, thread_pool_size: int = 10) -> None:
     accounts = [a for a in queries.get_aws_accounts() if a.get("garbageCollection")]
     settings = queries.get_app_interface_settings()
     with AWSApi(thread_pool_size, accounts, settings=settings) as aws:

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_iam_keys.py

@@ -1,28 +1,28 @@
 import logging
 import shutil
 import sys
+from collections.abc import Callable, Iterable, Mapping
 from typing import Any
 
 from reconcile import queries
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.defer import defer
-from reconcile.utils.state import (
-    State,
-    init_state,
-)
+from reconcile.utils.state import State, init_state
 from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
 
 QONTRACT_INTEGRATION = "aws-iam-keys"
 
 
-def filter_accounts(accounts, account_name):
+def filter_accounts(
+    accounts: Iterable[dict[str, Any]], account_name: str | None
+) -> list[dict[str, Any]]:
     accounts = [a for a in accounts if a.get("deleteKeys")]
     if account_name:
         accounts = [a for a in accounts if a["name"] == account_name]
     return accounts
 
 
-def get_keys_to_delete(accounts) -> dict[str, list[str]]:
+def get_keys_to_delete(accounts: Iterable[dict[str, Any]]) -> dict[str, list[str]]:
     return {
         account["name"]: account["deleteKeys"]
         for account in accounts
@@ -37,13 +37,17 @@ def should_run(state: State, keys_to_delete: dict[str, list[str]]) -> bool:
     return False
 
 
-def update_state(state: State, keys_to_update: dict[str, list[str]]):
+def update_state(state: State, keys_to_update: dict[str, list[str]]) -> None:
     for account_name, keys in keys_to_update.items():
         if state.get(account_name, []) != keys:
             state.add(account_name, keys, force=True)
 
 
-def init_tf_working_dirs(accounts, thread_pool_size, settings):
+def init_tf_working_dirs(
+    accounts: Iterable[dict[str, Any]],
+    thread_pool_size: int,
+    settings: Mapping[str, Any],
+) -> dict[str, str]:
     # copied here to avoid circular dependency
     QONTRACT_INTEGRATION = "terraform_resources"
     QONTRACT_TF_PREFIX = "qrtf"
@@ -64,19 +68,19 @@ def init_tf_working_dirs(accounts, thread_pool_size, settings):
     return ts.dump()
 
 
-def cleanup(working_dirs):
+def cleanup(working_dirs: Mapping[str, str]) -> None:
     for wd in working_dirs.values():
         shutil.rmtree(wd)
 
 
 @defer
 def run(
-    dry_run,
-    thread_pool_size=10,
-    disable_service_account_keys=False,
-    account_name=None,
-    defer=None,
-):
+    dry_run: bool,
+    thread_pool_size: int = 10,
+    disable_service_account_keys: bool = False,
+    account_name: str | None = None,
+    defer: Callable | None = None,
+) -> None:
     accounts = filter_accounts(
         queries.get_aws_accounts(terraform_state=True), account_name
     )
@@ -88,7 +92,8 @@ def run(
 
     settings = queries.get_app_interface_settings()
     state = init_state(integration=QONTRACT_INTEGRATION)
-    defer(state.cleanup)
+    if defer:
+        defer(state.cleanup)
     keys_to_delete = get_keys_to_delete(accounts)
     if not should_run(state, keys_to_delete):
         logging.debug("nothing to do here")
@@ -97,7 +102,9 @@ def run(
         return
 
     working_dirs = init_tf_working_dirs(accounts, thread_pool_size, settings)
-    defer(lambda: cleanup(working_dirs))
+    if defer:
+        defer(lambda: cleanup(working_dirs))
+
     with AWSApi(thread_pool_size, accounts, settings=settings) as aws:
         error, service_account_recycle_complete = aws.delete_keys(
             dry_run, keys_to_delete, working_dirs, disable_service_account_keys

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_iam_password_reset.py

@@ -1,6 +1,7 @@
 import logging
 import sys
 from collections.abc import (
+    Callable,
     Iterable,
     Mapping,
 )
@@ -44,12 +45,13 @@ class AwsAccountWithResets(BaseModel):
 
 
 @defer
-def run(dry_run, defer=None):
+def run(dry_run: bool, defer: Callable | None = None) -> None:
     accounts = queries.get_aws_accounts(reset_passwords=True)
     settings = queries.get_app_interface_settings()
     roles = queries.get_roles(aws=True, saas_files=False)
     state = init_state(integration=QONTRACT_INTEGRATION)
-    defer(state.cleanup)
+    if defer:
+        defer(state.cleanup)
 
     accounts_to_reset: list[AwsAccountWithResets] = []
 

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/aws_support_cases_sos.py

@@ -1,5 +1,7 @@
 import itertools
 import logging
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any
 
 from reconcile import (
     mr_client_gateway,
@@ -12,11 +14,11 @@ from reconcile.utils.mr import CreateDeleteAwsAccessKey
 QONTRACT_INTEGRATION = "aws-support-cases-sos"
 
 
-def filter_accounts(accounts):
+def filter_accounts(accounts: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
     return [a for a in accounts if a.get("premiumSupport")]
 
 
-def get_deleted_keys(accounts):
+def get_deleted_keys(accounts: Iterable[dict[str, Any]]) -> dict[str, list[str]]:
     return {
         account["name"]: account["deleteKeys"]
         for account in accounts
@@ -24,7 +26,9 @@ def get_deleted_keys(accounts):
     }
 
 
-def get_keys_to_delete(aws_support_cases):
+def get_keys_to_delete(
+    aws_support_cases: Mapping[str, Iterable[Mapping[str, Any]]],
+) -> list[dict[str, str]]:
     search_pattern = "We have become aware that the AWS Access Key "
     keys = []
     # ref:
@@ -43,10 +47,17 @@ def get_keys_to_delete(aws_support_cases):
 
 
 @defer
-def act(dry_run, gitlab_project_id, accounts, keys_to_delete, defer=None):
+def act(
+    dry_run: bool,
+    gitlab_project_id: str | None,
+    accounts: Iterable[dict[str, Any]],
+    keys_to_delete: list[dict[str, str]],
+    defer: Callable | None = None,
+) -> None:
     if not dry_run and keys_to_delete:
         mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
-        defer(mr_cli.cleanup)
+        if defer:
+            defer(mr_cli.cleanup)
 
     for k in keys_to_delete:
         account = k["account"]
@@ -59,7 +70,9 @@ def act(dry_run, gitlab_project_id, accounts, keys_to_delete, defer=None):
             mr.submit(cli=mr_cli)
 
 
-def run(dry_run, gitlab_project_id=None, thread_pool_size=10, enable_deletion=False):
+def run(
+    dry_run: bool, gitlab_project_id: str | None = None, thread_pool_size: int = 10
+) -> None:
     accounts = filter_accounts(queries.get_aws_accounts())
     settings = queries.get_app_interface_settings()
     deleted_keys = get_deleted_keys(accounts)

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/closedbox_endpoint_monitoring_base.py

@@ -63,7 +63,7 @@ class EndpointMonitoringProvider:
         return None
 
     @property
-    def metric_labels(self):
+    def metric_labels(self) -> dict[str, Any]:
         return json.loads(self.metricLabels) if self.metricLabels else {}
 
 
@@ -136,7 +136,7 @@ def run_for_provider(
     thread_pool_size: int,
     internal: bool,
     use_jump_host: bool,
-    defer=None,
+    defer: Callable | None = None,
 ) -> None:
     # prepare
     desired_endpoints = get_endpoints(provider)
@@ -152,7 +152,8 @@ def run_for_provider(
             integration_version=integration_version,
             override_managed_types=["Probe"],
         )
-        defer(oc_map.cleanup)
+        if defer:
+            defer(oc_map.cleanup)
 
         # reconcile
         for ep_mon_provider, endpoints in desired_endpoints.items():

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/cna/client.py

@@ -23,7 +23,7 @@ class CNAClient:
         cnas = self._ocm_client.get(api_path="/api/cna-management/v1/cnas")
         return cnas.get("items", [])
 
-    def create(self, asset: Asset, dry_run: bool = False):
+    def create(self, asset: Asset, dry_run: bool = False) -> None:
         if dry_run:
             logging.info("CREATE %s", asset)
             return
@@ -32,7 +32,7 @@ class CNAClient:
             data=asset.api_payload(),
         )
 
-    def delete(self, asset: Asset, dry_run: bool = False):
+    def delete(self, asset: Asset, dry_run: bool = False) -> None:
         if dry_run:
             logging.info("DELETE %s", asset)
             return
@@ -41,7 +41,7 @@ class CNAClient:
                 api_path=asset.href,
             )
 
-    def update(self, asset: Asset, dry_run: bool = False):
+    def update(self, asset: Asset, dry_run: bool = False) -> None:
         if dry_run:
             logging.info("UPDATE %s", asset)
             return

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/cna/integration.py

@@ -46,13 +46,13 @@ class CNAIntegration:
         namespaces: Iterable[NamespaceV1],
         desired_states: Mapping[str, State] | None = None,
         current_states: Mapping[str, State] | None = None,
-    ):
+    ) -> None:
         self._cna_clients = cna_clients
         self._namespaces = namespaces
         self._desired_states = desired_states or defaultdict(State)
         self._current_states = current_states or defaultdict(State)
 
-    def assemble_desired_states(self):
+    def assemble_desired_states(self) -> None:
         self._desired_states = defaultdict(State)
         for namespace in self._namespaces:
             for provider in namespace.external_resources or []:
@@ -64,7 +64,7 @@ class CNAIntegration:
                         asset_factory_from_schema(resource)
                     )
 
-    def assemble_current_states(self):
+    def assemble_current_states(self) -> None:
         self._current_states = defaultdict(State)
         for name, client in self._cna_clients.items():
             cnas = client.list_assets()
@@ -72,7 +72,7 @@ class CNAIntegration:
             state.add_raw_data(cnas)
             self._current_states[name] = state
 
-    def provision(self, dry_run: bool = False):
+    def provision(self, dry_run: bool = False) -> None:
         for provisioner_name, cna_client in self._cna_clients.items():
             desired_state = self._desired_states[provisioner_name]
             current_state = self._current_states[provisioner_name]
@@ -119,7 +119,6 @@ def run(
     dry_run: bool,
     # TODO: Threadpool not used yet - will be used once we understand scopes in more detail
     thread_pool_size: int,
-    defer=None,
 ) -> None:
     settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=settings.vault)

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/cna/state.py

@@ -50,7 +50,7 @@ class State:
         # pytest should show nice diff
         return str(self._assets)
 
-    def _validate_addition(self, asset: Asset):
+    def _validate_addition(self, asset: Asset) -> None:
         if asset.kind not in self._assets:
             raise CNAStateError(f"State doesn't know asset_kind {asset.kind}")
         if asset.name in self._assets[asset.kind]:
@@ -58,11 +58,11 @@ class State:
                 f"Duplicate asset name found in state: kind={asset.kind}, name={asset.name}"
             )
 
-    def add_asset(self, asset: Asset):
+    def add_asset(self, asset: Asset) -> None:
         self._validate_addition(asset=asset)
         self._assets[asset.kind][asset.name] = asset
 
-    def add_raw_data(self, data: Iterable[Mapping[str, Any]]):
+    def add_raw_data(self, data: Iterable[Mapping[str, Any]]) -> None:
         for cna in data:
             asset = asset_factory_from_raw_data(cna)
             self._validate_addition(asset=asset)

qontract_reconcile-0.10.2.dev137/reconcile/email_sender.py

@@ -0,0 +1,125 @@
+import logging
+import sys
+from collections.abc import Callable
+
+from reconcile import typed_queries
+from reconcile.gql_definitions.email_sender.apps import query as apps_query
+from reconcile.gql_definitions.email_sender.emails import (
+    AppInterfaceEmailAudienceV1,
+    AppInterfaceEmailV1,
+)
+from reconcile.gql_definitions.email_sender.emails import query as emails_query
+from reconcile.gql_definitions.email_sender.users import query as users_query
+from reconcile.gql_definitions.fragments.email_service import EmailServiceOwners
+from reconcile.gql_definitions.fragments.email_user import EmailUser
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.utils import gql
+from reconcile.utils.defer import defer
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.smtp_client import (
+    DEFAULT_SMTP_TIMEOUT,
+    SmtpClient,
+    get_smtp_server_connection,
+)
+from reconcile.utils.state import init_state
+
+QONTRACT_INTEGRATION = "email-sender"
+
+
+def collect_to(
+    to: AppInterfaceEmailAudienceV1,
+    all_users: list[EmailUser],
+    all_services: list[EmailServiceOwners],
+) -> set[str]:
+    """Collect audience to send email to from to object
+
+    Arguments:
+        to -- AppInterfaceEmailAudience_v1 object
+        all_users -- List of all app-interface users
+        all_services -- List of all app-interface apps/services with owners
+
+    Raises:
+        AttributeError: Unknown alias
+
+    Returns:
+        set -- Audience to send email to
+    """
+    audience: set[str] = set()
+
+    for alias in to.aliases or []:
+        match alias:
+            case "all-users":
+                to.users = all_users
+            case "all-service-owners":
+                to.services = all_services
+            case _:
+                raise AttributeError(f"unknown alias: {alias}")
+
+    for service in to.services or []:
+        audience.update(
+            service_owner.email for service_owner in service.service_owners or []
+        )
+
+    for account in to.aws_accounts or []:
+        audience.update(
+            account_owner.email for account_owner in account.account_owners or []
+        )
+
+    for role in to.roles or []:
+        audience.update(user.org_username for user in role.users or [])
+
+    # SmtpClient supports sending to org_username and email addresses
+    audience.update(user.org_username for user in to.users or [])
+
+    if to.clusters or to.namespaces:
+        raise NotImplementedError("clusters and namespaces are not implemented yet")
+    return audience
+
+
+def get_emails(query_func: Callable) -> list[AppInterfaceEmailV1]:
+    return emails_query(query_func).emails or []
+
+
+@defer
+def run(dry_run: bool, defer: Callable | None = None) -> None:
+    gql_api = gql.get_api()
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    state = init_state(integration=QONTRACT_INTEGRATION, secret_reader=secret_reader)
+    if defer:
+        defer(state.cleanup)
+
+    emails = get_emails(gql_api.query)
+    if not emails:
+        logging.info("no emails to send")
+        sys.exit(0)
+
+    all_users = users_query(gql_api.query).users or []
+    all_services = apps_query(gql_api.query).apps or []
+    smtp_settings = typed_queries.smtp.settings()
+    smtp_client = SmtpClient(
+        server=get_smtp_server_connection(
+            secret_reader=secret_reader,
+            secret=smtp_settings.credentials,
+        ),
+        mail_address=smtp_settings.mail_address,
+        timeout=smtp_settings.timeout or DEFAULT_SMTP_TIMEOUT,
+    )
+    # validate no 2 emails have the same name
+    email_names = {e.name for e in emails}
+    if len(emails) != len(email_names):
+        logging.error("email names must be unique.")
+        sys.exit(1)
+
+    emails_to_send = [e for e in emails if not state.exists(e.name)]
+    for email in emails_to_send:
+        logging.info(["send_email", email.name, email.subject])
+
+        if not dry_run:
+            names = collect_to(
+                email.q_to, all_users=all_users, all_services=all_services
+            )
+            smtp_client.send_mail(names, email.subject, email.body)
+            state.add(email.name)

{qontract_reconcile-0.10.2.dev135 → qontract_reconcile-0.10.2.dev137}/reconcile/external_resources/aws.py

@@ -264,6 +264,7 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             if s.provision_provider == "aws"
             and s.provider == "rds"
             and s.resource["replica_source"] == spec.identifier
+            and not s.marked_to_delete
         }