qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/aus/ocm_upgrade_scheduler_org.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from dataclasses import dataclass
 
 from reconcile.aus.healthchecks import (
     AUSClusterHealthCheckProvider,
@@ -13,6 +14,7 @@ from reconcile.aus.node_pool_spec import get_node_pool_specs_by_org_cluster
 from reconcile.aus.ocm_upgrade_scheduler import OCMClusterUpgradeSchedulerIntegration
 from reconcile.gql_definitions.fragments.aus_organization import AUSOCMOrganization
 from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
+from reconcile.utils.ocm.base import LabelContainer
 from reconcile.utils.ocm.clusters import (
     OCMCluster,
     discover_clusters_for_organizations,
@@ -22,6 +24,12 @@ from reconcile.utils.ocm_base_client import init_ocm_base_client
 QONTRACT_INTEGRATION = "ocm-upgrade-scheduler-org"
 
 
+@dataclass
+class ClusterUpgradeSpecWithLabels:
+    cluster: OCMCluster
+    cluster_labels: LabelContainer
+
+
 class OCMClusterUpgradeSchedulerOrgIntegration(OCMClusterUpgradeSchedulerIntegration):
     @property
     def name(self) -> str:
@@ -60,7 +68,10 @@ class OCMClusterUpgradeSchedulerOrgIntegration(OCMClusterUpgradeSchedulerIntegra
                     specs=self._build_cluster_upgrade_specs(
                         org=org,
                         clusters_by_name={
-                            c.ocm_cluster.name: c.ocm_cluster
+                            c.ocm_cluster.name: ClusterUpgradeSpecWithLabels(
+                                cluster=c.ocm_cluster,
+                                cluster_labels=c.labels,
+                            )
                             for c in clusters_by_org[org.org_id]
                         },
                         cluster_health_provider=build_cluster_health_providers_for_organization(
@@ -79,7 +90,7 @@ class OCMClusterUpgradeSchedulerOrgIntegration(OCMClusterUpgradeSchedulerIntegra
     def _build_cluster_upgrade_specs(
         self,
         org: AUSOCMOrganization,
-        clusters_by_name: dict[str, OCMCluster],
+        clusters_by_name: dict[str, ClusterUpgradeSpecWithLabels],
         cluster_health_provider: AUSClusterHealthCheckProvider,
         node_pool_specs_by_cluster_id: dict[str, list[NodePoolSpec]],
     ) -> list[ClusterUpgradeSpec]:
@@ -87,12 +98,16 @@ class OCMClusterUpgradeSchedulerOrgIntegration(OCMClusterUpgradeSchedulerIntegra
             ClusterUpgradeSpec(
                 org=org,
                 upgradePolicy=cluster.upgrade_policy,
-                cluster=clusters_by_name[cluster.name],
+                cluster=clusters_by_name[cluster.name].cluster,
+                cluster_labels=clusters_by_name[cluster.name].cluster_labels,
                 health=cluster_health_provider.cluster_health(
-                    cluster_external_id=clusters_by_name[cluster.name].external_id,
+                    cluster_external_id=clusters_by_name[
+                        cluster.name
+                    ].cluster.external_id,
                     org_id=org.org_id,
                 ),
-                nodePools=node_pool_specs_by_cluster_id.get(ocm_cluster.id) or [],
+                nodePools=node_pool_specs_by_cluster_id.get(ocm_cluster.cluster.id)
+                or [],
             )
             for cluster in org.upgrade_policy_clusters or []
             # clusters that are not in the UUID dict will be ignored because

reconcile/aus/version_gate_approver.py

@@ -14,9 +14,6 @@ from reconcile.gql_definitions.common.ocm_environments import (
 )
 from reconcile.utils import gql
 from reconcile.utils.grouping import group_by
-from reconcile.utils.jobcontroller.controller import (
-    build_job_controller,
-)
 from reconcile.utils.ocm.base import (
     ClusterDetails,
     LabelContainer,
@@ -63,19 +60,7 @@ class VersionGateApprover(QontractReconcileIntegration[VersionGateApproverParams
 
     def initialize_handlers(self, query_func: Callable) -> None:
         self.handlers: dict[str, GateHandler] = {
-            sts_version_gate_handler.GATE_LABEL: sts_version_gate_handler.STSGateHandler(
-                job_controller=build_job_controller(
-                    integration=QONTRACT_INTEGRATION,
-                    integration_version=QONTRACT_INTEGRATION_VERSION,
-                    cluster=self.params.job_controller_cluster,
-                    namespace=self.params.job_controller_namespace,
-                    secret_reader=self.secret_reader,
-                    dry_run=False,
-                ),
-                aws_iam_role=self.params.rosa_role,
-                rosa_job_service_account=self.params.rosa_job_service_account,
-                rosa_job_image=self.params.rosa_job_image,
-            ),
+            sts_version_gate_handler.GATE_LABEL: sts_version_gate_handler.STSGateHandler(),
             ocp_gate_handler.GATE_LABEL: ocp_gate_handler.OCPGateHandler(),
             ingress_gate_handler.GATE_LABEL: ingress_gate_handler.IngressGateHandler(),
         }

reconcile/aus/version_gates/sts_version_gate_handler.py

@@ -1,27 +1,16 @@
-import logging
-
 from reconcile.aus.version_gates.handler import GateHandler
-from reconcile.utils.jobcontroller.controller import K8sJobController
 from reconcile.utils.ocm.base import OCMCluster, OCMVersionGate
 from reconcile.utils.ocm_base_client import OCMBaseClient
-from reconcile.utils.rosa.rosa_cli import RosaCliError
-from reconcile.utils.rosa.session import RosaSession
 
 GATE_LABEL = "api.openshift.com/gate-sts"
 
 
 class STSGateHandler(GateHandler):
-    def __init__(
-        self,
-        job_controller: K8sJobController,
-        aws_iam_role: str,
-        rosa_job_service_account: str | None = None,
-        rosa_job_image: str | None = None,
-    ) -> None:
-        self.job_controller = job_controller
-        self.aws_iam_role = aws_iam_role
-        self.rosa_job_image = rosa_job_image
-        self.rosa_job_service_account = rosa_job_service_account
+    """
+    This handler is used to handle the STS version gate.
+    Right now we just ack all gate-sts gates
+    The actual job of role upgrade is now a part of AUS and is handled by the AUSSTSGateHandler.
+    """
 
     @staticmethod
     def gate_applicable_to_cluster(cluster: OCMCluster) -> bool:
@@ -41,60 +30,4 @@ class STSGateHandler(GateHandler):
         gate: OCMVersionGate,
         dry_run: bool,
     ) -> bool:
-        if (
-            not cluster.id
-            or not cluster.aws
-            or not cluster.aws.sts
-            or not cluster.is_sts()
-        ):
-            # checked already but mypy :/
-            return False
-
-        if cluster.is_rosa_hypershift():
-            # thanks to hypershift managed policies, there is nothing to do for us here
-            # returning True will ack the version gate
-            return True
-        if not cluster.is_rosa_classic():
-            # we manage roels only for rosa classic clusters
-            # returning here will prevent OSD STS clusters to be handled right now
-            logging.error(
-                f"Cluster {cluster.id} is not a ROSA cluster. "
-                "STS version gates are only handled for ROSA classic clusters."
-            )
-            return False
-
-        rosa = RosaSession(
-            aws_account_id=cluster.aws.aws_account_id,
-            aws_region=cluster.region.id,
-            aws_iam_role=self.aws_iam_role,
-            ocm_org_id=ocm_org_id,
-            ocm_api=ocm_api,
-            job_controller=self.job_controller,
-            image=self.rosa_job_image,
-            service_account=self.rosa_job_service_account,
-        )
-
-        try:
-            # account role handling
-            account_role_prefix = cluster.aws.account_role_prefix
-            if not account_role_prefix:
-                raise Exception(
-                    f"Can't upgrade account roles. Cluster {cluster.name} does not define spec.aws.account_role_prefix"
-                )
-            rosa.upgrade_account_roles(
-                role_prefix=account_role_prefix,
-                minor_version=gate.version_raw_id_prefix,
-                channel_group=cluster.version.channel_group,
-                dry_run=dry_run,
-            )
-
-            # operator role handling
-            rosa.upgrade_operator_roles(
-                cluster_id=cluster.id,
-                dry_run=dry_run,
-            )
-        except RosaCliError as e:
-            logging.error(f"Failed to upgrade roles for cluster {cluster.name}: {e}")
-            e.write_logs_to_logger(logging.error)
-            return False
         return True

reconcile/automated_actions/config/integration.py

@@ -7,7 +7,7 @@ from collections.abc import (
 from typing import Any
 
 import yaml
-from kubernetes.client import (  # type: ignore[attr-defined]
+from kubernetes.client import (
     ApiClient,
     V1ConfigMap,
     V1ObjectMeta,
@@ -20,6 +20,7 @@ from reconcile.gql_definitions.automated_actions.instance import (
     AutomatedActionExternalResourceFlushElastiCacheV1,
     AutomatedActionExternalResourceRdsRebootV1,
     AutomatedActionExternalResourceRdsSnapshotV1,
+    AutomatedActionOpenshiftTriggerCronjobV1,
     AutomatedActionOpenshiftWorkloadDeleteV1,
     AutomatedActionOpenshiftWorkloadRestartArgumentV1,
     AutomatedActionOpenshiftWorkloadRestartV1,
@@ -82,7 +83,7 @@ class AutomatedActionsConfigIntegration(
             query_func = gql.get_api().query
         return {
             "automated_actions_instances": [
-                c.dict() for c in self.get_automated_actions_instances(query_func)
+                c.model_dump() for c in self.get_automated_actions_instances(query_func)
             ]
         }
 
@@ -167,7 +168,7 @@ class AutomatedActionsConfigIntegration(
                 case AutomatedActionActionListV1():
                     # no special handling needed, just dump the values
                     parameters.extend(
-                        arg.dict(exclude_none=True, exclude_defaults=True)
+                        arg.model_dump(exclude_none=True, exclude_defaults=True)
                         for arg in action.action_list_arguments or []
                     )
                 case AutomatedActionExternalResourceFlushElastiCacheV1():
@@ -205,6 +206,17 @@ class AutomatedActionsConfigIntegration(
                                 "account": f"^{rds_snapshot_er.provisioner.name}$",
                                 "identifier": rds_snapshot_arg.identifier,
                             })
+                case AutomatedActionOpenshiftTriggerCronjobV1():
+                    parameters.extend(
+                        {
+                            # all parameter values are regexes in the OPA policy
+                            # therefore, cluster and namespace must be fixed to the current strings
+                            "cluster": f"^{arg.namespace.cluster.name}$",
+                            "namespace": f"^{arg.namespace.name}$",
+                            "cronjob": arg.cronjob,
+                        }
+                        for arg in action.openshift_trigger_cronjob_arguments
+                    )
                 case AutomatedActionOpenshiftWorkloadDeleteV1():
                     parameters.extend(
                         {
@@ -257,7 +269,7 @@ class AutomatedActionsConfigIntegration(
             {
                 "users": {user.username: sorted(user.roles) for user in users},
                 "roles": {
-                    role: [policy.dict() for policy in policies]
+                    role: [policy.model_dump() for policy in policies]
                     for role, policies in roles.items()
                 },
             },

reconcile/aws_account_manager/integration.py

@@ -1,5 +1,4 @@
 from collections.abc import Callable, Iterable
-from datetime import UTC, datetime
 from typing import Any
 
 import jinja2
@@ -26,6 +25,7 @@ from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql, metrics
 from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
 from reconcile.utils.aws_api_typed.iam import AWSAccessKey
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.runtime.integration import (
@@ -42,14 +42,14 @@ QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
 
 
 class AwsAccountMgmtIntegrationParams(PydanticRunParams):
-    account_name: str | None
+    account_name: str | None = None
     flavor: str
     organization_account_role: str = "OrganizationAccountAccessRole"
     default_tags: dict[str, str] = {}
     initial_user_name: str = "terraform"
     initial_user_policy_arn: str = "arn:aws:iam::aws:policy/AdministratorAccess"
     initial_user_secret_vault_path: str = (
-        "app-sre-v2/creds/terraform/{account_name}/config"
+        "app-sre-v2/creds/terraform/{account_name}/config"  # noqa: RUF027
     )
     # To avoid the accidental deletion of the resource file, explicitly set the
     # qontract.cli option in the integration extraArgs!
@@ -76,9 +76,9 @@ class AwsAccountMgmtIntegration(
             query_func, account_name=self.params.account_name
         )
         return {
-            "payer_accounts": [account.dict() for account in payer_accounts],
+            "payer_accounts": [account.model_dump() for account in payer_accounts],
             "non_organization_accounts": [
-                account.dict() for account in non_organization_accounts
+                account.model_dump() for account in non_organization_accounts
             ],
         }
 
@@ -98,10 +98,10 @@ class AwsAccountMgmtIntegration(
             lstrip_blocks=True,
             keep_trailing_newline=True,
         ).render({
-            "accountRequest": account_request.dict(by_alias=True),
+            "accountRequest": account_request.model_dump(by_alias=True),
             "uid": uid,
             "settings": settings,
-            "timestamp": int(datetime.now(tz=UTC).timestamp()),
+            "timestamp": int(utc_now().timestamp()),
         })
         return tmpl
 
@@ -128,12 +128,24 @@ class AwsAccountMgmtIntegration(
             for payer_account in payer_accounts
             for org_account in payer_account.organization_accounts or []
         }
-
         non_organization_accounts = [
             account
             for account in all_aws_accounts
             if account.name not in all_organization_account_names
         ]
+
+        # check account requests for invalid emails
+        used_emails: set[str] = {
+            owner.email
+            for account in all_aws_accounts
+            for owner in account.account_owners
+        }
+        for payer_account in payer_accounts:
+            for account_request in payer_account.account_requests or []:
+                if account_request.account_owner.email in used_emails:
+                    raise ValueError(
+                        f"Invalid email for account request {account_request.name} in payer account {payer_account.name}. Email {account_request.account_owner.email} is already used."
+                    )
         return payer_accounts, non_organization_accounts
 
     def save_access_key(self, account: str, access_key: AWSAccessKey) -> None:
@@ -187,7 +199,7 @@ class AwsAccountMgmtIntegration(
                     template=account_template,
                     account_request=account_request,
                     uid=uid,
-                    settings=self.params.dict(),
+                    settings=self.params.model_dump(),
                 ),
                 account_request_file_path=f"data/{account_request.path.strip('/')}",
             )

reconcile/aws_account_manager/reconciler.py

@@ -35,7 +35,7 @@ class Quota(Protocol):
     quota_code: str
     value: float
 
-    def dict(self) -> dict[str, Any]: ...
+    def model_dump(self) -> dict[str, Any]: ...
 
 
 class Contact(Protocol):
@@ -44,7 +44,7 @@ class Contact(Protocol):
     email: str
     phone_number: str
 
-    def dict(self) -> dict[str, Any]: ...
+    def model_dump(self) -> dict[str, Any]: ...
 
 
 class AWSReconciler:
@@ -167,7 +167,7 @@ class AWSReconciler:
         self, aws_api: AWSApi, name: str, quotas: Iterable[Quota]
     ) -> list[str] | None:
         """Request service quota changes."""
-        quotas_dict = [q.dict() for q in quotas]
+        quotas_dict = [q.model_dump() for q in quotas]
         with self.state.transaction(
             state_key(name, TASK_REQUEST_SERVICE_QUOTA)
         ) as _state:

reconcile/aws_account_manager/utils.py

@@ -24,7 +24,7 @@ def validate(account: AWSAccountV1) -> bool:
         raise ExceptionGroup("Multiple quotas are referenced in the account", errors)
 
     if account.organization_accounts or account.account_requests:
-        # it's payer account
+        # it's a "payer account"
         if not account.premium_support:
             raise ValueError(
                 f"Premium support is required for payer account {account.name}"

reconcile/aws_ami_cleanup/integration.py

@@ -26,6 +26,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.parse_dhms_duration import dhms_to_seconds
 from reconcile.utils.secret_reader import create_secret_reader
@@ -39,15 +40,12 @@ QONTRACT_INTEGRATION = "aws_ami_cleanup"
 MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
 
 
-class AWSAmi(BaseModel):
+class AWSAmi(BaseModel, frozen=True):
     name: str
     image_id: str
     creation_date: datetime
     snapshot_ids: list[str]
 
-    class Config:
-        frozen = True
-
 
 def get_aws_amis_from_launch_templates(ec2_client: EC2Client) -> set[str]:
     amis = set()
@@ -77,7 +75,7 @@ def get_aws_amis(
     owner: str,
     regex: str,
     age_in_seconds: int,
-    utc_now: datetime,
+    now: datetime,
 ) -> list[AWSAmi]:
     """Get amis that match regex older than given age"""
 
@@ -89,10 +87,8 @@ def get_aws_amis(
             if not re.search(pattern, image["Name"]):
                 continue
 
-            creation_date = datetime.strptime(
-                image["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
-            )
-            current_delta = utc_now - creation_date
+            creation_date = from_utc_iso_format(image["CreationDate"])
+            current_delta = now - creation_date
             delete_delta = timedelta(seconds=age_in_seconds)
 
             if current_delta < delete_delta:
@@ -135,7 +131,7 @@ def get_region(
 
 @defer
 def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) -> None:
-    utc_now = datetime.utcnow()
+    now = utc_now()
     gqlapi = gql.get_api()
     aws_accounts = aws_accounts_query(gqlapi.query).accounts
 
@@ -177,7 +173,7 @@ def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) ->
     # Build AWSApi object. We will use all those accounts listed in ami_accounts since
     # we will also need to look for used AMIs.
     accounts_dicted = [
-        account.dict(by_alias=True)
+        account.model_dump(by_alias=True)
         for account in aws_accounts or []
         if account.name in ami_accounts
     ]
@@ -222,7 +218,7 @@ def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) ->
                 owner=account.uid,
                 regex=cleanup_config.regex,
                 age_in_seconds=age_in_seconds,
-                utc_now=utc_now,
+                now=now,
             )
 
             for ami in aws_amis:

reconcile/aws_ami_share.py

@@ -1,17 +1,19 @@
 import logging
+import re
 from collections.abc import (
-    Callable,
     Iterable,
     Mapping,
 )
 from typing import Any
 
 from reconcile import queries
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.defer import defer
 
 QONTRACT_INTEGRATION = "aws-ami-share"
-MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
+
+MANAGED_TAG = {"managed_by_integration": QONTRACT_INTEGRATION}
 
 
 def filter_accounts(accounts: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -37,65 +39,70 @@ def get_region(
     return region
 
 
-@defer
-def run(dry_run: bool, defer: Callable | None = None) -> None:
+def share_ami(
+    dry_run: bool,
+    src_account: Mapping[str, Any],
+    share: Mapping[str, Any],
+    default_tags: dict[str, str],
+    aws_api: AWSApi,
+) -> None:
+    dst_account = share["account"]
+    regex = re.compile(share["regex"])
+    region = get_region(share, src_account, dst_account)
+    src_amis = aws_api.get_amis_details(src_account, src_account, regex, region)
+    dst_amis = aws_api.get_amis_details(dst_account, src_account, regex, region)
+
+    for ami_id, src_ami_tags in src_amis.items():
+        dst_ami_tags = dst_amis.get(ami_id)
+        if dst_ami_tags is None:
+            logging.info([
+                "share_ami",
+                src_account["name"],
+                dst_account["name"],
+                ami_id,
+            ])
+            if not dry_run:
+                aws_api.share_ami(src_account, dst_account["uid"], ami_id, region)
+        dst_account_tags = default_tags | get_aws_account_tags(
+            dst_account.get("organization", None)
+        )
+        desired_tags = src_ami_tags | dst_account_tags | MANAGED_TAG
+        current_tags = dst_ami_tags or {}
+
+        if desired_tags != current_tags:
+            logging.info([
+                "tag_shared_ami",
+                dst_account["name"],
+                ami_id,
+                desired_tags,
+            ])
+            if not dry_run:
+                aws_api.create_tags(dst_account, ami_id, desired_tags)
+
+
+def run(dry_run: bool) -> None:
     accounts = queries.get_aws_accounts(sharing=True)
     sharing_accounts = filter_accounts(accounts)
     settings = queries.get_app_interface_settings()
-    aws_api = AWSApi(1, sharing_accounts, settings=settings, init_users=False)
-    if defer:
-        defer(aws_api.cleanup)
-
-    for src_account in sharing_accounts:
-        sharing = src_account.get("sharing")
-        if not sharing:
-            continue
-        for share in sharing:
-            if share["provider"] != "ami":
-                continue
-            dst_account = share["account"]
-            regex = share["regex"]
-            region = get_region(share, src_account, dst_account)
-            src_amis = aws_api.get_amis_details(src_account, src_account, regex, region)
-            dst_amis = aws_api.get_amis_details(dst_account, src_account, regex, region)
-
-            for src_ami in src_amis:
-                src_ami_id = src_ami["image_id"]
-                found_dst_amis = [d for d in dst_amis if d["image_id"] == src_ami_id]
-                if not found_dst_amis:
-                    logging.info([
-                        "share_ami",
-                        src_account["name"],
-                        dst_account["name"],
-                        src_ami_id,
-                    ])
-                    if not dry_run:
-                        aws_api.share_ami(
-                            src_account, dst_account["uid"], src_ami_id, region
-                        )
-                    # we assume an unshared ami does not have tags
-                    found_dst_amis = [{"image_id": src_ami_id, "tags": []}]
-
-                dst_ami = found_dst_amis[0]
-                dst_ami_id = dst_ami["image_id"]
-                dst_ami_tags = dst_ami["tags"]
-                if MANAGED_TAG not in dst_ami_tags:
-                    logging.info([
-                        "tag_shared_ami",
-                        dst_account["name"],
-                        dst_ami_id,
-                        MANAGED_TAG,
-                    ])
-                    if not dry_run:
-                        aws_api.create_tag(dst_account, dst_ami_id, MANAGED_TAG)
-                src_ami_tags = src_ami["tags"]
-                for src_tag in src_ami_tags:
-                    if src_tag not in dst_ami_tags:
-                        logging.info([
-                            "tag_shared_ami",
-                            dst_account["name"],
-                            dst_ami_id,
-                            src_tag,
-                        ])
-                        if not dry_run:
-                            aws_api.create_tag(dst_account, dst_ami_id, src_tag)
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = {}
+
+    with AWSApi(
+        1,
+        sharing_accounts,
+        settings=settings,
+        init_users=False,
+    ) as aws_api:
+        for src_account in sharing_accounts:
+            for share in src_account.get("sharing") or []:
+                if share["provider"] == "ami":
+                    share_ami(
+                        dry_run=dry_run,
+                        src_account=src_account,
+                        share=share,
+                        default_tags=default_tags,
+                        aws_api=aws_api,
+                    )