qontract-reconcile 0.10.1rc675__py3-none-any.whl → 0.10.1rc677__py3-none-any.whl

{qontract_reconcile-0.10.1rc675.dist-info → qontract_reconcile-0.10.1rc677.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc675
+Version: 0.10.1rc677
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc675.dist-info → qontract_reconcile-0.10.1rc677.dist-info}/RECORD

@@ -141,7 +141,7 @@ reconcile/aws_cloudwatch_log_retention/integration.py,sha256=0UcSZIrGvnGY4m9fj87
 reconcile/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_saml_idp/integration.py,sha256=uqec-EnxnfGOgQtg33S-Q1wTCv0sVBHNo02aT94hXrw,4807
 reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aws_saml_roles/integration.py,sha256=kgAUaCfKA2ujvIym8sPSWoBf1EnRKi2vVbgKyi8eELU,5695
+reconcile/aws_saml_roles/integration.py,sha256=kC4Rnbuy07TMvZO4rjUEcQkJev10M0Ro6r7YXcB7j_c,9530
 reconcile/aws_version_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_version_sync/integration.py,sha256=0rgFLL2usTnngjIgFXIEfVXapFsDI1A493Mftqjfbk0,17292
 reconcile/aws_version_sync/utils.py,sha256=sVv-48PKi2VITlqqvmpbjnFDOPeGqfKzgkpIszlmjL0,1708
@@ -190,7 +190,7 @@ reconcile/gql_definitions/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 reconcile/gql_definitions/aws_saml_idp/aws_accounts.py,sha256=ZhsTyUmR-i-0eDXEEl2qZV64bIHMK25KrvlkqpzwO8Y,2652
 reconcile/gql_definitions/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/aws_saml_roles/aws_accounts.py,sha256=a6tHAPKyJ4cPOYKlDPcdnLyE6i0w6jewvlrNJ3be2yk,2702
-reconcile/gql_definitions/aws_saml_roles/aws_groups.py,sha256=IyVzp5d8bie1dD2XLLbdpI9d1hhz6aLkPZ-FjqSf2r0,2639
+reconcile/gql_definitions/aws_saml_roles/roles.py,sha256=2Bbk71rFZkB6x0TF5xe1B758jq9wUQ4JNe5-ym_u1so,2529
 reconcile/gql_definitions/aws_version_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/aws_version_sync/clusters.py,sha256=2TOJOFxpTkZ2HKuqAGUo6tQkzOINSfO-bYDINVpJeL0,2295
 reconcile/gql_definitions/aws_version_sync/namespaces.py,sha256=eBLyXlSjWdmEE-jY9M2Ocgk7JGi2OsWisTkjHLfgU_A,4311
@@ -616,7 +616,7 @@ reconcile/utils/state.py,sha256=zjsprjbOb0WddzmAvh8ACqAt0fcayrX2YPfz7qceRWw,1609
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
 reconcile/utils/terraform_client.py,sha256=7L55Rvxfzj3KtJH8AD8D8YRfBnFpHiTFqSa5e2_9jtk,32092
-reconcile/utils/terrascript_aws_client.py,sha256=Jxh6boKnGAKEM-3K1AFlhF6YneOQPVoBj3q98dITNFc,271891
+reconcile/utils/terrascript_aws_client.py,sha256=0kYizE48gVlcXxGdtKcdrbw5Xkb2uQ0ao1stov4lyhU,272099
 reconcile/utils/three_way_diff_strategy.py,sha256=nyqeQsLCoPI6e16k2CF3b9KNgQLU-rPf5RtfdUfVMwE,4468
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/unleash.py,sha256=1D56CsZfE3ShDtN3IErE1T2eeIwNmxhK-yYbCotJ99E,3601
@@ -723,8 +723,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=OvalpVRfY4pNmpMaWHHYqBjV68b1eGQjX8SCyTAXb1w,3501
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc675.dist-info/METADATA,sha256=otHpW3N3ydPZOwDWRdiqZjFC2DmYoGee9x0GfVeIzFU,2382
-qontract_reconcile-0.10.1rc675.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc675.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc675.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc675.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc677.dist-info/METADATA,sha256=jeb66UTo9ySLlZQEJ9PYl7pNa5qzudUnrbKuXHWuWLI,2382
+qontract_reconcile-0.10.1rc677.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc677.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
+qontract_reconcile-0.10.1rc677.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc677.dist-info/RECORD,,

reconcile/aws_saml_roles/integration.py

@@ -1,3 +1,4 @@
+import json
 import sys
 from collections.abc import (
     Callable,
@@ -7,7 +8,7 @@ from typing import (
     Any,
 )
 
-from pydantic import validator
+from pydantic import BaseModel, root_validator, validator
 
 from reconcile.gql_definitions.aws_saml_roles.aws_accounts import (
     AWSAccountV1,
@@ -15,15 +16,13 @@ from reconcile.gql_definitions.aws_saml_roles.aws_accounts import (
 from reconcile.gql_definitions.aws_saml_roles.aws_accounts import (
     query as aws_accounts_query,
 )
-from reconcile.gql_definitions.aws_saml_roles.aws_groups import (
-    AWSGroupV1,
-)
-from reconcile.gql_definitions.aws_saml_roles.aws_groups import (
-    query as aws_groups_query,
+from reconcile.gql_definitions.aws_saml_roles.roles import (
+    query as roles_query,
 )
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.aws_helper import unique_sso_aws_accounts
 from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.runtime.integration import (
@@ -54,6 +53,82 @@ class AwsSamlRolesIntegrationParams(PydanticRunParams):
         raise ValueError("max_session_duration_hours must be between 1 and 12 hours")
 
 
+class CustomPolicy(BaseModel):
+    name: str
+    policy: dict[str, Any]
+
+    @validator("name")
+    def name_size(cls, v: str) -> str:
+        """Check the policy name size.
+
+        See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
+        """
+        if len(v) > 128:
+            raise ValueError(
+                f"The policy name '{v}' is too long. The AWS policy name must be 128 characters or less."
+            )
+        return v
+
+    @validator("policy")
+    def policy_size(cls, v: dict[str, Any]) -> dict[str, Any]:
+        """Check the policy size.
+
+        See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
+        """
+        if len(json.dumps(v, separators=(",", ":"))) > 6144:
+            raise ValueError(
+                f"The policy document '{v}' is too large. AWS policy documents must be 6144 characters or less (w/o white spaces)."
+            )
+        return v
+
+
+class ManagedPolicy(BaseModel):
+    name: str
+
+
+class AwsRole(BaseModel):
+    name: str
+    account: str
+    custom_policies: list[CustomPolicy]
+    managed_policies: list[ManagedPolicy]
+
+    @validator("name")
+    def name_size(cls, v: str) -> str:
+        """Check the role name size.
+
+        See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
+        """
+        if len(v) > 64:
+            raise ValueError(
+                f"The role name '{v}' is too long. The AWS role name must be 64 characters or less."
+            )
+        return v
+
+    @root_validator
+    def validate_policies(cls, values: dict[str, Any]) -> dict[str, Any]:
+        """Check the policies.
+
+        See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
+        """
+        custom_policies = values.get("custom_policies", [])
+        managed_policies = values.get("managed_policies", [])
+        if len(custom_policies) + len(managed_policies) > 20:
+            raise ValueError(
+                f"The role '{values['name']}' has too many policies. AWS roles can have at most 20 policies (via quota increase). Please consider consolidating the policies."
+            )
+        cp_names = [cp.name for cp in custom_policies]
+        if len(set(cp_names)) != len(cp_names):
+            raise ValueError(
+                f"The role '{values['name']}' has duplicate custom policies."
+            )
+        mp_names = [mp.name for mp in managed_policies]
+        if len(set(mp_names)) != len(mp_names):
+            raise ValueError(
+                f"The role '{values['name']}' has duplicate managed policies."
+            )
+        return values
+
+
 class AwsSamlRolesIntegration(
     QontractReconcileIntegration[AwsSamlRolesIntegrationParams]
 ):
@@ -70,7 +145,7 @@ class AwsSamlRolesIntegration(
         if not query_func:
             query_func = gql.get_api().query
         return {
-            "aws_groups": [c.dict() for c in self.get_aws_groups(query_func)],
+            "roles": [c.dict() for c in self.get_roles(query_func)],
         }
 
     def get_aws_accounts(
@@ -85,40 +160,69 @@ class AwsSamlRolesIntegration(
             and (not account_name or account.name == account_name)
         ]
 
-    def get_aws_groups(
+    def get_roles(
         self, query_func: Callable, account_name: str | None = None
-    ) -> list[AWSGroupV1]:
-        """Get all AWS groups with SSO enabled."""
-        data = aws_groups_query(query_func)
-        return [
-            group
-            for group in data.aws_groups or []
-            if integration_is_enabled(self.name, group.account)
-            and (not account_name or group.account.name == account_name)
-            and group.account.sso
-            and group.roles
-            and group.policies
-            and any(role.users for role in group.roles)
-        ]
+    ) -> list[AwsRole]:
+        """Return all roles with AWS account relations."""
+        aws_roles = []
+        for role in roles_query(query_func).roles or []:
+            if not role.aws_groups and not role.user_policies:
+                continue
+
+            user_policies = role.user_policies or []
+            aws_groups = role.aws_groups or []
+            for sso_aws_account in unique_sso_aws_accounts(
+                integration=self.name,
+                accounts=[i.account for i in user_policies + aws_groups],
+                account_name=account_name,
+            ):
+                # AWS limits are checked via pydantic validators
+                custom_policies = [
+                    CustomPolicy(name=user_policy.name, policy=user_policy.policy)
+                    for user_policy in user_policies
+                    if user_policy.account.uid == sso_aws_account.uid
+                ]
+                managed_policies = [
+                    ManagedPolicy(name=p)
+                    for aws_group in aws_groups
+                    if aws_group.account.uid == sso_aws_account.uid
+                    for p in aws_group.policies or []
+                ]
+
+                aws_roles.append(
+                    AwsRole(
+                        name=f"{sso_aws_account.uid}-{role.name}",
+                        account=sso_aws_account.name,
+                        custom_policies=custom_policies,
+                        managed_policies=managed_policies,
+                    )
+                )
+
+        return aws_roles
 
     def populate_saml_iam_roles(
-        self, ts: TerrascriptClient, aws_groups: Iterable[AWSGroupV1]
+        self, ts: TerrascriptClient, aws_roles: Iterable[AwsRole]
     ) -> None:
         """Populate the SAML IAM roles."""
-        for group in aws_groups:
-            # aws groups without policies are filtered out by the query
-            # duplicated policies aren't allowed in the same role
-            policies = group.policies or []
-            if len(policies) != len(set(policies)):
-                raise ValueError(
-                    f"Group {group.name} has duplicated policies: {policies}"
-                )
-
+        unique_policies = {
+            (role.account, custom_policy.name): custom_policy.policy
+            for role in aws_roles
+            for custom_policy in role.custom_policies
+        }
+        # User policies are unique per account
+        for (account, policy), policy_doc in unique_policies.items():
+            ts.populate_iam_policy(
+                account=account,
+                name=policy,
+                policy=policy_doc,
+            )
+        for role in aws_roles:
             ts.populate_saml_iam_role(
-                account=group.account.name,
-                name=group.name,
+                account=role.account,
+                name=role.name,
                 saml_provider_name=self.params.saml_idp_name,
-                policies=policies,
+                aws_managed_policies=[p.name for p in role.managed_policies],
+                customer_managed_policies=[p.name for p in role.custom_policies],
                 max_session_duration_hours=self.params.max_session_duration_hours,
             )
 
@@ -130,9 +234,7 @@ class AwsSamlRolesIntegration(
             gql_api.query, account_name=self.params.account_name
         )
         aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
-        aws_groups = self.get_aws_groups(
-            gql_api.query, account_name=self.params.account_name
-        )
+        aws_roles = self.get_roles(gql_api.query, account_name=self.params.account_name)
 
         ts = TerrascriptClient(
             self.name.replace("-", "_"),
@@ -141,7 +243,7 @@ class AwsSamlRolesIntegration(
             aws_accounts_dict,
             secret_reader=self.secret_reader,
         )
-        self.populate_saml_iam_roles(ts, aws_groups)
+        self.populate_saml_iam_roles(ts, aws_roles)
         working_dirs = ts.dump(print_to_file=self.params.print_to_file)
 
         if self.params.print_to_file:

reconcile/gql_definitions/aws_saml_roles/{aws_groups.py → roles.py}

@@ -17,25 +17,35 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
+from reconcile.gql_definitions.fragments.aws_account_sso import AWSAccountSSO
+
 
 DEFINITION = """
-query AwsSamlRolesAwsGroupQuery {
-  aws_groups: awsgroups_v1 {
+fragment AWSAccountSSO on AWSAccount_v1 {
+  name
+  uid
+  sso
+  disable {
+    integrations
+  }
+}
+
+query AwsSamlRolesQuery {
+  roles: roles_v1 {
     name
-    account {
+    user_policies {
       name
-      uid
-      sso
-      disable {
-        integrations
+      policy
+      account {
+        ...AWSAccountSSO
       }
     }
-    roles {
-      users {
-        org_username
+    aws_groups {
+      account {
+        ...AWSAccountSSO
       }
+      policies
     }
-    policies
   }
 }
 """
@@ -47,37 +57,28 @@ class ConfiguredBaseModel(BaseModel):
         extra=Extra.forbid
 
 
-class DisableClusterAutomationsV1(ConfiguredBaseModel):
-    integrations: Optional[list[str]] = Field(..., alias="integrations")
-
-
-class AWSAccountV1(ConfiguredBaseModel):
+class AWSUserPolicyV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
-    uid: str = Field(..., alias="uid")
-    sso: Optional[bool] = Field(..., alias="sso")
-    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
+    policy: Json = Field(..., alias="policy")
+    account: AWSAccountSSO = Field(..., alias="account")
 
 
-class UserV1(ConfiguredBaseModel):
-    org_username: str = Field(..., alias="org_username")
+class AWSGroupV1(ConfiguredBaseModel):
+    account: AWSAccountSSO = Field(..., alias="account")
+    policies: Optional[list[str]] = Field(..., alias="policies")
 
 
 class RoleV1(ConfiguredBaseModel):
-    users: list[UserV1] = Field(..., alias="users")
-
-
-class AWSGroupV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
-    account: AWSAccountV1 = Field(..., alias="account")
-    roles: Optional[list[RoleV1]] = Field(..., alias="roles")
-    policies: Optional[list[str]] = Field(..., alias="policies")
+    user_policies: Optional[list[AWSUserPolicyV1]] = Field(..., alias="user_policies")
+    aws_groups: Optional[list[AWSGroupV1]] = Field(..., alias="aws_groups")
 
 
-class AwsSamlRolesAwsGroupQueryQueryData(ConfiguredBaseModel):
-    aws_groups: Optional[list[AWSGroupV1]] = Field(..., alias="aws_groups")
+class AwsSamlRolesQueryQueryData(ConfiguredBaseModel):
+    roles: Optional[list[RoleV1]] = Field(..., alias="roles")
 
 
-def query(query_func: Callable, **kwargs: Any) -> AwsSamlRolesAwsGroupQueryQueryData:
+def query(query_func: Callable, **kwargs: Any) -> AwsSamlRolesQueryQueryData:
     """
     This is a convenience function which queries and parses the data into
     concrete types. It should be compatible with most GQL clients.
@@ -90,7 +91,7 @@ def query(query_func: Callable, **kwargs: Any) -> AwsSamlRolesAwsGroupQueryQuery
         kwargs: optional arguments that will be passed to the query function
 
     Returns:
-        AwsSamlRolesAwsGroupQueryQueryData: queried data parsed into generated classes
+        AwsSamlRolesQueryQueryData: queried data parsed into generated classes
     """
     raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return AwsSamlRolesAwsGroupQueryQueryData(**raw_data)
+    return AwsSamlRolesQueryQueryData(**raw_data)

reconcile/utils/terrascript_aws_client.py

@@ -843,13 +843,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             for user_policy in user_policies:
                 policy_name = user_policy["name"]
                 account_name = user_policy["account"]["name"]
-                account_uid = user_policy["account"]["uid"]
                 for user in users:
-                    # replace known keys with values
                     user_name = self._get_aws_username(user)
-                    policy = user_policy["policy"]
-                    policy = policy.replace("${aws:username}", user_name)
-                    policy = policy.replace("${aws:accountid}", account_uid)
 
                     # Ref: terraform aws_iam_policy
                     tf_iam_user = self.get_tf_iam_user(user_name)
@@ -857,7 +852,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     tf_aws_iam_policy = aws_iam_policy(
                         identifier,
                         name=identifier,
-                        policy=policy,
+                        policy=user_policy["policy"],
                     )
                     self.add_resource(account_name, tf_aws_iam_policy)
                     # Ref: terraform aws_iam_user_policy_attachment
@@ -2569,18 +2564,29 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         self.add_resources(account, tf_resources)
 
+    def populate_iam_policy(self, account: str, name: str, policy: dict[str, Any]):
+        tf_aws_iam_policy = aws_iam_policy(
+            f"{account}-{name}", name=name, policy=json.dumps(policy)
+        )
+        self.add_resource(account, tf_aws_iam_policy)
+
     def populate_saml_iam_role(
         self,
         account: str,
         name: str,
         saml_provider_name: str,
-        policies: list[str],
+        aws_managed_policies: list[str],
+        customer_managed_policies: list[str] | None = None,
         max_session_duration_hours: int = 1,
     ) -> None:
         """Manage the an IAM role needed for SAML authentication."""
         managed_policy_arns = [
-            f"arn:{self._get_partition(account)}:" + f"iam::aws:policy/{policy}"
-            for policy in policies
+            f"arn:{self._get_partition(account)}:iam::aws:policy/{policy}"
+            for policy in aws_managed_policies
+        ]
+        managed_policy_arns += [
+            f"arn:{self._get_partition(account)}:iam::{self.uids[account]}:policy/{policy}"
+            for policy in customer_managed_policies or []
         ]
         assume_role_policy = {
             "Version": "2012-10-17",
@@ -2601,7 +2607,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         }
         role_tf_resource = aws_iam_role(
             f"{account}-{name}",
-            name=f"{self.uids[account]}-{name}",
+            name=name,
             assume_role_policy=json.dumps(assume_role_policy),
             managed_policy_arns=managed_policy_arns,
             max_session_duration=max_session_duration_hours * 3600,