qontract-reconcile 0.10.1rc736__py3-none-any.whl → 0.10.1rc737__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/RECORD +14 -7
- reconcile/cli.py +35 -0
- reconcile/gql_definitions/terraform_init/__init__.py +0 -0
- reconcile/gql_definitions/terraform_init/aws_accounts.py +93 -0
- reconcile/terraform_init/__init__.py +0 -0
- reconcile/terraform_init/integration.py +165 -0
- reconcile/terraform_init/merge_request.py +57 -0
- reconcile/terraform_init/merge_request_manager.py +102 -0
- reconcile/utils/aws_api_typed/api.py +10 -0
- reconcile/utils/aws_api_typed/s3.py +26 -0
- {qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/entry_points.txt +0 -0
- {qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/top_level.txt +0 -0
{qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: qontract-reconcile
|
3
|
-
Version: 0.10.
|
3
|
+
Version: 0.10.1rc737
|
4
4
|
Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
|
5
5
|
Home-page: https://github.com/app-sre/qontract-reconcile
|
6
6
|
Author: Red Hat App-SRE Team
|
{qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/RECORD
RENAMED
@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
|
|
10
10
|
reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
|
11
11
|
reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
|
12
12
|
reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
|
13
|
-
reconcile/cli.py,sha256=
|
13
|
+
reconcile/cli.py,sha256=Fra8s6kxIuCoLF-moSto81gDFoJ6Q-WrsaRcVxR-KRI,98281
|
14
14
|
reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
|
15
15
|
reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
|
16
16
|
reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
|
@@ -342,6 +342,8 @@ reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_re
|
|
342
342
|
reconcile/gql_definitions/terraform_cloudflare_users/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
343
343
|
reconcile/gql_definitions/terraform_cloudflare_users/app_interface_setting_cloudflare_and_vault.py,sha256=KFey-0ItgpGPeIwViGKqb55HAFJoKdi3eCNSIzf6Rc8,1960
|
344
344
|
reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.py,sha256=1KiTikTKSSRYmISN8tY29rJ_fVtgBnT7sK85IJjN420,4027
|
345
|
+
reconcile/gql_definitions/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
346
|
+
reconcile/gql_definitions/terraform_init/aws_accounts.py,sha256=OJ0hDbRachRaDkL-OGT6-byr9cKdBiQDnNCpwUe3oJ8,2674
|
345
347
|
reconcile/gql_definitions/terraform_repo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
346
348
|
reconcile/gql_definitions/terraform_repo/terraform_repo.py,sha256=pefTRhb0ZcWm_j6dYz6m6qNqZFteqgrQ25SgUqRAVaI,3173
|
347
349
|
reconcile/gql_definitions/terraform_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -425,6 +427,10 @@ reconcile/templating/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
|
|
425
427
|
reconcile/templating/lib/merge_request_manager.py,sha256=JUkfF3smaQ8onzKF5F7UpmA7MWaQpftANy6dDo1FCug,5464
|
426
428
|
reconcile/templating/lib/model.py,sha256=fb6FYYLQjmoh2DjVKO7TEWCuDPf1Q34xmOx0M9Z07ek,324
|
427
429
|
reconcile/templating/lib/rendering.py,sha256=_BVQ2gqip8K1AgLYfaTWh8NKJFTW6VjUZ6rBI_GH30E,5061
|
430
|
+
reconcile/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
431
|
+
reconcile/terraform_init/integration.py,sha256=xcFKTc_or3xB3kE_I3OECNkkgbwALIwwdiktnF-MyWI,6114
|
432
|
+
reconcile/terraform_init/merge_request.py,sha256=3CYtgSd7Q9zjKg4wsDz437EPCRfGeZZ8fZ0Y-ChKXJY,1475
|
433
|
+
reconcile/terraform_init/merge_request_manager.py,sha256=fMcT6hbdEF3nFATJpvr8BedvQHq_MzFkgVJSloBNwOQ,3101
|
428
434
|
reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
429
435
|
reconcile/test/conftest.py,sha256=rQousYrxUz-EwAIbsYO6bIwR1B4CrOz9y_zaUVo2lfI,4466
|
430
436
|
reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
|
@@ -664,9 +670,10 @@ reconcile/utils/acs/notifiers.py,sha256=LfWw9LGq7hA90A69n8Ie9f-NozvCGdYwXEXRLIc1
|
|
664
670
|
reconcile/utils/acs/policies.py,sha256=_jAz6cv8KRYtDsXjGoJgNbD8_9PUa5LSwwVlpK4A_cQ,5505
|
665
671
|
reconcile/utils/acs/rbac.py,sha256=ugsLM9Pb7FbUbdq85E3VzXGMaB9ZovXob7tdWCxwqZ8,8808
|
666
672
|
reconcile/utils/aws_api_typed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
667
|
-
reconcile/utils/aws_api_typed/api.py,sha256=
|
673
|
+
reconcile/utils/aws_api_typed/api.py,sha256=rGUh7-gc8AcjUFuLgQxKyPy1KGY7ffNe6zT1EDdurOM,8283
|
668
674
|
reconcile/utils/aws_api_typed/iam.py,sha256=ka46H2-SzTCgy6EJYapKTzyZK9vR1bkfD0wF8bDdy1Q,2201
|
669
675
|
reconcile/utils/aws_api_typed/organization.py,sha256=oXftcLVuSs9qej6efdssl38FvjeZaQC5R2Wj3NzxX4U,5529
|
676
|
+
reconcile/utils/aws_api_typed/s3.py,sha256=J2uOTtEFgMyKT22pa4DbFnV7zfg575m2DeidQaeselM,1034
|
670
677
|
reconcile/utils/aws_api_typed/service_quotas.py,sha256=OU1D8LCmMw1IT87nt45LqXhguzcWwC8AaBdDTI7tz98,3018
|
671
678
|
reconcile/utils/aws_api_typed/sts.py,sha256=5Sauncj9Fif3YDLkJYkBZrtOX0v0bGAqOmY0A5Bh9yA,1237
|
672
679
|
reconcile/utils/aws_api_typed/support.py,sha256=PH3UW96Ne4_8I1J-_Vqj-DsK73gYGeVdOH13eD1783c,2447
|
@@ -770,8 +777,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
|
|
770
777
|
tools/test/test_qontract_cli.py,sha256=w2l4BHB09k1d-BGJ1jBUNCqDv7zkqYrMHojQXg-21kQ,4155
|
771
778
|
tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
|
772
779
|
tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
|
773
|
-
qontract_reconcile-0.10.
|
774
|
-
qontract_reconcile-0.10.
|
775
|
-
qontract_reconcile-0.10.
|
776
|
-
qontract_reconcile-0.10.
|
777
|
-
qontract_reconcile-0.10.
|
780
|
+
qontract_reconcile-0.10.1rc737.dist-info/METADATA,sha256=cSDUzKY1nBbpIuGIWfYeQyeIlIvmyU8slP8iVQOiR7Q,2382
|
781
|
+
qontract_reconcile-0.10.1rc737.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
782
|
+
qontract_reconcile-0.10.1rc737.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
|
783
|
+
qontract_reconcile-0.10.1rc737.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
|
784
|
+
qontract_reconcile-0.10.1rc737.dist-info/RECORD,,
|
reconcile/cli.py
CHANGED
@@ -1032,6 +1032,41 @@ def aws_account_manager(
|
|
1032
1032
|
)
|
1033
1033
|
|
1034
1034
|
|
1035
|
+
@integration.command(short_help="Initialize AWS accounts for Terraform usage.")
|
1036
|
+
@account_name
|
1037
|
+
@click.option(
|
1038
|
+
"--state-tmpl-resource",
|
1039
|
+
help="Resource name of the state template-collection template in the app-interface.",
|
1040
|
+
required=True,
|
1041
|
+
default="/terraform-init/terraform-state.yml",
|
1042
|
+
)
|
1043
|
+
@click.option(
|
1044
|
+
"--template-collection-root-path",
|
1045
|
+
help="File path to the root directory to store new state template-collections.",
|
1046
|
+
required=True,
|
1047
|
+
default="data/templating/collections/terraform-init",
|
1048
|
+
)
|
1049
|
+
@click.pass_context
|
1050
|
+
def terraform_init(
|
1051
|
+
ctx, account_name, state_tmpl_resource, template_collection_root_path
|
1052
|
+
):
|
1053
|
+
from reconcile.terraform_init.integration import (
|
1054
|
+
TerraformInitIntegration,
|
1055
|
+
TerraformInitIntegrationParams,
|
1056
|
+
)
|
1057
|
+
|
1058
|
+
run_class_integration(
|
1059
|
+
integration=TerraformInitIntegration(
|
1060
|
+
TerraformInitIntegrationParams(
|
1061
|
+
account_name=account_name,
|
1062
|
+
state_tmpl_resource=state_tmpl_resource,
|
1063
|
+
template_collection_root_path=template_collection_root_path,
|
1064
|
+
)
|
1065
|
+
),
|
1066
|
+
ctx=ctx.obj,
|
1067
|
+
)
|
1068
|
+
|
1069
|
+
|
1035
1070
|
@integration.command(short_help="Manage Jenkins roles association via REST API.")
|
1036
1071
|
@click.pass_context
|
1037
1072
|
def jenkins_roles(ctx):
|
File without changes
|
@@ -0,0 +1,93 @@
|
|
1
|
+
"""
|
2
|
+
Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
|
3
|
+
"""
|
4
|
+
from collections.abc import Callable # noqa: F401 # pylint: disable=W0611
|
5
|
+
from datetime import datetime # noqa: F401 # pylint: disable=W0611
|
6
|
+
from enum import Enum # noqa: F401 # pylint: disable=W0611
|
7
|
+
from typing import ( # noqa: F401 # pylint: disable=W0611
|
8
|
+
Any,
|
9
|
+
Optional,
|
10
|
+
Union,
|
11
|
+
)
|
12
|
+
|
13
|
+
from pydantic import ( # noqa: F401 # pylint: disable=W0611
|
14
|
+
BaseModel,
|
15
|
+
Extra,
|
16
|
+
Field,
|
17
|
+
Json,
|
18
|
+
)
|
19
|
+
|
20
|
+
from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
|
21
|
+
|
22
|
+
|
23
|
+
DEFINITION = """
|
24
|
+
fragment VaultSecret on VaultSecret_v1 {
|
25
|
+
path
|
26
|
+
field
|
27
|
+
version
|
28
|
+
format
|
29
|
+
}
|
30
|
+
|
31
|
+
query TerraformInitAWSAccounts {
|
32
|
+
accounts: awsaccounts_v1 {
|
33
|
+
name
|
34
|
+
terraformUsername
|
35
|
+
terraformState {
|
36
|
+
region
|
37
|
+
}
|
38
|
+
resourcesDefaultRegion
|
39
|
+
automationToken {
|
40
|
+
...VaultSecret
|
41
|
+
}
|
42
|
+
disable {
|
43
|
+
integrations
|
44
|
+
}
|
45
|
+
}
|
46
|
+
}
|
47
|
+
"""
|
48
|
+
|
49
|
+
|
50
|
+
class ConfiguredBaseModel(BaseModel):
|
51
|
+
class Config:
|
52
|
+
smart_union=True
|
53
|
+
extra=Extra.forbid
|
54
|
+
|
55
|
+
|
56
|
+
class TerraformStateAWSV1(ConfiguredBaseModel):
|
57
|
+
region: str = Field(..., alias="region")
|
58
|
+
|
59
|
+
|
60
|
+
class DisableClusterAutomationsV1(ConfiguredBaseModel):
|
61
|
+
integrations: Optional[list[str]] = Field(..., alias="integrations")
|
62
|
+
|
63
|
+
|
64
|
+
class AWSAccountV1(ConfiguredBaseModel):
|
65
|
+
name: str = Field(..., alias="name")
|
66
|
+
terraform_username: Optional[str] = Field(..., alias="terraformUsername")
|
67
|
+
terraform_state: Optional[TerraformStateAWSV1] = Field(..., alias="terraformState")
|
68
|
+
resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
|
69
|
+
automation_token: VaultSecret = Field(..., alias="automationToken")
|
70
|
+
disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
|
71
|
+
|
72
|
+
|
73
|
+
class TerraformInitAWSAccountsQueryData(ConfiguredBaseModel):
|
74
|
+
accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
|
75
|
+
|
76
|
+
|
77
|
+
def query(query_func: Callable, **kwargs: Any) -> TerraformInitAWSAccountsQueryData:
|
78
|
+
"""
|
79
|
+
This is a convenience function which queries and parses the data into
|
80
|
+
concrete types. It should be compatible with most GQL clients.
|
81
|
+
You do not have to use it to consume the generated data classes.
|
82
|
+
Alternatively, you can also mime and alternate the behavior
|
83
|
+
of this function in the caller.
|
84
|
+
|
85
|
+
Parameters:
|
86
|
+
query_func (Callable): Function which queries your GQL Server
|
87
|
+
kwargs: optional arguments that will be passed to the query function
|
88
|
+
|
89
|
+
Returns:
|
90
|
+
TerraformInitAWSAccountsQueryData: queried data parsed into generated classes
|
91
|
+
"""
|
92
|
+
raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
|
93
|
+
return TerraformInitAWSAccountsQueryData(**raw_data)
|
File without changes
|
@@ -0,0 +1,165 @@
|
|
1
|
+
import logging
|
2
|
+
from collections.abc import Callable
|
3
|
+
from datetime import datetime, timezone
|
4
|
+
from typing import Any
|
5
|
+
|
6
|
+
import jinja2
|
7
|
+
|
8
|
+
from reconcile.gql_definitions.terraform_init.aws_accounts import AWSAccountV1
|
9
|
+
from reconcile.gql_definitions.terraform_init.aws_accounts import (
|
10
|
+
query as aws_accounts_query,
|
11
|
+
)
|
12
|
+
from reconcile.terraform_init.merge_request import Renderer, create_parser
|
13
|
+
from reconcile.terraform_init.merge_request_manager import MergeRequestManager, MrData
|
14
|
+
from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
|
15
|
+
from reconcile.typed_queries.github_orgs import get_github_orgs
|
16
|
+
from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
|
17
|
+
from reconcile.utils import gql
|
18
|
+
from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
|
19
|
+
from reconcile.utils.defer import defer
|
20
|
+
from reconcile.utils.disabled_integrations import integration_is_enabled
|
21
|
+
from reconcile.utils.runtime.integration import (
|
22
|
+
PydanticRunParams,
|
23
|
+
QontractReconcileIntegration,
|
24
|
+
)
|
25
|
+
from reconcile.utils.semver_helper import make_semver
|
26
|
+
from reconcile.utils.unleash import get_feature_toggle_state
|
27
|
+
from reconcile.utils.vcs import VCS
|
28
|
+
|
29
|
+
QONTRACT_INTEGRATION = "terraform-init"
|
30
|
+
QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
|
31
|
+
|
32
|
+
|
33
|
+
class TerraformInitIntegrationParams(PydanticRunParams):
|
34
|
+
account_name: str | None
|
35
|
+
state_tmpl_resource: str = "/terraform-init/terraform-state.yml"
|
36
|
+
template_collection_root_path: str = "data/templating/collections/terraform-init"
|
37
|
+
|
38
|
+
|
39
|
+
class TerraformInitIntegration(
|
40
|
+
QontractReconcileIntegration[TerraformInitIntegrationParams]
|
41
|
+
):
|
42
|
+
"""Initialize AWS accounts for Terraform usage."""
|
43
|
+
|
44
|
+
@property
|
45
|
+
def name(self) -> str:
|
46
|
+
return QONTRACT_INTEGRATION
|
47
|
+
|
48
|
+
def get_early_exit_desired_state(
|
49
|
+
self, query_func: Callable | None = None
|
50
|
+
) -> dict[str, Any]:
|
51
|
+
"""Return the desired state for early exit."""
|
52
|
+
if not query_func:
|
53
|
+
query_func = gql.get_api().query
|
54
|
+
return {
|
55
|
+
"accounts": [
|
56
|
+
account.dict() for account in self.get_aws_accounts(query_func)
|
57
|
+
],
|
58
|
+
}
|
59
|
+
|
60
|
+
def get_aws_accounts(
|
61
|
+
self, query_func: Callable, account_name: str | None = None
|
62
|
+
) -> list[AWSAccountV1]:
|
63
|
+
"""Return all AWS accounts with terraform username but no terraform state set."""
|
64
|
+
return [
|
65
|
+
account
|
66
|
+
for account in aws_accounts_query(query_func).accounts or []
|
67
|
+
if integration_is_enabled(self.name, account)
|
68
|
+
and (not account_name or account.name == account_name)
|
69
|
+
and account.terraform_username
|
70
|
+
and not account.terraform_state
|
71
|
+
]
|
72
|
+
|
73
|
+
def render_state_collection(
|
74
|
+
self, template: str, bucket_name: str, account: AWSAccountV1
|
75
|
+
) -> str:
|
76
|
+
return jinja2.Template(
|
77
|
+
template,
|
78
|
+
undefined=jinja2.StrictUndefined,
|
79
|
+
trim_blocks=True,
|
80
|
+
lstrip_blocks=True,
|
81
|
+
keep_trailing_newline=True,
|
82
|
+
).render({
|
83
|
+
"account_name": account.name,
|
84
|
+
"bucket_name": bucket_name,
|
85
|
+
"region": account.resources_default_region,
|
86
|
+
"timestamp": int(datetime.now(tz=timezone.utc).timestamp()),
|
87
|
+
})
|
88
|
+
|
89
|
+
def reconcile_account(
|
90
|
+
self,
|
91
|
+
account_aws_api: AWSApi,
|
92
|
+
merge_request_manager: MergeRequestManager,
|
93
|
+
dry_run: bool,
|
94
|
+
state_collection: str,
|
95
|
+
bucket_name: str,
|
96
|
+
account: AWSAccountV1,
|
97
|
+
) -> None:
|
98
|
+
logging.info("Creating bucket '%s' for account '%s'", bucket_name, account.name)
|
99
|
+
if not dry_run:
|
100
|
+
# the creation of the bucket is idempotent
|
101
|
+
account_aws_api.s3.create_bucket(
|
102
|
+
name=bucket_name, region=account.resources_default_region
|
103
|
+
)
|
104
|
+
merge_request_manager.create_merge_request(
|
105
|
+
data=MrData(
|
106
|
+
account=account.name,
|
107
|
+
content=state_collection,
|
108
|
+
path=f"{self.params.template_collection_root_path}/{account.name}.yml",
|
109
|
+
)
|
110
|
+
)
|
111
|
+
|
112
|
+
@defer
|
113
|
+
def run(self, dry_run: bool, defer: Callable | None = None) -> None:
|
114
|
+
"""Run the integration."""
|
115
|
+
gql_api = gql.get_api()
|
116
|
+
accounts = self.get_aws_accounts(
|
117
|
+
gql_api.query, account_name=self.params.account_name
|
118
|
+
)
|
119
|
+
if not accounts:
|
120
|
+
# nothing to do
|
121
|
+
return
|
122
|
+
|
123
|
+
vcs = VCS(
|
124
|
+
secret_reader=self.secret_reader,
|
125
|
+
github_orgs=get_github_orgs(),
|
126
|
+
gitlab_instances=get_gitlab_instances(),
|
127
|
+
app_interface_repo_url=get_app_interface_repo_url(),
|
128
|
+
dry_run=dry_run,
|
129
|
+
allow_deleting_mrs=False,
|
130
|
+
allow_opening_mrs=True,
|
131
|
+
)
|
132
|
+
if defer:
|
133
|
+
defer(vcs.cleanup)
|
134
|
+
merge_request_manager = MergeRequestManager(
|
135
|
+
vcs=vcs,
|
136
|
+
renderer=Renderer(),
|
137
|
+
parser=create_parser(),
|
138
|
+
auto_merge_enabled=get_feature_toggle_state(
|
139
|
+
integration_name=f"{self.name}-allow-auto-merge-mrs", default=False
|
140
|
+
),
|
141
|
+
)
|
142
|
+
state_template = gql_api.get_resource(path=self.params.state_tmpl_resource)[
|
143
|
+
"content"
|
144
|
+
]
|
145
|
+
for account in accounts:
|
146
|
+
secret = self.secret_reader.read_all_secret(account.automation_token)
|
147
|
+
with AWSApi(
|
148
|
+
AWSStaticCredentials(
|
149
|
+
access_key_id=secret["aws_access_key_id"],
|
150
|
+
secret_access_key=secret["aws_secret_access_key"],
|
151
|
+
region=account.resources_default_region,
|
152
|
+
)
|
153
|
+
) as account_aws_api:
|
154
|
+
bucket_name = f"terraform-{account.name}"
|
155
|
+
state_collection = self.render_state_collection(
|
156
|
+
state_template, bucket_name, account
|
157
|
+
)
|
158
|
+
self.reconcile_account(
|
159
|
+
account_aws_api,
|
160
|
+
merge_request_manager,
|
161
|
+
dry_run,
|
162
|
+
state_collection,
|
163
|
+
bucket_name,
|
164
|
+
account,
|
165
|
+
)
|
@@ -0,0 +1,57 @@
|
|
1
|
+
import re
|
2
|
+
import string
|
3
|
+
|
4
|
+
from pydantic import BaseModel
|
5
|
+
|
6
|
+
from reconcile.utils.merge_request_manager.parser import Parser
|
7
|
+
|
8
|
+
PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
|
9
|
+
VERSION = "1.0.0"
|
10
|
+
LABEL = "terraform-init"
|
11
|
+
|
12
|
+
VERSION_REF = "tf_init_version"
|
13
|
+
ACCOUNT_REF = "account"
|
14
|
+
COMPILED_REGEXES = {
|
15
|
+
i: re.compile(rf".*{i}: (.*)$", re.MULTILINE) for i in [VERSION_REF, ACCOUNT_REF]
|
16
|
+
}
|
17
|
+
|
18
|
+
DESC = string.Template(
|
19
|
+
f"""
|
20
|
+
This MR is triggered by app-interface's [terraform-init](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/terraform_init).
|
21
|
+
|
22
|
+
Please **do not remove** the **{LABEL}** label from this MR!
|
23
|
+
|
24
|
+
Parts of this description are used by integration to manage the MR.
|
25
|
+
|
26
|
+
{PROMOTION_DATA_SEPARATOR}
|
27
|
+
|
28
|
+
* {VERSION_REF}: {VERSION}
|
29
|
+
* {ACCOUNT_REF}: $account
|
30
|
+
"""
|
31
|
+
)
|
32
|
+
|
33
|
+
|
34
|
+
class Info(BaseModel):
|
35
|
+
account: str
|
36
|
+
|
37
|
+
|
38
|
+
def create_parser() -> Parser:
|
39
|
+
"""Create a parser for MRs created by terraform-init."""
|
40
|
+
|
41
|
+
return Parser[Info](
|
42
|
+
klass=Info,
|
43
|
+
compiled_regexes=COMPILED_REGEXES,
|
44
|
+
version_ref=VERSION_REF,
|
45
|
+
expected_version=VERSION,
|
46
|
+
data_separator=PROMOTION_DATA_SEPARATOR,
|
47
|
+
)
|
48
|
+
|
49
|
+
|
50
|
+
class Renderer:
|
51
|
+
"""This class is only concerned with rendering text for MRs."""
|
52
|
+
|
53
|
+
def render_description(self, account: str) -> str:
|
54
|
+
return DESC.safe_substitute(account=account)
|
55
|
+
|
56
|
+
def render_title(self, account: str) -> str:
|
57
|
+
return f"[auto] Terraform State settings for AWS account {account}"
|
@@ -0,0 +1,102 @@
|
|
1
|
+
import logging
|
2
|
+
|
3
|
+
from gitlab.exceptions import GitlabGetError
|
4
|
+
from pydantic import BaseModel
|
5
|
+
|
6
|
+
from reconcile.terraform_init.merge_request import (
|
7
|
+
LABEL,
|
8
|
+
Info,
|
9
|
+
Renderer,
|
10
|
+
)
|
11
|
+
from reconcile.utils.gitlab_api import GitLabApi
|
12
|
+
from reconcile.utils.merge_request_manager.merge_request_manager import (
|
13
|
+
MergeRequestManagerBase,
|
14
|
+
)
|
15
|
+
from reconcile.utils.merge_request_manager.parser import Parser
|
16
|
+
from reconcile.utils.mr.base import MergeRequestBase
|
17
|
+
from reconcile.utils.mr.labels import AUTO_MERGE
|
18
|
+
from reconcile.utils.vcs import VCS
|
19
|
+
|
20
|
+
|
21
|
+
class TerraformInitMR(MergeRequestBase):
|
22
|
+
name = "TerraformInit"
|
23
|
+
|
24
|
+
def __init__(
|
25
|
+
self, title: str, description: str, path: str, content: str, labels: list[str]
|
26
|
+
):
|
27
|
+
super().__init__()
|
28
|
+
self._title = title
|
29
|
+
self._description = description
|
30
|
+
self._path = path
|
31
|
+
self._content = content
|
32
|
+
self.labels = labels
|
33
|
+
|
34
|
+
@property
|
35
|
+
def title(self) -> str:
|
36
|
+
return self._title
|
37
|
+
|
38
|
+
@property
|
39
|
+
def description(self) -> str:
|
40
|
+
return self._description
|
41
|
+
|
42
|
+
def process(self, gitlab_cli: GitLabApi) -> None:
|
43
|
+
gitlab_cli.create_file(
|
44
|
+
branch_name=self.branch,
|
45
|
+
file_path=self._path,
|
46
|
+
commit_message="add terraform state template collection",
|
47
|
+
content=self._content,
|
48
|
+
)
|
49
|
+
|
50
|
+
|
51
|
+
class MrData(BaseModel):
|
52
|
+
account: str
|
53
|
+
content: str
|
54
|
+
path: str
|
55
|
+
|
56
|
+
|
57
|
+
class MergeRequestManager(MergeRequestManagerBase[Info]):
|
58
|
+
"""Manager for the merge requests.
|
59
|
+
|
60
|
+
This class is responsible for housekeeping (closing old/bad MRs) and
|
61
|
+
opening new MRs.
|
62
|
+
"""
|
63
|
+
|
64
|
+
def __init__(
|
65
|
+
self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
|
66
|
+
):
|
67
|
+
super().__init__(vcs, parser, LABEL)
|
68
|
+
self._renderer = renderer
|
69
|
+
self._auto_merge_enabled = auto_merge_enabled
|
70
|
+
|
71
|
+
def create_merge_request(self, data: MrData) -> None:
|
72
|
+
"""Open a new MR, if not already present, for an AWS account and close any outdated before."""
|
73
|
+
if not self._housekeeping_ran:
|
74
|
+
self.housekeeping()
|
75
|
+
|
76
|
+
if self._merge_request_already_exists({"account": data.account}):
|
77
|
+
logging.info("MR already exists for %s", data.account)
|
78
|
+
return None
|
79
|
+
|
80
|
+
try:
|
81
|
+
self._vcs.get_file_content_from_app_interface_master(file_path=data.path)
|
82
|
+
# the file exists, nothing to do
|
83
|
+
return None
|
84
|
+
except GitlabGetError as e:
|
85
|
+
if e.response_code != 404:
|
86
|
+
raise
|
87
|
+
|
88
|
+
description = self._renderer.render_description(account=data.account)
|
89
|
+
title = self._renderer.render_title(account=data.account)
|
90
|
+
logging.info("Open MR for %s", data.account)
|
91
|
+
mr_labels = [LABEL]
|
92
|
+
if self._auto_merge_enabled:
|
93
|
+
mr_labels.append(AUTO_MERGE)
|
94
|
+
self._vcs.open_app_interface_merge_request(
|
95
|
+
mr=TerraformInitMR(
|
96
|
+
path=data.path,
|
97
|
+
title=title,
|
98
|
+
description=description,
|
99
|
+
content=data.content,
|
100
|
+
labels=mr_labels,
|
101
|
+
)
|
102
|
+
)
|
@@ -11,11 +11,13 @@ from pydantic import BaseModel
|
|
11
11
|
|
12
12
|
import reconcile.utils.aws_api_typed.iam
|
13
13
|
import reconcile.utils.aws_api_typed.organization
|
14
|
+
import reconcile.utils.aws_api_typed.s3
|
14
15
|
import reconcile.utils.aws_api_typed.service_quotas
|
15
16
|
import reconcile.utils.aws_api_typed.sts
|
16
17
|
import reconcile.utils.aws_api_typed.support
|
17
18
|
from reconcile.utils.aws_api_typed.iam import AWSApiIam
|
18
19
|
from reconcile.utils.aws_api_typed.organization import AWSApiOrganizations
|
20
|
+
from reconcile.utils.aws_api_typed.s3 import AWSApiS3
|
19
21
|
from reconcile.utils.aws_api_typed.service_quotas import AWSApiServiceQuotas
|
20
22
|
from reconcile.utils.aws_api_typed.sts import AWSApiSts
|
21
23
|
from reconcile.utils.aws_api_typed.support import AWSApiSupport
|
@@ -161,6 +163,9 @@ class AWSApi:
|
|
161
163
|
case reconcile.utils.aws_api_typed.organization.AWSApiOrganizations:
|
162
164
|
client = self.session.client("organizations")
|
163
165
|
api = api_cls(client)
|
166
|
+
case reconcile.utils.aws_api_typed.s3.AWSApiS3:
|
167
|
+
client = self.session.client("s3")
|
168
|
+
api = api_cls(client)
|
164
169
|
case reconcile.utils.aws_api_typed.service_quotas.AWSApiServiceQuotas:
|
165
170
|
client = self.session.client("service-quotas")
|
166
171
|
api = api_cls(client)
|
@@ -186,6 +191,11 @@ class AWSApi:
|
|
186
191
|
"""Return an AWS Organizations Api client."""
|
187
192
|
return self._init_sub_api(AWSApiOrganizations)
|
188
193
|
|
194
|
+
@cached_property
|
195
|
+
def s3(self) -> AWSApiS3:
|
196
|
+
"""Return an AWS S3 Api client."""
|
197
|
+
return self._init_sub_api(AWSApiS3)
|
198
|
+
|
189
199
|
@cached_property
|
190
200
|
def service_quotas(self) -> AWSApiServiceQuotas:
|
191
201
|
"""Return an AWS Service Quotas Api client."""
|
@@ -0,0 +1,26 @@
|
|
1
|
+
from typing import TYPE_CHECKING
|
2
|
+
|
3
|
+
if TYPE_CHECKING:
|
4
|
+
from mypy_boto3_s3 import S3Client
|
5
|
+
from mypy_boto3_s3.literals import BucketLocationConstraintType
|
6
|
+
else:
|
7
|
+
S3Client = BucketLocationConstraintType = object
|
8
|
+
|
9
|
+
|
10
|
+
class AWSApiS3:
|
11
|
+
def __init__(self, client: S3Client) -> None:
|
12
|
+
self.client = client
|
13
|
+
|
14
|
+
def create_bucket(self, name: str, region: str) -> str:
|
15
|
+
"""Create an S3 bucket without any ACLs therefore the creator will be the owner and returns the ARN."""
|
16
|
+
bucket_kwargs = {}
|
17
|
+
if region != "us-east-1":
|
18
|
+
# you can't specify the location if it's us-east-1 :(
|
19
|
+
# see valid values "LocationConstraint" here: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucketConfiguration.html
|
20
|
+
bucket_kwargs = {
|
21
|
+
"CreateBucketConfiguration": {
|
22
|
+
"LocationConstraint": region,
|
23
|
+
},
|
24
|
+
}
|
25
|
+
self.client.create_bucket(Bucket=name, **bucket_kwargs) # type: ignore
|
26
|
+
return f"arn:aws:s3:::{name}"
|
File without changes
|
File without changes
|
{qontract_reconcile-0.10.1rc736.dist-info → qontract_reconcile-0.10.1rc737.dist-info}/top_level.txt
RENAMED
File without changes
|