atlas-init 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +3 -3
- atlas_init/atlas_init.yaml +18 -1
- atlas_init/cli.py +62 -70
- atlas_init/cli_cfn/app.py +40 -117
- atlas_init/cli_cfn/{cfn.py → aws.py} +129 -14
- atlas_init/cli_cfn/cfn_parameter_finder.py +89 -6
- atlas_init/cli_cfn/example.py +203 -0
- atlas_init/cli_cfn/files.py +63 -0
- atlas_init/cli_helper/run.py +18 -2
- atlas_init/cli_helper/tf_runner.py +4 -6
- atlas_init/cli_root/__init__.py +0 -0
- atlas_init/cli_root/trigger.py +153 -0
- atlas_init/cli_tf/app.py +211 -4
- atlas_init/cli_tf/changelog.py +103 -0
- atlas_init/cli_tf/debug_logs.py +221 -0
- atlas_init/cli_tf/debug_logs_test_data.py +253 -0
- atlas_init/cli_tf/github_logs.py +229 -0
- atlas_init/cli_tf/go_test_run.py +194 -0
- atlas_init/cli_tf/go_test_run_format.py +31 -0
- atlas_init/cli_tf/go_test_summary.py +144 -0
- atlas_init/cli_tf/hcl/__init__.py +0 -0
- atlas_init/cli_tf/hcl/cli.py +161 -0
- atlas_init/cli_tf/hcl/cluster_mig.py +348 -0
- atlas_init/cli_tf/hcl/parser.py +140 -0
- atlas_init/cli_tf/schema.py +222 -18
- atlas_init/cli_tf/schema_go_parser.py +236 -0
- atlas_init/cli_tf/schema_table.py +150 -0
- atlas_init/cli_tf/schema_table_models.py +155 -0
- atlas_init/cli_tf/schema_v2.py +599 -0
- atlas_init/cli_tf/schema_v2_api_parsing.py +298 -0
- atlas_init/cli_tf/schema_v2_sdk.py +361 -0
- atlas_init/cli_tf/schema_v3.py +222 -0
- atlas_init/cli_tf/schema_v3_sdk.py +279 -0
- atlas_init/cli_tf/schema_v3_sdk_base.py +68 -0
- atlas_init/cli_tf/schema_v3_sdk_create.py +216 -0
- atlas_init/humps.py +253 -0
- atlas_init/repos/cfn.py +6 -1
- atlas_init/repos/path.py +3 -3
- atlas_init/settings/config.py +14 -4
- atlas_init/settings/env_vars.py +16 -1
- atlas_init/settings/path.py +12 -1
- atlas_init/settings/rich_utils.py +2 -0
- atlas_init/terraform.yaml +77 -1
- atlas_init/tf/.terraform.lock.hcl +59 -83
- atlas_init/tf/always.tf +7 -0
- atlas_init/tf/main.tf +3 -0
- atlas_init/tf/modules/aws_s3/provider.tf +1 -1
- atlas_init/tf/modules/aws_vars/aws_vars.tf +2 -0
- atlas_init/tf/modules/aws_vpc/provider.tf +4 -1
- atlas_init/tf/modules/cfn/cfn.tf +47 -33
- atlas_init/tf/modules/cfn/kms.tf +54 -0
- atlas_init/tf/modules/cfn/resource_actions.yaml +1 -0
- atlas_init/tf/modules/cfn/variables.tf +31 -0
- atlas_init/tf/modules/cloud_provider/cloud_provider.tf +1 -0
- atlas_init/tf/modules/cloud_provider/provider.tf +1 -1
- atlas_init/tf/modules/cluster/cluster.tf +34 -24
- atlas_init/tf/modules/cluster/provider.tf +1 -1
- atlas_init/tf/modules/federated_vars/federated_vars.tf +3 -0
- atlas_init/tf/modules/federated_vars/provider.tf +1 -1
- atlas_init/tf/modules/project_extra/project_extra.tf +15 -1
- atlas_init/tf/modules/stream_instance/stream_instance.tf +1 -1
- atlas_init/tf/modules/vpc_peering/vpc_peering.tf +1 -1
- atlas_init/tf/modules/vpc_privatelink/versions.tf +1 -1
- atlas_init/tf/outputs.tf +11 -3
- atlas_init/tf/providers.tf +2 -1
- atlas_init/tf/variables.tf +12 -0
- atlas_init/typer_app.py +76 -0
- {atlas_init-0.1.1.dist-info → atlas_init-0.1.4.dist-info}/METADATA +36 -18
- atlas_init-0.1.4.dist-info/RECORD +91 -0
- {atlas_init-0.1.1.dist-info → atlas_init-0.1.4.dist-info}/WHEEL +1 -1
- atlas_init-0.1.1.dist-info/RECORD +0 -62
- /atlas_init/tf/modules/aws_vpc/{aws-vpc.tf → aws_vpc.tf} +0 -0
- {atlas_init-0.1.1.dist-info → atlas_init-0.1.4.dist-info}/entry_points.txt +0 -0
@@ -6,16 +6,22 @@ from collections.abc import Sequence
|
|
6
6
|
from concurrent.futures import ThreadPoolExecutor, wait
|
7
7
|
from datetime import UTC, datetime
|
8
8
|
from functools import lru_cache, total_ordering
|
9
|
+
from pathlib import Path
|
9
10
|
|
10
11
|
import botocore.exceptions
|
12
|
+
import humanize
|
13
|
+
import typer
|
11
14
|
from boto3.session import Session
|
12
15
|
from model_lib import Event
|
13
16
|
from mypy_boto3_cloudformation import CloudFormationClient
|
14
|
-
from mypy_boto3_cloudformation.type_defs import ParameterTypeDef
|
17
|
+
from mypy_boto3_cloudformation.type_defs import ListTypesOutputTypeDef, ParameterTypeDef
|
15
18
|
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
|
19
|
+
from zero_3rdparty.datetime_utils import utc_now
|
16
20
|
from zero_3rdparty.iter_utils import group_by_once
|
17
21
|
|
22
|
+
from atlas_init.cli_helper.run import run_command_is_ok
|
18
23
|
from atlas_init.cloud.aws import REGIONS, PascalAlias, region_continent
|
24
|
+
from atlas_init.settings.interactive import confirm
|
19
25
|
|
20
26
|
logger = logging.getLogger(__name__)
|
21
27
|
EARLY_DATETIME = datetime(year=1990, month=1, day=1, tzinfo=UTC)
|
@@ -76,12 +82,12 @@ def deactivate_type(type_name: str, region: str):
|
|
76
82
|
client.deactivate_type(TypeName=type_name, Type="RESOURCE")
|
77
83
|
|
78
84
|
|
79
|
-
def delete_role_stack(type_name: str, region_name: str) -> None:
|
85
|
+
def delete_role_stack(type_name: str, region_name: str, role_arn: str = "") -> None:
|
80
86
|
stack_name = type_name.replace("::", "-").lower() + "-role-stack"
|
81
|
-
delete_stack(region_name, stack_name)
|
87
|
+
delete_stack(region_name, stack_name, role_arn)
|
82
88
|
|
83
89
|
|
84
|
-
def delete_stack(region_name: str, stack_name: str):
|
90
|
+
def delete_stack(region_name: str, stack_name: str, role_arn: str = ""):
|
85
91
|
client = cloud_formation_client(region_name)
|
86
92
|
logger.warning(f"deleting stack {stack_name} in region={region_name}")
|
87
93
|
try:
|
@@ -91,7 +97,7 @@ def delete_stack(region_name: str, stack_name: str):
|
|
91
97
|
logger.warning(f"stack {stack_name} not found")
|
92
98
|
return
|
93
99
|
raise
|
94
|
-
client.delete_stack(StackName=stack_name)
|
100
|
+
client.delete_stack(StackName=stack_name, RoleARN=role_arn)
|
95
101
|
wait_on_stack_ok(stack_name, region_name, expect_not_found=True)
|
96
102
|
|
97
103
|
|
@@ -110,7 +116,9 @@ def create_stack(
|
|
110
116
|
Parameters=parameters,
|
111
117
|
RoleARN=role_arn,
|
112
118
|
)
|
113
|
-
logger.info(
|
119
|
+
logger.info(
|
120
|
+
f"stack with name: {stack_name} created in {region_name} has id: {stack_id['StackId']} role_arn:{role_arn}"
|
121
|
+
)
|
114
122
|
wait_on_stack_ok(stack_name, region_name, timeout_seconds=timeout_seconds)
|
115
123
|
|
116
124
|
|
@@ -146,7 +154,9 @@ class StackInProgressError(StackBaseError):
|
|
146
154
|
|
147
155
|
|
148
156
|
class StackError(StackBaseError):
|
149
|
-
|
157
|
+
def __init__(self, status: str, timestamp: datetime, status_reason: str, reasons: str) -> None:
|
158
|
+
super().__init__(status, timestamp, status_reason)
|
159
|
+
self.reasons = reasons
|
150
160
|
|
151
161
|
|
152
162
|
@total_ordering
|
@@ -184,10 +194,20 @@ class StackEvents(Event):
|
|
184
194
|
|
185
195
|
def last_reason(self) -> str:
|
186
196
|
for event in sorted(self.stack_events, reverse=True):
|
187
|
-
if reason := event.resource_status_reason:
|
197
|
+
if reason := event.resource_status_reason.strip():
|
188
198
|
return reason
|
189
199
|
return ""
|
190
200
|
|
201
|
+
def multiple_reasons(self, max_reasons: int = 5) -> str:
|
202
|
+
reasons = []
|
203
|
+
for event in sorted(self.stack_events, reverse=True):
|
204
|
+
if reason := event.resource_status_reason.strip():
|
205
|
+
reason_number = len(reasons) + 1
|
206
|
+
reasons.append(f"{reason_number}. {reason}")
|
207
|
+
if reason_number >= max_reasons:
|
208
|
+
break
|
209
|
+
return "\n".join(reasons)
|
210
|
+
|
191
211
|
|
192
212
|
def wait_on_stack_ok(
|
193
213
|
stack_name: str,
|
@@ -229,15 +249,21 @@ def wait_on_stack_ok(
|
|
229
249
|
current_event.resource_status,
|
230
250
|
current_event.timestamp,
|
231
251
|
current_event.resource_status_reason,
|
252
|
+
reasons=parsed.multiple_reasons(),
|
232
253
|
)
|
233
254
|
status = current_event.resource_status
|
234
255
|
logger.info(f"stack is ready {stack_name} {status} ✅")
|
235
256
|
if "ROLLBACK" in status:
|
236
|
-
last_reason = parsed.
|
257
|
+
last_reason = parsed.multiple_reasons()
|
237
258
|
logger.warning(f"stack did rollback, got: {current_event!r}\n{last_reason}")
|
259
|
+
|
238
260
|
return None
|
239
261
|
|
240
|
-
|
262
|
+
try:
|
263
|
+
return _wait_on_stack_ok()
|
264
|
+
except StackError as e:
|
265
|
+
logger.warning(f"stack error {stack_name} {e.status} {e.status_reason}\n{e.reasons}")
|
266
|
+
raise typer.Exit(1) from None
|
241
267
|
|
242
268
|
|
243
269
|
def print_version_regions(type_name: str) -> None:
|
@@ -257,7 +283,7 @@ def get_last_version_all_regions(type_name: str) -> dict[str | None, list[str]]:
|
|
257
283
|
futures = {}
|
258
284
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
259
285
|
for region in REGIONS:
|
260
|
-
future = pool.submit(get_last_cfn_type, type_name, region)
|
286
|
+
future = pool.submit(get_last_cfn_type, type_name, region, is_third_party=True)
|
261
287
|
futures[future] = region
|
262
288
|
done, not_done = wait(futures.keys(), timeout=300)
|
263
289
|
for f in not_done:
|
@@ -287,8 +313,18 @@ class CfnTypeDetails(Event):
|
|
287
313
|
raise TypeError
|
288
314
|
return self.last_updated < other.last_updated
|
289
315
|
|
316
|
+
def seconds_since_update(self) -> float:
|
317
|
+
return (utc_now() - self.last_updated).total_seconds()
|
318
|
+
|
290
319
|
|
291
|
-
def
|
320
|
+
def publish_cfn_type(region: str):
|
321
|
+
client: CloudFormationClient = cloud_formation_client(region)
|
322
|
+
client.publish_type()
|
323
|
+
|
324
|
+
|
325
|
+
def get_last_cfn_type(
|
326
|
+
type_name: str, region: str, *, is_third_party: bool = False, force_version: str = ""
|
327
|
+
) -> None | CfnTypeDetails:
|
292
328
|
client: CloudFormationClient = cloud_formation_client(region)
|
293
329
|
prefix = type_name
|
294
330
|
logger.info(f"finding public 3rd party for '{prefix}' in {region}")
|
@@ -302,7 +338,7 @@ def get_last_cfn_type(type_name: str, region: str, *, is_third_party: bool = Fal
|
|
302
338
|
}
|
303
339
|
next_token = ""
|
304
340
|
for _ in range(100):
|
305
|
-
types_response = client.list_types(**kwargs) # type: ignore
|
341
|
+
types_response: ListTypesOutputTypeDef = client.list_types(**kwargs) # type: ignore
|
306
342
|
next_token = types_response.get("NextToken", "")
|
307
343
|
kwargs["NextToken"] = next_token
|
308
344
|
for t in types_response["TypeSummaries"]:
|
@@ -325,6 +361,12 @@ def get_last_cfn_type(type_name: str, region: str, *, is_third_party: bool = Fal
|
|
325
361
|
if not type_details:
|
326
362
|
logger.warning(f"no version for {type_name} in region {region}")
|
327
363
|
return None
|
364
|
+
if force_version:
|
365
|
+
for detail in type_details:
|
366
|
+
if detail.version == force_version:
|
367
|
+
return detail
|
368
|
+
versions = [d.version for d in type_details]
|
369
|
+
raise ValueError(f"unable to find version {force_version} for {type_name}, got {versions}")
|
328
370
|
return sorted(type_details)[-1]
|
329
371
|
|
330
372
|
|
@@ -335,4 +377,77 @@ def activate_resource_type(details: CfnTypeDetails, region: str, execution_role_
|
|
335
377
|
PublicTypeArn=details.type_arn,
|
336
378
|
ExecutionRoleArn=execution_role_arn,
|
337
379
|
)
|
338
|
-
logger.info(f"activate response: {response}")
|
380
|
+
logger.info(f"activate response: {response} role={execution_role_arn}")
|
381
|
+
|
382
|
+
|
383
|
+
def ensure_resource_type_activated(
|
384
|
+
type_name: str,
|
385
|
+
region: str,
|
386
|
+
force_deregister: bool,
|
387
|
+
is_interactive: bool,
|
388
|
+
resource_path: Path,
|
389
|
+
cfn_execution_role: str,
|
390
|
+
force_version: str = "",
|
391
|
+
) -> None:
|
392
|
+
cfn_type_details = get_last_cfn_type(type_name, region, is_third_party=False)
|
393
|
+
logger.info(f"found cfn_type_details {cfn_type_details} for {type_name}")
|
394
|
+
is_third_party = False
|
395
|
+
if cfn_type_details is None:
|
396
|
+
cfn_type_details = get_last_cfn_type(type_name, region, is_third_party=True)
|
397
|
+
if cfn_type_details:
|
398
|
+
is_third_party = True
|
399
|
+
logger.warning(f"found 3rd party extension for cfn type {type_name} active")
|
400
|
+
if force_version:
|
401
|
+
if cfn_type_details and cfn_type_details.version == force_version:
|
402
|
+
logger.info(f"version {force_version} already active")
|
403
|
+
return
|
404
|
+
force_deregister = True
|
405
|
+
if cfn_type_details is not None and (cfn_type_details.seconds_since_update() > 3600 * 24 or force_deregister):
|
406
|
+
outdated_warning = f"more than {humanize.naturaldelta(cfn_type_details.seconds_since_update())} since last update to {type_name} {cfn_type_details.version}"
|
407
|
+
logger.warning(outdated_warning)
|
408
|
+
if force_deregister or confirm(
|
409
|
+
f"{outdated_warning}, should deregister?",
|
410
|
+
is_interactive=is_interactive,
|
411
|
+
default=True,
|
412
|
+
):
|
413
|
+
if is_third_party:
|
414
|
+
deactivate_third_party_type(type_name, region)
|
415
|
+
else:
|
416
|
+
deregister_cfn_resource_type(type_name, deregister=True, region_filter=region)
|
417
|
+
cfn_type_details = None
|
418
|
+
|
419
|
+
# assert cfn_type_details, f"no cfn_type_details found for {type_name}"
|
420
|
+
# client = cloud_formation_client(region)
|
421
|
+
# response = client.activate_type(
|
422
|
+
# Type="RESOURCE",
|
423
|
+
# # PublicTypeArn=cfn_type_details.type_arn.replace(":type/", "::type/"),
|
424
|
+
# PublicTypeArn="arn:aws:cloudformation:eu-south-2:358363220050::type/resource/MongoDB-Atlas-ResourcePolicy/00000001",
|
425
|
+
# ExecutionRoleArn=cfn_execution_role,
|
426
|
+
# LoggingConfig={"LogRoleArn": cfn_execution_role, "LogGroupName": "/apix/espen/cfn-test1"},
|
427
|
+
# )
|
428
|
+
# logger.info(f"activate response: {response}")
|
429
|
+
submit_cmd = f"cfn submit --verbose --set-default --region {region} --role-arn {cfn_execution_role}"
|
430
|
+
if (
|
431
|
+
not force_version
|
432
|
+
and cfn_type_details is None
|
433
|
+
and confirm(
|
434
|
+
f"No existing {type_name} found, ok to run:\n{submit_cmd}\nsubmit?",
|
435
|
+
is_interactive=is_interactive,
|
436
|
+
default=True,
|
437
|
+
)
|
438
|
+
):
|
439
|
+
assert run_command_is_ok(cmd=submit_cmd.split(), env=None, cwd=resource_path, logger=logger)
|
440
|
+
cfn_type_details = get_last_cfn_type(type_name, region, is_third_party=False)
|
441
|
+
if cfn_type_details is None:
|
442
|
+
third_party = get_last_cfn_type(type_name, region, is_third_party=True, force_version=force_version)
|
443
|
+
assert third_party, f"unable to find 3rd party type for {type_name}"
|
444
|
+
last_updated = third_party.last_updated
|
445
|
+
if confirm(
|
446
|
+
f"No existing {type_name} found, ok to activate 3rd party: :\n'{third_party.version} ({humanize.naturalday(last_updated), {last_updated.isoformat()}})'\n?",
|
447
|
+
is_interactive=is_interactive,
|
448
|
+
default=True,
|
449
|
+
):
|
450
|
+
activate_resource_type(third_party, region, cfn_execution_role)
|
451
|
+
cfn_type_details = third_party
|
452
|
+
assert cfn_type_details, f"no cfn_type_details found for {type_name}"
|
453
|
+
# TODO: validate the active type details uses the execution role
|
@@ -7,9 +7,11 @@ from mypy_boto3_cloudformation.type_defs import ParameterTypeDef
|
|
7
7
|
from pydantic import ConfigDict, Field
|
8
8
|
from rich import prompt
|
9
9
|
from zero_3rdparty.dict_nested import read_nested
|
10
|
+
from zero_3rdparty.file_utils import clean_dir
|
10
11
|
|
12
|
+
from atlas_init.cli_cfn.files import create_sample_file, default_log_group_name
|
11
13
|
from atlas_init.cloud.aws import PascalAlias
|
12
|
-
from atlas_init.repos.cfn import cfn_examples_dir, cfn_type_normalized
|
14
|
+
from atlas_init.repos.cfn import CfnType, cfn_examples_dir, cfn_type_normalized
|
13
15
|
from atlas_init.settings.path import DEFAULT_TF_PATH
|
14
16
|
|
15
17
|
logger = logging.getLogger(__name__)
|
@@ -46,13 +48,15 @@ class TemplatePathNotFoundError(Exception):
|
|
46
48
|
self.examples_dir = examples_dir
|
47
49
|
|
48
50
|
|
49
|
-
def infer_template_path(repo_path: Path, type_name: str, stack_name: str) -> Path:
|
51
|
+
def infer_template_path(repo_path: Path, type_name: str, stack_name: str, example_name: str = "") -> Path:
|
50
52
|
examples_dir = cfn_examples_dir(repo_path)
|
51
53
|
template_paths: list[Path] = []
|
52
54
|
type_setting = f'"Type": "{type_name}"'
|
53
55
|
for p in examples_dir.rglob("*.json"):
|
56
|
+
if example_name and example_name != p.stem:
|
57
|
+
continue
|
54
58
|
if type_setting in p.read_text():
|
55
|
-
logger.info(f"found template @ {p}")
|
59
|
+
logger.info(f"found template @ '{p.stem}': {p.parent}")
|
56
60
|
template_paths.append(p)
|
57
61
|
if not template_paths:
|
58
62
|
raise TemplatePathNotFoundError(type_name, examples_dir)
|
@@ -78,6 +82,14 @@ parameters_exported_env_vars = {
|
|
78
82
|
"KeyId": "MONGODB_ATLAS_ORG_API_KEY_ID",
|
79
83
|
"TeamId": "MONGODB_ATLAS_TEAM_ID",
|
80
84
|
"ProjectId": "MONGODB_ATLAS_PROJECT_ID",
|
85
|
+
"AWSVpcId": "AWS_VPC_ID",
|
86
|
+
"MongoDBAtlasProjectId": "MONGODB_ATLAS_PROJECT_ID",
|
87
|
+
"AWSSubnetId": "AWS_SUBNET_ID",
|
88
|
+
"AWSRegion": "AWS_REGION",
|
89
|
+
"AppId": "MONGODB_REALM_APP_ID",
|
90
|
+
"FunctionId": "MONGODB_REALM_FUNCTION_ID",
|
91
|
+
"FunctionName": "MONGODB_REALM_FUNCTION_NAME",
|
92
|
+
"ServiceId": "MONGODB_REALM_SERVICE_ID",
|
81
93
|
}
|
82
94
|
|
83
95
|
STACK_NAME_PARAM = "$STACK_NAME_PARAM$"
|
@@ -91,6 +103,13 @@ type_names_defaults: dict[str, dict[str, str]] = {
|
|
91
103
|
STACK_NAME_PARAM: "ClusterName",
|
92
104
|
"ProjectName": "Cluster-CFN-Example",
|
93
105
|
},
|
106
|
+
"resourcepolicy": {
|
107
|
+
STACK_NAME_PARAM: "Name",
|
108
|
+
"Policies": 'forbid (principal, action == cloud::Action::"project.edit",resource) when {context.project.ipAccessList.contains(ip("0.0.0.0/0"))};',
|
109
|
+
},
|
110
|
+
"trigger": {
|
111
|
+
STACK_NAME_PARAM: "TriggerName",
|
112
|
+
},
|
94
113
|
}
|
95
114
|
|
96
115
|
|
@@ -114,6 +133,11 @@ class CfnTemplate(Entity):
|
|
114
133
|
parameters: dict[str, CfnParameter]
|
115
134
|
resources: dict[str, CfnResource]
|
116
135
|
|
136
|
+
@classmethod
|
137
|
+
def read_template_types(cls, template_path: Path, prefix: str = CfnType.MONGODB_ATLAS_CFN_TYPE_PREFIX) -> set[str]:
|
138
|
+
cfn_template = parse_model(template_path, t=CfnTemplate)
|
139
|
+
return {r.type for r in cfn_template.resources.values() if r.type.startswith(prefix)}
|
140
|
+
|
117
141
|
def find_resource(self, type_name: str) -> CfnResource:
|
118
142
|
for r in self.resources.values():
|
119
143
|
if r.type == type_name:
|
@@ -128,6 +152,31 @@ class CfnTemplate(Entity):
|
|
128
152
|
resource = self.find_resource(type_name)
|
129
153
|
resource.properties.update(resources)
|
130
154
|
|
155
|
+
def get_resource_properties(self, type_name: str, parameters: list[ParameterTypeDef]) -> dict:
|
156
|
+
resource = self.find_resource(type_name)
|
157
|
+
properties = resource.properties
|
158
|
+
for param in parameters:
|
159
|
+
key = param.get("ParameterKey")
|
160
|
+
assert key
|
161
|
+
if key not in properties:
|
162
|
+
key_found = next(
|
163
|
+
(
|
164
|
+
maybe_key
|
165
|
+
for maybe_key, value in properties.items()
|
166
|
+
if isinstance(value, dict) and value.get("Ref") == key
|
167
|
+
),
|
168
|
+
None,
|
169
|
+
)
|
170
|
+
err_msg = f"unable to find parameter {key} in resource {type_name}, can happen if there are template parameters not used for {type_name}"
|
171
|
+
if key_found is None:
|
172
|
+
logger.warning(err_msg)
|
173
|
+
continue
|
174
|
+
key = key_found
|
175
|
+
param_value = param.get("ParameterValue", "")
|
176
|
+
assert param_value
|
177
|
+
properties[key] = param_value
|
178
|
+
return properties
|
179
|
+
|
131
180
|
|
132
181
|
def updated_template_path(path: Path) -> Path:
|
133
182
|
old_stem = path.stem
|
@@ -147,9 +196,11 @@ def decode_parameters(
|
|
147
196
|
if resource_params:
|
148
197
|
cfn_template.add_resource_params(type_name, resource_params)
|
149
198
|
template_path = updated_template_path(template_path)
|
150
|
-
logger.info(f"updating template {template_path}")
|
199
|
+
logger.info(f"updating template {template_path} with {resource_params}")
|
151
200
|
raw_dict = cfn_template.model_dump(by_alias=True, exclude_unset=True)
|
152
|
-
|
201
|
+
file_extension = template_path.suffix.lstrip(".")
|
202
|
+
dump_format = "pretty_json" if file_extension == "json" else file_extension
|
203
|
+
template_str = dump(raw_dict, format=dump_format)
|
153
204
|
template_path.write_text(template_str)
|
154
205
|
parameters_dict: dict[str, Any] = {}
|
155
206
|
type_defaults = type_names_defaults.get(cfn_template.normalized_type_name(type_name), {})
|
@@ -178,9 +229,41 @@ def decode_parameters(
|
|
178
229
|
|
179
230
|
if force_params:
|
180
231
|
logger.warning(f"overiding params: {force_params} for {stack_name}")
|
181
|
-
parameters_dict
|
232
|
+
parameters_dict |= force_params
|
182
233
|
unknown_params = {key for key, value in parameters_dict.items() if value == "UNKNOWN"}
|
183
234
|
parameters: list[ParameterTypeDef] = [
|
184
235
|
{"ParameterKey": key, "ParameterValue": value} for key, value in parameters_dict.items()
|
185
236
|
]
|
186
237
|
return template_path, parameters, unknown_params
|
238
|
+
|
239
|
+
|
240
|
+
def dump_resource_to_file(
|
241
|
+
inputs_dir: Path,
|
242
|
+
template_path: Path,
|
243
|
+
type_name: str,
|
244
|
+
parameters: list[ParameterTypeDef],
|
245
|
+
) -> Path:
|
246
|
+
cfn_template = parse_model(template_path, t=CfnTemplate)
|
247
|
+
properties = cfn_template.get_resource_properties(type_name, parameters)
|
248
|
+
clean_dir(inputs_dir, recreate=True)
|
249
|
+
dest_path = inputs_dir / "inputs_1_create.json"
|
250
|
+
dest_json = dump(properties, "pretty_json")
|
251
|
+
dest_path.write_text(dest_json)
|
252
|
+
return dest_path
|
253
|
+
|
254
|
+
|
255
|
+
def dump_sample_file(
|
256
|
+
samples_dir: Path,
|
257
|
+
template_path: Path,
|
258
|
+
type_name: str,
|
259
|
+
parameters: list[ParameterTypeDef],
|
260
|
+
):
|
261
|
+
cfn_template = parse_model(template_path, t=CfnTemplate)
|
262
|
+
samples_path = samples_dir / template_path.stem / "create.json"
|
263
|
+
create_sample_file(
|
264
|
+
samples_path,
|
265
|
+
default_log_group_name(CfnType.resource_name(type_name)),
|
266
|
+
cfn_template.get_resource_properties(type_name, parameters),
|
267
|
+
prev_resource_state={},
|
268
|
+
)
|
269
|
+
return samples_path
|
@@ -0,0 +1,203 @@
|
|
1
|
+
import logging
|
2
|
+
from pathlib import Path
|
3
|
+
from typing import Any
|
4
|
+
|
5
|
+
import typer
|
6
|
+
from pydantic import field_validator, model_validator
|
7
|
+
from rich import prompt
|
8
|
+
|
9
|
+
from atlas_init.cli_args import parse_key_values_any
|
10
|
+
from atlas_init.cli_cfn.aws import (
|
11
|
+
create_stack,
|
12
|
+
ensure_resource_type_activated,
|
13
|
+
update_stack,
|
14
|
+
)
|
15
|
+
from atlas_init.cli_cfn.aws import delete_stack as delete_stack_aws
|
16
|
+
from atlas_init.cli_cfn.cfn_parameter_finder import (
|
17
|
+
CfnTemplate,
|
18
|
+
check_execution_role,
|
19
|
+
decode_parameters,
|
20
|
+
dump_resource_to_file,
|
21
|
+
dump_sample_file,
|
22
|
+
infer_template_path,
|
23
|
+
)
|
24
|
+
from atlas_init.repos.cfn import CfnType, Operation, infer_cfn_type_name
|
25
|
+
from atlas_init.repos.path import Repo, find_paths
|
26
|
+
from atlas_init.settings.env_vars import AtlasInitSettings, init_settings
|
27
|
+
|
28
|
+
logger = logging.getLogger(__name__)
|
29
|
+
|
30
|
+
|
31
|
+
class CfnExampleInputs(CfnType):
|
32
|
+
stack_name: str
|
33
|
+
operation: Operation
|
34
|
+
example_name: str
|
35
|
+
resource_params: dict[str, Any] | None = None
|
36
|
+
stack_timeout_s: int
|
37
|
+
delete_stack_first: bool
|
38
|
+
reg_version: str = ""
|
39
|
+
force_deregister: bool
|
40
|
+
force_keep: bool
|
41
|
+
execution_role: str
|
42
|
+
export_example_to_inputs: bool
|
43
|
+
export_example_to_samples: bool
|
44
|
+
register_all_types_in_example: bool
|
45
|
+
|
46
|
+
@field_validator("resource_params", mode="before")
|
47
|
+
@classmethod
|
48
|
+
def validate_resource_params(cls, v):
|
49
|
+
return parse_key_values_any(v) if isinstance(v, list) else v
|
50
|
+
|
51
|
+
@model_validator(mode="after")
|
52
|
+
def check(self):
|
53
|
+
assert self.region_filter, "region is required"
|
54
|
+
assert self.execution_role.startswith("arn:aws:iam::"), f"invalid execution role: {self.execution_role}"
|
55
|
+
assert self.region
|
56
|
+
return self
|
57
|
+
|
58
|
+
@property
|
59
|
+
def is_export(self) -> bool:
|
60
|
+
return self.export_example_to_inputs or self.export_example_to_samples
|
61
|
+
|
62
|
+
@property
|
63
|
+
def region(self) -> str:
|
64
|
+
region = self.region_filter
|
65
|
+
assert isinstance(region, str), "region is required"
|
66
|
+
return region
|
67
|
+
|
68
|
+
|
69
|
+
def example_cmd(
|
70
|
+
type_name: str = typer.Option("", "-n", "--type-name", help="inferred from your cwd if not provided"),
|
71
|
+
region: str = typer.Option("", help="inferred from your atlas_init cfn region if not provided"),
|
72
|
+
stack_name: str = typer.Option(
|
73
|
+
"",
|
74
|
+
help="inferred from your atlas_init cfn profile name and example if not provided",
|
75
|
+
),
|
76
|
+
operation: str = typer.Argument(...),
|
77
|
+
example_name: str = typer.Option("", "-e", "--example-name", help="example filestem"),
|
78
|
+
resource_params: list[str] = typer.Option(
|
79
|
+
..., "-r", "--resource-param", default_factory=list, help="key=value, can be set many times"
|
80
|
+
),
|
81
|
+
stack_timeout_s: int = typer.Option(3600, "-t", "--stack-timeout-s"),
|
82
|
+
delete_first: bool = typer.Option(False, "-d", "--delete-first", help="Delete existing stack first"),
|
83
|
+
reg_version: str = typer.Option("", "--reg-version", help="Register a specific version"),
|
84
|
+
force_deregister: bool = typer.Option(False, "--dereg", help="Force deregister CFN Type"),
|
85
|
+
force_keep: bool = typer.Option(False, "--noreg", help="Force keep CFN Type (do not prompt)"),
|
86
|
+
execution_role: str = typer.Option("", "--execution-role", help="Execution role to use, otherwise inferred"),
|
87
|
+
export_example_to_inputs: bool = typer.Option(
|
88
|
+
False, "-o", "--export-example-to-inputs", help="Export example to inputs"
|
89
|
+
),
|
90
|
+
export_example_to_samples: bool = typer.Option(
|
91
|
+
False, "-s", "--export-example-to-samples", help="Export example to samples"
|
92
|
+
),
|
93
|
+
register_all_types_in_example: bool = typer.Option(False, "--reg-all", help="Check all types"),
|
94
|
+
):
|
95
|
+
settings = init_settings()
|
96
|
+
assert settings.cfn_config, "no cfn config found, re-run atlas_init apply with CFN flags"
|
97
|
+
repo_path, resource_path, _ = find_paths(Repo.CFN)
|
98
|
+
env_vars_generated = settings.load_env_vars_generated()
|
99
|
+
inputs = CfnExampleInputs(
|
100
|
+
type_name=type_name or infer_cfn_type_name(),
|
101
|
+
example_name=example_name,
|
102
|
+
delete_stack_first=delete_first,
|
103
|
+
region_filter=region or settings.cfn_region,
|
104
|
+
stack_name=stack_name or f"{settings.cfn_profile}-{example_name or 'atlas-init'}",
|
105
|
+
operation=operation, # type: ignore
|
106
|
+
resource_params=resource_params, # type: ignore
|
107
|
+
stack_timeout_s=stack_timeout_s,
|
108
|
+
force_deregister=force_deregister,
|
109
|
+
reg_version=reg_version,
|
110
|
+
force_keep=force_keep,
|
111
|
+
execution_role=execution_role or check_execution_role(repo_path, env_vars_generated),
|
112
|
+
export_example_to_inputs=export_example_to_inputs,
|
113
|
+
export_example_to_samples=export_example_to_samples,
|
114
|
+
register_all_types_in_example=register_all_types_in_example,
|
115
|
+
)
|
116
|
+
example_handler(inputs, repo_path, resource_path, settings)
|
117
|
+
|
118
|
+
|
119
|
+
def example_handler(inputs: CfnExampleInputs, repo_path: Path, resource_path: Path, settings: AtlasInitSettings):
|
120
|
+
logger.info(
|
121
|
+
f"about to {inputs.operation} stack {inputs.stack_name} for {inputs.type_name} in {inputs.region_filter} params: {inputs.resource_params}"
|
122
|
+
)
|
123
|
+
type_name = inputs.type_name
|
124
|
+
stack_name = inputs.stack_name
|
125
|
+
env_vars_generated = settings.load_env_vars_generated()
|
126
|
+
region = inputs.region
|
127
|
+
operation = inputs.operation
|
128
|
+
stack_timeout_s = inputs.stack_timeout_s
|
129
|
+
delete_first = inputs.delete_stack_first
|
130
|
+
force_deregister = inputs.force_deregister
|
131
|
+
execution_role = inputs.execution_role
|
132
|
+
logger.info(f"using execution role: {execution_role}")
|
133
|
+
if not inputs.is_export and not inputs.force_keep:
|
134
|
+
ensure_resource_type_activated(
|
135
|
+
type_name,
|
136
|
+
region,
|
137
|
+
force_deregister,
|
138
|
+
settings.is_interactive,
|
139
|
+
resource_path,
|
140
|
+
execution_role,
|
141
|
+
force_version=inputs.reg_version,
|
142
|
+
)
|
143
|
+
if not inputs.is_export and (operation == Operation.DELETE or delete_first):
|
144
|
+
delete_stack_aws(region, stack_name, execution_role)
|
145
|
+
if not delete_first:
|
146
|
+
return
|
147
|
+
template_path = infer_template_path(repo_path, type_name, stack_name, inputs.example_name)
|
148
|
+
template_path, parameters, not_found = decode_parameters(
|
149
|
+
exported_env_vars=env_vars_generated,
|
150
|
+
template_path=template_path,
|
151
|
+
stack_name=stack_name,
|
152
|
+
force_params=inputs.resource_params,
|
153
|
+
resource_params=inputs.resource_params,
|
154
|
+
type_name=type_name,
|
155
|
+
)
|
156
|
+
if inputs.register_all_types_in_example:
|
157
|
+
extra_example_types = [t for t in CfnTemplate.read_template_types(template_path) if t != type_name]
|
158
|
+
for extra_type in extra_example_types:
|
159
|
+
logger.info(f"extra type {extra_type} in example {template_path}")
|
160
|
+
ensure_resource_type_activated(
|
161
|
+
extra_type,
|
162
|
+
region,
|
163
|
+
force_deregister,
|
164
|
+
settings.is_interactive,
|
165
|
+
resource_path,
|
166
|
+
execution_role,
|
167
|
+
)
|
168
|
+
logger.info(f"parameters: {parameters}")
|
169
|
+
if not_found:
|
170
|
+
# TODO: support specifying these extra
|
171
|
+
logger.critical(f"need to fill out parameters manually: {not_found} for {type_name}")
|
172
|
+
raise typer.Exit(1)
|
173
|
+
if not prompt.Confirm("parameters 👆looks good?")():
|
174
|
+
raise typer.Abort
|
175
|
+
if inputs.export_example_to_inputs:
|
176
|
+
out_inputs = dump_resource_to_file(resource_path / "inputs", template_path, type_name, parameters)
|
177
|
+
logger.info(f"dumped to {out_inputs} ✅")
|
178
|
+
return
|
179
|
+
if inputs.export_example_to_samples:
|
180
|
+
samples_dir = resource_path / "samples"
|
181
|
+
samples_path = dump_sample_file(samples_dir, template_path, type_name, parameters)
|
182
|
+
logger.info(f"dumped to {samples_path} ✅")
|
183
|
+
return
|
184
|
+
if operation == Operation.CREATE:
|
185
|
+
create_stack(
|
186
|
+
stack_name,
|
187
|
+
template_str=template_path.read_text(),
|
188
|
+
region_name=region,
|
189
|
+
role_arn=execution_role,
|
190
|
+
parameters=parameters,
|
191
|
+
timeout_seconds=stack_timeout_s,
|
192
|
+
)
|
193
|
+
elif operation == Operation.UPDATE:
|
194
|
+
update_stack(
|
195
|
+
stack_name,
|
196
|
+
template_str=template_path.read_text(),
|
197
|
+
region_name=region,
|
198
|
+
parameters=parameters,
|
199
|
+
role_arn=execution_role,
|
200
|
+
timeout_seconds=stack_timeout_s,
|
201
|
+
)
|
202
|
+
else:
|
203
|
+
raise NotImplementedError
|
@@ -0,0 +1,63 @@
|
|
1
|
+
import contextlib
|
2
|
+
import logging
|
3
|
+
import re
|
4
|
+
from collections.abc import Iterable
|
5
|
+
from pathlib import Path
|
6
|
+
|
7
|
+
import stringcase
|
8
|
+
from model_lib import Entity, dump, parse_model
|
9
|
+
from pydantic import ConfigDict, Field, ValidationError
|
10
|
+
from zero_3rdparty import file_utils
|
11
|
+
|
12
|
+
logger = logging.getLogger(__name__)
|
13
|
+
|
14
|
+
|
15
|
+
def create_sample_file(
|
16
|
+
samples_file: Path,
|
17
|
+
log_group_name: str,
|
18
|
+
resource_state: dict,
|
19
|
+
prev_resource_state: dict | None = None,
|
20
|
+
):
|
21
|
+
logger.info(f"adding sample @ {samples_file}")
|
22
|
+
assert isinstance(resource_state, dict)
|
23
|
+
new_json = dump(
|
24
|
+
{
|
25
|
+
"providerLogGroupName": log_group_name,
|
26
|
+
"previousResourceState": prev_resource_state or {},
|
27
|
+
"desiredResourceState": resource_state,
|
28
|
+
},
|
29
|
+
"pretty_json",
|
30
|
+
)
|
31
|
+
file_utils.ensure_parents_write_text(samples_file, new_json)
|
32
|
+
|
33
|
+
|
34
|
+
CamelAlias = ConfigDict(alias_generator=stringcase.camelcase, populate_by_name=True)
|
35
|
+
|
36
|
+
|
37
|
+
class CfnSchema(Entity):
|
38
|
+
model_config = CamelAlias
|
39
|
+
|
40
|
+
description: str
|
41
|
+
type_name: str = Field(pattern=r"^[a-zA-Z0-9]{2,64}::[a-zA-Z0-9]{2,64}::[a-zA-Z0-9]{2,64}$")
|
42
|
+
|
43
|
+
|
44
|
+
def iterate_schemas(resource_root: Path) -> Iterable[tuple[Path, CfnSchema]]:
|
45
|
+
for path in resource_root.rglob("*.json"):
|
46
|
+
if path.parent.parent != resource_root:
|
47
|
+
continue
|
48
|
+
with contextlib.suppress(ValidationError):
|
49
|
+
yield path, parse_model(path, t=CfnSchema)
|
50
|
+
|
51
|
+
|
52
|
+
_md_patterns = [
|
53
|
+
re.compile(r"\[[^\]]+\]\([^\)]+\)"),
|
54
|
+
re.compile(r"`\S+`"),
|
55
|
+
]
|
56
|
+
|
57
|
+
|
58
|
+
def has_md_link(text: str) -> bool:
|
59
|
+
return any(bool(pattern.findall(text)) for pattern in _md_patterns)
|
60
|
+
|
61
|
+
|
62
|
+
def default_log_group_name(resource_name: str) -> str:
|
63
|
+
return f"mongodb-atlas-{resource_name}-logs"
|