atlas-init 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +1 -1
- atlas_init/cli_root/aws_clean.py +108 -0
- atlas_init/cli_tf/hcl/modifier2.py +51 -1
- atlas_init/tf_ext/paths.py +18 -1
- atlas_init/tf_ext/run_tf.py +20 -0
- atlas_init/tf_ext/settings.py +8 -0
- atlas_init/tf_ext/tf_mod_gen.py +4 -19
- atlas_init/tf_ext/tf_ws.py +269 -0
- atlas_init/tf_ext/typer_app.py +2 -1
- atlas_init/typer_app.py +2 -1
- {atlas_init-0.9.0.dist-info → atlas_init-0.10.0.dist-info}/METADATA +2 -1
- {atlas_init-0.9.0.dist-info → atlas_init-0.10.0.dist-info}/RECORD +15 -12
- {atlas_init-0.9.0.dist-info → atlas_init-0.10.0.dist-info}/WHEEL +0 -0
- {atlas_init-0.9.0.dist-info → atlas_init-0.10.0.dist-info}/entry_points.txt +0 -0
- {atlas_init-0.9.0.dist-info → atlas_init-0.10.0.dist-info}/licenses/LICENSE +0 -0
atlas_init/__init__.py
CHANGED
@@ -0,0 +1,108 @@
|
|
1
|
+
import logging
|
2
|
+
from datetime import datetime, timedelta
|
3
|
+
from pathlib import Path
|
4
|
+
|
5
|
+
import humanize
|
6
|
+
import typer
|
7
|
+
from ask_shell import run_and_wait
|
8
|
+
from boto3.session import Session
|
9
|
+
from model_lib import Entity
|
10
|
+
from mypy_boto3_iam import IAMClient
|
11
|
+
from mypy_boto3_iam.type_defs import RoleTypeDef
|
12
|
+
from zero_3rdparty.datetime_utils import utc_now
|
13
|
+
|
14
|
+
from atlas_init.cloud.aws import PascalAlias
|
15
|
+
from atlas_init.settings.env_vars import init_settings
|
16
|
+
from atlas_init.typer_app import app_command
|
17
|
+
|
18
|
+
logger = logging.getLogger(__name__)
|
19
|
+
|
20
|
+
|
21
|
+
class LastUsed(Entity):
|
22
|
+
model_config = PascalAlias
|
23
|
+
last_used_date: datetime
|
24
|
+
|
25
|
+
|
26
|
+
class IAMRole(Entity):
|
27
|
+
model_config = PascalAlias
|
28
|
+
arn: str
|
29
|
+
role_name: str
|
30
|
+
create_date: datetime
|
31
|
+
role_last_used: LastUsed | None = None
|
32
|
+
|
33
|
+
|
34
|
+
@app_command()
|
35
|
+
def aws_clean(
|
36
|
+
skip_iam_roles: bool = typer.Option(False, help="skip iam roles"),
|
37
|
+
iam_role_prefix_name: str = typer.Option(
|
38
|
+
"mongodb-atlas-test-acc-tf-",
|
39
|
+
help="prefix name of iam roles to clean",
|
40
|
+
),
|
41
|
+
):
|
42
|
+
init_settings()
|
43
|
+
if skip_iam_roles:
|
44
|
+
return
|
45
|
+
client: IAMClient = Session().client("iam") # pyright: ignore[reportAssignmentType]
|
46
|
+
all_roles: list[RoleTypeDef] = []
|
47
|
+
aws_account_id = run_and_wait("aws sts get-caller-identity --query Account --output text").stdout_one_line
|
48
|
+
|
49
|
+
roles_response = client.list_roles()
|
50
|
+
|
51
|
+
all_roles.extend(roles_response["Roles"])
|
52
|
+
marker = roles_response.get("Marker", "")
|
53
|
+
while marker:
|
54
|
+
roles_response = client.list_roles(Marker=marker)
|
55
|
+
all_roles.extend(roles_response["Roles"])
|
56
|
+
marker = roles_response.get("Marker", "")
|
57
|
+
total_roles = len(all_roles)
|
58
|
+
logger.info(f"found {total_roles} roles")
|
59
|
+
roles_parsed: list[IAMRole] = []
|
60
|
+
delete_if_created_before = utc_now() - timedelta(days=5)
|
61
|
+
delete_count = 0
|
62
|
+
role_names: list[str] = []
|
63
|
+
for role in all_roles:
|
64
|
+
parsed = IAMRole.model_validate(role)
|
65
|
+
roles_parsed.append(parsed)
|
66
|
+
role_name = parsed.role_name
|
67
|
+
role_names.append(role_name)
|
68
|
+
if not role_name.startswith(iam_role_prefix_name):
|
69
|
+
continue
|
70
|
+
# want to delete 'mongodb-atlas-test-acc-tf-1345851232260229574'
|
71
|
+
# want to keep 'mongodb-atlas-test-acc-tf-7973337217371171538-git-ear'?
|
72
|
+
if role_name[-1].isdigit() and parsed.create_date < delete_if_created_before:
|
73
|
+
logger.info(f"role: {parsed.arn} will be deleted")
|
74
|
+
delete_count += 1
|
75
|
+
delete_role(client, role_name)
|
76
|
+
else:
|
77
|
+
logger.info(f"skipping role: {parsed.arn}, created: {humanize.naturaltime(parsed.create_date)}")
|
78
|
+
logger.info(f"deleted {delete_count}/{total_roles} roles")
|
79
|
+
out_path = Path(f"aws_roles_{aws_account_id}.txt")
|
80
|
+
out_path.write_text("\n".join(sorted(role_names)))
|
81
|
+
|
82
|
+
|
83
|
+
def delete_role(client: IAMClient, role_name: str):
|
84
|
+
try:
|
85
|
+
attached_policies = client.list_attached_role_policies(RoleName=role_name)
|
86
|
+
for policy in attached_policies["AttachedPolicies"]:
|
87
|
+
policy_arn = policy.get("PolicyArn")
|
88
|
+
if policy_arn:
|
89
|
+
logger.info(f"detaching managed policy: {policy_arn} from role: {role_name}")
|
90
|
+
client.detach_role_policy(RoleName=role_name, PolicyArn=policy_arn)
|
91
|
+
except Exception as e:
|
92
|
+
logger.warning(f"failed to detach managed policies from role {role_name}: {e}")
|
93
|
+
|
94
|
+
# Second, delete all inline policies
|
95
|
+
try:
|
96
|
+
inline_policies = client.list_role_policies(RoleName=role_name)
|
97
|
+
for policy_name in inline_policies["PolicyNames"]:
|
98
|
+
logger.info(f"deleting inline policy: {policy_name} from role: {role_name}")
|
99
|
+
client.delete_role_policy(RoleName=role_name, PolicyName=policy_name)
|
100
|
+
except Exception as e:
|
101
|
+
logger.warning(f"failed to delete inline policies from role {role_name}: {e}")
|
102
|
+
|
103
|
+
# Finally, delete the role
|
104
|
+
try:
|
105
|
+
client.delete_role(RoleName=role_name)
|
106
|
+
logger.info(f"deleted role: {role_name}")
|
107
|
+
except Exception as e:
|
108
|
+
logger.error(f"failed to delete role {role_name}: {e}")
|
@@ -2,10 +2,12 @@ from collections import defaultdict
|
|
2
2
|
import logging
|
3
3
|
from contextlib import suppress
|
4
4
|
from pathlib import Path
|
5
|
-
from typing import NamedTuple
|
5
|
+
from typing import Any, NamedTuple
|
6
6
|
from lark import Token, Transformer, Tree, UnexpectedToken, v_args
|
7
7
|
from hcl2.transformer import Attribute, DictTransformer
|
8
8
|
from hcl2.api import reverse_transform, writes, parses
|
9
|
+
from model_lib import Entity
|
10
|
+
from pydantic import field_validator
|
9
11
|
import rich
|
10
12
|
|
11
13
|
logger = logging.getLogger(__name__)
|
@@ -58,6 +60,54 @@ def attribute_transfomer(attr_name: str, obj_key: str, new_value: str) -> tuple[
|
|
58
60
|
return AttributeTransformer(with_meta=True), changes
|
59
61
|
|
60
62
|
|
63
|
+
_unset = object()
|
64
|
+
|
65
|
+
|
66
|
+
class TFVar(Entity):
|
67
|
+
name: str
|
68
|
+
description: str | None = ""
|
69
|
+
default: Any = _unset
|
70
|
+
type: str = ""
|
71
|
+
sensitive: bool = False
|
72
|
+
|
73
|
+
@field_validator("default", mode="before")
|
74
|
+
def unpack_token(cls, v: Any) -> Any:
|
75
|
+
if isinstance(v, Token):
|
76
|
+
return v.value.strip('"')
|
77
|
+
return v
|
78
|
+
|
79
|
+
|
80
|
+
def variable_reader_typed(tree: Tree) -> dict[str, TFVar]:
|
81
|
+
variables: dict[str, TFVar] = {}
|
82
|
+
|
83
|
+
class TFVarReader(DictTransformer):
|
84
|
+
def __init__(self, with_meta: bool = False, *, name: str):
|
85
|
+
super().__init__(with_meta)
|
86
|
+
self.kwargs: dict[str, Any] = {
|
87
|
+
"name": name,
|
88
|
+
}
|
89
|
+
|
90
|
+
def attribute(self, args: list) -> Attribute:
|
91
|
+
if len(args) == 3:
|
92
|
+
name, _, value = args
|
93
|
+
self.kwargs[name] = value
|
94
|
+
return super().attribute(args)
|
95
|
+
|
96
|
+
class BlockReader(Transformer):
|
97
|
+
@v_args(tree=True)
|
98
|
+
def block(self, block_tree: Tree) -> Tree:
|
99
|
+
current_block_name = _identifier_name(block_tree)
|
100
|
+
if current_block_name == "variable":
|
101
|
+
variable_name = token_name(block_tree.children[1])
|
102
|
+
reader = TFVarReader(name=variable_name)
|
103
|
+
reader.transform(block_tree)
|
104
|
+
variables[variable_name] = TFVar(**reader.kwargs)
|
105
|
+
return block_tree
|
106
|
+
|
107
|
+
BlockReader().transform(tree)
|
108
|
+
return variables
|
109
|
+
|
110
|
+
|
61
111
|
def variable_reader(tree: Tree) -> dict[str, str | None]:
|
62
112
|
"""
|
63
113
|
Reads the variable names from a parsed HCL2 tree.
|
atlas_init/tf_ext/paths.py
CHANGED
@@ -9,7 +9,14 @@ from model_lib import Entity
|
|
9
9
|
from pydantic import Field, RootModel
|
10
10
|
from zero_3rdparty.file_utils import iter_paths
|
11
11
|
|
12
|
-
from atlas_init.cli_tf.hcl.modifier2 import
|
12
|
+
from atlas_init.cli_tf.hcl.modifier2 import (
|
13
|
+
TFVar,
|
14
|
+
resource_types_vars_usage,
|
15
|
+
safe_parse,
|
16
|
+
variable_reader,
|
17
|
+
variable_reader_typed,
|
18
|
+
variable_usages,
|
19
|
+
)
|
13
20
|
from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME, DEFAULT_EXTERNAL_SUBSTRINGS, DEFAULT_INTERNAL_SUBSTRINGS
|
14
21
|
|
15
22
|
logger = logging.getLogger(__name__)
|
@@ -32,6 +39,16 @@ def get_example_directories(repo_path: Path, skip_names: list[str]):
|
|
32
39
|
return example_dirs
|
33
40
|
|
34
41
|
|
42
|
+
def find_variables_typed(variables_tf: Path) -> dict[str, TFVar]:
|
43
|
+
if not variables_tf.exists():
|
44
|
+
return {}
|
45
|
+
tree = safe_parse(variables_tf)
|
46
|
+
if not tree:
|
47
|
+
logger.warning(f"Failed to parse {variables_tf}")
|
48
|
+
return {}
|
49
|
+
return variable_reader_typed(tree)
|
50
|
+
|
51
|
+
|
35
52
|
def find_variables(variables_tf: Path) -> dict[str, str | None]:
|
36
53
|
if not variables_tf.exists():
|
37
54
|
return {}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
from ask_shell import new_task, run_and_wait
|
2
|
+
from pathlib import Path
|
3
|
+
|
4
|
+
|
5
|
+
def validate_tf_workspace(
|
6
|
+
tf_workdir: Path, *, tf_cli_config_file: Path | None = None, env_extra: dict[str, str] | None = None
|
7
|
+
):
|
8
|
+
terraform_commands = [
|
9
|
+
"terraform init",
|
10
|
+
"terraform fmt .",
|
11
|
+
"terraform validate .",
|
12
|
+
]
|
13
|
+
env_extra = env_extra or {}
|
14
|
+
if tf_cli_config_file:
|
15
|
+
env_extra["TF_CLI_CONFIG_FILE"] = str(tf_cli_config_file)
|
16
|
+
with new_task("Terraform Module Validate Checks", total=len(terraform_commands)) as task:
|
17
|
+
for command in terraform_commands:
|
18
|
+
attempts = 3 if command == "terraform init" else 1 # terraform init can fail due to network issues
|
19
|
+
run_and_wait(command, cwd=tf_workdir, env=env_extra, attempts=attempts)
|
20
|
+
task.update(advance=1)
|
atlas_init/tf_ext/settings.py
CHANGED
@@ -175,6 +175,14 @@ class TfExtSettings(StaticSettings):
|
|
175
175
|
def provider_cache_dir(self, provider_name: str) -> Path:
|
176
176
|
return self.cache_root / "provider_cache" / provider_name
|
177
177
|
|
178
|
+
@property
|
179
|
+
def variable_plan_resolvers_file_path(self) -> Path:
|
180
|
+
return self.static_root / "variable_plan_resolvers.yaml"
|
181
|
+
|
182
|
+
@property
|
183
|
+
def variable_plan_resolvers_dumped_file_path(self) -> Path:
|
184
|
+
return self.static_root / "variable_plan_resolvers_dumped.yaml"
|
185
|
+
|
178
186
|
|
179
187
|
def init_tf_ext_settings(*, allow_empty_out_path: bool = False) -> TfExtSettings:
|
180
188
|
settings = TfExtSettings.from_env()
|
atlas_init/tf_ext/tf_mod_gen.py
CHANGED
@@ -32,6 +32,7 @@ from atlas_init.tf_ext.plan_diffs import (
|
|
32
32
|
read_variables_path,
|
33
33
|
)
|
34
34
|
from atlas_init.tf_ext.provider_schema import AtlasSchemaInfo, ResourceSchema, parse_atlas_schema
|
35
|
+
from atlas_init.tf_ext.run_tf import validate_tf_workspace
|
35
36
|
from atlas_init.tf_ext.schema_to_dataclass import convert_and_format
|
36
37
|
from atlas_init.tf_ext.settings import TfExtSettings
|
37
38
|
|
@@ -131,29 +132,13 @@ def generate_resource_module(config: ModuleGenConfig, resource_type: str, atlas_
|
|
131
132
|
def finalize_and_validate_module(config: ModuleGenConfig) -> Path:
|
132
133
|
dump_versions_tf(config.module_out_path, skip_python=config.skip_python)
|
133
134
|
logger.info(f"Module dumped to {config.module_out_path}, running checks")
|
134
|
-
|
135
|
+
validate_tf_workspace(config.module_out_path, tf_cli_config_file=config.settings.tf_cli_config_file)
|
135
136
|
return config.module_out_path
|
136
137
|
|
137
138
|
|
138
139
|
OUT_BINARY_PATH = "tfplan.binary"
|
139
140
|
|
140
141
|
|
141
|
-
def validate_module(tf_workdir: Path, *, tf_cli_config_file: Path | None = None):
|
142
|
-
terraform_commands = [
|
143
|
-
"terraform init",
|
144
|
-
"terraform fmt .",
|
145
|
-
"terraform validate .",
|
146
|
-
]
|
147
|
-
env_extra = {}
|
148
|
-
if tf_cli_config_file:
|
149
|
-
env_extra["TF_CLI_CONFIG_FILE"] = str(tf_cli_config_file)
|
150
|
-
with new_task("Terraform Module Validate Checks", total=len(terraform_commands)) as task:
|
151
|
-
for command in terraform_commands:
|
152
|
-
attempts = 3 if command == "terraform init" else 1 # terraform init can fail due to network issues
|
153
|
-
run_and_wait(command, cwd=tf_workdir, env=env_extra, attempts=attempts)
|
154
|
-
task.update(advance=1)
|
155
|
-
|
156
|
-
|
157
142
|
def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Path | None = None) -> Path:
|
158
143
|
path = config.module_out_path
|
159
144
|
if (examples_test := config.examples_test_path) and examples_test.exists():
|
@@ -165,7 +150,7 @@ def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Pat
|
|
165
150
|
if examples_generated:
|
166
151
|
with run_pool("Validating examples", total=len(examples_generated), exit_wait_timeout=60) as pool:
|
167
152
|
for example_path in examples_generated:
|
168
|
-
pool.submit(
|
153
|
+
pool.submit(validate_tf_workspace, example_path)
|
169
154
|
|
170
155
|
attribute_descriptions = parse_attribute_descriptions(config.settings)
|
171
156
|
settings = config.settings
|
@@ -225,7 +210,7 @@ def example_plan_checks(config: ModuleGenConfig, timeout_all_seconds: int = 60)
|
|
225
210
|
with TemporaryDirectory() as temp_dir:
|
226
211
|
stored_plan = Path(temp_dir) / "plan.json"
|
227
212
|
tf_dir = config.example_path(check.example_name)
|
228
|
-
|
213
|
+
validate_tf_workspace(tf_dir)
|
229
214
|
var_arg = f" -var-file={variables_path}" if variables_path else ""
|
230
215
|
run_and_wait(f"terraform plan -out={OUT_BINARY_PATH}{var_arg}", cwd=tf_dir)
|
231
216
|
run_and_wait(f"terraform show -json {OUT_BINARY_PATH} > {stored_plan}", cwd=tf_dir)
|
@@ -0,0 +1,269 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from enum import StrEnum
|
3
|
+
import fnmatch
|
4
|
+
import logging
|
5
|
+
from collections import defaultdict
|
6
|
+
from pathlib import Path
|
7
|
+
from typing import Any, Literal, Self
|
8
|
+
|
9
|
+
import typer
|
10
|
+
import humanize
|
11
|
+
from ask_shell import run_and_wait, run_pool
|
12
|
+
from model_lib import Entity, dump, parse_model
|
13
|
+
from pydantic import ConfigDict, Field
|
14
|
+
from zero_3rdparty.file_utils import ensure_parents_write_text, iter_paths_and_relative
|
15
|
+
import stringcase
|
16
|
+
|
17
|
+
from atlas_init.cli_tf.hcl.modifier2 import TFVar
|
18
|
+
from atlas_init.settings.env_vars import init_settings
|
19
|
+
from atlas_init.settings.env_vars_generated import AtlasSettingsWithProject, AWSSettings
|
20
|
+
from atlas_init.tf_ext.paths import find_variables_typed
|
21
|
+
from atlas_init.tf_ext.settings import TfExtSettings, init_tf_ext_settings
|
22
|
+
from atlas_init.tf_ext.tf_mod_gen import validate_tf_workspace
|
23
|
+
|
24
|
+
logger = logging.getLogger(__name__)
|
25
|
+
LOCKFILE_NAME = ".terraform.tfstate.lock.info"
|
26
|
+
PascalAlias = ConfigDict(alias_generator=stringcase.pascalcase, populate_by_name=True)
|
27
|
+
|
28
|
+
|
29
|
+
class Lockfile(Entity):
|
30
|
+
model_config = PascalAlias
|
31
|
+
created: datetime
|
32
|
+
path: str
|
33
|
+
operation: str
|
34
|
+
|
35
|
+
def __str__(self) -> str:
|
36
|
+
return (
|
37
|
+
f"lockfile for state {self.path} created={humanize.naturaltime(self.created)}, operation={self.operation})"
|
38
|
+
)
|
39
|
+
|
40
|
+
|
41
|
+
class ResolvedEnvVar(Entity):
|
42
|
+
var_matches: list[str] = Field(default_factory=list)
|
43
|
+
name: str
|
44
|
+
value: str
|
45
|
+
sensitive: bool = False
|
46
|
+
type: Literal["env"] = "env"
|
47
|
+
|
48
|
+
def can_resolve(self, variable: TFVar) -> bool:
|
49
|
+
return any(fnmatch.fnmatch(variable.name, var) for var in self.var_matches)
|
50
|
+
|
51
|
+
|
52
|
+
class ResolvedStringVar(Entity):
|
53
|
+
var_matches: list[str] = Field(default_factory=list)
|
54
|
+
value: str = ""
|
55
|
+
sensitive: bool = False
|
56
|
+
type: Literal["string"] = "string"
|
57
|
+
|
58
|
+
def can_resolve(self, variable: TFVar) -> bool:
|
59
|
+
if variable.type and variable.type != self.type:
|
60
|
+
return False
|
61
|
+
return any(fnmatch.fnmatch(variable.name, var) for var in self.var_matches)
|
62
|
+
|
63
|
+
|
64
|
+
ResolverVar = ResolvedStringVar | ResolvedEnvVar
|
65
|
+
|
66
|
+
|
67
|
+
def as_tfvars_env(resolver_vars: dict[str, ResolverVar]) -> tuple[dict[str, Any], dict[str, Any]]:
|
68
|
+
env_vars = {}
|
69
|
+
tf_vars = {}
|
70
|
+
for var_name, var in resolver_vars.items():
|
71
|
+
match var:
|
72
|
+
case ResolvedEnvVar(name=name, value=value):
|
73
|
+
env_vars[name] = value
|
74
|
+
case ResolvedStringVar(value=value):
|
75
|
+
tf_vars[var_name] = value
|
76
|
+
return tf_vars, env_vars
|
77
|
+
|
78
|
+
|
79
|
+
class _MissingResolverVarsError(Exception):
|
80
|
+
def __init__(self, missing_vars: list[str], path: Path, rel_path: str):
|
81
|
+
self.missing_vars = missing_vars
|
82
|
+
self.path = path
|
83
|
+
self.rel_path = rel_path
|
84
|
+
super().__init__(f"Missing variables: {missing_vars} for path: {path} with rel_path: {rel_path}")
|
85
|
+
|
86
|
+
def __str__(self) -> str:
|
87
|
+
return f"Missing variables: {self.missing_vars} for path: {self.path} with rel_path: {self.rel_path}"
|
88
|
+
|
89
|
+
|
90
|
+
class VariablesPlanResolver(Entity):
|
91
|
+
paths: dict[str, list[ResolverVar]]
|
92
|
+
|
93
|
+
def merge(self, other: Self) -> Self:
|
94
|
+
merged = defaultdict(list)
|
95
|
+
for path, vars in self.paths.items():
|
96
|
+
merged[path].extend(vars)
|
97
|
+
for path, vars in other.paths.items():
|
98
|
+
merged[path].extend(vars)
|
99
|
+
return type(self)(paths=merged)
|
100
|
+
|
101
|
+
def variable_path_matches(self, path: Path, rel_path: str) -> list[ResolverVar]:
|
102
|
+
resolved = []
|
103
|
+
for path_pattern, vars in self.paths.items():
|
104
|
+
if fnmatch.fnmatch(rel_path, path_pattern):
|
105
|
+
resolved.extend(vars)
|
106
|
+
return resolved
|
107
|
+
|
108
|
+
def resolve_vars(self, path: Path, rel_path: str) -> dict[str, ResolverVar]:
|
109
|
+
variables = find_variables_typed(path / "variables.tf")
|
110
|
+
resolved_vars: dict[str, ResolverVar] = {}
|
111
|
+
for var in variables.values():
|
112
|
+
for resolver_var in self.variable_path_matches(path, rel_path):
|
113
|
+
if resolver_var.can_resolve(var):
|
114
|
+
resolved_vars[var.name] = resolver_var
|
115
|
+
if missing_vars := set(variables.keys()) - set(resolved_vars.keys()):
|
116
|
+
raise _MissingResolverVarsError(sorted(missing_vars), path, rel_path)
|
117
|
+
return resolved_vars
|
118
|
+
|
119
|
+
|
120
|
+
def update_dumped_vars(path: Path) -> VariablesPlanResolver:
|
121
|
+
assert init_settings(AWSSettings, AtlasSettingsWithProject), "Settings must be initialized"
|
122
|
+
project_settings = AtlasSettingsWithProject.from_env()
|
123
|
+
dumped_vars = VariablesPlanResolver(
|
124
|
+
paths={
|
125
|
+
"*": [
|
126
|
+
ResolvedStringVar(
|
127
|
+
var_matches=["project*"],
|
128
|
+
value=project_settings.MONGODB_ATLAS_PROJECT_ID,
|
129
|
+
sensitive=False,
|
130
|
+
),
|
131
|
+
ResolvedStringVar(
|
132
|
+
var_matches=["org*"],
|
133
|
+
value=project_settings.MONGODB_ATLAS_ORG_ID,
|
134
|
+
sensitive=False,
|
135
|
+
),
|
136
|
+
ResolvedEnvVar(
|
137
|
+
var_matches=["atlas_private_key"],
|
138
|
+
sensitive=True,
|
139
|
+
value=project_settings.MONGODB_ATLAS_PRIVATE_KEY,
|
140
|
+
name="MONGODB_ATLAS_PRIVATE_KEY",
|
141
|
+
),
|
142
|
+
ResolvedEnvVar(
|
143
|
+
var_matches=["atlas_public_key"],
|
144
|
+
sensitive=True,
|
145
|
+
value=project_settings.MONGODB_ATLAS_PUBLIC_KEY,
|
146
|
+
name="MONGODB_ATLAS_PUBLIC_KEY",
|
147
|
+
),
|
148
|
+
ResolvedEnvVar(
|
149
|
+
var_matches=["atlas_base_url"],
|
150
|
+
sensitive=False,
|
151
|
+
value=project_settings.MONGODB_ATLAS_BASE_URL,
|
152
|
+
name="MONGODB_ATLAS_BASE_URL",
|
153
|
+
),
|
154
|
+
]
|
155
|
+
}
|
156
|
+
)
|
157
|
+
yaml = dump(dumped_vars, "yaml")
|
158
|
+
ensure_parents_write_text(path, yaml)
|
159
|
+
return dumped_vars
|
160
|
+
|
161
|
+
|
162
|
+
_ignored_workspace_dirs = [
|
163
|
+
".terraform",
|
164
|
+
]
|
165
|
+
|
166
|
+
|
167
|
+
class TFWorkspacRunConfig(Entity):
|
168
|
+
path: Path
|
169
|
+
rel_path: str
|
170
|
+
resolved_vars: dict[str, Any]
|
171
|
+
resolved_env_vars: dict[str, Any]
|
172
|
+
|
173
|
+
def tf_data_dir(self, settings: TfExtSettings) -> Path:
|
174
|
+
repo_out = settings.repo_out
|
175
|
+
assert self.path.is_relative_to(repo_out.base), f"path {self.path} is not relative to {repo_out.base}"
|
176
|
+
relative_repo_path = str(self.path.relative_to(repo_out.base))
|
177
|
+
return settings.static_root / "tf-ws-check" / relative_repo_path / ".terraform"
|
178
|
+
|
179
|
+
def tf_vars_path_json(self, settings: TfExtSettings) -> Path:
|
180
|
+
return self.tf_data_dir(settings) / "vars.auto.tfvars.json"
|
181
|
+
|
182
|
+
|
183
|
+
class TFWsCommands(StrEnum):
|
184
|
+
VALIDATE = "validate"
|
185
|
+
PLAN = "plan"
|
186
|
+
APPLY = "apply"
|
187
|
+
DESTROY = "destroy"
|
188
|
+
|
189
|
+
|
190
|
+
def tf_ws(
|
191
|
+
command: TFWsCommands = typer.Argument("plan", help="The command to run in the workspace"),
|
192
|
+
root_path: Path = typer.Option(
|
193
|
+
...,
|
194
|
+
"-p",
|
195
|
+
"--root-path",
|
196
|
+
help="Path to the root directory, will recurse and look for **/main.tf",
|
197
|
+
default_factory=Path.cwd,
|
198
|
+
),
|
199
|
+
):
|
200
|
+
settings = init_tf_ext_settings()
|
201
|
+
variable_resolvers = update_dumped_vars(settings.variable_plan_resolvers_dumped_file_path)
|
202
|
+
manual_path = settings.variable_plan_resolvers_file_path
|
203
|
+
if manual_path.exists():
|
204
|
+
manual_resolvers = parse_model(manual_path, t=VariablesPlanResolver)
|
205
|
+
variable_resolvers = variable_resolvers.merge(manual_resolvers)
|
206
|
+
|
207
|
+
def include_path(rel_path: str) -> bool:
|
208
|
+
return all(f"/{dir}/" not in rel_path for dir in _ignored_workspace_dirs)
|
209
|
+
|
210
|
+
paths = sorted(
|
211
|
+
(path.parent, rel_path)
|
212
|
+
for path, rel_path in iter_paths_and_relative(root_path, "main.tf", only_files=True)
|
213
|
+
if include_path(rel_path)
|
214
|
+
)
|
215
|
+
run_configs = []
|
216
|
+
missing_vars_errors = []
|
217
|
+
for path, rel_path in paths:
|
218
|
+
try:
|
219
|
+
resolver_vars = variable_resolvers.resolve_vars(path, rel_path)
|
220
|
+
resolved_vars, resolved_env_vars = as_tfvars_env(resolver_vars)
|
221
|
+
run_configs.append(
|
222
|
+
TFWorkspacRunConfig(
|
223
|
+
path=path, rel_path=rel_path, resolved_vars=resolved_vars, resolved_env_vars=resolved_env_vars
|
224
|
+
)
|
225
|
+
)
|
226
|
+
except _MissingResolverVarsError as e:
|
227
|
+
missing_vars_errors.append(e)
|
228
|
+
continue
|
229
|
+
if missing_vars_errors:
|
230
|
+
missing_vars_formatted = "\n".join(str(e) for e in missing_vars_errors)
|
231
|
+
logger.warning(f"Missing variables:\n{missing_vars_formatted}")
|
232
|
+
|
233
|
+
run_count = len(run_configs)
|
234
|
+
assert run_count > 0, f"No run configs found from {root_path}"
|
235
|
+
|
236
|
+
def run_cmd(run_config: TFWorkspacRunConfig):
|
237
|
+
tf_vars_str = dump(run_config.resolved_vars, "pretty_json")
|
238
|
+
tf_vars_path = run_config.tf_vars_path_json(settings)
|
239
|
+
ensure_parents_write_text(tf_vars_path, tf_vars_str)
|
240
|
+
env_extra = run_config.resolved_env_vars | {"TF_DATA_DIR": str(run_config.tf_data_dir(settings))}
|
241
|
+
|
242
|
+
lockfile_path = run_config.path / LOCKFILE_NAME
|
243
|
+
if lockfile_path.exists():
|
244
|
+
lockfile = parse_model(lockfile_path, t=Lockfile, format="json")
|
245
|
+
logger.warning(f"Lockfile exists for {run_config.path}, skipping: {lockfile}")
|
246
|
+
return
|
247
|
+
|
248
|
+
validate_tf_workspace(run_config.path, tf_cli_config_file=settings.tf_cli_config_file, env_extra=env_extra)
|
249
|
+
if command == TFWsCommands.VALIDATE:
|
250
|
+
return
|
251
|
+
command_extra = ""
|
252
|
+
if command in {TFWsCommands.APPLY, TFWsCommands.DESTROY}:
|
253
|
+
command_extra = " -auto-approve"
|
254
|
+
|
255
|
+
run_and_wait(
|
256
|
+
f"terraform {command} -var-file={tf_vars_path}{command_extra}",
|
257
|
+
cwd=run_config.path,
|
258
|
+
env=env_extra,
|
259
|
+
user_input=run_count == 1,
|
260
|
+
)
|
261
|
+
|
262
|
+
with run_pool(f"{command} in TF Workspaces", total=run_count, max_concurrent_submits=9) as pool:
|
263
|
+
futures = {pool.submit(run_cmd, run_config): run_config for run_config in run_configs}
|
264
|
+
for future, run_config in futures.items():
|
265
|
+
try:
|
266
|
+
future.result()
|
267
|
+
except Exception as e:
|
268
|
+
logger.error(f"Error running {command} for {run_config.path}: {e}")
|
269
|
+
continue
|
atlas_init/tf_ext/typer_app.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
from ask_shell import configure_logging
|
2
2
|
from typer import Typer
|
3
3
|
|
4
|
-
from atlas_init.tf_ext import api_call, settings, tf_desc_gen, tf_example_readme, tf_mod_gen_provider
|
4
|
+
from atlas_init.tf_ext import api_call, settings, tf_desc_gen, tf_example_readme, tf_mod_gen_provider, tf_ws
|
5
5
|
|
6
6
|
|
7
7
|
def typer_main():
|
@@ -21,6 +21,7 @@ def typer_main():
|
|
21
21
|
app.command(name="mod-gen-provider")(tf_mod_gen_provider.tf_mod_gen_provider_resource_modules)
|
22
22
|
app.command(name="check-env-vars")(settings.init_tf_ext_settings)
|
23
23
|
app.command(name="example-readme")(tf_example_readme.tf_example_readme)
|
24
|
+
app.command(name="ws")(tf_ws.tf_ws)
|
24
25
|
configure_logging(app)
|
25
26
|
app()
|
26
27
|
|
atlas_init/typer_app.py
CHANGED
@@ -58,11 +58,12 @@ app_command = partial(
|
|
58
58
|
|
59
59
|
|
60
60
|
def extra_root_commands():
|
61
|
-
from atlas_init.cli_root import go_test, trigger, mms_released
|
61
|
+
from atlas_init.cli_root import go_test, trigger, mms_released, aws_clean
|
62
62
|
|
63
63
|
assert trigger
|
64
64
|
assert go_test
|
65
65
|
assert mms_released
|
66
|
+
assert aws_clean
|
66
67
|
|
67
68
|
|
68
69
|
@app.callback(invoke_without_command=True)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: atlas-init
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.10.0
|
4
4
|
Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
|
5
5
|
Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
|
6
6
|
Project-URL: Source, https://github.com/EspenAlbert/atlas-init
|
@@ -20,6 +20,7 @@ Requires-Dist: inflection==0.5.1
|
|
20
20
|
Requires-Dist: model-lib
|
21
21
|
Requires-Dist: motor==3.7.1
|
22
22
|
Requires-Dist: mypy-boto3-cloudformation==1.37.22
|
23
|
+
Requires-Dist: mypy-boto3-iam>=1.40.0
|
23
24
|
Requires-Dist: orjson==3.10.13
|
24
25
|
Requires-Dist: pydantic-settings==2.7.1
|
25
26
|
Requires-Dist: pydot==4.0.1
|
@@ -1,11 +1,11 @@
|
|
1
|
-
atlas_init/__init__.py,sha256=
|
1
|
+
atlas_init/__init__.py,sha256=Q4aM4T7e9LAIWZNe0514lJ1Mi_S3ixIJVWFvbzSGLTg,214
|
2
2
|
atlas_init/__main__.py,sha256=dY1dWWvwxRZMmnOFla6RSfti-hMeLeKdoXP7SVYqMUc,52
|
3
3
|
atlas_init/atlas_init.yaml,sha256=aCCg6PJgLcqh75v3tdEEG4UPrcJlbmbRaLj0c7PCumE,2373
|
4
4
|
atlas_init/cli.py,sha256=0b4_osi67Oj-hZnrtK70IbSKQDREjwUFv1_30_A03Zg,9447
|
5
5
|
atlas_init/cli_args.py,sha256=4lbEz6IVf0kHSULLXQ05SLmB4gvLfminbxFJ3qVTFNU,1202
|
6
6
|
atlas_init/humps.py,sha256=l0ZXXuI34wwd9TskXhCjULfGbUyK-qNmiyC6_2ow6kU,7339
|
7
7
|
atlas_init/terraform.yaml,sha256=qPrnbzBEP-JAQVkYadHsggRnDmshrOJyiv0ckyZCxwY,2734
|
8
|
-
atlas_init/typer_app.py,sha256=
|
8
|
+
atlas_init/typer_app.py,sha256=rEEDe871M4D1eVciDHMQhv59k8GdvG5-0pBwHH0h9OE,3991
|
9
9
|
atlas_init/cli_cfn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
10
|
atlas_init/cli_cfn/app.py,sha256=N0z4LtAyfnKAZHlrWNQ0m5TbmarECrwXBH1qp-0xkrA,6025
|
11
11
|
atlas_init/cli_cfn/aws.py,sha256=KtJWJmYDknPFtd4j6evMFRwmnFGcLYUFHArV6J49TjI,17911
|
@@ -21,6 +21,7 @@ atlas_init/cli_helper/sdk.py,sha256=exh58-VZwxtosaxM269C62EEy1VnpJPOVziPDPkGsmE,
|
|
21
21
|
atlas_init/cli_helper/sdk_auto_changes.py,sha256=oWyXw7P0PdO28hclRvza_RcIVXAyzu0lCYTJTNBDMeo,189
|
22
22
|
atlas_init/cli_helper/tf_runner.py,sha256=V8pfxPSDBSeQWkR27kI8JLu-MEadUR9TGnoXBjSP4G4,3610
|
23
23
|
atlas_init/cli_root/__init__.py,sha256=Mf0wqy4kqq8pmbjLa98zOGuUWv0bLk2OYGc1n1_ZmZ4,223
|
24
|
+
atlas_init/cli_root/aws_clean.py,sha256=n5bTjEhV5whVVN3W6Lk91Mb5LFwcR0ZBLiJS4V4H-Ak,4061
|
24
25
|
atlas_init/cli_root/go_test.py,sha256=roQIOS-qVfNhJMztR-V3hjtxFMf7-Ioy3e1ffqtTRyo,4601
|
25
26
|
atlas_init/cli_root/mms_released.py,sha256=gaUWzY4gqb1Tuo7u-8HqkOC6pW8QSjcwTz2UJMCV3Cw,1765
|
26
27
|
atlas_init/cli_root/trigger.py,sha256=vT32qeWq946r8UaU8ZUKtWafpU0kodbUJpQXctFGjjk,8526
|
@@ -57,7 +58,7 @@ atlas_init/cli_tf/hcl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
57
58
|
atlas_init/cli_tf/hcl/cli.py,sha256=6V1kU_a1c4LA3rS7sWN821gQex00fb70AUyd07xO0es,5760
|
58
59
|
atlas_init/cli_tf/hcl/cluster_mig.py,sha256=kMb_0V_XWr_iQj-mZZ-mmzIvYOLfuC4FYGYYSe9VKkQ,12496
|
59
60
|
atlas_init/cli_tf/hcl/modifier.py,sha256=imcKuQZLkzeD_-_DKAUjwbW_zKbPvAElmVDIjKaQfrI,8199
|
60
|
-
atlas_init/cli_tf/hcl/modifier2.py,sha256=
|
61
|
+
atlas_init/cli_tf/hcl/modifier2.py,sha256=jszWiCzwcKJquG8bb9SMqH8TA3Y3jRVRGORHgGW7GkI,9317
|
61
62
|
atlas_init/cli_tf/hcl/parser.py,sha256=wqj0YIn9nyEfjRqZnM7FH4yL43-K9ANvRiy9RCahImc,4833
|
62
63
|
atlas_init/cloud/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
63
64
|
atlas_init/cloud/aws.py,sha256=AXVobJ724S6OeEs_uXH9dvecc_klnXqejRnI7KaLyzo,4935
|
@@ -133,23 +134,25 @@ atlas_init/tf_ext/gen_versions.py,sha256=yYX4jIgQfAZuG2wfu6OsIbDAqt9g8btiLcgKm7D
|
|
133
134
|
atlas_init/tf_ext/models.py,sha256=vS725jiEc0t61IO1Db0mpxWVQPGaJKIGeZLOb2SOf4I,4430
|
134
135
|
atlas_init/tf_ext/models_module.py,sha256=JngaAxN-54hYPffx1rZ-_j-hYT_GELq7lf24e-EQHb0,18276
|
135
136
|
atlas_init/tf_ext/newres.py,sha256=quMSLlkJRuvA3attTvJ-DQNSwRPFyT_XJ32ucmEhA-s,3104
|
136
|
-
atlas_init/tf_ext/paths.py,sha256=
|
137
|
+
atlas_init/tf_ext/paths.py,sha256=9MK7UdiHs9ViUmUOa1MhXsZjhlGIH1MHe53_s2yztxg,5264
|
137
138
|
atlas_init/tf_ext/plan_diffs.py,sha256=Sc-VFrq2k7p01ZZzgtAfqRXCog2h6bbFlonGfaxSzog,5237
|
138
139
|
atlas_init/tf_ext/provider_schema.py,sha256=6XrJ3UHjpQ_8yBqWeZ1b4xIPLOqPL9L1ljn4mGF2Cj4,6908
|
139
140
|
atlas_init/tf_ext/py_gen.py,sha256=orZLX2c-tw9eJb8MyTunhkdfqs-f6uOngu5RlLUb9uI,11243
|
141
|
+
atlas_init/tf_ext/run_tf.py,sha256=iMaNbjsq4sQe9cesP-uI9En3FifPUaqXFQFbm7K20kM,818
|
140
142
|
atlas_init/tf_ext/schema_to_dataclass.py,sha256=kqg0OxqYZwmNGvlayZkXPwU2CueiHsnAAgLyoXkDUtY,19797
|
141
|
-
atlas_init/tf_ext/settings.py,sha256=
|
143
|
+
atlas_init/tf_ext/settings.py,sha256=_NVOt_464XyWq69sT-dwA1hfJ9Ag1H2QssE2datx_rU,7102
|
142
144
|
atlas_init/tf_ext/tf_dep.py,sha256=UFrNmolw3qW02E7mq23vVGY0C8HOSSY3-6AHmSEqYno,13453
|
143
145
|
atlas_init/tf_ext/tf_desc_gen.py,sha256=G6Hv9kKJDNbNL8nRRkPA7w8c7MCDgijjdTv-2BjKrZ0,2575
|
144
146
|
atlas_init/tf_ext/tf_desc_update.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
145
147
|
atlas_init/tf_ext/tf_example_readme.py,sha256=A8yBJq9Ts0NzBnurWbbmADrfursjhxfdWX3ZxkYZ5cs,14992
|
146
|
-
atlas_init/tf_ext/tf_mod_gen.py,sha256=
|
148
|
+
atlas_init/tf_ext/tf_mod_gen.py,sha256=J3BDPHoTqScZ0CmNiYpE_sJaV17jFkMrfgtgejGzhi4,11536
|
147
149
|
atlas_init/tf_ext/tf_mod_gen_provider.py,sha256=M7jJeH0VR1uN440d2H5-9lnj0s0d9QtaMeYsLwWW3Ak,5866
|
148
150
|
atlas_init/tf_ext/tf_modules.py,sha256=h_E8Po1fmOcIjU_d92dCjKZytBij4N9Mux70yLW7xVQ,16248
|
149
151
|
atlas_init/tf_ext/tf_vars.py,sha256=A__zcIBNvJ5y0l5F5G7KMQsYCuyr3iBZtRzXm4e6DQU,7875
|
150
|
-
atlas_init/tf_ext/
|
151
|
-
atlas_init
|
152
|
-
atlas_init-0.
|
153
|
-
atlas_init-0.
|
154
|
-
atlas_init-0.
|
155
|
-
atlas_init-0.
|
152
|
+
atlas_init/tf_ext/tf_ws.py,sha256=Wm17_Zgn4KrnWFpuShuEPabE8M4TtOxoG3EM3gCOr1Y,10099
|
153
|
+
atlas_init/tf_ext/typer_app.py,sha256=wzC6pwt7W0u2Ce3xcWVYRbFeP8C8pLw1Gy5BPeWQjZo,1110
|
154
|
+
atlas_init-0.10.0.dist-info/METADATA,sha256=JwoMAGhhoJc1VD5usMTGOZvRIgRLHyH1CuCjw9sx2Us,5902
|
155
|
+
atlas_init-0.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
156
|
+
atlas_init-0.10.0.dist-info/entry_points.txt,sha256=l38KdfCjY2v5q8Ves1qkWNvPTPND6Tp2EKX-RL-MN3c,200
|
157
|
+
atlas_init-0.10.0.dist-info/licenses/LICENSE,sha256=aKnucPyXnK1A-aXn4vac71zRpcB5BXjDyl4PDyi_hZg,1069
|
158
|
+
atlas_init-0.10.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|