atlas-init 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +2 -3
- atlas_init/cli.py +2 -1
- atlas_init/cli_cfn/app.py +4 -7
- atlas_init/cli_cfn/cfn_parameter_finder.py +3 -30
- atlas_init/cli_cfn/contract.py +9 -10
- atlas_init/cli_cfn/example.py +8 -8
- atlas_init/cli_helper/go.py +18 -14
- atlas_init/cli_helper/tf_runner.py +14 -11
- atlas_init/cli_root/trigger.py +24 -11
- atlas_init/cli_tf/app.py +1 -1
- atlas_init/cli_tf/github_logs.py +4 -16
- atlas_init/cli_tf/hcl/modifier.py +115 -14
- atlas_init/cli_tf/mock_tf_log.py +3 -2
- atlas_init/cli_tf/schema_v3.py +2 -2
- atlas_init/cli_tf/schema_v3_sdk_base.py +1 -1
- atlas_init/settings/env_vars.py +130 -166
- atlas_init/settings/env_vars_generated.py +40 -9
- atlas_init/settings/env_vars_modules.py +71 -0
- atlas_init/settings/path.py +3 -9
- atlas_init/typer_app.py +3 -3
- {atlas_init-0.4.3.dist-info → atlas_init-0.4.5.dist-info}/METADATA +2 -2
- {atlas_init-0.4.3.dist-info → atlas_init-0.4.5.dist-info}/RECORD +25 -29
- atlas_init/cli_tf/example_update_test/test_update_example.tf +0 -23
- atlas_init/cli_tf/example_update_test.py +0 -96
- atlas_init/cli_tf/hcl/modifier_test/test_process_variables_output_.tf +0 -25
- atlas_init/cli_tf/hcl/modifier_test/test_process_variables_variable_.tf +0 -24
- atlas_init/cli_tf/hcl/modifier_test.py +0 -95
- {atlas_init-0.4.3.dist-info → atlas_init-0.4.5.dist-info}/WHEEL +0 -0
- {atlas_init-0.4.3.dist-info → atlas_init-0.4.5.dist-info}/entry_points.txt +0 -0
- {atlas_init-0.4.3.dist-info → atlas_init-0.4.5.dist-info}/licenses/LICENSE +0 -0
atlas_init/__init__.py
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
from pathlib import Path
|
2
2
|
|
3
|
-
VERSION = "0.4.
|
3
|
+
VERSION = "0.4.5"
|
4
4
|
|
5
5
|
|
6
6
|
def running_in_repo() -> bool:
|
7
|
-
|
8
|
-
git_config = maybe_git_directory / "config"
|
7
|
+
git_config = Path(__file__).parent.parent / ".git/config"
|
9
8
|
return git_config.exists() and "atlas-init" in git_config.read_text()
|
atlas_init/cli.py
CHANGED
@@ -45,6 +45,7 @@ from atlas_init.settings.env_vars import (
|
|
45
45
|
active_suites,
|
46
46
|
init_settings,
|
47
47
|
)
|
48
|
+
from atlas_init.settings.env_vars_generated import AWSSettings, AtlasSettings
|
48
49
|
from atlas_init.settings.path import (
|
49
50
|
CwdIsNoRepoPathError,
|
50
51
|
dump_vscode_dotenv,
|
@@ -79,7 +80,7 @@ def apply(context: typer.Context, *, skip_outputs: bool = False):
|
|
79
80
|
|
80
81
|
|
81
82
|
def _plan_or_apply(extra_args: list[str], command: Literal["plan", "apply"], *, skip_outputs: bool) -> list[TestSuite]: # type: ignore
|
82
|
-
settings = init_settings()
|
83
|
+
settings = init_settings(AtlasSettings, AWSSettings)
|
83
84
|
logger.info(f"using the '{command}' command, extra args: {extra_args}")
|
84
85
|
try:
|
85
86
|
suites = active_suites(settings)
|
atlas_init/cli_cfn/app.py
CHANGED
@@ -14,9 +14,6 @@ from atlas_init.cli_cfn.aws import (
|
|
14
14
|
from atlas_init.cli_cfn.aws import (
|
15
15
|
delete_stack as delete_stack_aws,
|
16
16
|
)
|
17
|
-
from atlas_init.cli_cfn.cfn_parameter_finder import (
|
18
|
-
read_execution_role,
|
19
|
-
)
|
20
17
|
from atlas_init.cli_cfn.contract import contract_test_cmd
|
21
18
|
from atlas_init.cli_cfn.example import example_cmd
|
22
19
|
from atlas_init.cli_cfn.files import (
|
@@ -31,6 +28,7 @@ from atlas_init.repos.cfn import (
|
|
31
28
|
)
|
32
29
|
from atlas_init.repos.path import Repo, current_dir, find_paths, resource_root
|
33
30
|
from atlas_init.settings.env_vars import active_suites, init_settings
|
31
|
+
from atlas_init.settings.env_vars_modules import TFModuleCfn
|
34
32
|
|
35
33
|
app = typer.Typer(no_args_is_help=True)
|
36
34
|
app.command(name="example")(example_cmd)
|
@@ -55,8 +53,8 @@ def reg(
|
|
55
53
|
if local:
|
56
54
|
deregister_cfn_resource_type(type_name, deregister=not dry_run, region_filter=region)
|
57
55
|
logger.info(f"ready to activate {type_name}")
|
58
|
-
|
59
|
-
cfn_execution_role =
|
56
|
+
init_settings(TFModuleCfn)
|
57
|
+
cfn_execution_role = TFModuleCfn.from_env().CFN_EXAMPLE_EXECUTION_ROLE
|
60
58
|
last_third_party = get_last_cfn_type(type_name, region, is_third_party=True)
|
61
59
|
assert last_third_party, f"no 3rd party extension found for {type_name} in {region}"
|
62
60
|
if dry_run:
|
@@ -102,7 +100,6 @@ def inputs(
|
|
102
100
|
cwd = current_dir()
|
103
101
|
suite = suites[0]
|
104
102
|
assert suite.cwd_is_repo_go_pkg(cwd, repo_alias="cfn")
|
105
|
-
env_extra = settings.load_env_vars_full()
|
106
103
|
CREATE_FILENAME = "cfn-test-create-inputs.sh" # noqa: N806
|
107
104
|
create_dirs = ["test/contract-testing", "test"]
|
108
105
|
parent_dir = None
|
@@ -115,7 +112,7 @@ def inputs(
|
|
115
112
|
if not run_command_is_ok(
|
116
113
|
cwd=cwd,
|
117
114
|
cmd=f"./{parent_dir}/{CREATE_FILENAME}" + " ".join(context.args),
|
118
|
-
env={**os.environ}
|
115
|
+
env={**os.environ},
|
119
116
|
logger=logger,
|
120
117
|
):
|
121
118
|
logger.critical("failed to create cfn contract input files")
|
@@ -1,47 +1,21 @@
|
|
1
1
|
import logging
|
2
|
+
import os
|
2
3
|
from pathlib import Path
|
3
4
|
from typing import Any
|
4
5
|
|
5
|
-
from model_lib import Entity, dump, parse_model
|
6
|
+
from model_lib import Entity, dump, parse_model
|
6
7
|
from mypy_boto3_cloudformation.type_defs import ParameterTypeDef
|
7
8
|
from pydantic import ConfigDict, Field
|
8
9
|
from rich import prompt
|
9
|
-
from zero_3rdparty.dict_nested import read_nested
|
10
10
|
from zero_3rdparty.file_utils import clean_dir
|
11
11
|
|
12
12
|
from atlas_init.cli_cfn.files import create_sample_file, default_log_group_name
|
13
13
|
from atlas_init.cloud.aws import PascalAlias
|
14
14
|
from atlas_init.repos.cfn import CfnType, cfn_examples_dir, cfn_type_normalized
|
15
|
-
from atlas_init.settings.path import DEFAULT_TF_PATH
|
16
15
|
|
17
16
|
logger = logging.getLogger(__name__)
|
18
17
|
|
19
18
|
|
20
|
-
def read_execution_role(loaded_env_vars: dict[str, str]) -> str:
|
21
|
-
return loaded_env_vars["CFN_EXAMPLE_EXECUTION_ROLE"]
|
22
|
-
|
23
|
-
|
24
|
-
def check_execution_role(repo_path: Path, loaded_env_vars: dict[str, str]) -> str:
|
25
|
-
execution_role = cfn_examples_dir(repo_path) / "execution-role.yaml"
|
26
|
-
execution_raw = parse_payload(execution_role)
|
27
|
-
actions_expected = read_nested(
|
28
|
-
execution_raw,
|
29
|
-
"Resources.ExecutionRole.Properties.Policies.[0].PolicyDocument.Statement.[0].Action",
|
30
|
-
)
|
31
|
-
actions_found = parse_payload(DEFAULT_TF_PATH / "modules/cfn/resource_actions.yaml")
|
32
|
-
if diff := set(actions_expected) ^ set(actions_found):
|
33
|
-
raise ValueError(f"non-matching execution role actions: {sorted(diff)}")
|
34
|
-
services_found = parse_payload(DEFAULT_TF_PATH / "modules/cfn/assume_role_services.yaml")
|
35
|
-
services_expected = read_nested(
|
36
|
-
execution_raw,
|
37
|
-
"Resources.ExecutionRole.Properties.AssumeRolePolicyDocument.Statement.[0].Principal.Service",
|
38
|
-
)
|
39
|
-
if diff := set(services_found) ^ set(services_expected):
|
40
|
-
raise ValueError(f"non-matching execution role services: {sorted(diff)}")
|
41
|
-
logger.info(f"execution role is up to date with {execution_role}")
|
42
|
-
return read_execution_role(loaded_env_vars)
|
43
|
-
|
44
|
-
|
45
19
|
class TemplatePathNotFoundError(Exception):
|
46
20
|
def __init__(self, type_name: str, examples_dir: Path) -> None:
|
47
21
|
self.type_name = type_name
|
@@ -185,7 +159,6 @@ def updated_template_path(path: Path) -> Path:
|
|
185
159
|
|
186
160
|
|
187
161
|
def decode_parameters(
|
188
|
-
exported_env_vars: dict[str, str],
|
189
162
|
template_path: Path,
|
190
163
|
type_name: str,
|
191
164
|
stack_name: str,
|
@@ -213,7 +186,7 @@ def decode_parameters(
|
|
213
186
|
parameters_dict[param_name] = type_default
|
214
187
|
continue
|
215
188
|
if env_key := parameters_exported_env_vars.get(param_name): # noqa: SIM102
|
216
|
-
if env_value :=
|
189
|
+
if env_value := os.environ.get(env_key):
|
217
190
|
logger.info(f"using {env_key} to fill parameter: {param_name}")
|
218
191
|
parameters_dict[param_name] = env_value
|
219
192
|
continue
|
atlas_init/cli_cfn/contract.py
CHANGED
@@ -16,6 +16,7 @@ from atlas_init.cli_helper.run_manager import RunManager
|
|
16
16
|
from atlas_init.cli_root import is_dry_run
|
17
17
|
from atlas_init.repos.path import Repo, ResourcePaths, find_paths
|
18
18
|
from atlas_init.settings.env_vars import AtlasInitSettings, init_settings
|
19
|
+
from atlas_init.settings.env_vars_generated import AWSSettings
|
19
20
|
|
20
21
|
logger = logging.getLogger(__name__)
|
21
22
|
|
@@ -52,7 +53,6 @@ class RunContractTestOutput(Entity):
|
|
52
53
|
|
53
54
|
class CreateContractTestInputs(Entity):
|
54
55
|
resource_path: Path
|
55
|
-
env_vars_generated: dict[str, str]
|
56
56
|
log_group_name: str
|
57
57
|
|
58
58
|
|
@@ -111,22 +111,21 @@ def contract_test(
|
|
111
111
|
resource_paths: ResourcePaths | None = None,
|
112
112
|
only_names: list[str] | None = None,
|
113
113
|
):
|
114
|
-
settings = settings or init_settings()
|
114
|
+
settings = settings or init_settings(AWSSettings)
|
115
115
|
resource_paths = resource_paths or find_paths(Repo.CFN)
|
116
116
|
resource_name = resource_paths.resource_name
|
117
|
-
generated_env_vars = settings.load_env_vars_full()
|
118
117
|
create_inputs = CreateContractTestInputs(
|
119
118
|
resource_path=resource_paths.resource_path,
|
120
|
-
env_vars_generated=generated_env_vars,
|
121
119
|
log_group_name=f"mongodb-atlas-{resource_name}-logs",
|
122
120
|
)
|
123
121
|
create_response = create_contract_test_inputs(create_inputs)
|
124
122
|
create_response.log_input_files(logger)
|
123
|
+
aws_settings = AWSSettings.from_env()
|
125
124
|
run_contract_test = RunContractTest(
|
126
125
|
resource_path=resource_paths.resource_path,
|
127
126
|
repo_path=resource_paths.repo_path,
|
128
|
-
aws_profile=
|
129
|
-
cfn_region=settings.cfn_region,
|
127
|
+
aws_profile=aws_settings.AWS_PROFILE,
|
128
|
+
cfn_region=settings.cfn_region(aws_settings.AWS_REGION),
|
130
129
|
only_names=only_names,
|
131
130
|
)
|
132
131
|
if run_contract_test.skip_build:
|
@@ -162,7 +161,7 @@ def create_contract_test_inputs(
|
|
162
161
|
input_files = []
|
163
162
|
for template in sorted(test_dir.glob("*.template.json")):
|
164
163
|
template_file = template.read_text()
|
165
|
-
template_file = file_replacements(template_file,
|
164
|
+
template_file = file_replacements(template_file, template.name)
|
166
165
|
inputs_file = inputs_dir / template.name.replace(".template", "")
|
167
166
|
ensure_parents_write_text(inputs_file, template_file)
|
168
167
|
input_files.append(inputs_file)
|
@@ -171,11 +170,11 @@ def create_contract_test_inputs(
|
|
171
170
|
return CreateContractTestInputsResponse(input_files=input_files, sample_files=sample_files)
|
172
171
|
|
173
172
|
|
174
|
-
def file_replacements(text: str,
|
173
|
+
def file_replacements(text: str, file_name: str) -> str:
|
175
174
|
for match in re.finditer(r"\${(\w+)}", text):
|
176
175
|
var_name = match.group(1)
|
177
|
-
if
|
178
|
-
text = text.replace(match.group(0),
|
176
|
+
if env_value := os.environ.get(var_name):
|
177
|
+
text = text.replace(match.group(0), env_value)
|
179
178
|
else:
|
180
179
|
logger.warning(f"found placeholder {match.group(0)} in {file_name} but no replacement")
|
181
180
|
return text
|
atlas_init/cli_cfn/example.py
CHANGED
@@ -15,7 +15,6 @@ from atlas_init.cli_cfn.aws import (
|
|
15
15
|
from atlas_init.cli_cfn.aws import delete_stack as delete_stack_aws
|
16
16
|
from atlas_init.cli_cfn.cfn_parameter_finder import (
|
17
17
|
CfnTemplate,
|
18
|
-
check_execution_role,
|
19
18
|
decode_parameters,
|
20
19
|
dump_resource_to_file,
|
21
20
|
dump_sample_file,
|
@@ -24,6 +23,8 @@ from atlas_init.cli_cfn.cfn_parameter_finder import (
|
|
24
23
|
from atlas_init.repos.cfn import CfnType, Operation, infer_cfn_type_name
|
25
24
|
from atlas_init.repos.path import Repo, find_paths
|
26
25
|
from atlas_init.settings.env_vars import AtlasInitSettings, init_settings
|
26
|
+
from atlas_init.settings.env_vars_generated import AWSSettings
|
27
|
+
from atlas_init.settings.env_vars_modules import TFModuleCfn
|
27
28
|
|
28
29
|
logger = logging.getLogger(__name__)
|
29
30
|
|
@@ -99,23 +100,24 @@ def example_cmd(
|
|
99
100
|
),
|
100
101
|
register_all_types_in_example: bool = typer.Option(False, "--reg-all", help="Check all types"),
|
101
102
|
):
|
102
|
-
settings = init_settings()
|
103
|
+
settings = init_settings(TFModuleCfn, AWSSettings)
|
104
|
+
cfn_settings = TFModuleCfn.from_env()
|
105
|
+
aws_settings = AWSSettings.from_env()
|
103
106
|
assert settings.tf_vars, "no cfn config found, re-run atlas_init apply with CFN flags"
|
104
107
|
repo_path, resource_path, _ = find_paths(Repo.CFN)
|
105
|
-
env_vars_generated = settings.load_env_vars_full()
|
106
108
|
inputs = CfnExampleInputs(
|
107
109
|
type_name=type_name or infer_cfn_type_name(),
|
108
110
|
example_name=example_name,
|
109
111
|
delete_stack_first=delete_first,
|
110
|
-
region_filter=region or settings.cfn_region,
|
111
|
-
stack_name=stack_name or f"{
|
112
|
+
region_filter=region or settings.cfn_region(aws_settings.AWS_REGION),
|
113
|
+
stack_name=stack_name or f"{cfn_settings.MONGODB_ATLAS_PROFILE}-{example_name or 'atlas-init'}",
|
112
114
|
operation=operation, # type: ignore
|
113
115
|
resource_params=resource_params, # type: ignore
|
114
116
|
stack_timeout_s=stack_timeout_s,
|
115
117
|
force_deregister=force_deregister,
|
116
118
|
reg_version=reg_version,
|
117
119
|
force_keep=force_keep,
|
118
|
-
execution_role=execution_role or
|
120
|
+
execution_role=execution_role or cfn_settings.CFN_EXAMPLE_EXECUTION_ROLE,
|
119
121
|
export_example_to_inputs=export_example_to_inputs,
|
120
122
|
export_example_to_samples=export_example_to_samples,
|
121
123
|
register_all_types_in_example=register_all_types_in_example,
|
@@ -134,7 +136,6 @@ def example_handler(
|
|
134
136
|
)
|
135
137
|
type_name = inputs.type_name
|
136
138
|
stack_name = inputs.stack_name
|
137
|
-
env_vars_generated = settings.load_env_vars_full()
|
138
139
|
region = inputs.region
|
139
140
|
operation = inputs.operation
|
140
141
|
stack_timeout_s = inputs.stack_timeout_s
|
@@ -158,7 +159,6 @@ def example_handler(
|
|
158
159
|
return
|
159
160
|
template_path = infer_template_path(repo_path, type_name, stack_name, inputs.example_name)
|
160
161
|
template_path, parameters, not_found = decode_parameters(
|
161
|
-
exported_env_vars=env_vars_generated,
|
162
162
|
template_path=template_path,
|
163
163
|
stack_name=stack_name,
|
164
164
|
force_params=inputs.resource_params,
|
atlas_init/cli_helper/go.py
CHANGED
@@ -16,7 +16,7 @@ from atlas_init.cli_tf.go_test_run import (
|
|
16
16
|
)
|
17
17
|
from atlas_init.settings.config import TestSuite
|
18
18
|
from atlas_init.settings.env_vars import AtlasInitSettings
|
19
|
-
from atlas_init.settings.path import
|
19
|
+
from atlas_init.settings.path import load_dotenv
|
20
20
|
|
21
21
|
logger = logging.getLogger(__name__)
|
22
22
|
|
@@ -51,6 +51,7 @@ def env_vars_for_capture(mode: GoTestCaptureMode) -> dict[str, str]:
|
|
51
51
|
|
52
52
|
|
53
53
|
class GoTestResult(Entity):
|
54
|
+
logs_dir: Path
|
54
55
|
runs: dict[str, list[GoTestRun]] = Field(default_factory=dict)
|
55
56
|
failure_names: set[str] = Field(default_factory=set)
|
56
57
|
|
@@ -66,7 +67,7 @@ class GoTestResult(Entity):
|
|
66
67
|
if prev_test_results:
|
67
68
|
logger.warning(f"2nd time test results for {test_name}")
|
68
69
|
for result in test_results:
|
69
|
-
log_path = _log_path(test_name)
|
70
|
+
log_path = _log_path(self.logs_dir, test_name)
|
70
71
|
result.log_path = log_path
|
71
72
|
prev_test_results.extend(test_results)
|
72
73
|
return all(run.is_pass for run in test_results)
|
@@ -95,7 +96,8 @@ def run_go_tests(
|
|
95
96
|
)
|
96
97
|
if ci_value := test_env.pop("CI", None):
|
97
98
|
logger.warning(f"popped CI={ci_value}")
|
98
|
-
|
99
|
+
logs_dir = settings.go_test_logs_dir
|
100
|
+
results = GoTestResult(logs_dir=logs_dir)
|
99
101
|
commands_to_run: dict[str, str] = {}
|
100
102
|
for group in groups:
|
101
103
|
if group.sequential_tests:
|
@@ -118,6 +120,7 @@ def run_go_tests(
|
|
118
120
|
return _run_tests(
|
119
121
|
results,
|
120
122
|
repo_path,
|
123
|
+
logs_dir,
|
121
124
|
commands_to_run,
|
122
125
|
test_env,
|
123
126
|
test_timeout_s=timeout_minutes * 60,
|
@@ -168,9 +171,9 @@ def resolve_env_vars(
|
|
168
171
|
skip_os: bool = False,
|
169
172
|
) -> dict[str, str]:
|
170
173
|
if env_vars == GoEnvVars.manual:
|
171
|
-
test_env_vars = settings.
|
174
|
+
test_env_vars = settings.manual_env_vars
|
172
175
|
elif env_vars == GoEnvVars.vscode:
|
173
|
-
test_env_vars =
|
176
|
+
test_env_vars = load_dotenv(settings.env_vars_vs_code)
|
174
177
|
else:
|
175
178
|
raise NotImplementedError(f"don't know how to load env_vars={env_vars}")
|
176
179
|
test_env_vars |= {
|
@@ -188,6 +191,7 @@ def resolve_env_vars(
|
|
188
191
|
def _run_tests(
|
189
192
|
results: GoTestResult,
|
190
193
|
repo_path: Path,
|
194
|
+
logs_dir: Path,
|
191
195
|
commands_to_run: dict[str, str],
|
192
196
|
test_env: dict[str, str],
|
193
197
|
test_timeout_s: int = 301 * 60,
|
@@ -199,11 +203,11 @@ def _run_tests(
|
|
199
203
|
actual_workers = min(max_workers, len(commands_to_run)) or 1
|
200
204
|
with ThreadPoolExecutor(max_workers=actual_workers) as pool:
|
201
205
|
for name, command in sorted(commands_to_run.items()):
|
202
|
-
log_path = _log_path(name)
|
206
|
+
log_path = _log_path(logs_dir, name)
|
203
207
|
if log_path.exists() and log_path.read_text():
|
204
208
|
if re_run:
|
205
209
|
logger.info(f"moving existing logs of {name} to old dir")
|
206
|
-
move_logs_to_dir({name}, dir_name="old")
|
210
|
+
move_logs_to_dir(logs_dir, {name}, dir_name="old")
|
207
211
|
else:
|
208
212
|
logger.info(f"skipping {name} because log exists")
|
209
213
|
continue
|
@@ -229,7 +233,7 @@ def _run_tests(
|
|
229
233
|
continue
|
230
234
|
context = GoTestContext(
|
231
235
|
name=name,
|
232
|
-
html_url=f"file://{_log_path(name)}",
|
236
|
+
html_url=f"file://{_log_path(logs_dir, name)}",
|
233
237
|
steps=[GoTestContextStep(name="local-run")],
|
234
238
|
)
|
235
239
|
try:
|
@@ -250,14 +254,14 @@ def _run_tests(
|
|
250
254
|
if not results.add_test_results_all_pass(name, parsed_tests):
|
251
255
|
results.failure_names.add(name)
|
252
256
|
if failure_names := results.failure_names:
|
253
|
-
move_logs_to_dir(failure_names)
|
257
|
+
move_logs_to_dir(logs_dir, failure_names)
|
254
258
|
logger.error(f"failed to run tests: {sorted(failure_names)}")
|
255
259
|
return results
|
256
260
|
|
257
261
|
|
258
|
-
def move_logs_to_dir(names: set[str], dir_name: str = "failures"):
|
259
|
-
new_dir =
|
260
|
-
for log in
|
262
|
+
def move_logs_to_dir(logs_dir: Path, names: set[str], dir_name: str = "failures"):
|
263
|
+
new_dir = logs_dir / dir_name
|
264
|
+
for log in logs_dir.glob("*.log"):
|
261
265
|
if log.stem in names:
|
262
266
|
text = log.read_text()
|
263
267
|
assert "\n" in text
|
@@ -266,5 +270,5 @@ def move_logs_to_dir(names: set[str], dir_name: str = "failures"):
|
|
266
270
|
log.rename(new_dir / f"{ts}.{log.name}")
|
267
271
|
|
268
272
|
|
269
|
-
def _log_path(name: str) -> Path:
|
270
|
-
return
|
273
|
+
def _log_path(logs_dir: Path, name: str) -> Path:
|
274
|
+
return logs_dir / f"{name}.log"
|
@@ -13,6 +13,7 @@ from atlas_init.cli_helper.run import (
|
|
13
13
|
)
|
14
14
|
from atlas_init.settings.config import TerraformVars, TestSuite
|
15
15
|
from atlas_init.settings.env_vars import AtlasInitSettings
|
16
|
+
from atlas_init.settings.env_vars_generated import AWSSettings, AtlasSettings
|
16
17
|
|
17
18
|
logger = logging.getLogger(__name__)
|
18
19
|
|
@@ -20,17 +21,19 @@ logger = logging.getLogger(__name__)
|
|
20
21
|
def get_tf_vars(settings: AtlasInitSettings, active_groups: list[TestSuite]) -> dict[str, Any]: # type: ignore
|
21
22
|
tf_vars = TerraformVars() # type: ignore
|
22
23
|
tf_vars = sum((group.vars for group in active_groups), start=tf_vars)
|
24
|
+
aws_settings = AWSSettings.from_env()
|
25
|
+
atlas_settings = AtlasSettings.from_env()
|
23
26
|
return {
|
24
|
-
"atlas_public_key":
|
25
|
-
"atlas_private_key":
|
26
|
-
"atlas_base_url":
|
27
|
-
"is_mongodbgov_cloud":
|
28
|
-
"org_id":
|
29
|
-
"aws_region":
|
27
|
+
"atlas_public_key": atlas_settings.MONGODB_ATLAS_PUBLIC_KEY,
|
28
|
+
"atlas_private_key": atlas_settings.MONGODB_ATLAS_PRIVATE_KEY,
|
29
|
+
"atlas_base_url": atlas_settings.MONGODB_ATLAS_BASE_URL,
|
30
|
+
"is_mongodbgov_cloud": atlas_settings.is_mongodbgov_cloud,
|
31
|
+
"org_id": atlas_settings.MONGODB_ATLAS_ORG_ID,
|
32
|
+
"aws_region": aws_settings.AWS_REGION,
|
30
33
|
"project_name": settings.project_name,
|
31
34
|
"out_dir": settings.profile_dir,
|
32
35
|
"extra_env_vars": settings.manual_env_vars,
|
33
|
-
**settings.tf_vars(),
|
36
|
+
**settings.tf_vars(aws_settings.AWS_REGION),
|
34
37
|
**tf_vars.as_configs(),
|
35
38
|
}
|
36
39
|
|
@@ -56,7 +59,7 @@ class state_copier: # noqa: N801
|
|
56
59
|
|
57
60
|
|
58
61
|
def run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[str]):
|
59
|
-
with state_copier(settings.tf_state_path, settings.
|
62
|
+
with state_copier(settings.tf_state_path, settings.atlas_init_tf_src_path):
|
60
63
|
_run_terraform(settings, command, extra_args)
|
61
64
|
|
62
65
|
|
@@ -71,7 +74,7 @@ def _run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[s
|
|
71
74
|
"terraform",
|
72
75
|
" ".join(command_parts),
|
73
76
|
env=os.environ | {"TF_DATA_DIR": settings.tf_data_dir},
|
74
|
-
cwd=settings.
|
77
|
+
cwd=settings.atlas_init_tf_src_path,
|
75
78
|
logger=logger,
|
76
79
|
)
|
77
80
|
if not is_ok:
|
@@ -88,10 +91,10 @@ def dump_tf_vars(settings: AtlasInitSettings, tf_vars: dict[str, Any]):
|
|
88
91
|
|
89
92
|
|
90
93
|
def export_outputs(settings: AtlasInitSettings) -> None:
|
91
|
-
with state_copier(settings.tf_state_path, settings.
|
94
|
+
with state_copier(settings.tf_state_path, settings.atlas_init_tf_src_path):
|
92
95
|
result = run_command_receive_result(
|
93
96
|
"terraform output -json",
|
94
|
-
settings.
|
97
|
+
settings.atlas_init_tf_src_path,
|
95
98
|
logger,
|
96
99
|
env=os.environ | {"TF_DATA_DIR": settings.tf_data_dir},
|
97
100
|
)
|
atlas_init/cli_root/trigger.py
CHANGED
@@ -4,12 +4,13 @@ import requests
|
|
4
4
|
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
|
5
5
|
from zero_3rdparty.id_creator import simple_id
|
6
6
|
|
7
|
-
from atlas_init.settings.env_vars import init_settings
|
7
|
+
from atlas_init.settings.env_vars import env_vars_cls_or_none, init_settings
|
8
8
|
from atlas_init.settings.env_vars_generated import (
|
9
|
-
|
9
|
+
AtlasSettingsWithProject,
|
10
|
+
AWSSettings,
|
10
11
|
RealmSettings,
|
11
|
-
TFModuleCluster,
|
12
12
|
)
|
13
|
+
from atlas_init.settings.env_vars_modules import TFModuleCluster
|
13
14
|
from atlas_init.settings.path import dump_dotenv
|
14
15
|
from atlas_init.typer_app import app_command
|
15
16
|
|
@@ -22,12 +23,15 @@ def trigger_app():
|
|
22
23
|
|
23
24
|
|
24
25
|
def create_realm_app():
|
25
|
-
settings = init_settings()
|
26
|
-
|
27
|
-
|
28
|
-
|
26
|
+
settings = init_settings(AtlasSettingsWithProject, TFModuleCluster, AWSSettings)
|
27
|
+
atlas_settings = AtlasSettingsWithProject.from_env()
|
28
|
+
cluster_settings = TFModuleCluster.from_env()
|
29
|
+
aws_settings = AWSSettings.from_env()
|
30
|
+
project_id = atlas_settings.MONGODB_ATLAS_PROJECT_ID
|
31
|
+
base_url = atlas_settings.realm_url
|
32
|
+
cluster_name = cluster_settings.MONGODB_ATLAS_CLUSTER_NAME
|
29
33
|
auth_headers = login_to_realm(settings, base_url)
|
30
|
-
realm_settings =
|
34
|
+
realm_settings = env_vars_cls_or_none(RealmSettings, dotenv_path=settings.env_vars_trigger)
|
31
35
|
if realm_settings and function_exists(
|
32
36
|
base_url,
|
33
37
|
auth_headers,
|
@@ -44,7 +48,7 @@ def create_realm_app():
|
|
44
48
|
app_id = apps[0]["_id"]
|
45
49
|
else:
|
46
50
|
logger.info("no apps found, creating one")
|
47
|
-
app = create_app(base_url, auth_headers, project_id, cluster_name,
|
51
|
+
app = create_app(base_url, auth_headers, project_id, cluster_name, aws_settings.AWS_REGION)
|
48
52
|
logger.info(f"created app: {app}")
|
49
53
|
app_id = app["_id"]
|
50
54
|
logger.info(f"using app_id: {app_id}")
|
@@ -109,7 +113,11 @@ _cloud_deployment_regions = {
|
|
109
113
|
|
110
114
|
|
111
115
|
def create_app(
|
112
|
-
base_url: str,
|
116
|
+
base_url: str,
|
117
|
+
auth_headers: dict[str, str],
|
118
|
+
project_id: str,
|
119
|
+
cluster_name: str,
|
120
|
+
aws_region: str,
|
113
121
|
) -> dict:
|
114
122
|
provider_region = f"aws-{aws_region}"
|
115
123
|
location = _cloud_deployment_regions.get(provider_region)
|
@@ -229,7 +237,12 @@ class _RetryPostRequestError(Exception):
|
|
229
237
|
reraise=True,
|
230
238
|
)
|
231
239
|
def _request_post_call(
|
232
|
-
url: str,
|
240
|
+
url: str,
|
241
|
+
data: dict,
|
242
|
+
headers: dict[str, str],
|
243
|
+
timeout: int,
|
244
|
+
*,
|
245
|
+
log_data_on_failure: bool = False,
|
233
246
|
) -> dict:
|
234
247
|
response = requests.post(url, json=data, headers=headers, timeout=timeout)
|
235
248
|
if response.status_code >= 500: # noqa: PLR2004
|
atlas_init/cli_tf/app.py
CHANGED
@@ -63,7 +63,7 @@ def schema(
|
|
63
63
|
schema_out_path = settings.schema_out_path_computed
|
64
64
|
schema_out_path.mkdir(exist_ok=True)
|
65
65
|
|
66
|
-
schema_parsed = parse_py_terraform_schema(settings.
|
66
|
+
schema_parsed = parse_py_terraform_schema(settings.atlas_init_tf_schema_config_path)
|
67
67
|
generator_config = dump_generator_config(schema_parsed)
|
68
68
|
generator_config_path = schema_out_path / "generator_config.yaml"
|
69
69
|
generator_config_path.write_text(generator_config)
|
atlas_init/cli_tf/github_logs.py
CHANGED
@@ -20,17 +20,13 @@ from atlas_init.cli_tf.go_test_run import GoTestRun, parse
|
|
20
20
|
from atlas_init.repos.path import (
|
21
21
|
GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS,
|
22
22
|
)
|
23
|
-
from atlas_init.settings.
|
24
|
-
DEFAULT_GITHUB_CI_RUN_LOGS,
|
25
|
-
DEFAULT_GITHUB_SUMMARY_DIR,
|
26
|
-
)
|
23
|
+
from atlas_init.settings.env_vars import init_settings
|
27
24
|
|
28
25
|
logger = logging.getLogger(__name__)
|
29
26
|
|
30
27
|
GH_TOKEN_ENV_NAME = "GH_TOKEN" # noqa: S105 #nosec
|
31
|
-
GITHUB_CI_RUN_LOGS_ENV_NAME = "GITHUB_CI_RUN_LOGS"
|
32
28
|
GITHUB_CI_SUMMARY_DIR_ENV_NAME = "GITHUB_CI_SUMMARY_DIR_ENV_NAME"
|
33
|
-
REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME
|
29
|
+
REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME]
|
34
30
|
MAX_DOWNLOADS = 5
|
35
31
|
|
36
32
|
|
@@ -154,19 +150,11 @@ def download_job_safely(workflow_dir: Path, job: WorkflowJob) -> Path | None:
|
|
154
150
|
|
155
151
|
|
156
152
|
def logs_dir() -> Path:
|
157
|
-
|
158
|
-
if not logs_dir_str:
|
159
|
-
logger.info(f"using {DEFAULT_GITHUB_CI_RUN_LOGS} to store github ci logs!")
|
160
|
-
return DEFAULT_GITHUB_CI_RUN_LOGS
|
161
|
-
return Path(logs_dir_str)
|
153
|
+
return init_settings().github_ci_run_logs
|
162
154
|
|
163
155
|
|
164
156
|
def summary_dir(summary_name: str) -> Path:
|
165
|
-
|
166
|
-
if not summary_dir_str:
|
167
|
-
logger.info(f"using {DEFAULT_GITHUB_SUMMARY_DIR / summary_name} to store summaries")
|
168
|
-
return DEFAULT_GITHUB_SUMMARY_DIR / summary_name
|
169
|
-
return Path(summary_dir_str) / summary_name
|
157
|
+
return init_settings().github_ci_summary_dir / summary_name
|
170
158
|
|
171
159
|
|
172
160
|
def workflow_logs_dir(workflow: WorkflowRun) -> Path:
|