atlas-init 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +3 -7
- atlas_init/cli.py +2 -1
- atlas_init/cli_cfn/app.py +3 -5
- atlas_init/cli_cfn/cfn_parameter_finder.py +1 -28
- atlas_init/cli_cfn/contract.py +5 -3
- atlas_init/cli_cfn/example.py +8 -6
- atlas_init/cli_helper/go.py +18 -14
- atlas_init/cli_helper/tf_runner.py +14 -11
- atlas_init/cli_root/trigger.py +21 -8
- atlas_init/cli_tf/app.py +1 -1
- atlas_init/cli_tf/debug_logs.py +4 -4
- atlas_init/cli_tf/example_update.py +3 -3
- atlas_init/cli_tf/github_logs.py +4 -16
- atlas_init/cli_tf/hcl/modifier.py +115 -14
- atlas_init/cli_tf/mock_tf_log.py +4 -3
- atlas_init/cli_tf/schema_v2.py +2 -2
- atlas_init/cli_tf/schema_v2_api_parsing.py +3 -3
- atlas_init/cli_tf/schema_v3.py +2 -2
- atlas_init/cli_tf/schema_v3_sdk_base.py +1 -1
- atlas_init/settings/env_vars.py +119 -142
- atlas_init/settings/env_vars_generated.py +40 -9
- atlas_init/settings/env_vars_modules.py +71 -0
- atlas_init/settings/path.py +4 -10
- atlas_init/typer_app.py +3 -3
- {atlas_init-0.4.2.dist-info → atlas_init-0.4.4.dist-info}/METADATA +6 -5
- {atlas_init-0.4.2.dist-info → atlas_init-0.4.4.dist-info}/RECORD +29 -32
- atlas_init-0.4.4.dist-info/licenses/LICENSE +21 -0
- atlas_init/cli_tf/example_update_test/test_update_example.tf +0 -23
- atlas_init/cli_tf/example_update_test.py +0 -96
- atlas_init/cli_tf/hcl/modifier_test/test_process_variables_output_.tf +0 -25
- atlas_init/cli_tf/hcl/modifier_test/test_process_variables_variable_.tf +0 -24
- atlas_init/cli_tf/hcl/modifier_test.py +0 -95
- {atlas_init-0.4.2.dist-info → atlas_init-0.4.4.dist-info}/WHEEL +0 -0
- {atlas_init-0.4.2.dist-info → atlas_init-0.4.4.dist-info}/entry_points.txt +0 -0
    
        atlas_init/__init__.py
    CHANGED
    
    | @@ -1,12 +1,8 @@ | |
| 1 1 | 
             
            from pathlib import Path
         | 
| 2 2 |  | 
| 3 | 
            -
            VERSION = "0.4. | 
| 3 | 
            +
            VERSION = "0.4.4"
         | 
| 4 4 |  | 
| 5 5 |  | 
| 6 6 | 
             
            def running_in_repo() -> bool:
         | 
| 7 | 
            -
                 | 
| 8 | 
            -
                 | 
| 9 | 
            -
                    return False
         | 
| 10 | 
            -
                git_directory = py_directory.parent / ".git"
         | 
| 11 | 
            -
                git_config = git_directory / "config"
         | 
| 12 | 
            -
                return git_directory.exists() and git_config.exists() and "atlas-init" in git_config.read_text()
         | 
| 7 | 
            +
                git_config = Path(__file__).parent.parent / ".git/config"
         | 
| 8 | 
            +
                return git_config.exists() and "atlas-init" in git_config.read_text()
         | 
    
        atlas_init/cli.py
    CHANGED
    
    | @@ -45,6 +45,7 @@ from atlas_init.settings.env_vars import ( | |
| 45 45 | 
             
                active_suites,
         | 
| 46 46 | 
             
                init_settings,
         | 
| 47 47 | 
             
            )
         | 
| 48 | 
            +
            from atlas_init.settings.env_vars_generated import AWSSettings, AtlasSettings
         | 
| 48 49 | 
             
            from atlas_init.settings.path import (
         | 
| 49 50 | 
             
                CwdIsNoRepoPathError,
         | 
| 50 51 | 
             
                dump_vscode_dotenv,
         | 
| @@ -79,7 +80,7 @@ def apply(context: typer.Context, *, skip_outputs: bool = False): | |
| 79 80 |  | 
| 80 81 |  | 
| 81 82 | 
             
            def _plan_or_apply(extra_args: list[str], command: Literal["plan", "apply"], *, skip_outputs: bool) -> list[TestSuite]:  # type: ignore
         | 
| 82 | 
            -
                settings = init_settings()
         | 
| 83 | 
            +
                settings = init_settings(AtlasSettings, AWSSettings)
         | 
| 83 84 | 
             
                logger.info(f"using the '{command}' command, extra args: {extra_args}")
         | 
| 84 85 | 
             
                try:
         | 
| 85 86 | 
             
                    suites = active_suites(settings)
         | 
    
        atlas_init/cli_cfn/app.py
    CHANGED
    
    | @@ -14,9 +14,6 @@ from atlas_init.cli_cfn.aws import ( | |
| 14 14 | 
             
            from atlas_init.cli_cfn.aws import (
         | 
| 15 15 | 
             
                delete_stack as delete_stack_aws,
         | 
| 16 16 | 
             
            )
         | 
| 17 | 
            -
            from atlas_init.cli_cfn.cfn_parameter_finder import (
         | 
| 18 | 
            -
                read_execution_role,
         | 
| 19 | 
            -
            )
         | 
| 20 17 | 
             
            from atlas_init.cli_cfn.contract import contract_test_cmd
         | 
| 21 18 | 
             
            from atlas_init.cli_cfn.example import example_cmd
         | 
| 22 19 | 
             
            from atlas_init.cli_cfn.files import (
         | 
| @@ -31,6 +28,7 @@ from atlas_init.repos.cfn import ( | |
| 31 28 | 
             
            )
         | 
| 32 29 | 
             
            from atlas_init.repos.path import Repo, current_dir, find_paths, resource_root
         | 
| 33 30 | 
             
            from atlas_init.settings.env_vars import active_suites, init_settings
         | 
| 31 | 
            +
            from atlas_init.settings.env_vars_modules import TFModuleCfn
         | 
| 34 32 |  | 
| 35 33 | 
             
            app = typer.Typer(no_args_is_help=True)
         | 
| 36 34 | 
             
            app.command(name="example")(example_cmd)
         | 
| @@ -55,8 +53,8 @@ def reg( | |
| 55 53 | 
             
                    if local:
         | 
| 56 54 | 
             
                        deregister_cfn_resource_type(type_name, deregister=not dry_run, region_filter=region)
         | 
| 57 55 | 
             
                logger.info(f"ready to activate {type_name}")
         | 
| 58 | 
            -
                settings = init_settings()
         | 
| 59 | 
            -
                cfn_execution_role =  | 
| 56 | 
            +
                settings = init_settings(TFModuleCfn)
         | 
| 57 | 
            +
                cfn_execution_role = settings.env_vars_cls(TFModuleCfn).CFN_EXAMPLE_EXECUTION_ROLE
         | 
| 60 58 | 
             
                last_third_party = get_last_cfn_type(type_name, region, is_third_party=True)
         | 
| 61 59 | 
             
                assert last_third_party, f"no 3rd party extension found for {type_name} in {region}"
         | 
| 62 60 | 
             
                if dry_run:
         | 
| @@ -2,46 +2,19 @@ import logging | |
| 2 2 | 
             
            from pathlib import Path
         | 
| 3 3 | 
             
            from typing import Any
         | 
| 4 4 |  | 
| 5 | 
            -
            from model_lib import Entity, dump, parse_model | 
| 5 | 
            +
            from model_lib import Entity, dump, parse_model
         | 
| 6 6 | 
             
            from mypy_boto3_cloudformation.type_defs import ParameterTypeDef
         | 
| 7 7 | 
             
            from pydantic import ConfigDict, Field
         | 
| 8 8 | 
             
            from rich import prompt
         | 
| 9 | 
            -
            from zero_3rdparty.dict_nested import read_nested
         | 
| 10 9 | 
             
            from zero_3rdparty.file_utils import clean_dir
         | 
| 11 10 |  | 
| 12 11 | 
             
            from atlas_init.cli_cfn.files import create_sample_file, default_log_group_name
         | 
| 13 12 | 
             
            from atlas_init.cloud.aws import PascalAlias
         | 
| 14 13 | 
             
            from atlas_init.repos.cfn import CfnType, cfn_examples_dir, cfn_type_normalized
         | 
| 15 | 
            -
            from atlas_init.settings.path import DEFAULT_TF_PATH
         | 
| 16 14 |  | 
| 17 15 | 
             
            logger = logging.getLogger(__name__)
         | 
| 18 16 |  | 
| 19 17 |  | 
| 20 | 
            -
            def read_execution_role(loaded_env_vars: dict[str, str]) -> str:
         | 
| 21 | 
            -
                return loaded_env_vars["CFN_EXAMPLE_EXECUTION_ROLE"]
         | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
            def check_execution_role(repo_path: Path, loaded_env_vars: dict[str, str]) -> str:
         | 
| 25 | 
            -
                execution_role = cfn_examples_dir(repo_path) / "execution-role.yaml"
         | 
| 26 | 
            -
                execution_raw = parse_payload(execution_role)
         | 
| 27 | 
            -
                actions_expected = read_nested(
         | 
| 28 | 
            -
                    execution_raw,
         | 
| 29 | 
            -
                    "Resources.ExecutionRole.Properties.Policies.[0].PolicyDocument.Statement.[0].Action",
         | 
| 30 | 
            -
                )
         | 
| 31 | 
            -
                actions_found = parse_payload(DEFAULT_TF_PATH / "modules/cfn/resource_actions.yaml")
         | 
| 32 | 
            -
                if diff := set(actions_expected) ^ set(actions_found):
         | 
| 33 | 
            -
                    raise ValueError(f"non-matching execution role actions: {sorted(diff)}")
         | 
| 34 | 
            -
                services_found = parse_payload(DEFAULT_TF_PATH / "modules/cfn/assume_role_services.yaml")
         | 
| 35 | 
            -
                services_expected = read_nested(
         | 
| 36 | 
            -
                    execution_raw,
         | 
| 37 | 
            -
                    "Resources.ExecutionRole.Properties.AssumeRolePolicyDocument.Statement.[0].Principal.Service",
         | 
| 38 | 
            -
                )
         | 
| 39 | 
            -
                if diff := set(services_found) ^ set(services_expected):
         | 
| 40 | 
            -
                    raise ValueError(f"non-matching execution role services: {sorted(diff)}")
         | 
| 41 | 
            -
                logger.info(f"execution role is up to date with {execution_role}")
         | 
| 42 | 
            -
                return read_execution_role(loaded_env_vars)
         | 
| 43 | 
            -
             | 
| 44 | 
            -
             | 
| 45 18 | 
             
            class TemplatePathNotFoundError(Exception):
         | 
| 46 19 | 
             
                def __init__(self, type_name: str, examples_dir: Path) -> None:
         | 
| 47 20 | 
             
                    self.type_name = type_name
         | 
    
        atlas_init/cli_cfn/contract.py
    CHANGED
    
    | @@ -16,6 +16,7 @@ from atlas_init.cli_helper.run_manager import RunManager | |
| 16 16 | 
             
            from atlas_init.cli_root import is_dry_run
         | 
| 17 17 | 
             
            from atlas_init.repos.path import Repo, ResourcePaths, find_paths
         | 
| 18 18 | 
             
            from atlas_init.settings.env_vars import AtlasInitSettings, init_settings
         | 
| 19 | 
            +
            from atlas_init.settings.env_vars_generated import AWSSettings
         | 
| 19 20 |  | 
| 20 21 | 
             
            logger = logging.getLogger(__name__)
         | 
| 21 22 |  | 
| @@ -111,7 +112,7 @@ def contract_test( | |
| 111 112 | 
             
                resource_paths: ResourcePaths | None = None,
         | 
| 112 113 | 
             
                only_names: list[str] | None = None,
         | 
| 113 114 | 
             
            ):
         | 
| 114 | 
            -
                settings = settings or init_settings()
         | 
| 115 | 
            +
                settings = settings or init_settings(AWSSettings)
         | 
| 115 116 | 
             
                resource_paths = resource_paths or find_paths(Repo.CFN)
         | 
| 116 117 | 
             
                resource_name = resource_paths.resource_name
         | 
| 117 118 | 
             
                generated_env_vars = settings.load_env_vars_full()
         | 
| @@ -122,11 +123,12 @@ def contract_test( | |
| 122 123 | 
             
                )
         | 
| 123 124 | 
             
                create_response = create_contract_test_inputs(create_inputs)
         | 
| 124 125 | 
             
                create_response.log_input_files(logger)
         | 
| 126 | 
            +
                aws_settings = AWSSettings.from_env()
         | 
| 125 127 | 
             
                run_contract_test = RunContractTest(
         | 
| 126 128 | 
             
                    resource_path=resource_paths.resource_path,
         | 
| 127 129 | 
             
                    repo_path=resource_paths.repo_path,
         | 
| 128 | 
            -
                    aws_profile= | 
| 129 | 
            -
                    cfn_region=settings.cfn_region,
         | 
| 130 | 
            +
                    aws_profile=aws_settings.AWS_PROFILE,
         | 
| 131 | 
            +
                    cfn_region=settings.cfn_region(aws_settings.AWS_REGION),
         | 
| 130 132 | 
             
                    only_names=only_names,
         | 
| 131 133 | 
             
                )
         | 
| 132 134 | 
             
                if run_contract_test.skip_build:
         | 
    
        atlas_init/cli_cfn/example.py
    CHANGED
    
    | @@ -15,7 +15,6 @@ from atlas_init.cli_cfn.aws import ( | |
| 15 15 | 
             
            from atlas_init.cli_cfn.aws import delete_stack as delete_stack_aws
         | 
| 16 16 | 
             
            from atlas_init.cli_cfn.cfn_parameter_finder import (
         | 
| 17 17 | 
             
                CfnTemplate,
         | 
| 18 | 
            -
                check_execution_role,
         | 
| 19 18 | 
             
                decode_parameters,
         | 
| 20 19 | 
             
                dump_resource_to_file,
         | 
| 21 20 | 
             
                dump_sample_file,
         | 
| @@ -24,6 +23,8 @@ from atlas_init.cli_cfn.cfn_parameter_finder import ( | |
| 24 23 | 
             
            from atlas_init.repos.cfn import CfnType, Operation, infer_cfn_type_name
         | 
| 25 24 | 
             
            from atlas_init.repos.path import Repo, find_paths
         | 
| 26 25 | 
             
            from atlas_init.settings.env_vars import AtlasInitSettings, init_settings
         | 
| 26 | 
            +
            from atlas_init.settings.env_vars_generated import AWSSettings
         | 
| 27 | 
            +
            from atlas_init.settings.env_vars_modules import TFModuleCfn
         | 
| 27 28 |  | 
| 28 29 | 
             
            logger = logging.getLogger(__name__)
         | 
| 29 30 |  | 
| @@ -99,23 +100,24 @@ def example_cmd( | |
| 99 100 | 
             
                ),
         | 
| 100 101 | 
             
                register_all_types_in_example: bool = typer.Option(False, "--reg-all", help="Check all types"),
         | 
| 101 102 | 
             
            ):
         | 
| 102 | 
            -
                settings = init_settings()
         | 
| 103 | 
            +
                settings = init_settings(TFModuleCfn, AWSSettings)
         | 
| 104 | 
            +
                cfn_settings = TFModuleCfn.from_env()
         | 
| 105 | 
            +
                aws_settings = AWSSettings.from_env()
         | 
| 103 106 | 
             
                assert settings.tf_vars, "no cfn config found, re-run atlas_init apply with CFN flags"
         | 
| 104 107 | 
             
                repo_path, resource_path, _ = find_paths(Repo.CFN)
         | 
| 105 | 
            -
                env_vars_generated = settings.load_env_vars_full()
         | 
| 106 108 | 
             
                inputs = CfnExampleInputs(
         | 
| 107 109 | 
             
                    type_name=type_name or infer_cfn_type_name(),
         | 
| 108 110 | 
             
                    example_name=example_name,
         | 
| 109 111 | 
             
                    delete_stack_first=delete_first,
         | 
| 110 | 
            -
                    region_filter=region or settings.cfn_region,
         | 
| 111 | 
            -
                    stack_name=stack_name or f"{ | 
| 112 | 
            +
                    region_filter=region or settings.cfn_region(aws_settings.AWS_REGION),
         | 
| 113 | 
            +
                    stack_name=stack_name or f"{cfn_settings.MONGODB_ATLAS_PROFILE}-{example_name or 'atlas-init'}",
         | 
| 112 114 | 
             
                    operation=operation,  # type: ignore
         | 
| 113 115 | 
             
                    resource_params=resource_params,  # type: ignore
         | 
| 114 116 | 
             
                    stack_timeout_s=stack_timeout_s,
         | 
| 115 117 | 
             
                    force_deregister=force_deregister,
         | 
| 116 118 | 
             
                    reg_version=reg_version,
         | 
| 117 119 | 
             
                    force_keep=force_keep,
         | 
| 118 | 
            -
                    execution_role=execution_role or  | 
| 120 | 
            +
                    execution_role=execution_role or cfn_settings.CFN_EXAMPLE_EXECUTION_ROLE,
         | 
| 119 121 | 
             
                    export_example_to_inputs=export_example_to_inputs,
         | 
| 120 122 | 
             
                    export_example_to_samples=export_example_to_samples,
         | 
| 121 123 | 
             
                    register_all_types_in_example=register_all_types_in_example,
         | 
    
        atlas_init/cli_helper/go.py
    CHANGED
    
    | @@ -16,7 +16,7 @@ from atlas_init.cli_tf.go_test_run import ( | |
| 16 16 | 
             
            )
         | 
| 17 17 | 
             
            from atlas_init.settings.config import TestSuite
         | 
| 18 18 | 
             
            from atlas_init.settings.env_vars import AtlasInitSettings
         | 
| 19 | 
            -
            from atlas_init.settings.path import  | 
| 19 | 
            +
            from atlas_init.settings.path import load_dotenv
         | 
| 20 20 |  | 
| 21 21 | 
             
            logger = logging.getLogger(__name__)
         | 
| 22 22 |  | 
| @@ -51,6 +51,7 @@ def env_vars_for_capture(mode: GoTestCaptureMode) -> dict[str, str]: | |
| 51 51 |  | 
| 52 52 |  | 
| 53 53 | 
             
            class GoTestResult(Entity):
         | 
| 54 | 
            +
                logs_dir: Path
         | 
| 54 55 | 
             
                runs: dict[str, list[GoTestRun]] = Field(default_factory=dict)
         | 
| 55 56 | 
             
                failure_names: set[str] = Field(default_factory=set)
         | 
| 56 57 |  | 
| @@ -66,7 +67,7 @@ class GoTestResult(Entity): | |
| 66 67 | 
             
                    if prev_test_results:
         | 
| 67 68 | 
             
                        logger.warning(f"2nd time test results for {test_name}")
         | 
| 68 69 | 
             
                    for result in test_results:
         | 
| 69 | 
            -
                        log_path = _log_path(test_name)
         | 
| 70 | 
            +
                        log_path = _log_path(self.logs_dir, test_name)
         | 
| 70 71 | 
             
                        result.log_path = log_path
         | 
| 71 72 | 
             
                    prev_test_results.extend(test_results)
         | 
| 72 73 | 
             
                    return all(run.is_pass for run in test_results)
         | 
| @@ -95,7 +96,8 @@ def run_go_tests( | |
| 95 96 | 
             
                )
         | 
| 96 97 | 
             
                if ci_value := test_env.pop("CI", None):
         | 
| 97 98 | 
             
                    logger.warning(f"popped CI={ci_value}")
         | 
| 98 | 
            -
                 | 
| 99 | 
            +
                logs_dir = settings.go_test_logs_dir
         | 
| 100 | 
            +
                results = GoTestResult(logs_dir=logs_dir)
         | 
| 99 101 | 
             
                commands_to_run: dict[str, str] = {}
         | 
| 100 102 | 
             
                for group in groups:
         | 
| 101 103 | 
             
                    if group.sequential_tests:
         | 
| @@ -118,6 +120,7 @@ def run_go_tests( | |
| 118 120 | 
             
                return _run_tests(
         | 
| 119 121 | 
             
                    results,
         | 
| 120 122 | 
             
                    repo_path,
         | 
| 123 | 
            +
                    logs_dir,
         | 
| 121 124 | 
             
                    commands_to_run,
         | 
| 122 125 | 
             
                    test_env,
         | 
| 123 126 | 
             
                    test_timeout_s=timeout_minutes * 60,
         | 
| @@ -137,7 +140,7 @@ def group_commands_for_mode( | |
| 137 140 | 
             
            ) -> dict[str, str]:
         | 
| 138 141 | 
             
                commands_to_run: dict[str, str] = {}
         | 
| 139 142 | 
             
                if mode == GoTestMode.package:
         | 
| 140 | 
            -
                    name_regex = f | 
| 143 | 
            +
                    name_regex = f"^({'|'.join(names)})$" if names else "^TestAcc*"
         | 
| 141 144 | 
             
                    for pkg_url in group.package_url_tests(repo_path):
         | 
| 142 145 | 
             
                        command = f"go test {pkg_url} -v -run {name_regex} -timeout {timeout_minutes}m"
         | 
| 143 146 | 
             
                        if not group.sequential_tests:
         | 
| @@ -170,7 +173,7 @@ def resolve_env_vars( | |
| 170 173 | 
             
                if env_vars == GoEnvVars.manual:
         | 
| 171 174 | 
             
                    test_env_vars = settings.load_profile_manual_env_vars(skip_os_update=True)
         | 
| 172 175 | 
             
                elif env_vars == GoEnvVars.vscode:
         | 
| 173 | 
            -
                    test_env_vars =  | 
| 176 | 
            +
                    test_env_vars = load_dotenv(settings.env_vars_vs_code)
         | 
| 174 177 | 
             
                else:
         | 
| 175 178 | 
             
                    raise NotImplementedError(f"don't know how to load env_vars={env_vars}")
         | 
| 176 179 | 
             
                test_env_vars |= {
         | 
| @@ -188,6 +191,7 @@ def resolve_env_vars( | |
| 188 191 | 
             
            def _run_tests(
         | 
| 189 192 | 
             
                results: GoTestResult,
         | 
| 190 193 | 
             
                repo_path: Path,
         | 
| 194 | 
            +
                logs_dir: Path,
         | 
| 191 195 | 
             
                commands_to_run: dict[str, str],
         | 
| 192 196 | 
             
                test_env: dict[str, str],
         | 
| 193 197 | 
             
                test_timeout_s: int = 301 * 60,
         | 
| @@ -199,11 +203,11 @@ def _run_tests( | |
| 199 203 | 
             
                actual_workers = min(max_workers, len(commands_to_run)) or 1
         | 
| 200 204 | 
             
                with ThreadPoolExecutor(max_workers=actual_workers) as pool:
         | 
| 201 205 | 
             
                    for name, command in sorted(commands_to_run.items()):
         | 
| 202 | 
            -
                        log_path = _log_path(name)
         | 
| 206 | 
            +
                        log_path = _log_path(logs_dir, name)
         | 
| 203 207 | 
             
                        if log_path.exists() and log_path.read_text():
         | 
| 204 208 | 
             
                            if re_run:
         | 
| 205 209 | 
             
                                logger.info(f"moving existing logs of {name} to old dir")
         | 
| 206 | 
            -
                                move_logs_to_dir({name}, dir_name="old")
         | 
| 210 | 
            +
                                move_logs_to_dir(logs_dir, {name}, dir_name="old")
         | 
| 207 211 | 
             
                            else:
         | 
| 208 212 | 
             
                                logger.info(f"skipping {name} because log exists")
         | 
| 209 213 | 
             
                                continue
         | 
| @@ -229,7 +233,7 @@ def _run_tests( | |
| 229 233 | 
             
                        continue
         | 
| 230 234 | 
             
                    context = GoTestContext(
         | 
| 231 235 | 
             
                        name=name,
         | 
| 232 | 
            -
                        html_url=f"file://{_log_path(name)}",
         | 
| 236 | 
            +
                        html_url=f"file://{_log_path(logs_dir, name)}",
         | 
| 233 237 | 
             
                        steps=[GoTestContextStep(name="local-run")],
         | 
| 234 238 | 
             
                    )
         | 
| 235 239 | 
             
                    try:
         | 
| @@ -250,14 +254,14 @@ def _run_tests( | |
| 250 254 | 
             
                    if not results.add_test_results_all_pass(name, parsed_tests):
         | 
| 251 255 | 
             
                        results.failure_names.add(name)
         | 
| 252 256 | 
             
                if failure_names := results.failure_names:
         | 
| 253 | 
            -
                    move_logs_to_dir(failure_names)
         | 
| 257 | 
            +
                    move_logs_to_dir(logs_dir, failure_names)
         | 
| 254 258 | 
             
                    logger.error(f"failed to run tests: {sorted(failure_names)}")
         | 
| 255 259 | 
             
                return results
         | 
| 256 260 |  | 
| 257 261 |  | 
| 258 | 
            -
            def move_logs_to_dir(names: set[str], dir_name: str = "failures"):
         | 
| 259 | 
            -
                new_dir =  | 
| 260 | 
            -
                for log in  | 
| 262 | 
            +
            def move_logs_to_dir(logs_dir: Path, names: set[str], dir_name: str = "failures"):
         | 
| 263 | 
            +
                new_dir = logs_dir / dir_name
         | 
| 264 | 
            +
                for log in logs_dir.glob("*.log"):
         | 
| 261 265 | 
             
                    if log.stem in names:
         | 
| 262 266 | 
             
                        text = log.read_text()
         | 
| 263 267 | 
             
                        assert "\n" in text
         | 
| @@ -266,5 +270,5 @@ def move_logs_to_dir(names: set[str], dir_name: str = "failures"): | |
| 266 270 | 
             
                        log.rename(new_dir / f"{ts}.{log.name}")
         | 
| 267 271 |  | 
| 268 272 |  | 
| 269 | 
            -
            def _log_path(name: str) -> Path:
         | 
| 270 | 
            -
                return  | 
| 273 | 
            +
            def _log_path(logs_dir: Path, name: str) -> Path:
         | 
| 274 | 
            +
                return logs_dir / f"{name}.log"
         | 
| @@ -13,6 +13,7 @@ from atlas_init.cli_helper.run import ( | |
| 13 13 | 
             
            )
         | 
| 14 14 | 
             
            from atlas_init.settings.config import TerraformVars, TestSuite
         | 
| 15 15 | 
             
            from atlas_init.settings.env_vars import AtlasInitSettings
         | 
| 16 | 
            +
            from atlas_init.settings.env_vars_generated import AWSSettings, AtlasSettings
         | 
| 16 17 |  | 
| 17 18 | 
             
            logger = logging.getLogger(__name__)
         | 
| 18 19 |  | 
| @@ -20,17 +21,19 @@ logger = logging.getLogger(__name__) | |
| 20 21 | 
             
            def get_tf_vars(settings: AtlasInitSettings, active_groups: list[TestSuite]) -> dict[str, Any]:  # type: ignore
         | 
| 21 22 | 
             
                tf_vars = TerraformVars()  # type: ignore
         | 
| 22 23 | 
             
                tf_vars = sum((group.vars for group in active_groups), start=tf_vars)
         | 
| 24 | 
            +
                aws_settings = AWSSettings.from_env()
         | 
| 25 | 
            +
                atlas_settings = AtlasSettings.from_env()
         | 
| 23 26 | 
             
                return {
         | 
| 24 | 
            -
                    "atlas_public_key":  | 
| 25 | 
            -
                    "atlas_private_key":  | 
| 26 | 
            -
                    "atlas_base_url":  | 
| 27 | 
            -
                    "is_mongodbgov_cloud":  | 
| 28 | 
            -
                    "org_id":  | 
| 29 | 
            -
                    "aws_region":  | 
| 27 | 
            +
                    "atlas_public_key": atlas_settings.MONGODB_ATLAS_PUBLIC_KEY,
         | 
| 28 | 
            +
                    "atlas_private_key": atlas_settings.MONGODB_ATLAS_PRIVATE_KEY,
         | 
| 29 | 
            +
                    "atlas_base_url": atlas_settings.MONGODB_ATLAS_BASE_URL,
         | 
| 30 | 
            +
                    "is_mongodbgov_cloud": atlas_settings.is_mongodbgov_cloud,
         | 
| 31 | 
            +
                    "org_id": atlas_settings.MONGODB_ATLAS_ORG_ID,
         | 
| 32 | 
            +
                    "aws_region": aws_settings.AWS_REGION,
         | 
| 30 33 | 
             
                    "project_name": settings.project_name,
         | 
| 31 34 | 
             
                    "out_dir": settings.profile_dir,
         | 
| 32 35 | 
             
                    "extra_env_vars": settings.manual_env_vars,
         | 
| 33 | 
            -
                    **settings.tf_vars(),
         | 
| 36 | 
            +
                    **settings.tf_vars(aws_settings.AWS_REGION),
         | 
| 34 37 | 
             
                    **tf_vars.as_configs(),
         | 
| 35 38 | 
             
                }
         | 
| 36 39 |  | 
| @@ -56,7 +59,7 @@ class state_copier:  # noqa: N801 | |
| 56 59 |  | 
| 57 60 |  | 
| 58 61 | 
             
            def run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[str]):
         | 
| 59 | 
            -
                with state_copier(settings.tf_state_path, settings. | 
| 62 | 
            +
                with state_copier(settings.tf_state_path, settings.atlas_init_tf_src_path):
         | 
| 60 63 | 
             
                    _run_terraform(settings, command, extra_args)
         | 
| 61 64 |  | 
| 62 65 |  | 
| @@ -71,7 +74,7 @@ def _run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[s | |
| 71 74 | 
             
                    "terraform",
         | 
| 72 75 | 
             
                    " ".join(command_parts),
         | 
| 73 76 | 
             
                    env=os.environ | {"TF_DATA_DIR": settings.tf_data_dir},
         | 
| 74 | 
            -
                    cwd=settings. | 
| 77 | 
            +
                    cwd=settings.atlas_init_tf_src_path,
         | 
| 75 78 | 
             
                    logger=logger,
         | 
| 76 79 | 
             
                )
         | 
| 77 80 | 
             
                if not is_ok:
         | 
| @@ -88,10 +91,10 @@ def dump_tf_vars(settings: AtlasInitSettings, tf_vars: dict[str, Any]): | |
| 88 91 |  | 
| 89 92 |  | 
| 90 93 | 
             
            def export_outputs(settings: AtlasInitSettings) -> None:
         | 
| 91 | 
            -
                with state_copier(settings.tf_state_path, settings. | 
| 94 | 
            +
                with state_copier(settings.tf_state_path, settings.atlas_init_tf_src_path):
         | 
| 92 95 | 
             
                    result = run_command_receive_result(
         | 
| 93 96 | 
             
                        "terraform output -json",
         | 
| 94 | 
            -
                        settings. | 
| 97 | 
            +
                        settings.atlas_init_tf_src_path,
         | 
| 95 98 | 
             
                        logger,
         | 
| 96 99 | 
             
                        env=os.environ | {"TF_DATA_DIR": settings.tf_data_dir},
         | 
| 97 100 | 
             
                    )
         | 
    
        atlas_init/cli_root/trigger.py
    CHANGED
    
    | @@ -6,10 +6,11 @@ from zero_3rdparty.id_creator import simple_id | |
| 6 6 |  | 
| 7 7 | 
             
            from atlas_init.settings.env_vars import init_settings
         | 
| 8 8 | 
             
            from atlas_init.settings.env_vars_generated import (
         | 
| 9 | 
            -
                 | 
| 9 | 
            +
                AtlasSettingsWithProject,
         | 
| 10 | 
            +
                AWSSettings,
         | 
| 10 11 | 
             
                RealmSettings,
         | 
| 11 | 
            -
                TFModuleCluster,
         | 
| 12 12 | 
             
            )
         | 
| 13 | 
            +
            from atlas_init.settings.env_vars_modules import TFModuleCluster
         | 
| 13 14 | 
             
            from atlas_init.settings.path import dump_dotenv
         | 
| 14 15 | 
             
            from atlas_init.typer_app import app_command
         | 
| 15 16 |  | 
| @@ -23,11 +24,14 @@ def trigger_app(): | |
| 23 24 |  | 
| 24 25 | 
             
            def create_realm_app():
         | 
| 25 26 | 
             
                settings = init_settings()
         | 
| 26 | 
            -
                 | 
| 27 | 
            -
                 | 
| 28 | 
            -
                 | 
| 27 | 
            +
                atlas_settings = settings.env_vars_cls(AtlasSettingsWithProject)
         | 
| 28 | 
            +
                cluster_settings = settings.env_vars_cls(TFModuleCluster)
         | 
| 29 | 
            +
                project_id = atlas_settings.MONGODB_ATLAS_PROJECT_ID
         | 
| 30 | 
            +
                base_url = atlas_settings.realm_url
         | 
| 31 | 
            +
                cluster_name = cluster_settings.MONGODB_ATLAS_CLUSTER_NAME
         | 
| 29 32 | 
             
                auth_headers = login_to_realm(settings, base_url)
         | 
| 30 33 | 
             
                realm_settings = settings.env_vars_cls_or_none(RealmSettings, path=settings.env_vars_trigger)
         | 
| 34 | 
            +
                aws_settings = settings.env_vars_cls(AWSSettings)
         | 
| 31 35 | 
             
                if realm_settings and function_exists(
         | 
| 32 36 | 
             
                    base_url,
         | 
| 33 37 | 
             
                    auth_headers,
         | 
| @@ -44,7 +48,7 @@ def create_realm_app(): | |
| 44 48 | 
             
                    app_id = apps[0]["_id"]
         | 
| 45 49 | 
             
                else:
         | 
| 46 50 | 
             
                    logger.info("no apps found, creating one")
         | 
| 47 | 
            -
                    app = create_app(base_url, auth_headers, project_id, cluster_name,  | 
| 51 | 
            +
                    app = create_app(base_url, auth_headers, project_id, cluster_name, aws_settings.AWS_REGION)
         | 
| 48 52 | 
             
                    logger.info(f"created app: {app}")
         | 
| 49 53 | 
             
                    app_id = app["_id"]
         | 
| 50 54 | 
             
                logger.info(f"using app_id: {app_id}")
         | 
| @@ -109,7 +113,11 @@ _cloud_deployment_regions = { | |
| 109 113 |  | 
| 110 114 |  | 
| 111 115 | 
             
            def create_app(
         | 
| 112 | 
            -
                base_url: str, | 
| 116 | 
            +
                base_url: str,
         | 
| 117 | 
            +
                auth_headers: dict[str, str],
         | 
| 118 | 
            +
                project_id: str,
         | 
| 119 | 
            +
                cluster_name: str,
         | 
| 120 | 
            +
                aws_region: str,
         | 
| 113 121 | 
             
            ) -> dict:
         | 
| 114 122 | 
             
                provider_region = f"aws-{aws_region}"
         | 
| 115 123 | 
             
                location = _cloud_deployment_regions.get(provider_region)
         | 
| @@ -229,7 +237,12 @@ class _RetryPostRequestError(Exception): | |
| 229 237 | 
             
                reraise=True,
         | 
| 230 238 | 
             
            )
         | 
| 231 239 | 
             
            def _request_post_call(
         | 
| 232 | 
            -
                url: str, | 
| 240 | 
            +
                url: str,
         | 
| 241 | 
            +
                data: dict,
         | 
| 242 | 
            +
                headers: dict[str, str],
         | 
| 243 | 
            +
                timeout: int,
         | 
| 244 | 
            +
                *,
         | 
| 245 | 
            +
                log_data_on_failure: bool = False,
         | 
| 233 246 | 
             
            ) -> dict:
         | 
| 234 247 | 
             
                response = requests.post(url, json=data, headers=headers, timeout=timeout)
         | 
| 235 248 | 
             
                if response.status_code >= 500:  # noqa: PLR2004
         | 
    
        atlas_init/cli_tf/app.py
    CHANGED
    
    | @@ -63,7 +63,7 @@ def schema( | |
| 63 63 | 
             
                schema_out_path = settings.schema_out_path_computed
         | 
| 64 64 | 
             
                schema_out_path.mkdir(exist_ok=True)
         | 
| 65 65 |  | 
| 66 | 
            -
                schema_parsed = parse_py_terraform_schema(settings. | 
| 66 | 
            +
                schema_parsed = parse_py_terraform_schema(settings.atlas_init_tf_schema_config_path)
         | 
| 67 67 | 
             
                generator_config = dump_generator_config(schema_parsed)
         | 
| 68 68 | 
             
                generator_config_path = schema_out_path / "generator_config.yaml"
         | 
| 69 69 | 
             
                generator_config_path.write_text(generator_config)
         | 
    
        atlas_init/cli_tf/debug_logs.py
    CHANGED
    
    | @@ -225,7 +225,7 @@ def match_request( | |
| 225 225 | 
             
                            step_number=step_number,
         | 
| 226 226 | 
             
                        )
         | 
| 227 227 | 
             
                remaining_responses = [resp for i, resp in enumerate(responses_list) if i not in used_responses]
         | 
| 228 | 
            -
                err_msg = f"Could not match request {request.path} ({ref}) with any response\n\n{request}\n\n\nThere are #{len(remaining_responses)} responses left that doesn't match\n{'-'*80}\n{'\n'.join(r.text for r in remaining_responses)}"
         | 
| 228 | 
            +
                err_msg = f"Could not match request {request.path} ({ref}) with any response\n\n{request}\n\n\nThere are #{len(remaining_responses)} responses left that doesn't match\n{'-' * 80}\n{'\n'.join(r.text for r in remaining_responses)}"
         | 
| 229 229 | 
             
                raise ValueError(err_msg)
         | 
| 230 230 |  | 
| 231 231 |  | 
| @@ -260,9 +260,9 @@ def parse_raw_req_responses( | |
| 260 260 | 
             
                        in_response = False
         | 
| 261 261 | 
             
                assert not in_request, "Request not closed"
         | 
| 262 262 | 
             
                assert not in_response, "Response not closed"
         | 
| 263 | 
            -
                assert (
         | 
| 264 | 
            -
                    request_count  | 
| 265 | 
            -
                ) | 
| 263 | 
            +
                assert request_count == response_count, (
         | 
| 264 | 
            +
                    f"Mismatch in request and response count: {request_count} != {response_count}"
         | 
| 265 | 
            +
                )
         | 
| 266 266 | 
             
                parsed_requests = {}
         | 
| 267 267 | 
             
                for ref, request_lines in requests.items():
         | 
| 268 268 | 
             
                    parsed_requests[ref] = parse_request(request_lines)
         | 
| @@ -65,9 +65,9 @@ def update_examples(event_in: UpdateExamples) -> UpdateExamplesOutput: | |
| 65 65 | 
             
                if event_in.skip_tf_fmt:
         | 
| 66 66 | 
             
                    logger.info("skipping terraform fmt")
         | 
| 67 67 | 
             
                else:
         | 
| 68 | 
            -
                    assert run_binary_command_is_ok(
         | 
| 69 | 
            -
                        "terraform | 
| 70 | 
            -
                    ) | 
| 68 | 
            +
                    assert run_binary_command_is_ok("terraform", "fmt -recursive", cwd=event_in.examples_base_dir, logger=logger), (
         | 
| 69 | 
            +
                        "terraform fmt failed"
         | 
| 70 | 
            +
                    )
         | 
| 71 71 | 
             
                return UpdateExamplesOutput(
         | 
| 72 72 | 
             
                    before_var_descriptions=flatten_descriptions(existing_var_descriptions),
         | 
| 73 73 | 
             
                    before_output_descriptions=flatten_descriptions(existing_output_descriptions),
         | 
    
        atlas_init/cli_tf/github_logs.py
    CHANGED
    
    | @@ -20,17 +20,13 @@ from atlas_init.cli_tf.go_test_run import GoTestRun, parse | |
| 20 20 | 
             
            from atlas_init.repos.path import (
         | 
| 21 21 | 
             
                GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS,
         | 
| 22 22 | 
             
            )
         | 
| 23 | 
            -
            from atlas_init.settings. | 
| 24 | 
            -
                DEFAULT_GITHUB_CI_RUN_LOGS,
         | 
| 25 | 
            -
                DEFAULT_GITHUB_SUMMARY_DIR,
         | 
| 26 | 
            -
            )
         | 
| 23 | 
            +
            from atlas_init.settings.env_vars import init_settings
         | 
| 27 24 |  | 
| 28 25 | 
             
            logger = logging.getLogger(__name__)
         | 
| 29 26 |  | 
| 30 27 | 
             
            GH_TOKEN_ENV_NAME = "GH_TOKEN"  # noqa: S105 #nosec
         | 
| 31 | 
            -
            GITHUB_CI_RUN_LOGS_ENV_NAME = "GITHUB_CI_RUN_LOGS"
         | 
| 32 28 | 
             
            GITHUB_CI_SUMMARY_DIR_ENV_NAME = "GITHUB_CI_SUMMARY_DIR_ENV_NAME"
         | 
| 33 | 
            -
            REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME | 
| 29 | 
            +
            REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME]
         | 
| 34 30 | 
             
            MAX_DOWNLOADS = 5
         | 
| 35 31 |  | 
| 36 32 |  | 
| @@ -154,19 +150,11 @@ def download_job_safely(workflow_dir: Path, job: WorkflowJob) -> Path | None: | |
| 154 150 |  | 
| 155 151 |  | 
| 156 152 | 
             
            def logs_dir() -> Path:
         | 
| 157 | 
            -
                 | 
| 158 | 
            -
                if not logs_dir_str:
         | 
| 159 | 
            -
                    logger.info(f"using {DEFAULT_GITHUB_CI_RUN_LOGS} to store github ci logs!")
         | 
| 160 | 
            -
                    return DEFAULT_GITHUB_CI_RUN_LOGS
         | 
| 161 | 
            -
                return Path(logs_dir_str)
         | 
| 153 | 
            +
                return init_settings().github_ci_run_logs
         | 
| 162 154 |  | 
| 163 155 |  | 
| 164 156 | 
             
            def summary_dir(summary_name: str) -> Path:
         | 
| 165 | 
            -
                 | 
| 166 | 
            -
                if not summary_dir_str:
         | 
| 167 | 
            -
                    logger.info(f"using {DEFAULT_GITHUB_SUMMARY_DIR / summary_name} to store summaries")
         | 
| 168 | 
            -
                    return DEFAULT_GITHUB_SUMMARY_DIR / summary_name
         | 
| 169 | 
            -
                return Path(summary_dir_str) / summary_name
         | 
| 157 | 
            +
                return init_settings().github_ci_summary_dir / summary_name
         | 
| 170 158 |  | 
| 171 159 |  | 
| 172 160 | 
             
            def workflow_logs_dir(workflow: WorkflowRun) -> Path:
         |