atlas-init 0.3.7__py3-none-any.whl → 0.4.1__py3-none-any.whl

atlas_init/cli_tf/debug_logs_test_data.py

@@ -246,22 +246,32 @@ def create_mock_data(
         request_path = rt.request.path
         method = rt.request.method
         spec_path = find_normalized_path(request_path, api_spec_paths[method])
-        rt_variables = spec_path.variables(request_path)
-        normalized_path = spec_path.path
-        try:
-            mock_data.update_variables(rt_variables)
-        except VariablesChangedError as e:
-            for change in e.changes:
-                rt_variables.pop(change.var_name)
-                rt_variables[change.new_var_name] = change.new
-            normalized_path = normalize_text(request_path, rt_variables)
-        for modifier in modifiers:
-            if modifier.match(rt, normalized_path):
-                modifier.modification(rt)
-        normalized_text = normalize_text(rt.request.text, mock_data.variables, expect_json=True)
-        normalized_response_text = normalize_text(rt.response.text, mock_data.variables, expect_json=True)
+        normalized_path, normalized_text, normalized_response_text = normalize_rt(modifiers, mock_data, rt, spec_path)
         mock_data.add_roundtrip(rt, normalized_path, normalized_text, normalized_response_text, is_diff(rt))
     mock_data.replace_text_variables()
     if prune_duplicates:
         mock_data.prune_duplicate_responses()  # better to keep duplicates to stay KISS
     return mock_data
+
+
+def normalize_rt(
+    modifiers: list[RTModifier],
+    mock_data: MockRequestData,
+    rt: SDKRoundtrip,
+    spec_path: ApiSpecPath,
+):
+    request_path = rt.request.path
+    rt_variables = spec_path.variables(request_path)
+    try:
+        mock_data.update_variables(rt_variables)
+    except VariablesChangedError as e:
+        for change in e.changes:
+            rt_variables.pop(change.var_name)
+            rt_variables[change.new_var_name] = change.new
+    normalized_path = normalize_text(request_path, rt_variables)
+    for modifier in modifiers:
+        if modifier.match(rt, normalized_path):
+            modifier.modification(rt)
+    normalized_text = normalize_text(rt.request.text, mock_data.variables, expect_json=True)
+    normalized_response_text = normalize_text(rt.response.text, mock_data.variables, expect_json=True)
+    return normalized_path, normalized_text, normalized_response_text

atlas_init/cli_tf/github_logs.py

@@ -27,7 +27,7 @@ from atlas_init.settings.path import (
 
 logger = logging.getLogger(__name__)
 
-GH_TOKEN_ENV_NAME = "GH_TOKEN"  # noqa: S105
+GH_TOKEN_ENV_NAME = "GH_TOKEN"  # noqa: S105 #nosec
 GITHUB_CI_RUN_LOGS_ENV_NAME = "GITHUB_CI_RUN_LOGS"
 GITHUB_CI_SUMMARY_DIR_ENV_NAME = "GITHUB_CI_SUMMARY_DIR_ENV_NAME"
 REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME, GITHUB_CI_RUN_LOGS_ENV_NAME]
@@ -88,7 +88,7 @@ def find_test_runs(
     repository = tf_repo()
     for workflow in repository.get_workflow_runs(
         created=f">{since.strftime('%Y-%m-%d')}",
-        branch=branch,
+        branch=branch,  # type: ignore
         exclude_pull_requests=True,  # type: ignore
     ):
         if not include_workflow(workflow):
@@ -127,6 +127,8 @@ def find_job_test_runs(workflow_dir: Path, job: WorkflowJob) -> list[GoTestRun]:
 
 
 def parse_job_logs(job: WorkflowJob, logs_path: Path) -> list[GoTestRun]:
+    if job.conclusion in {"skipped", "cancelled", None}:
+        return []
     step, logs_lines = select_step_and_log_content(job, logs_path)
     return list(parse(logs_lines, job, step))
 
@@ -151,7 +153,7 @@ def download_job_safely(workflow_dir: Path, job: WorkflowJob) -> Path | None:
 def logs_dir() -> Path:
     logs_dir_str = os.environ.get(GITHUB_CI_RUN_LOGS_ENV_NAME)
     if not logs_dir_str:
-        logger.warning(f"using {DEFAULT_GITHUB_CI_RUN_LOGS} to store github ci logs!")
+        logger.info(f"using {DEFAULT_GITHUB_CI_RUN_LOGS} to store github ci logs!")
         return DEFAULT_GITHUB_CI_RUN_LOGS
     return Path(logs_dir_str)
 
@@ -159,7 +161,7 @@ def logs_dir() -> Path:
 def summary_dir(summary_name: str) -> Path:
     summary_dir_str = os.environ.get(GITHUB_CI_SUMMARY_DIR_ENV_NAME)
     if not summary_dir_str:
-        logger.warning(f"using {DEFAULT_GITHUB_SUMMARY_DIR / summary_name} to store summaries")
+        logger.info(f"using {DEFAULT_GITHUB_SUMMARY_DIR / summary_name} to store summaries")
         return DEFAULT_GITHUB_SUMMARY_DIR / summary_name
     return Path(summary_dir_str) / summary_name
 
@@ -222,7 +224,8 @@ def select_step_and_log_content(job: WorkflowJob, logs_path: Path) -> tuple[int,
 
 def test_step(steps: list[WorkflowStep]) -> int:
     for i, step in enumerate(steps, 1):
-        if "test" in step.name.lower():
+        name_lower = step.name.lower()
+        if "acceptance test" in name_lower and "mocked" not in name_lower:
             return i
     last_step = len(steps)
     logger.warning(f"using {last_step} as final step, unable to find 'test' in {steps}")

atlas_init/cli_tf/go_test_run.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 class GoTestStatus(StrEnum):
     RUN = "RUN"
-    PASS = "PASS"  # noqa: S105
+    PASS = "PASS"  # noqa: S105 #nosec
     FAIL = "FAIL"
     SKIP = "SKIP"
 

atlas_init/cli_tf/hcl/parser.py

@@ -108,7 +108,7 @@ def iter_blocks(block: Block, level: int | None = None) -> Iterable[Block]:
                     hcl="\n".join(block_lines),
                 )
     if line_level_start_names.get(level) is not None:
-        raise ValueError(f"Unfinished block @ {line_nr} in {block.name} at level {level}")
+        raise ValueError(f"Unfinished block @ {line_nr} in {block.name} at level {level}")  # pyright: ignore
 
 
 def hcl_attrs(block: Block) -> dict[str, str]:

atlas_init/cli_tf/mock_tf_log.py

@@ -128,7 +128,7 @@ def mock_tf_log_cmd(
     log_diff_roundtrips: bool = typer.Option(
         False, "-l", "--log-diff-roundtrips", help="print out the roundtrips used in diffs"
     ),
-    package_name: str = typer.Option("", "-p", "--package-name", help="the package name to use for modifiers"),
+    package_name: str = typer.Option("-p", "--package-name", prompt=True, help="the package name to use for modifiers"),
 ):
     cwd = Path.cwd()
     default_testdir = cwd / "testdata"

atlas_init/cli_tf/schema_table.py

@@ -1,6 +1,4 @@
-# import typer
-
-
+# pyright: reportIncompatibleMethodOverride=none
 from collections import defaultdict
 from collections.abc import Iterable
 from functools import total_ordering

atlas_init/cli_tf/schema_v3.py

@@ -210,7 +210,7 @@ class Schema(BaseModelLocal):
 
 
 class Resource(BaseModelLocal):
-    schema: Schema
+    schema: Schema  # pyright: ignore
     name: SnakeCaseString
 
     @property

atlas_init/cloud/aws.py

@@ -1,13 +1,19 @@
 import logging
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, wait
+from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import Annotated, TypeVar
 
 import stringcase
 from pydantic import AfterValidator, ConfigDict
+from zero_3rdparty.file_utils import copy, file_modified_time, iter_paths_and_relative
 from zero_3rdparty.iter_utils import flat_map
 from zero_3rdparty.object_name import as_name
 
+from atlas_init.cli_helper.run import run_binary_command_is_ok
+from atlas_init.cli_root import is_dry_run
+
 logger = logging.getLogger(__name__)
 PascalAlias = ConfigDict(alias_generator=stringcase.pascalcase, populate_by_name=True)
 REGIONS = "af-south-1,ap-east-1,ap-northeast-1,ap-northeast-2,ap-northeast-3,ap-south-1,ap-southeast-1,ap-southeast-2,ap-southeast-3,ca-central-1,eu-central-1,eu-north-1,eu-south-1,eu-west-1,eu-west-2,eu-west-3,me-south-1,sa-east-1,us-east-1,us-east-2,us-west-1,us-west-2,ap-south-2,ap-southeast-4,eu-central-2,eu-south-2,me-central-1,il-central-1".split(
@@ -62,3 +68,60 @@ def run_in_regions(call: Callable[[str], T], regions: list[str] | None = None) -
         except Exception:
             logger.exception(f"failed to call {name} in region = {region}, error 👆")
     return region_responses
+
+
+def upload_to_s3(profile_path: Path, s3_bucket: str, s3_prefix: str = ""):
+    profiles_path = profile_path.parent
+    assert profiles_path.name == "profiles"
+    excluded = [".DS_Store", ".terraform/*"]
+    excluded_str = " ".join([f'--exclude "{pattern}"' for pattern in excluded])
+    dest_path = _s3_path(s3_bucket, profile_path.name, "", s3_prefix=s3_prefix)
+    assert run_binary_command_is_ok(
+        "aws",
+        f"s3 sync {profile_path.name} {dest_path} {excluded_str}",
+        profiles_path,
+        logger=logger,
+        dry_run=is_dry_run(),
+    )
+
+
+def _s3_path(s3_bucket: str, profile_name: str, rel_path: str, s3_prefix: str = "") -> str:
+    return f"s3://{s3_bucket}//{s3_prefix}profiles/{profile_name}/{rel_path}"
+
+
+def download_from_s3(profile_path: Path, s3_bucket: str, s3_prefix: str = ""):
+    profiles_path = profile_path.parent
+    assert profiles_path.name == "profiles"
+    src_path = _s3_path(s3_bucket, profile_path.name, "", s3_prefix)
+    copy_dir = f"{profile_path.name}_copy/"
+    with TemporaryDirectory() as tmp_dir:
+        copy_dir = Path(tmp_dir) / f"safe-{profile_path.name}"
+        assert run_binary_command_is_ok(
+            "aws",
+            f"s3 sync {src_path} {copy_dir}",
+            profiles_path,
+            logger=logger,
+            dry_run=is_dry_run(),
+        )
+        copy_new_files(copy_dir, profile_path)
+
+
+_aws_keys = (
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_PROFILE",
+)
+
+
+def copy_new_files(src_dir: Path, dest_dir: Path):
+    for src_path, rel_path in iter_paths_and_relative(src_dir, "*", only_files=True):
+        dest_path = dest_dir / rel_path
+        if not dest_path.exists() or file_modified_time(src_path) > file_modified_time(dest_path):
+            dest_path.parent.mkdir(parents=True, exist_ok=True)
+            if src_path.name == ".env-manual":
+                if dest_path.exists():
+                    continue  # never overwrite the manual file
+                lines_no_aws = [line for line in src_path.read_text().splitlines() if not line.startswith(_aws_keys)]
+                dest_path.write_text("\n".join(lines_no_aws) + "\n")
+            else:
+                copy(src_path, dest_path)

atlas_init/settings/config.py

@@ -65,6 +65,11 @@ class TerraformVars(Entity):
         return config
 
 
+class PyHook(Entity):
+    name: str
+    locate: str
+
+
 @dump_ignore_falsy
 @total_ordering
 class TestSuite(Entity):
@@ -75,6 +80,7 @@ class TestSuite(Entity):
     repo_go_packages: dict[str, list[str]] = Field(default_factory=dict)
     repo_globs: dict[str, list[str]] = Field(default_factory=dict)
     vars: TerraformVars = Field(default_factory=TerraformVars)  # type: ignore
+    post_apply_hooks: list[PyHook] = Field(default_factory=list)
 
     def __lt__(self, other) -> bool:
         if not isinstance(other, TestSuite):  # type: ignore

atlas_init/settings/env_vars.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import logging
 import os
+from contextlib import suppress
 from functools import cached_property
 from pathlib import Path
-from typing import Any, NamedTuple
+from typing import Any, NamedTuple, TypeVar
 
-import typer
-from model_lib import field_names, parse_payload
-from pydantic import field_validator, model_validator
+from model_lib import parse_payload
+from pydantic import ValidationError, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from atlas_init.cloud.aws import AwsRegion
@@ -19,11 +19,13 @@ from atlas_init.settings.config import (
 from atlas_init.settings.config import (
     active_suites as config_active_suites,
 )
+from atlas_init.settings.env_vars_generated import AtlasSettings
 from atlas_init.settings.path import (
     DEFAULT_CONFIG_PATH,
     DEFAULT_PROFILES_PATH,
     DEFAULT_SCHEMA_CONFIG_PATH,
     DEFAULT_TF_PATH,
+    dump_dotenv,
     load_dotenv,
     repo_path_rel_path,
 )
@@ -31,25 +33,36 @@ from atlas_init.settings.path import (
 logger = logging.getLogger(__name__)
 ENV_PREFIX = "ATLAS_INIT_"
 DEFAULT_PROFILE = "default"
+ENV_S3_PROFILE_BUCKET = f"{ENV_PREFIX}S3_PROFILE_BUCKET"
+ENV_PROJECT_NAME = f"{ENV_PREFIX}PROJECT_NAME"
+ENV_PROFILE = f"{ENV_PREFIX}PROFILE"
+ENV_PROFILES_PATH = f"{ENV_PREFIX}PROFILES_PATH"
+ENV_TEST_SUITES = f"{ENV_PREFIX}TEST_SUITES"
+ENV_CLIPBOARD_COPY = f"{ENV_PREFIX}CLIPBOARD_COPY"
 REQUIRED_FIELDS = [
     "MONGODB_ATLAS_ORG_ID",
     "MONGODB_ATLAS_PRIVATE_KEY",
     "MONGODB_ATLAS_PUBLIC_KEY",
 ]
+FILENAME_ENV_MANUAL = ".env-manual"
+T = TypeVar("T")
 
 
-class ExternalSettings(BaseSettings):
-    model_config = SettingsConfigDict(env_prefix="")
+class ExternalSettings(AtlasSettings):
+    model_config = SettingsConfigDict(env_prefix="", extra="ignore")
 
     TF_CLI_CONFIG_FILE: str = ""
     AWS_PROFILE: str = ""
     AWS_REGION: AwsRegion = "us-east-1"
-    MONGODB_ATLAS_ORG_ID: str
-    MONGODB_ATLAS_PRIVATE_KEY: str
-    MONGODB_ATLAS_PUBLIC_KEY: str
-    MONGODB_ATLAS_BASE_URL: str = "https://cloud-dev.mongodb.com/"
     non_interactive: bool = False
 
+    @property
+    def realm_url(self) -> str:
+        assert not self.is_mongodbgov_cloud, "realm_url is not supported for mongodbgov cloud"
+        if "cloud-dev." in self.MONGODB_ATLAS_BASE_URL:
+            return "https://services.cloud-dev.mongodb.com/"
+        return "https://services.cloud.mongodb.com/"
+
     @property
     def is_interactive(self) -> bool:
         return not self.non_interactive
@@ -59,49 +72,10 @@ class ExternalSettings(BaseSettings):
         return "mongodbgov" in self.MONGODB_ATLAS_BASE_URL
 
 
-def as_env_var_name(field_name: str) -> str:
-    names = set(field_names(AtlasInitSettings))
-    assert (
-        field_name in names or field_name.lower() in names
-    ), f"unknown field name for {AtlasInitSettings}: {field_name}"
-    external_settings_names = set(field_names(ExternalSettings))
-    if field_name in external_settings_names:
-        return field_name.upper()
-    return f"{ENV_PREFIX}{field_name}".upper()
-
-
-def dump_manual_dotenv_from_env(path: Path) -> None:
-    env_vars: dict[str, str] = {}
-    names = field_names(AtlasInitSettings)
-    ext_settings_names = field_names(ExternalSettings)
-    path_settings_names = field_names(AtlasInitPaths)
-    names = set(names + ext_settings_names + path_settings_names)
-    os_env = os.environ
-    for name in sorted(names):
-        env_name = as_env_var_name(name)
-        if env_name.lower() in os_env or env_name.upper() in os_env:
-            env_value = os_env.get(env_name.upper()) or os_env.get(env_name.lower())
-            if env_value:
-                env_vars[env_name] = env_value
-
-    content = "\n".join(f"{k}={v}" for k, v in env_vars.items())
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(content)
-
-
-def env_var_names(field_name: str) -> list[str]:
-    return [f"{ENV_PREFIX}{name}" for name in (field_name, field_name.lower(), field_name.upper())]
-
-
-def read_from_env(field_name: str, default: str = "") -> str:
-    assert as_env_var_name(field_name)
-    for name in [field_name, field_name.lower(), field_name.upper()]:
+def read_from_env(env_key: str, default: str = "") -> str:
+    for name in [env_key, env_key.lower(), env_key.upper()]:
         if name in os.environ:
             return os.environ[name]
-        prefix_name = f"{ENV_PREFIX}{name}"
-        if prefix_name in os.environ:
-            return os.environ[prefix_name]
-    logger.info(f"field not found in env: {field_name}, using default: {default}")
     return default
 
 
@@ -114,6 +88,7 @@ class AtlasInitPaths(BaseSettings):
     profiles_path: Path = DEFAULT_PROFILES_PATH
     tf_schema_config_path: Path = DEFAULT_SCHEMA_CONFIG_PATH
     schema_out_path: Path | None = None
+    s3_profile_bucket: str = ""
 
     @property
     def schema_out_path_computed(self) -> Path:
@@ -125,14 +100,12 @@ class AtlasInitPaths(BaseSettings):
 
     @property
     def env_file_manual(self) -> Path:
-        return self.profile_dir / ".env-manual"
+        return self.profile_dir / FILENAME_ENV_MANUAL
 
     @property
     def manual_env_vars(self) -> dict[str, str]:
         env_manual_path = self.env_file_manual
-        if env_manual_path.exists():
-            return load_dotenv(env_manual_path)
-        return {}
+        return load_dotenv(env_manual_path) if env_manual_path.exists() else {}
 
     @property
     def env_vars_generated(self) -> Path:
@@ -165,23 +138,40 @@ class AtlasInitPaths(BaseSettings):
     def load_env_vars(self, path: Path) -> dict[str, str]:
         return load_dotenv(path)
 
-    def load_env_vars_generated(self) -> dict[str, str]:
-        env_path = self.env_vars_generated
+    def load_env_vars_full(self) -> dict[str, str]:
+        env_path = self.env_vars_vs_code
         assert env_path.exists(), f"no env-vars exist {env_path} have you forgotten apply?"
         return load_dotenv(env_path)
 
+    def env_vars_cls_or_none(self, t: type[T], *, path: Path | None = None) -> T | None:
+        with suppress(ValidationError):
+            return self.env_vars_cls(t, path=path)
+
+    def env_vars_cls(self, t: type[T], *, path: Path | None = None) -> T:
+        path = path or self.env_vars_vs_code
+        env_vars = self.load_env_vars(path) if path.exists() else {}
+        return t(**env_vars)
+
     def load_profile_manual_env_vars(self, *, skip_os_update: bool = False) -> dict[str, str]:
         # sourcery skip: dict-assign-update-to-union
         manual_env_vars = self.manual_env_vars
         if manual_env_vars:
             if skip_os_update:
                 return manual_env_vars
-            logger.warning(f"loading manual env-vars from {self.env_file_manual}")
-            os.environ.update(manual_env_vars)
+            if new_updates := {k: v for k, v in manual_env_vars.items() if k not in os.environ}:
+                logger.info(f"loading manual env-vars {','.join(new_updates)}")
+                os.environ.update(new_updates)
         else:
-            logger.warning(f"no {self.env_file_manual}")
+            logger.warning(f"no {self.env_file_manual} exists")
         return manual_env_vars
 
+    def include_extra_env_vars_in_vscode(self, extra_env_vars: dict[str, str]) -> None:
+        extra_name = ", ".join(extra_env_vars.keys())
+        original_env_vars = self.load_env_vars(self.env_vars_vs_code)
+        new_env_vars = original_env_vars | extra_env_vars
+        dump_dotenv(self.env_vars_vs_code, new_env_vars)
+        logger.info(f"done {self.env_vars_vs_code} updated with {extra_name} env-vars ✅")
+
 
 class EnvVarsCheck(NamedTuple):
     missing: list[str]
@@ -196,42 +186,36 @@ class AtlasInitSettings(AtlasInitPaths, ExternalSettings):
     cfn_use_kms_key: bool = False
     project_name: str = ""
 
-    skip_copy: bool = False
+    cliboard_copy: str = ""
     test_suites: str = ""
 
     @classmethod
     def check_env_vars(
         cls,
         profile: str = DEFAULT_PROFILE,
-        required_extra_fields: list[str] | None = None,
-        explicit_env_vars: dict[str, str] | None = None,
+        required_env_vars: list[str] | None = None,
     ) -> EnvVarsCheck:
-        """side effect of loading env-vars and set profile"""
-        os.environ[as_env_var_name("profile")] = profile
-        required_extra_fields = required_extra_fields or []
-        explicit_env_vars = explicit_env_vars or {}
-        path_settings = AtlasInitPaths()
-        manual_env_vars = path_settings.load_profile_manual_env_vars()
+        required_env_vars = required_env_vars or []
+        path_settings = AtlasInitPaths(profile=profile)
+        manual_env_vars = path_settings.manual_env_vars
         ambiguous: list[str] = []
-        for env_name, env_value in explicit_env_vars.items():
-            manual_value = manual_env_vars.get(env_name)
-            if manual_value and manual_value != env_value:
+        for env_name, manual_value in manual_env_vars.items():
+            env_value = read_from_env(env_name)
+            if env_value and manual_value != env_value:
                 ambiguous.append(env_name)
-            else:
-                os.environ[env_name] = env_value
         missing_env_vars = sorted(
-            as_env_var_name(field_name)
-            for field_name in REQUIRED_FIELDS + required_extra_fields
-            if read_from_env(field_name) == ""
+            env_name
+            for env_name in REQUIRED_FIELDS + required_env_vars
+            if read_from_env(env_name) == "" and env_name not in manual_env_vars
         )
         return EnvVarsCheck(missing=missing_env_vars, ambiguous=sorted(ambiguous))
 
     @classmethod
-    def safe_settings(cls) -> AtlasInitSettings:
-        """loads .env_manual before creating the settings"""
-        path_settings = AtlasInitPaths()
-        path_settings.load_profile_manual_env_vars()
-        ext_settings = ExternalSettings()  # type: ignore
+    def safe_settings(cls, profile: str, *, ext_settings: ExternalSettings | None = None) -> AtlasInitSettings:
+        """side effect of loading manual env-vars and set profile"""
+        os.environ[ENV_PROFILE] = profile
+        AtlasInitPaths(profile=profile).load_profile_manual_env_vars()
+        ext_settings = ext_settings or ExternalSettings()  # type: ignore
         path_settings = AtlasInitPaths()
         return cls(**path_settings.model_dump(), **ext_settings.model_dump())
 
@@ -257,34 +241,63 @@ class AtlasInitSettings(AtlasInitPaths, ExternalSettings):
     def test_suites_parsed(self) -> list[str]:
         return [t for t in self.test_suites.split(",") if t]
 
-    def cfn_config(self) -> dict[str, Any]:
+    def tf_vars(self) -> dict[str, Any]:
+        variables = {}
         if self.cfn_profile:
-            return {
-                "cfn_config": {
-                    "profile": self.cfn_profile,
-                    "region": self.cfn_region,
-                    "use_kms_key": self.cfn_use_kms_key,
-                }
+            variables["cfn_config"] = {
+                "profile": self.cfn_profile,
+                "region": self.cfn_region,
+                "use_kms_key": self.cfn_use_kms_key,
             }
-        return {}
+        if self.s3_profile_bucket:
+            variables["use_aws_s3"] = True
+        return variables
 
 
-def active_suites(settings: AtlasInitSettings) -> list[TestSuite]:
+def active_suites(settings: AtlasInitSettings) -> list[TestSuite]:  # type: ignore
     repo_path, cwd_rel_path = repo_path_rel_path()
     return config_active_suites(settings.config, repo_path, cwd_rel_path, settings.test_suites_parsed)
 
 
-def init_settings() -> AtlasInitSettings:
+_sentinel = object()
+PLACEHOLDER_VALUE = "PLACEHOLDER"
+
+
+class EnvVarsError(Exception):
+    def __init__(self, missing: list[str], ambiguous: list[str]):
+        self.missing = missing
+        self.ambiguous = ambiguous
+        super().__init__(f"missing: {missing}, ambiguous: {ambiguous}")
+
+    def __str__(self) -> str:
+        return f"missing: {self.missing}, ambiguous: {self.ambiguous}"
+
+
+def init_settings(
+    required_env_vars: list[str] | object = _sentinel,
+    *,
+    non_required: bool = False,
+) -> AtlasInitSettings:
+    if required_env_vars is _sentinel:
+        required_env_vars = [ENV_PROJECT_NAME]
+    if non_required:
+        required_env_vars = []
+    profile = os.getenv("ATLAS_INIT_PROFILE", DEFAULT_PROFILE)
     missing_env_vars, ambiguous_env_vars = AtlasInitSettings.check_env_vars(
-        os.getenv("ATLAS_INIT_PROFILE", DEFAULT_PROFILE),
-        required_extra_fields=["project_name"],
+        profile,
+        required_env_vars=required_env_vars,  # type: ignore
     )
-    if missing_env_vars:
-        typer.echo(f"missing env_vars: {missing_env_vars}")
+    if missing_env_vars and not non_required:
+        logger.warning(f"missing env_vars: {missing_env_vars}")
     if ambiguous_env_vars:
-        typer.echo(
-            f"amiguous env_vars: {ambiguous_env_vars} (specified both in cli & in .env-manual file with different values)"
+        logger.warning(
+            f"amiguous env_vars: {ambiguous_env_vars} (specified both in cli/env & in .env-manual file with different values)"
         )
+    ext_settings = None
+    if non_required and missing_env_vars:
+        placeholders = {k: PLACEHOLDER_VALUE for k in missing_env_vars}
+        missing_env_vars = []
+        ext_settings = ExternalSettings(**placeholders)  # type: ignore
     if missing_env_vars or ambiguous_env_vars:
-        raise typer.Exit(1)
-    return AtlasInitSettings.safe_settings()
+        raise EnvVarsError(missing_env_vars, ambiguous_env_vars)
+    return AtlasInitSettings.safe_settings(profile, ext_settings=ext_settings)

atlas_init/settings/env_vars_generated.py

@@ -0,0 +1,34 @@
+import random
+
+from pydantic import ConfigDict, Field
+from pydantic_settings import BaseSettings
+
+
+class _EnvVarsGenerated(BaseSettings):
+    model_config = ConfigDict(extra="ignore")  # type: ignore
+
+
+class AtlasSettings(_EnvVarsGenerated):
+    MONGODB_ATLAS_ORG_ID: str
+    MONGODB_ATLAS_PRIVATE_KEY: str
+    MONGODB_ATLAS_PUBLIC_KEY: str
+    MONGODB_ATLAS_BASE_URL: str = "https://cloud-dev.mongodb.com/"
+
+
+class RealmSettings(_EnvVarsGenerated):
+    MONGODB_REALM_APP_ID: str
+    MONGODB_REALM_SERVICE_ID: str
+    MONGODB_REALM_FUNCTION_ID: str
+    MONGODB_REALM_FUNCTION_NAME: str
+    MONGODB_REALM_BASE_URL: str
+    RANDOM_INT_100K: str = Field(default_factory=lambda: str(random.randint(0, 100_000)))  # noqa: S311 # not used for cryptographic purposes # nosec
+
+
+class EnvVarsGenerated(AtlasSettings):
+    MONGODB_ATLAS_PROJECT_ID: str
+
+
+class TFModuleCluster(_EnvVarsGenerated):
+    MONGODB_ATLAS_CLUSTER_NAME: str
+    MONGODB_ATLAS_CONTAINER_ID: str
+    MONGODB_URL: str

atlas_init/settings/rich_utils.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Literal
 
+import typer
 from pydantic import BaseModel
 from rich.logging import RichHandler
 
@@ -44,13 +45,20 @@ def hide_secrets(handler: logging.Handler, secrets_dict: dict[str, str]) -> None
     handler.addFilter(SecretsHider(list(secrets_to_hide), name="secrets-hider"))
 
 
-def configure_logging(log_level: str = "INFO") -> logging.Handler:
+def configure_logging(
+    app: typer.Typer, log_level: str = "INFO", *, is_running_in_repo: bool = False
+) -> logging.Handler:
     _LogLevel(log_level=log_level)  # type: ignore
-    handler = RichHandler(rich_tracebacks=False)
+    handler = RichHandler(rich_tracebacks=False, level=log_level)
     logging.basicConfig(
-        level=logging.getLevelName(log_level),
+        level=log_level,
         format="%(message)s",
         datefmt="[%X]",
         handlers=[handler],
     )
+    if not is_running_in_repo or handler.level >= logging.WARNING:
+        logging.warning("using basic tracebacks/errors")
+        app.pretty_exceptions_enable = False
+        app.pretty_exceptions_show_locals = False
+
     return handler

atlas_init/tf/modules/cfn/cfn.tf

@@ -11,7 +11,7 @@ locals {
   resource_actions_yaml = file("${path.module}/resource_actions.yaml")
   services              = yamldecode(local.services_yaml)
   resource_actions      = yamldecode(local.resource_actions_yaml)
-  role_name             = "cfn-execution-role-${var.cfn_profile}"
+  role_name             = "mongodb-atlas-cfn-${var.cfn_profile}"
   iam_policy_statement = {
     Sid      = "Original"
     Action   = local.resource_actions