atlas-init 0.8.1__tar.gz → 0.10.0__tar.gz

{atlas_init-0.8.1 → atlas_init-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-init
-Version: 0.8.1
+Version: 0.10.0
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init
@@ -20,6 +20,7 @@ Requires-Dist: inflection==0.5.1
 Requires-Dist: model-lib
 Requires-Dist: motor==3.7.1
 Requires-Dist: mypy-boto3-cloudformation==1.37.22
+Requires-Dist: mypy-boto3-iam>=1.40.0
 Requires-Dist: orjson==3.10.13
 Requires-Dist: pydantic-settings==2.7.1
 Requires-Dist: pydot==4.0.1

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.8.1"
+VERSION = "0.10.0"
 
 
 def running_in_repo() -> bool:

atlas_init-0.10.0/atlas_init/cli_root/aws_clean.py

@@ -0,0 +1,108 @@
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import humanize
+import typer
+from ask_shell import run_and_wait
+from boto3.session import Session
+from model_lib import Entity
+from mypy_boto3_iam import IAMClient
+from mypy_boto3_iam.type_defs import RoleTypeDef
+from zero_3rdparty.datetime_utils import utc_now
+
+from atlas_init.cloud.aws import PascalAlias
+from atlas_init.settings.env_vars import init_settings
+from atlas_init.typer_app import app_command
+
+logger = logging.getLogger(__name__)
+
+
+class LastUsed(Entity):
+    model_config = PascalAlias
+    last_used_date: datetime
+
+
+class IAMRole(Entity):
+    model_config = PascalAlias
+    arn: str
+    role_name: str
+    create_date: datetime
+    role_last_used: LastUsed | None = None
+
+
+@app_command()
+def aws_clean(
+    skip_iam_roles: bool = typer.Option(False, help="skip iam roles"),
+    iam_role_prefix_name: str = typer.Option(
+        "mongodb-atlas-test-acc-tf-",
+        help="prefix name of iam roles to clean",
+    ),
+):
+    init_settings()
+    if skip_iam_roles:
+        return
+    client: IAMClient = Session().client("iam")  # pyright: ignore[reportAssignmentType]
+    all_roles: list[RoleTypeDef] = []
+    aws_account_id = run_and_wait("aws sts get-caller-identity --query Account --output text").stdout_one_line
+
+    roles_response = client.list_roles()
+
+    all_roles.extend(roles_response["Roles"])
+    marker = roles_response.get("Marker", "")
+    while marker:
+        roles_response = client.list_roles(Marker=marker)
+        all_roles.extend(roles_response["Roles"])
+        marker = roles_response.get("Marker", "")
+    total_roles = len(all_roles)
+    logger.info(f"found {total_roles} roles")
+    roles_parsed: list[IAMRole] = []
+    delete_if_created_before = utc_now() - timedelta(days=5)
+    delete_count = 0
+    role_names: list[str] = []
+    for role in all_roles:
+        parsed = IAMRole.model_validate(role)
+        roles_parsed.append(parsed)
+        role_name = parsed.role_name
+        role_names.append(role_name)
+        if not role_name.startswith(iam_role_prefix_name):
+            continue
+        # want to delete 'mongodb-atlas-test-acc-tf-1345851232260229574'
+        # want to keep  'mongodb-atlas-test-acc-tf-7973337217371171538-git-ear'?
+        if role_name[-1].isdigit() and parsed.create_date < delete_if_created_before:
+            logger.info(f"role: {parsed.arn} will be deleted")
+            delete_count += 1
+            delete_role(client, role_name)
+        else:
+            logger.info(f"skipping role: {parsed.arn}, created: {humanize.naturaltime(parsed.create_date)}")
+    logger.info(f"deleted {delete_count}/{total_roles} roles")
+    out_path = Path(f"aws_roles_{aws_account_id}.txt")
+    out_path.write_text("\n".join(sorted(role_names)))
+
+
+def delete_role(client: IAMClient, role_name: str):
+    try:
+        attached_policies = client.list_attached_role_policies(RoleName=role_name)
+        for policy in attached_policies["AttachedPolicies"]:
+            policy_arn = policy.get("PolicyArn")
+            if policy_arn:
+                logger.info(f"detaching managed policy: {policy_arn} from role: {role_name}")
+                client.detach_role_policy(RoleName=role_name, PolicyArn=policy_arn)
+    except Exception as e:
+        logger.warning(f"failed to detach managed policies from role {role_name}: {e}")
+
+    # Second, delete all inline policies
+    try:
+        inline_policies = client.list_role_policies(RoleName=role_name)
+        for policy_name in inline_policies["PolicyNames"]:
+            logger.info(f"deleting inline policy: {policy_name} from role: {role_name}")
+            client.delete_role_policy(RoleName=role_name, PolicyName=policy_name)
+    except Exception as e:
+        logger.warning(f"failed to delete inline policies from role {role_name}: {e}")
+
+    # Finally, delete the role
+    try:
+        client.delete_role(RoleName=role_name)
+        logger.info(f"deleted role: {role_name}")
+    except Exception as e:
+        logger.error(f"failed to delete role {role_name}: {e}")

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/cli_root/trigger.py

@@ -30,7 +30,7 @@ def create_realm_app():
     project_id = atlas_settings.MONGODB_ATLAS_PROJECT_ID
     base_url = atlas_settings.realm_url
     cluster_name = cluster_settings.MONGODB_ATLAS_CLUSTER_NAME
-    auth_headers = login_to_realm(settings, base_url)
+    auth_headers = login_to_realm(atlas_settings, base_url)
     realm_settings = env_vars_cls_or_none(RealmSettings, dotenv_path=settings.env_vars_trigger)
     if realm_settings and function_exists(
         base_url,

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/cli_tf/hcl/modifier2.py

@@ -2,10 +2,12 @@ from collections import defaultdict
 import logging
 from contextlib import suppress
 from pathlib import Path
-from typing import NamedTuple
+from typing import Any, NamedTuple
 from lark import Token, Transformer, Tree, UnexpectedToken, v_args
 from hcl2.transformer import Attribute, DictTransformer
 from hcl2.api import reverse_transform, writes, parses
+from model_lib import Entity
+from pydantic import field_validator
 import rich
 
 logger = logging.getLogger(__name__)
@@ -58,6 +60,54 @@ def attribute_transfomer(attr_name: str, obj_key: str, new_value: str) -> tuple[
     return AttributeTransformer(with_meta=True), changes
 
 
+_unset = object()
+
+
+class TFVar(Entity):
+    name: str
+    description: str | None = ""
+    default: Any = _unset
+    type: str = ""
+    sensitive: bool = False
+
+    @field_validator("default", mode="before")
+    def unpack_token(cls, v: Any) -> Any:
+        if isinstance(v, Token):
+            return v.value.strip('"')
+        return v
+
+
+def variable_reader_typed(tree: Tree) -> dict[str, TFVar]:
+    variables: dict[str, TFVar] = {}
+
+    class TFVarReader(DictTransformer):
+        def __init__(self, with_meta: bool = False, *, name: str):
+            super().__init__(with_meta)
+            self.kwargs: dict[str, Any] = {
+                "name": name,
+            }
+
+        def attribute(self, args: list) -> Attribute:
+            if len(args) == 3:
+                name, _, value = args
+                self.kwargs[name] = value
+            return super().attribute(args)
+
+    class BlockReader(Transformer):
+        @v_args(tree=True)
+        def block(self, block_tree: Tree) -> Tree:
+            current_block_name = _identifier_name(block_tree)
+            if current_block_name == "variable":
+                variable_name = token_name(block_tree.children[1])
+                reader = TFVarReader(name=variable_name)
+                reader.transform(block_tree)
+                variables[variable_name] = TFVar(**reader.kwargs)
+            return block_tree
+
+    BlockReader().transform(tree)
+    return variables
+
+
 def variable_reader(tree: Tree) -> dict[str, str | None]:
     """
     Reads the variable names from a parsed HCL2 tree.

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/settings/rich_utils.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Literal
 
+from rich.console import Console
+from rich.tree import Tree
 import typer
 from pydantic import BaseModel
 from rich.logging import RichHandler
@@ -62,3 +64,23 @@ def configure_logging(
         app.pretty_exceptions_show_locals = False
 
     return handler
+
+
+# https://github.com/Textualize/rich/blob/8c4d3d1d50047e3aaa4140d0ffc1e0c9f1df5af4/tests/test_live.py#L11
+def create_capture_console(*, width: int = 60, height: int = 80, force_terminal: bool = True) -> Console:
+    return Console(
+        width=width,
+        height=height,
+        force_terminal=force_terminal,
+        legacy_windows=False,
+        color_system=None,  # use no color system to reduce complexity of output,
+        _environ={},
+    )
+
+
+def tree_text(tree: Tree) -> str:
+    console = create_capture_console()
+    console.width = 10_000
+    console.begin_capture()
+    console.print(tree)
+    return "\n".join(line.rstrip() for line in console.end_capture().splitlines())

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/tf_ext/gen_examples.py

@@ -19,10 +19,12 @@ def _examples_casted(examples: dict) -> dict[str, ResourceAbs]:
     return examples
 
 
-def read_example_dirs(module_path: Path) -> list[Path]:
+def read_example_dirs(examples_dir: Path) -> list[Path]:
+    if not examples_dir.exists():
+        return []
     return sorted(
         example_dir
-        for example_dir in (module_path / "examples").glob("*")
+        for example_dir in examples_dir.glob("*")
         if example_dir.is_dir()
         and len(example_dir.name) > 2
         and example_dir.name[:2].isdigit()

atlas_init-0.8.1/atlas_init/tf_ext/gen_module_readme.py → atlas_init-0.10.0/atlas_init/tf_ext/gen_readme.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
 import logging
 from enum import StrEnum
+from pathlib import Path
+from typing import Callable, TypeAlias
 
 from ask_shell import run_and_wait
 from zero_3rdparty.file_utils import ensure_parents_write_text, update_between_markers
 
 from atlas_init.tf_ext.gen_examples import read_example_dirs
-from atlas_init.tf_ext.models_module import ModuleGenConfig
+from atlas_init.tf_ext.models_module import EXAMPLES_DIRNAME, README_FILENAME, TERRAFORM_DOCS_CONFIG_FILENAME
 
 logger = logging.getLogger(__name__)
 _readme_disclaimer = """\
@@ -13,12 +16,13 @@ _readme_disclaimer = """\
 This Module is not meant for external consumption.
 It is part of a development PoC.
 Any usage problems will not be supported.
-However, if you have any ideas or feedback feel free to open a Github Issue!
+However, if you have any ideas or feedback, feel free to open a Github Issue!
 """
 
 
 class ReadmeMarkers(StrEnum):
     DISCLAIMER = "DISCLAIMER"
+    MODULES = "MODULES"
     EXAMPLE = "TF_EXAMPLES"
     TF_DOCS = "TF_DOCS"
 
@@ -42,9 +46,23 @@ class ReadmeMarkers(StrEnum):
     def example_boilerplate(cls) -> str:
         return "\n".join(cls.marker_lines(marker_name) for marker_name in list(cls))
 
+    @classmethod
+    def readme_generators(cls) -> ReadmeGenerators:
+        return [
+            (cls.DISCLAIMER, lambda _: _readme_disclaimer),
+            (cls.EXAMPLE, lambda workspace: read_examples(workspace / EXAMPLES_DIRNAME)),
+        ]
+
+
+ReadmeGenerators: TypeAlias = list[tuple[ReadmeMarkers, Callable[[Path], str]]]
 
-def read_examples(module: ModuleGenConfig) -> str:
-    example_dirs = read_example_dirs(module.module_out_path)
+
+def read_examples(examples_dir: Path) -> str:
+    example_dirs = read_example_dirs(examples_dir)
+    if not example_dirs:
+        return ""
+    # ensure the examples are formatted first
+    run_and_wait("terraform fmt -recursive .", cwd=examples_dir.parent, allow_non_zero_exit=True, ansi_content=False)
     content = ["# Examples"]
     for example_dir in example_dirs:
         example_name = example_dir.name
@@ -80,10 +98,10 @@ sort:
 """
 
 
-def terraform_docs_config_content(module: ModuleGenConfig) -> str:
+def terraform_docs_config_content(readme_path: Path) -> str:
     config = _static_terraform_config
     for replacement_in, replacement_out in [
-        ("FILENAME", module.readme_path().name),
+        ("FILENAME", readme_path.name),
         ("START_MARKER", ReadmeMarkers.as_start(ReadmeMarkers.TF_DOCS)),
         ("END_MARKER", ReadmeMarkers.as_end(ReadmeMarkers.TF_DOCS)),
     ]:
@@ -91,36 +109,38 @@ def terraform_docs_config_content(module: ModuleGenConfig) -> str:
     return config
 
 
-def generate_readme(module: ModuleGenConfig) -> str:
-    readme_path = module.readme_path()
+def generate_and_write_readme(terraform_workdir: Path, *, generators: ReadmeGenerators | None = None) -> str:
+    generators = generators or ReadmeMarkers.readme_generators()
+    readme_path = terraform_workdir / README_FILENAME
     assert readme_path.exists(), (
         f"{readme_path} does not exist, currently a boilerplate is expected, consider adding to {readme_path}\n{ReadmeMarkers.example_boilerplate()}"
     )
-    update_between_markers(
-        readme_path,
-        _readme_disclaimer,
-        ReadmeMarkers.as_start(ReadmeMarkers.DISCLAIMER),
-        ReadmeMarkers.as_end(ReadmeMarkers.DISCLAIMER),
-    )
-    run_and_wait("terraform fmt -recursive .", cwd=module.module_out_path, allow_non_zero_exit=True, ansi_content=False)
-    example_section = read_examples(module)
-    update_between_markers(
-        readme_path,
-        example_section,
-        ReadmeMarkers.as_start(ReadmeMarkers.EXAMPLE),
-        ReadmeMarkers.as_end(ReadmeMarkers.EXAMPLE),
-    )
-    docs_config_path = module.terraform_docs_config_path()
+    for marker, generator in generators:
+        content = generator(terraform_workdir)
+        if not content:
+            continue
+        update_between_markers(
+            readme_path,
+            content,
+            ReadmeMarkers.as_start(marker),
+            ReadmeMarkers.as_end(marker),
+        )
+    generate_terraform_docs(readme_path)
+    logger.info(f"updated {readme_path}")
+    return readme_path.read_text()
+
+
+def generate_terraform_docs(readme_path: Path) -> None:
+    docs_config_path = readme_path.parent / TERRAFORM_DOCS_CONFIG_FILENAME
     if docs_config_path.exists():
         logger.warning(f"{docs_config_path} already exists, skipping generation")
     else:
-        config_content = terraform_docs_config_content(module)
+        config_content = terraform_docs_config_content(readme_path)
         ensure_parents_write_text(docs_config_path, config_content)
         logger.info(f"generated {docs_config_path}")
-    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=module.module_out_path)
+    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=readme_path.parent)
     readme_content = _default_link_updater(readme_path.read_text())
     ensure_parents_write_text(readme_path, readme_content)
-    return readme_path.read_text()
 
 
 def _default_link_updater(readme_content: str) -> str:  # can be a global replacer for now

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/tf_ext/models.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from typing import Self
+from dataclasses import dataclass, field
+from typing import Iterable, Self
 
 from model_lib import Entity
 from pydantic import Field, RootModel, model_validator
@@ -30,6 +31,31 @@ def choose_next_emoji() -> str:
     return emoji
 
 
+@dataclass
+class EmojiCounter:
+    counter: int = 0
+    existing_emojis: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def emoji_to_names(self) -> dict[str, str]:
+        return dict(zip(self.existing_emojis.values(), self.existing_emojis.keys()))
+
+    def emoji_name(self) -> Iterable[tuple[str, str]]:
+        src = self.emoji_to_names
+        for emoji in _emojii_list:
+            if emoji in src:
+                yield emoji, src[emoji]
+            else:
+                break
+
+    def get_emoji(self, name: str) -> str:
+        if existing := self.existing_emojis.get(name):
+            return existing
+        emoji = self.existing_emojis[name] = _emojii_list[self.counter]
+        self.counter += 1
+        return emoji
+
+
 class ModuleState(Entity):
     resource_types: set[str] = Field(default_factory=set, description="Set of resource types in the module.")
 

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/tf_ext/models_module.py

@@ -24,6 +24,9 @@ from atlas_init.tf_ext.py_gen import (
 from atlas_init.tf_ext.settings import RepoOut, TfExtSettings
 
 ResourceTypeT: TypeAlias = str
+TERRAFORM_DOCS_CONFIG_FILENAME: str = ".terraform-docs.yml"
+README_FILENAME: str = "README.md"
+EXAMPLES_DIRNAME: str = "examples"
 
 
 @dataclass
@@ -253,12 +256,13 @@ class ModuleGenConfig(Entity):
                 return resource_type
         raise ValueError(f"Could not resolve resource type for path {path}")
 
+    @property
     def readme_path(self) -> Path:
-        return self.module_out_path / "README.md"
+        return self.module_out_path / README_FILENAME
 
     @property
     def examples_path(self) -> Path:
-        return self.module_out_path / "examples"
+        return self.module_out_path / EXAMPLES_DIRNAME
 
     def example_name(self, name: str, example_nr: int) -> str:
         return f"{example_nr:02d}_{name}"
@@ -267,7 +271,7 @@ class ModuleGenConfig(Entity):
         return self.examples_path / name
 
     def terraform_docs_config_path(self) -> Path:
-        return self.module_out_path / ".terraform-docs.yml"
+        return self.module_out_path / TERRAFORM_DOCS_CONFIG_FILENAME
 
 
 @dataclass

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/tf_ext/paths.py

@@ -9,7 +9,14 @@ from model_lib import Entity
 from pydantic import Field, RootModel
 from zero_3rdparty.file_utils import iter_paths
 
-from atlas_init.cli_tf.hcl.modifier2 import resource_types_vars_usage, safe_parse, variable_reader, variable_usages
+from atlas_init.cli_tf.hcl.modifier2 import (
+    TFVar,
+    resource_types_vars_usage,
+    safe_parse,
+    variable_reader,
+    variable_reader_typed,
+    variable_usages,
+)
 from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME, DEFAULT_EXTERNAL_SUBSTRINGS, DEFAULT_INTERNAL_SUBSTRINGS
 
 logger = logging.getLogger(__name__)
@@ -32,6 +39,16 @@ def get_example_directories(repo_path: Path, skip_names: list[str]):
     return example_dirs
 
 
+def find_variables_typed(variables_tf: Path) -> dict[str, TFVar]:
+    if not variables_tf.exists():
+        return {}
+    tree = safe_parse(variables_tf)
+    if not tree:
+        logger.warning(f"Failed to parse {variables_tf}")
+        return {}
+    return variable_reader_typed(tree)
+
+
 def find_variables(variables_tf: Path) -> dict[str, str | None]:
     if not variables_tf.exists():
         return {}

atlas_init-0.10.0/atlas_init/tf_ext/run_tf.py

@@ -0,0 +1,20 @@
+from ask_shell import new_task, run_and_wait
+from pathlib import Path
+
+
+def validate_tf_workspace(
+    tf_workdir: Path, *, tf_cli_config_file: Path | None = None, env_extra: dict[str, str] | None = None
+):
+    terraform_commands = [
+        "terraform init",
+        "terraform fmt .",
+        "terraform validate .",
+    ]
+    env_extra = env_extra or {}
+    if tf_cli_config_file:
+        env_extra["TF_CLI_CONFIG_FILE"] = str(tf_cli_config_file)
+    with new_task("Terraform Module Validate Checks", total=len(terraform_commands)) as task:
+        for command in terraform_commands:
+            attempts = 3 if command == "terraform init" else 1  # terraform init can fail due to network issues
+            run_and_wait(command, cwd=tf_workdir, env=env_extra, attempts=attempts)
+            task.update(advance=1)

{atlas_init-0.8.1 → atlas_init-0.10.0}/atlas_init/tf_ext/settings.py

@@ -175,6 +175,14 @@ class TfExtSettings(StaticSettings):
     def provider_cache_dir(self, provider_name: str) -> Path:
         return self.cache_root / "provider_cache" / provider_name
 
+    @property
+    def variable_plan_resolvers_file_path(self) -> Path:
+        return self.static_root / "variable_plan_resolvers.yaml"
+
+    @property
+    def variable_plan_resolvers_dumped_file_path(self) -> Path:
+        return self.static_root / "variable_plan_resolvers_dumped.yaml"
+
 
 def init_tf_ext_settings(*, allow_empty_out_path: bool = False) -> TfExtSettings:
     settings = TfExtSettings.from_env()