atlas-init 0.7.0__tar.gz → 0.8.1__tar.gz

{atlas_init-0.7.0 → atlas_init-0.8.1}/.gitignore

@@ -26,4 +26,6 @@ demo/
 tf_log/
 downloads
 htmlcov
-tf_dep/
+tf_dep/
+static
+cache

{atlas_init-0.7.0 → atlas_init-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-init
-Version: 0.7.0
+Version: 0.8.1
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init
@@ -12,10 +12,11 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Requires-Dist: appdirs==1.4.4
-Requires-Dist: ask-shell>=0.0.4
+Requires-Dist: ask-shell>=0.0.5
 Requires-Dist: boto3==1.35.92
 Requires-Dist: gitpython==3.1.42
 Requires-Dist: humanize==4.9.0
+Requires-Dist: inflection==0.5.1
 Requires-Dist: model-lib
 Requires-Dist: motor==3.7.1
 Requires-Dist: mypy-boto3-cloudformation==1.37.22
@@ -30,7 +31,7 @@ Requires-Dist: rich==14.0.0
 Requires-Dist: stringcase==1.2.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: typer>=0.15.1
-Requires-Dist: zero-3rdparty
+Requires-Dist: zero-3rdparty==1.0.0b5
 Description-Content-Type: text/markdown
 
 # Atlas Init - A CLI for developing integrations with MongoDB Atlas

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.7.0"
+VERSION = "0.8.1"
 
 
 def running_in_repo() -> bool:

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/atlas_init.yaml

@@ -94,6 +94,7 @@ test_suites:
   vars:
     cluster_info_m10: true
     stream_instance: true
+    use_aws_vpc: true
 - name: trigger
   repo_go_packages:
     cfn:

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/cli_tf/example_update.py

@@ -2,26 +2,30 @@ import logging
 from collections import defaultdict
 from functools import total_ordering
 from pathlib import Path
+from typing import Callable
 
 import typer
 from model_lib import Entity, Event, dump, parse_payload
-from pydantic import Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from atlas_init.cli_helper.run import run_binary_command_is_ok
 from atlas_init.cli_tf.hcl.modifier import (
     BLOCK_TYPE_OUTPUT,
     BLOCK_TYPE_VARIABLE,
+    NewDescription,
     update_descriptions,
 )
 
 logger = logging.getLogger(__name__)
 
 
-class UpdateExamples(Entity):
+class UpdateExamples(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
     examples_base_dir: Path
-    var_descriptions: dict[str, str]
+    var_descriptions: dict[str, str] = Field(default_factory=dict)
     output_descriptions: dict[str, str] = Field(default_factory=dict)
     skip_tf_fmt: bool = False
+    new_description_call: Callable[[str, str, Path], str] | None = None  # Protocol not supported for Pydantic
 
 
 @total_ordering
@@ -50,16 +54,24 @@ class UpdateExamplesOutput(Entity):
 
 def update_examples(event_in: UpdateExamples) -> UpdateExamplesOutput:
     changes = []
+
+    def get_description(name: str, old_description: str, path: Path) -> str:
+        return event_in.var_descriptions.get(name, "")
+
     existing_var_descriptions = update_block_descriptions(
         event_in.examples_base_dir,
         changes,
-        event_in.var_descriptions,
+        event_in.new_description_call or get_description,  # type: ignore
         BLOCK_TYPE_VARIABLE,
     )
+
+    def get_output_description(name: str, old_description: str, path: Path) -> str:
+        return event_in.output_descriptions.get(name, "")
+
     existing_output_descriptions = update_block_descriptions(
         event_in.examples_base_dir,
         changes,
-        event_in.output_descriptions,
+        event_in.new_description_call or get_output_description,  # type: ignore
         BLOCK_TYPE_OUTPUT,
     )
     if event_in.skip_tf_fmt:
@@ -85,14 +97,14 @@ def flatten_descriptions(descriptions: dict[str, list[str]]) -> dict[str, str]:
 def update_block_descriptions(
     base_dir: Path,
     changes: list[TFConfigDescriptionChange],
-    new_names: dict[str, str],
+    get_description: NewDescription,
     block_type: str,
 ):
     all_existing_descriptions = defaultdict(list)
     in_files = sorted(base_dir.rglob("*.tf"))
     for tf_file in in_files:
         logger.info(f"looking for {block_type} in {tf_file}")
-        new_tf, existing_descriptions = update_descriptions(tf_file, new_names, block_type=block_type)
+        new_tf, existing_descriptions = update_descriptions(tf_file, get_description, block_type=block_type)
         if not existing_descriptions:  # probably no variables in the file
             continue
         for name, descriptions in existing_descriptions.items():
@@ -101,7 +113,7 @@ def update_block_descriptions(
                     path=tf_file,
                     name=name,
                     before=description,
-                    after=new_names.get(name, ""),
+                    after=get_description(name, description, tf_file),
                     block_type=block_type,
                 )
                 for description in descriptions

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/cli_tf/hcl/modifier.py

@@ -1,8 +1,9 @@
+from __future__ import annotations
 import logging
 from collections import defaultdict
 from copy import deepcopy
 from pathlib import Path
-from typing import Callable
+from typing import Callable, Protocol
 
 import hcl2
 from lark import Token, Tree
@@ -40,14 +41,16 @@ def is_block_type(tree: Tree, block_type: str) -> bool:
         return False
 
 
-def update_description(tree: Tree, new_descriptions: dict[str, str], existing_names: dict[str, list[str]]) -> Tree:
+def update_description(
+    path: Path, tree: Tree, get_new_description: NewDescription, existing_names: dict[str, list[str]]
+) -> Tree:
     new_children = tree.children.copy()
     variable_body = new_children[2]
     assert variable_body.data == "body"
     name = token_name(new_children[1])
     old_description = read_description_attribute(variable_body)
     existing_names[name].append(old_description)
-    new_description = new_descriptions.get(name, "")
+    new_description = get_new_description(name, old_description, path)
     if not new_description:
         debug_log(f"no description found for variable {name}", 0)
         return tree
@@ -60,6 +63,8 @@ def token_name(token: Token | Tree) -> str:
         return token.value.strip('"')
     if isinstance(token, Tree) and token.data == "identifier":
         return token.children[0].value.strip('"')  # type: ignore
+    if isinstance(token, Tree) and isinstance(token.data, Token) and token.data.value == "heredoc_template_trim":
+        return token.children[0].value.strip('"')  # type: ignore
     err_msg = f"unexpected token type {type(token)} for token name"
     raise ValueError(err_msg)
 
@@ -103,10 +108,11 @@ def read_description_attribute(tree: Tree) -> str:
 
 
 def create_description_attribute(description_value: str) -> Tree:
+    token_value = f"<<-EOT\n{description_value}\nEOT\n" if "\n" in description_value else f'"{description_value}"'
     children = [
         Tree(Token("RULE", "identifier"), [Token("NAME", "description")]),
         Token("EQ", " ="),
-        Tree(Token("RULE", "expr_term"), [Token("STRING_LIT", f'"{description_value}"')]),
+        Tree(Token("RULE", "expr_term"), [Token("STRING_LIT", token_value)]),
     ]
     return Tree(Token("RULE", "attribute"), children)
 
@@ -129,9 +135,14 @@ def process_generic(
     return Tree(node.data, new_children)
 
 
+class NewDescription(Protocol):
+    def __call__(self, name: str, old_description: str, path: Path) -> str: ...
+
+
 def process_descriptions(
+    path: Path,
     node: Tree,
-    name_updates: dict[str, str],
+    new_description: NewDescription,
     existing_names: dict[str, list[str]],
     depth=0,
     *,
@@ -141,7 +152,7 @@ def process_descriptions(
         return is_block_type(tree, block_type)
 
     def tree_call(tree: Tree) -> Tree:
-        return update_description(tree, name_updates, existing_names)
+        return update_description(path, tree, new_description, existing_names)
 
     return process_generic(
         node,
@@ -151,14 +162,17 @@ def process_descriptions(
     )
 
 
-def update_descriptions(tf_path: Path, new_names: dict[str, str], block_type: str) -> tuple[str, dict[str, list[str]]]:
+def update_descriptions(
+    tf_path: Path, new_description: NewDescription, block_type: str
+) -> tuple[str, dict[str, list[str]]]:
     tree = safe_parse(tf_path)
     if tree is None:
         return "", {}
     existing_descriptions = defaultdict(list)
     new_tree = process_descriptions(
+        tf_path,
         tree,
-        new_names,
+        new_description,
         existing_descriptions,
         block_type=block_type,
     )

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/settings/env_vars.py

@@ -243,7 +243,9 @@ def detect_ambiguous_env_vars(manual_env_vars: dict[str, str]) -> list[str]:
 
 def find_missing_env_vars(required_env_vars: list[str], manual_env_vars: dict[str, str]) -> list[str]:
     return sorted(
-        env_name for env_name in required_env_vars if read_from_env(env_name) == "" and env_name not in manual_env_vars
+        env_name
+        for env_name in required_env_vars
+        if read_from_env(env_name) == "" and env_name not in manual_env_vars and env_name
     )
 
 
@@ -255,7 +257,15 @@ def init_settings(
     profile_env_vars = settings.manual_env_vars
     vscode_env_vars = settings.env_vars_vs_code
     if vscode_env_vars.exists():
-        profile_env_vars |= load_dotenv(vscode_env_vars)
+        skip_generated_vars: set[str] = set()
+        if "AWS_PROFILE" in profile_env_vars:
+            skip_generated_vars |= {
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+            }  # avoid generated env-vars overwriting AWS PROFILE
+        profile_env_vars |= {
+            key: value for key, value in load_dotenv(vscode_env_vars).items() if key not in skip_generated_vars
+        }
     required_env_vars = collect_required_env_vars(list(settings_classes))
     ambiguous = [] if skip_ambiguous_check else detect_ambiguous_env_vars(profile_env_vars)
     missing_env_vars = find_missing_env_vars(required_env_vars, profile_env_vars)

{atlas_init-0.7.0 → atlas_init-0.8.1}/atlas_init/tf_ext/api_call.py

@@ -1,18 +1,18 @@
-from collections import defaultdict
-from concurrent.futures import Future, as_completed
-from functools import lru_cache
 import json
 import logging
 import os
+from collections import defaultdict
+from concurrent.futures import Future, as_completed
+from functools import lru_cache
 from pathlib import Path
 
+import requests
+import typer
 from ask_shell import new_task, print_to_live, run_pool
 from model_lib import dump, parse_model
 from pydantic import BaseModel, Field, model_validator
-import requests
-from rich.markdown import Markdown
-import typer
 from requests.auth import HTTPDigestAuth
+from rich.markdown import Markdown
 from zero_3rdparty.file_utils import ensure_parents_write_text
 from zero_3rdparty.str_utils import ensure_prefix, ensure_suffix, instance_repr
 
@@ -27,7 +27,7 @@ from atlas_init.settings.env_vars_modules import (
     TFModuleStream_Instance,
 )
 from atlas_init.settings.path import load_dotenv
-from atlas_init.tf_ext.settings import TfDepSettings
+from atlas_init.tf_ext.settings import TfExtSettings
 
 logger = logging.getLogger(__name__)
 
@@ -259,7 +259,7 @@ def api_config(
     md_content = "\n".join(md_report)
     md = Markdown(md_content)
     print_to_live(md)
-    output_path = TfDepSettings.from_env().pagination_output_path(query_args_str)
+    output_path = TfExtSettings.from_env().pagination_output_path(query_args_str)
     ensure_parents_write_text(output_path, md_content)
     logger.info(f"Pagination report saved to {output_path}")
     return md
@@ -288,7 +288,7 @@ def api(
 
 
 def dump_config_path(query_args: dict[str, str]) -> Path:
-    settings = TfDepSettings.from_env()
+    settings = TfExtSettings.from_env()
     latest_api_spec = resolve_admin_api_path()
     model = parse_model(latest_api_spec, t=OpenapiSchema)
     paginated_paths: list[ApiCall] = []

atlas_init-0.8.1/atlas_init/tf_ext/args.py

@@ -0,0 +1,32 @@
+import typer
+
+
+def default_skippped_directories() -> list[str]:
+    return [
+        "prometheus-and-teams",  #  Provider registry.terraform.io/hashicorp/template v2.2.0 does not have a package available for your current platform, darwin_arm64.
+    ]
+
+
+REPO_PATH_ATLAS_ARG = typer.Argument(..., help="Path to the mongodbatlas-terraform-provider repository")
+SKIP_EXAMPLES_DIRS_OPTION = typer.Option(
+    ...,
+    "--skip-examples",
+    help="Skip example directories with these names",
+    default_factory=default_skippped_directories,
+    show_default=True,
+)
+TF_CLI_CONFIG_FILE_ENV_NAME = "TF_CLI_CONFIG_FILE"
+TF_CLI_CONFIG_FILE_ARG = typer.Option(
+    "",
+    "-tf-cli",
+    "--tf-cli-config-file",
+    envvar=TF_CLI_CONFIG_FILE_ENV_NAME,
+    help="Terraform CLI config file",
+)
+ENV_NAME_REPO_PATH_ATLAS_PROVIDER = "REPO_PATH_ATLAS_PROVIDER"
+TF_REPO_PATH_ATLAS = typer.Option(
+    "",
+    "--tf-repo-path-atlas",
+    help="Path to the mongodbatlas-terraform-provider repository",
+    envvar=ENV_NAME_REPO_PATH_ATLAS_PROVIDER,
+)

atlas_init-0.8.1/atlas_init/tf_ext/gen_examples.py

@@ -0,0 +1,141 @@
+from contextlib import suppress
+from dataclasses import asdict
+from functools import singledispatch
+from pathlib import Path
+
+from zero_3rdparty import humps
+from zero_3rdparty.file_utils import clean_dir, ensure_parents_write_text
+
+from atlas_init.tf_ext.gen_resource_variables import generate_resource_variables
+from atlas_init.tf_ext.gen_versions import dump_versions_tf
+from atlas_init.tf_ext.models_module import ModuleGenConfig, ResourceAbs, ResourceGenConfig, ResourceTypePythonModule
+from atlas_init.tf_ext.py_gen import import_module_by_using_parents
+
+VARIABLE_PLACEHOLDER = "var."
+INDENT = "    "
+
+
+def _examples_casted(examples: dict) -> dict[str, ResourceAbs]:
+    return examples
+
+
+def read_example_dirs(module_path: Path) -> list[Path]:
+    return sorted(
+        example_dir
+        for example_dir in (module_path / "examples").glob("*")
+        if example_dir.is_dir()
+        and len(example_dir.name) > 2
+        and example_dir.name[:2].isdigit()
+        and (example_dir / "main.tf").exists()
+    )
+
+
+def generate_module_examples(
+    config: ModuleGenConfig,
+    module: ResourceTypePythonModule,
+    resource_type: str,
+    *,
+    skip_clean_dir: bool = False,
+) -> list[Path]:
+    test_path = config.examples_test_path
+    imported_module = import_module_by_using_parents(test_path)
+    examples = getattr(imported_module, "EXAMPLES")
+    assert isinstance(examples, dict), f"{imported_module} does not have an EXAMPLES attribute"
+    examples_parsed = _examples_casted(examples)
+    examples_generated: list[Path] = []
+    for example_name, example in examples_parsed.items():
+        dumped_resource = {k: v for k, v in asdict(example).items() if v is not None}
+        variables = {
+            k: f"{v}{k}"
+            for k, v in dumped_resource.items()
+            if isinstance(v, str) and v.startswith(VARIABLE_PLACEHOLDER)
+        }
+        dumped_resource |= variables
+        variable_names = set(variables.keys())
+        ignored_names = set(module.all_field_names) - variable_names
+        ignored_names |= module.all_skip_variables
+        resource_cls = module.resource_ext or module.resource
+        assert resource_cls, f"{module} does not have a resource class"
+        example_path = config.example_path(example_name)
+        if not skip_clean_dir and example_path.exists():
+            clean_dir(example_path)
+
+        variables_tf = generate_resource_variables(
+            resource_cls,
+            ResourceGenConfig(
+                name=resource_type, skip_variables_extra=ignored_names, required_variables=variable_names
+            ),
+        )
+        ensure_parents_write_text(example_path / "variables.tf", variables_tf)
+        variables_str = "\n".join(f"{k} = {dump_variable(v)}" for k, v in dumped_resource.items() if can_dump(v))
+        example_main = example_main_tf(config, variables_str)
+        ensure_parents_write_text(example_path / "main.tf", example_main)
+        dump_versions_tf(example_path, skip_python=config.skip_python, minimal=True)
+        examples_generated.append(example_path)
+    return examples_generated
+
+
+class NotDumpableError(Exception):
+    def __init__(self, value: object) -> None:
+        super().__init__(f"Cannot dump variable {value!r}")
+
+
+def can_dump(variable: object) -> bool:
+    with suppress(NotDumpableError):
+        dump_variable(variable)
+        return True
+    return False
+
+
+@singledispatch
+def dump_variable(variable: object) -> str:
+    raise NotDumpableError(variable)
+
+
+@dump_variable.register
+def dump_variable_str(variable: str) -> str:
+    if "." in variable:
+        return variable
+    return f'"{variable}"'
+
+
+@dump_variable.register
+def dump_variable_int(variable: int) -> str:
+    return str(variable)
+
+
+@dump_variable.register
+def dump_variable_float(variable: float) -> str:
+    return str(variable)
+
+
+@dump_variable.register
+def dump_variable_bool(variable: bool) -> str:
+    return "true" if variable else "false"
+
+
+@dump_variable.register
+def dump_variable_list(variable: list) -> str:
+    return f"[\n{', '.join(f'{INDENT}{dump_variable(nested)}' for nested in variable if can_dump(nested))}\n]"
+
+
+@dump_variable.register
+def dump_variable_set(variable: set) -> str:
+    return f"[\n{', '.join(f'{INDENT}{dump_variable(nested)}' for nested in variable if can_dump(nested))}\n]"
+
+
+@dump_variable.register
+def dump_variable_dict(variable: dict) -> str:
+    return "{\n" + "\n".join(f"{INDENT}{k} = {dump_variable(v)}" for k, v in variable.items() if can_dump(v)) + "\n}"
+
+
+def example_main_tf(module: ModuleGenConfig, variables: str) -> str:
+    variables_indented = "\n".join(f"{INDENT}{var}" for var in variables.split("\n"))
+    module_name_snake = humps.dekebabize(module.name)
+    return f"""\
+module "{module_name_snake}" {{
+    source = "../.."
+
+{variables_indented}
+}}
+"""

atlas_init-0.8.1/atlas_init/tf_ext/gen_module_readme.py

@@ -0,0 +1,131 @@
+import logging
+from enum import StrEnum
+
+from ask_shell import run_and_wait
+from zero_3rdparty.file_utils import ensure_parents_write_text, update_between_markers
+
+from atlas_init.tf_ext.gen_examples import read_example_dirs
+from atlas_init.tf_ext.models_module import ModuleGenConfig
+
+logger = logging.getLogger(__name__)
+_readme_disclaimer = """\
+## Disclaimer
+This Module is not meant for external consumption.
+It is part of a development PoC.
+Any usage problems will not be supported.
+However, if you have any ideas or feedback feel free to open a Github Issue!
+"""
+
+
+class ReadmeMarkers(StrEnum):
+    DISCLAIMER = "DISCLAIMER"
+    EXAMPLE = "TF_EXAMPLES"
+    TF_DOCS = "TF_DOCS"
+
+    @classmethod
+    def as_start(cls, marker_name: str) -> str:
+        return f"<!-- BEGIN_{marker_name} -->"
+
+    @classmethod
+    def as_end(cls, marker_name: str) -> str:
+        return f"<!-- END_{marker_name} -->"
+
+    @classmethod
+    def marker_lines(cls, marker_name: str) -> str:
+        return f"""\
+{cls.as_start(marker_name)}
+
+{cls.as_end(marker_name)}
+"""
+
+    @classmethod
+    def example_boilerplate(cls) -> str:
+        return "\n".join(cls.marker_lines(marker_name) for marker_name in list(cls))
+
+
+def read_examples(module: ModuleGenConfig) -> str:
+    example_dirs = read_example_dirs(module.module_out_path)
+    content = ["# Examples"]
+    for example_dir in example_dirs:
+        example_name = example_dir.name
+        header_name = example_name.replace("_", " ").replace("-", " ").title()
+        main_path = example_dir / "main.tf"
+        assert main_path.exists(), f"{main_path} does not exist, every example must have a main.tf"
+        content.extend(
+            [
+                f"## [{header_name}](./examples/{example_name})",
+                "",
+                "```terraform",
+                main_path.read_text(),
+                "```",
+                "",
+                "",
+            ]
+        )
+    return "\n".join(content)
+
+
+_static_terraform_config = """\
+formatter: markdown document
+output:
+  file: "FILENAME"
+  mode: inject
+  template: |-
+    START_MARKER
+    {{ .Content }}
+    END_MARKER
+sort:
+  enabled: true
+  by: required
+"""
+
+
+def terraform_docs_config_content(module: ModuleGenConfig) -> str:
+    config = _static_terraform_config
+    for replacement_in, replacement_out in [
+        ("FILENAME", module.readme_path().name),
+        ("START_MARKER", ReadmeMarkers.as_start(ReadmeMarkers.TF_DOCS)),
+        ("END_MARKER", ReadmeMarkers.as_end(ReadmeMarkers.TF_DOCS)),
+    ]:
+        config = config.replace(replacement_in, replacement_out)
+    return config
+
+
+def generate_readme(module: ModuleGenConfig) -> str:
+    readme_path = module.readme_path()
+    assert readme_path.exists(), (
+        f"{readme_path} does not exist, currently a boilerplate is expected, consider adding to {readme_path}\n{ReadmeMarkers.example_boilerplate()}"
+    )
+    update_between_markers(
+        readme_path,
+        _readme_disclaimer,
+        ReadmeMarkers.as_start(ReadmeMarkers.DISCLAIMER),
+        ReadmeMarkers.as_end(ReadmeMarkers.DISCLAIMER),
+    )
+    run_and_wait("terraform fmt -recursive .", cwd=module.module_out_path, allow_non_zero_exit=True, ansi_content=False)
+    example_section = read_examples(module)
+    update_between_markers(
+        readme_path,
+        example_section,
+        ReadmeMarkers.as_start(ReadmeMarkers.EXAMPLE),
+        ReadmeMarkers.as_end(ReadmeMarkers.EXAMPLE),
+    )
+    docs_config_path = module.terraform_docs_config_path()
+    if docs_config_path.exists():
+        logger.warning(f"{docs_config_path} already exists, skipping generation")
+    else:
+        config_content = terraform_docs_config_content(module)
+        ensure_parents_write_text(docs_config_path, config_content)
+        logger.info(f"generated {docs_config_path}")
+    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=module.module_out_path)
+    readme_content = _default_link_updater(readme_path.read_text())
+    ensure_parents_write_text(readme_path, readme_content)
+    return readme_path.read_text()
+
+
+def _default_link_updater(readme_content: str) -> str:  # can be a global replacer for now
+    for replace_in, replace_out in {
+        "docs/resources/advanced_cluster": r"docs/resources/advanced_cluster%2520%2528preview%2520provider%25202.0.0%2529"
+    }.items():
+        readme_content = readme_content.replace(replace_in, replace_out)
+    return readme_content