atlas-init 0.8.0__tar.gz → 0.9.0__tar.gz

{atlas_init-0.8.0 → atlas_init-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-init
-Version: 0.8.0
+Version: 0.9.0
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.8.0"
+VERSION = "0.9.0"
 
 
 def running_in_repo() -> bool:

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/cli_root/trigger.py

@@ -30,7 +30,7 @@ def create_realm_app():
     project_id = atlas_settings.MONGODB_ATLAS_PROJECT_ID
     base_url = atlas_settings.realm_url
     cluster_name = cluster_settings.MONGODB_ATLAS_CLUSTER_NAME
-    auth_headers = login_to_realm(settings, base_url)
+    auth_headers = login_to_realm(atlas_settings, base_url)
     realm_settings = env_vars_cls_or_none(RealmSettings, dotenv_path=settings.env_vars_trigger)
     if realm_settings and function_exists(
         base_url,

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/settings/rich_utils.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Literal
 
+from rich.console import Console
+from rich.tree import Tree
 import typer
 from pydantic import BaseModel
 from rich.logging import RichHandler
@@ -62,3 +64,23 @@ def configure_logging(
         app.pretty_exceptions_show_locals = False
 
     return handler
+
+
+# https://github.com/Textualize/rich/blob/8c4d3d1d50047e3aaa4140d0ffc1e0c9f1df5af4/tests/test_live.py#L11
+def create_capture_console(*, width: int = 60, height: int = 80, force_terminal: bool = True) -> Console:
+    return Console(
+        width=width,
+        height=height,
+        force_terminal=force_terminal,
+        legacy_windows=False,
+        color_system=None,  # use no color system to reduce complexity of output,
+        _environ={},
+    )
+
+
+def tree_text(tree: Tree) -> str:
+    console = create_capture_console()
+    console.width = 10_000
+    console.begin_capture()
+    console.print(tree)
+    return "\n".join(line.rstrip() for line in console.end_capture().splitlines())

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/tf_ext/gen_examples.py

@@ -19,10 +19,12 @@ def _examples_casted(examples: dict) -> dict[str, ResourceAbs]:
     return examples
 
 
-def read_example_dirs(module_path: Path) -> list[Path]:
+def read_example_dirs(examples_dir: Path) -> list[Path]:
+    if not examples_dir.exists():
+        return []
     return sorted(
         example_dir
-        for example_dir in (module_path / "examples").glob("*")
+        for example_dir in examples_dir.glob("*")
         if example_dir.is_dir()
         and len(example_dir.name) > 2
         and example_dir.name[:2].isdigit()

atlas_init-0.8.0/atlas_init/tf_ext/gen_module_readme.py → atlas_init-0.9.0/atlas_init/tf_ext/gen_readme.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
 import logging
 from enum import StrEnum
+from pathlib import Path
+from typing import Callable, TypeAlias
 
 from ask_shell import run_and_wait
 from zero_3rdparty.file_utils import ensure_parents_write_text, update_between_markers
 
 from atlas_init.tf_ext.gen_examples import read_example_dirs
-from atlas_init.tf_ext.models_module import ModuleGenConfig
+from atlas_init.tf_ext.models_module import EXAMPLES_DIRNAME, README_FILENAME, TERRAFORM_DOCS_CONFIG_FILENAME
 
 logger = logging.getLogger(__name__)
 _readme_disclaimer = """\
@@ -13,12 +16,13 @@ _readme_disclaimer = """\
 This Module is not meant for external consumption.
 It is part of a development PoC.
 Any usage problems will not be supported.
-However, if you have any ideas or feedback feel free to open a Github Issue!
+However, if you have any ideas or feedback, feel free to open a Github Issue!
 """
 
 
 class ReadmeMarkers(StrEnum):
     DISCLAIMER = "DISCLAIMER"
+    MODULES = "MODULES"
     EXAMPLE = "TF_EXAMPLES"
     TF_DOCS = "TF_DOCS"
 
@@ -42,9 +46,23 @@ class ReadmeMarkers(StrEnum):
     def example_boilerplate(cls) -> str:
         return "\n".join(cls.marker_lines(marker_name) for marker_name in list(cls))
 
+    @classmethod
+    def readme_generators(cls) -> ReadmeGenerators:
+        return [
+            (cls.DISCLAIMER, lambda _: _readme_disclaimer),
+            (cls.EXAMPLE, lambda workspace: read_examples(workspace / EXAMPLES_DIRNAME)),
+        ]
+
+
+ReadmeGenerators: TypeAlias = list[tuple[ReadmeMarkers, Callable[[Path], str]]]
 
-def read_examples(module: ModuleGenConfig) -> str:
-    example_dirs = read_example_dirs(module.module_out_path)
+
+def read_examples(examples_dir: Path) -> str:
+    example_dirs = read_example_dirs(examples_dir)
+    if not example_dirs:
+        return ""
+    # ensure the examples are formatted first
+    run_and_wait("terraform fmt -recursive .", cwd=examples_dir.parent, allow_non_zero_exit=True, ansi_content=False)
     content = ["# Examples"]
     for example_dir in example_dirs:
         example_name = example_dir.name
@@ -80,10 +98,10 @@ sort:
 """
 
 
-def terraform_docs_config_content(module: ModuleGenConfig) -> str:
+def terraform_docs_config_content(readme_path: Path) -> str:
     config = _static_terraform_config
     for replacement_in, replacement_out in [
-        ("FILENAME", module.readme_path().name),
+        ("FILENAME", readme_path.name),
         ("START_MARKER", ReadmeMarkers.as_start(ReadmeMarkers.TF_DOCS)),
         ("END_MARKER", ReadmeMarkers.as_end(ReadmeMarkers.TF_DOCS)),
     ]:
@@ -91,36 +109,38 @@ def terraform_docs_config_content(module: ModuleGenConfig) -> str:
     return config
 
 
-def generate_readme(module: ModuleGenConfig) -> str:
-    readme_path = module.readme_path()
+def generate_and_write_readme(terraform_workdir: Path, *, generators: ReadmeGenerators | None = None) -> str:
+    generators = generators or ReadmeMarkers.readme_generators()
+    readme_path = terraform_workdir / README_FILENAME
     assert readme_path.exists(), (
         f"{readme_path} does not exist, currently a boilerplate is expected, consider adding to {readme_path}\n{ReadmeMarkers.example_boilerplate()}"
     )
-    update_between_markers(
-        readme_path,
-        _readme_disclaimer,
-        ReadmeMarkers.as_start(ReadmeMarkers.DISCLAIMER),
-        ReadmeMarkers.as_end(ReadmeMarkers.DISCLAIMER),
-    )
-    run_and_wait("terraform fmt -recursive .", cwd=module.module_out_path, allow_non_zero_exit=True, ansi_content=False)
-    example_section = read_examples(module)
-    update_between_markers(
-        readme_path,
-        example_section,
-        ReadmeMarkers.as_start(ReadmeMarkers.EXAMPLE),
-        ReadmeMarkers.as_end(ReadmeMarkers.EXAMPLE),
-    )
-    docs_config_path = module.terraform_docs_config_path()
+    for marker, generator in generators:
+        content = generator(terraform_workdir)
+        if not content:
+            continue
+        update_between_markers(
+            readme_path,
+            content,
+            ReadmeMarkers.as_start(marker),
+            ReadmeMarkers.as_end(marker),
+        )
+    generate_terraform_docs(readme_path)
+    logger.info(f"updated {readme_path}")
+    return readme_path.read_text()
+
+
+def generate_terraform_docs(readme_path: Path) -> None:
+    docs_config_path = readme_path.parent / TERRAFORM_DOCS_CONFIG_FILENAME
     if docs_config_path.exists():
         logger.warning(f"{docs_config_path} already exists, skipping generation")
     else:
-        config_content = terraform_docs_config_content(module)
+        config_content = terraform_docs_config_content(readme_path)
         ensure_parents_write_text(docs_config_path, config_content)
         logger.info(f"generated {docs_config_path}")
-    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=module.module_out_path)
+    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=readme_path.parent)
     readme_content = _default_link_updater(readme_path.read_text())
     ensure_parents_write_text(readme_path, readme_content)
-    return readme_path.read_text()
 
 
 def _default_link_updater(readme_content: str) -> str:  # can be a global replacer for now

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/tf_ext/gen_resource_variables.py

@@ -128,9 +128,11 @@ def generate_resource_variables(
     resource: type[ResourceAbs] | None, resource_config: ResourceGenConfig, extra_skipped: set[str] | None = None
 ) -> str:
     extra_skipped = extra_skipped or set()
-    required_variables = resource_config.required_variables
     if resource is None:
         return ""
+    required_variables = set(resource_config.required_variables)
+    if not resource_config.use_opt_in_required_variables:
+        required_variables |= getattr(resource, ResourceAbs.REQUIRED_ATTRIBUTES_NAME, set())
     out = []
     hints = get_type_hints(resource)
     ignored_names = (
@@ -145,7 +147,8 @@ def generate_resource_variables(
         return format_tf_content(f'''variable "{resource_config.name}" {{
   type     = {tf_type}
 }}\n''')
-    for f in fields(resource):  # type: ignore
+    fields_sorted = sorted(fields(resource), key=lambda f: (0 if f.name in required_variables else 1, f.name))
+    for f in fields_sorted:  # type: ignore
         field_name = f.name
         if field_name.isupper() or field_name in ignored_names:
             continue

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/tf_ext/models.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from typing import Self
+from dataclasses import dataclass, field
+from typing import Iterable, Self
 
 from model_lib import Entity
 from pydantic import Field, RootModel, model_validator
@@ -30,6 +31,31 @@ def choose_next_emoji() -> str:
     return emoji
 
 
+@dataclass
+class EmojiCounter:
+    counter: int = 0
+    existing_emojis: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def emoji_to_names(self) -> dict[str, str]:
+        return dict(zip(self.existing_emojis.values(), self.existing_emojis.keys()))
+
+    def emoji_name(self) -> Iterable[tuple[str, str]]:
+        src = self.emoji_to_names
+        for emoji in _emojii_list:
+            if emoji in src:
+                yield emoji, src[emoji]
+            else:
+                break
+
+    def get_emoji(self, name: str) -> str:
+        if existing := self.existing_emojis.get(name):
+            return existing
+        emoji = self.existing_emojis[name] = _emojii_list[self.counter]
+        self.counter += 1
+        return emoji
+
+
 class ModuleState(Entity):
     resource_types: set[str] = Field(default_factory=set, description="Set of resource types in the module.")
 

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/tf_ext/models_module.py

@@ -24,6 +24,9 @@ from atlas_init.tf_ext.py_gen import (
 from atlas_init.tf_ext.settings import RepoOut, TfExtSettings
 
 ResourceTypeT: TypeAlias = str
+TERRAFORM_DOCS_CONFIG_FILENAME: str = ".terraform-docs.yml"
+README_FILENAME: str = "README.md"
+EXAMPLES_DIRNAME: str = "examples"
 
 
 @dataclass
@@ -74,6 +77,7 @@ def as_import_line(name: str) -> str:
 class ResourceGenConfig(Entity):
     name: str
     use_single_variable: bool = False
+    use_opt_in_required_variables: bool = False
     required_variables: set[str] = PydanticField(default_factory=set)
     skip_variables_extra: set[str] = PydanticField(default_factory=set)
     attribute_default_hcl_strings: dict[str, str] = PydanticField(default_factory=dict)
@@ -252,12 +256,13 @@ class ModuleGenConfig(Entity):
                 return resource_type
         raise ValueError(f"Could not resolve resource type for path {path}")
 
+    @property
     def readme_path(self) -> Path:
-        return self.module_out_path / "README.md"
+        return self.module_out_path / README_FILENAME
 
     @property
     def examples_path(self) -> Path:
-        return self.module_out_path / "examples"
+        return self.module_out_path / EXAMPLES_DIRNAME
 
     def example_name(self, name: str, example_nr: int) -> str:
         return f"{example_nr:02d}_{name}"
@@ -266,7 +271,7 @@ class ModuleGenConfig(Entity):
         return self.examples_path / name
 
     def terraform_docs_config_path(self) -> Path:
-        return self.module_out_path / ".terraform-docs.yml"
+        return self.module_out_path / TERRAFORM_DOCS_CONFIG_FILENAME
 
 
 @dataclass

{atlas_init-0.8.0 → atlas_init-0.9.0}/atlas_init/tf_ext/tf_dep.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+from functools import total_ordering
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import Iterable, NamedTuple
+from threading import RLock
+from typing import Callable, Iterable, NamedTuple
 
 import pydot
 from ask_shell import ShellError, new_task, run_and_wait
 from ask_shell._run import stop_runs_and_pool
 from ask_shell.run_pool import run_pool
 from model_lib import Entity, dump
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 from typer import Typer
 from zero_3rdparty.file_utils import ensure_parents_write_text
@@ -57,13 +59,25 @@ def tf_dep_graph(
     example_dirs = get_example_directories(repo_path, skip_names)
     logger.info(f"example_dirs: \n{'\n'.join(str(d) for d in sorted(example_dirs))}")
     with new_task("Find terraform graphs", total=len(example_dirs)) as task:
-        atlas_graph = parse_graphs(example_dirs, task)
+        atlas_graph = create_atlas_graph(example_dirs, task)
     with new_task("Dump graph"):
         graph_yaml = atlas_graph.dump_yaml()
         ensure_parents_write_text(settings.atlas_graph_path, graph_yaml)
         logger.info(f"Atlas graph dumped to {settings.atlas_graph_path}")
 
 
+def create_atlas_graph(example_dirs: list[Path], task: new_task) -> AtlasGraph:
+    atlas_graph = AtlasGraph()
+
+    def on_graph(example_dir: Path, graph: pydot.Dot):
+        atlas_graph.add_edges(graph.get_edges())
+        atlas_graph.add_variable_edges(example_dir)
+
+    parse_graphs(on_graph, example_dirs, task)
+
+    return atlas_graph
+
+
 def print_edges(graph: pydot.Dot):
     edges = graph.get_edges()
     for edge in edges:
@@ -79,9 +93,15 @@ class ResourceParts(NamedTuple):
         return self.resource_type.split("_")[0]
 
 
+@total_ordering
 class ResourceRef(BaseModel):
     full_ref: str
 
+    @model_validator(mode="after")
+    def ensure_plain(self):
+        self.full_ref = plain_name(self.full_ref)
+        return self
+
     def _resource_parts(self) -> ResourceParts:
         match self.full_ref.split("."):
             case [resource_type, resource_name] if "_" in resource_type:
@@ -106,6 +126,11 @@ class ResourceRef(BaseModel):
     def is_module(self) -> bool:
         return self.full_ref.startswith(MODULE_PREFIX)
 
+    @property
+    def module_name(self) -> str:
+        assert self.is_module, f"ResourceRef {self.full_ref} is not a module"
+        return self.full_ref.removeprefix(MODULE_PREFIX).split(".")[0]
+
     @property
     def is_data(self) -> bool:
         return self.full_ref.startswith(DATA_PREFIX)
@@ -114,6 +139,22 @@ class ResourceRef(BaseModel):
     def resource_type(self) -> str:
         return self._resource_parts().resource_type
 
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            raise TypeError(f"cannot compare {type(self)} with {type(other)}")
+        return self.full_ref < other.full_ref
+
+    def __hash__(self) -> int:
+        return hash(self.full_ref)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            return NotImplemented
+        return self.full_ref == other.full_ref
+
+    def __str__(self) -> str:
+        return self.full_ref
+
 
 class EdgeParsed(BaseModel):
     parent: ResourceRef
@@ -149,7 +190,15 @@ class EdgeParsed(BaseModel):
 
 
 def edge_plain(edge_endpoint: pydot.EdgeEndpoint) -> str:
-    return str(edge_endpoint).strip('"').strip()
+    return plain_name(str(edge_endpoint))
+
+
+def node_plain(node: pydot.Node) -> str:
+    return plain_name(node.get_name())
+
+
+def plain_name(name: str) -> str:
+    return name.strip('"').strip()
 
 
 def edge_src_dest(edge: pydot.Edge) -> tuple[str, str]:
@@ -235,34 +284,27 @@ class AtlasGraph(Entity):
                         self.parent_child_edges[parent_type].add(child_type)
 
 
-def parse_graphs(example_dirs: list[Path], task: new_task, max_workers: int = 16, max_dirs: int = 9999) -> AtlasGraph:
-    atlas_graph = AtlasGraph()
+def parse_graphs(
+    on_graph: Callable[[Path, pydot.Dot], None], example_dirs: list[Path], task: new_task, max_dirs: int = 1_000
+) -> None:
     with run_pool("parse example graphs", total=len(example_dirs)) as executor:
         futures = {
             executor.submit(parse_graph, example_dir): example_dir
             for i, example_dir in enumerate(example_dirs)
             if i < max_dirs
         }
-        graphs = {}
-        for future in futures:
+        for future, example_dir in futures.items():
             try:
-                example_dir, graph_output = future.result()
+                _, graph = future.result()
             except ShellError as e:
-                logger.error(f"Error parsing graph for {futures[future]}: {e}")
+                logger.error(f"Error parsing graph for {example_dir}: {e}")
                 continue
             except KeyboardInterrupt:
                 logger.error("KeyboardInterrupt received, stopping graph parsing.")
                 stop_runs_and_pool("KeyboardInterrupt", immediate=True)
                 break
-            try:
-                graph = graphs[example_dir] = parse_graph_output(example_dir, graph_output)
-            except GraphParseError as e:
-                logger.error(e)
-                continue
-            atlas_graph.add_edges(graph.get_edges())
-            atlas_graph.add_variable_edges(example_dir)
+            on_graph(example_dir, graph)
             task.update(advance=1)
-    return atlas_graph
 
 
 class GraphParseError(Exception):
@@ -271,9 +313,13 @@ class GraphParseError(Exception):
         super().__init__(f"Failed to parse graph for {example_dir}: {message}")
 
 
+_lock = RLock()
+
+
 def parse_graph_output(example_dir: Path, graph_output: str, verbose: bool = False) -> pydot.Dot:
     assert graph_output, f"Graph output is empty for {example_dir}"
-    dots = pydot.graph_from_dot_data(graph_output)  # not thread safe, so we use the main thread here instead
+    with _lock:
+        dots = pydot.graph_from_dot_data(graph_output)
     if not dots:
         raise GraphParseError(example_dir, f"No graphs found in the output:\n{graph_output}")
     assert len(dots) == 1, f"Expected one graph for {example_dir}, got {len(dots)}"
@@ -297,10 +343,10 @@ class EmptyGraphOutputError(Exception):
 @retry(
     stop=stop_after_attempt(3),
     wait=wait_fixed(1),
-    retry=retry_if_exception_type(EmptyGraphOutputError),
+    retry=retry_if_exception_type((EmptyGraphOutputError, GraphParseError)),
     reraise=True,
 )
-def parse_graph(example_dir: Path) -> tuple[Path, str]:
+def parse_graph(example_dir: Path) -> tuple[Path, pydot.Dot]:
     env_vars = {
         "MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER": "true" if is_v2_example_dir(example_dir) else "false",
     }
@@ -309,7 +355,8 @@ def parse_graph(example_dir: Path) -> tuple[Path, str]:
         run_and_wait("terraform init", cwd=example_dir, env=env_vars)
     run = run_and_wait("terraform graph", cwd=example_dir, env=env_vars)
     if graph_output := run.stdout_one_line:
-        return example_dir, graph_output
+        graph = parse_graph_output(example_dir, graph_output)  # just to make sure we get no errors
+        return example_dir, graph
     raise EmptyGraphOutputError(example_dir)