atlas-init 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.8.1"
+VERSION = "0.9.0"
 
 
 def running_in_repo() -> bool:

atlas_init/cli_root/trigger.py

@@ -30,7 +30,7 @@ def create_realm_app():
     project_id = atlas_settings.MONGODB_ATLAS_PROJECT_ID
     base_url = atlas_settings.realm_url
     cluster_name = cluster_settings.MONGODB_ATLAS_CLUSTER_NAME
-    auth_headers = login_to_realm(settings, base_url)
+    auth_headers = login_to_realm(atlas_settings, base_url)
     realm_settings = env_vars_cls_or_none(RealmSettings, dotenv_path=settings.env_vars_trigger)
     if realm_settings and function_exists(
         base_url,

atlas_init/settings/rich_utils.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Literal
 
+from rich.console import Console
+from rich.tree import Tree
 import typer
 from pydantic import BaseModel
 from rich.logging import RichHandler
@@ -62,3 +64,23 @@ def configure_logging(
         app.pretty_exceptions_show_locals = False
 
     return handler
+
+
+# https://github.com/Textualize/rich/blob/8c4d3d1d50047e3aaa4140d0ffc1e0c9f1df5af4/tests/test_live.py#L11
+def create_capture_console(*, width: int = 60, height: int = 80, force_terminal: bool = True) -> Console:
+    return Console(
+        width=width,
+        height=height,
+        force_terminal=force_terminal,
+        legacy_windows=False,
+        color_system=None,  # use no color system to reduce complexity of output,
+        _environ={},
+    )
+
+
+def tree_text(tree: Tree) -> str:
+    console = create_capture_console()
+    console.width = 10_000
+    console.begin_capture()
+    console.print(tree)
+    return "\n".join(line.rstrip() for line in console.end_capture().splitlines())

atlas_init/tf_ext/gen_examples.py

@@ -19,10 +19,12 @@ def _examples_casted(examples: dict) -> dict[str, ResourceAbs]:
     return examples
 
 
-def read_example_dirs(module_path: Path) -> list[Path]:
+def read_example_dirs(examples_dir: Path) -> list[Path]:
+    if not examples_dir.exists():
+        return []
     return sorted(
         example_dir
-        for example_dir in (module_path / "examples").glob("*")
+        for example_dir in examples_dir.glob("*")
         if example_dir.is_dir()
         and len(example_dir.name) > 2
         and example_dir.name[:2].isdigit()

atlas_init/tf_ext/{gen_module_readme.py → gen_readme.py}

@@ -1,11 +1,14 @@
+from __future__ import annotations
 import logging
 from enum import StrEnum
+from pathlib import Path
+from typing import Callable, TypeAlias
 
 from ask_shell import run_and_wait
 from zero_3rdparty.file_utils import ensure_parents_write_text, update_between_markers
 
 from atlas_init.tf_ext.gen_examples import read_example_dirs
-from atlas_init.tf_ext.models_module import ModuleGenConfig
+from atlas_init.tf_ext.models_module import EXAMPLES_DIRNAME, README_FILENAME, TERRAFORM_DOCS_CONFIG_FILENAME
 
 logger = logging.getLogger(__name__)
 _readme_disclaimer = """\
@@ -13,12 +16,13 @@ _readme_disclaimer = """\
 This Module is not meant for external consumption.
 It is part of a development PoC.
 Any usage problems will not be supported.
-However, if you have any ideas or feedback feel free to open a Github Issue!
+However, if you have any ideas or feedback, feel free to open a Github Issue!
 """
 
 
 class ReadmeMarkers(StrEnum):
     DISCLAIMER = "DISCLAIMER"
+    MODULES = "MODULES"
     EXAMPLE = "TF_EXAMPLES"
     TF_DOCS = "TF_DOCS"
 
@@ -42,9 +46,23 @@ class ReadmeMarkers(StrEnum):
     def example_boilerplate(cls) -> str:
         return "\n".join(cls.marker_lines(marker_name) for marker_name in list(cls))
 
+    @classmethod
+    def readme_generators(cls) -> ReadmeGenerators:
+        return [
+            (cls.DISCLAIMER, lambda _: _readme_disclaimer),
+            (cls.EXAMPLE, lambda workspace: read_examples(workspace / EXAMPLES_DIRNAME)),
+        ]
+
+
+ReadmeGenerators: TypeAlias = list[tuple[ReadmeMarkers, Callable[[Path], str]]]
 
-def read_examples(module: ModuleGenConfig) -> str:
-    example_dirs = read_example_dirs(module.module_out_path)
+
+def read_examples(examples_dir: Path) -> str:
+    example_dirs = read_example_dirs(examples_dir)
+    if not example_dirs:
+        return ""
+    # ensure the examples are formatted first
+    run_and_wait("terraform fmt -recursive .", cwd=examples_dir.parent, allow_non_zero_exit=True, ansi_content=False)
     content = ["# Examples"]
     for example_dir in example_dirs:
         example_name = example_dir.name
@@ -80,10 +98,10 @@ sort:
 """
 
 
-def terraform_docs_config_content(module: ModuleGenConfig) -> str:
+def terraform_docs_config_content(readme_path: Path) -> str:
     config = _static_terraform_config
     for replacement_in, replacement_out in [
-        ("FILENAME", module.readme_path().name),
+        ("FILENAME", readme_path.name),
         ("START_MARKER", ReadmeMarkers.as_start(ReadmeMarkers.TF_DOCS)),
         ("END_MARKER", ReadmeMarkers.as_end(ReadmeMarkers.TF_DOCS)),
     ]:
@@ -91,36 +109,38 @@ def terraform_docs_config_content(module: ModuleGenConfig) -> str:
     return config
 
 
-def generate_readme(module: ModuleGenConfig) -> str:
-    readme_path = module.readme_path()
+def generate_and_write_readme(terraform_workdir: Path, *, generators: ReadmeGenerators | None = None) -> str:
+    generators = generators or ReadmeMarkers.readme_generators()
+    readme_path = terraform_workdir / README_FILENAME
     assert readme_path.exists(), (
         f"{readme_path} does not exist, currently a boilerplate is expected, consider adding to {readme_path}\n{ReadmeMarkers.example_boilerplate()}"
     )
-    update_between_markers(
-        readme_path,
-        _readme_disclaimer,
-        ReadmeMarkers.as_start(ReadmeMarkers.DISCLAIMER),
-        ReadmeMarkers.as_end(ReadmeMarkers.DISCLAIMER),
-    )
-    run_and_wait("terraform fmt -recursive .", cwd=module.module_out_path, allow_non_zero_exit=True, ansi_content=False)
-    example_section = read_examples(module)
-    update_between_markers(
-        readme_path,
-        example_section,
-        ReadmeMarkers.as_start(ReadmeMarkers.EXAMPLE),
-        ReadmeMarkers.as_end(ReadmeMarkers.EXAMPLE),
-    )
-    docs_config_path = module.terraform_docs_config_path()
+    for marker, generator in generators:
+        content = generator(terraform_workdir)
+        if not content:
+            continue
+        update_between_markers(
+            readme_path,
+            content,
+            ReadmeMarkers.as_start(marker),
+            ReadmeMarkers.as_end(marker),
+        )
+    generate_terraform_docs(readme_path)
+    logger.info(f"updated {readme_path}")
+    return readme_path.read_text()
+
+
+def generate_terraform_docs(readme_path: Path) -> None:
+    docs_config_path = readme_path.parent / TERRAFORM_DOCS_CONFIG_FILENAME
     if docs_config_path.exists():
         logger.warning(f"{docs_config_path} already exists, skipping generation")
     else:
-        config_content = terraform_docs_config_content(module)
+        config_content = terraform_docs_config_content(readme_path)
         ensure_parents_write_text(docs_config_path, config_content)
         logger.info(f"generated {docs_config_path}")
-    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=module.module_out_path)
+    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=readme_path.parent)
     readme_content = _default_link_updater(readme_path.read_text())
     ensure_parents_write_text(readme_path, readme_content)
-    return readme_path.read_text()
 
 
 def _default_link_updater(readme_content: str) -> str:  # can be a global replacer for now

atlas_init/tf_ext/models.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from typing import Self
+from dataclasses import dataclass, field
+from typing import Iterable, Self
 
 from model_lib import Entity
 from pydantic import Field, RootModel, model_validator
@@ -30,6 +31,31 @@ def choose_next_emoji() -> str:
     return emoji
 
 
+@dataclass
+class EmojiCounter:
+    counter: int = 0
+    existing_emojis: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def emoji_to_names(self) -> dict[str, str]:
+        return dict(zip(self.existing_emojis.values(), self.existing_emojis.keys()))
+
+    def emoji_name(self) -> Iterable[tuple[str, str]]:
+        src = self.emoji_to_names
+        for emoji in _emojii_list:
+            if emoji in src:
+                yield emoji, src[emoji]
+            else:
+                break
+
+    def get_emoji(self, name: str) -> str:
+        if existing := self.existing_emojis.get(name):
+            return existing
+        emoji = self.existing_emojis[name] = _emojii_list[self.counter]
+        self.counter += 1
+        return emoji
+
+
 class ModuleState(Entity):
     resource_types: set[str] = Field(default_factory=set, description="Set of resource types in the module.")
 

atlas_init/tf_ext/models_module.py

@@ -24,6 +24,9 @@ from atlas_init.tf_ext.py_gen import (
 from atlas_init.tf_ext.settings import RepoOut, TfExtSettings
 
 ResourceTypeT: TypeAlias = str
+TERRAFORM_DOCS_CONFIG_FILENAME: str = ".terraform-docs.yml"
+README_FILENAME: str = "README.md"
+EXAMPLES_DIRNAME: str = "examples"
 
 
 @dataclass
@@ -253,12 +256,13 @@ class ModuleGenConfig(Entity):
                 return resource_type
         raise ValueError(f"Could not resolve resource type for path {path}")
 
+    @property
     def readme_path(self) -> Path:
-        return self.module_out_path / "README.md"
+        return self.module_out_path / README_FILENAME
 
     @property
     def examples_path(self) -> Path:
-        return self.module_out_path / "examples"
+        return self.module_out_path / EXAMPLES_DIRNAME
 
     def example_name(self, name: str, example_nr: int) -> str:
         return f"{example_nr:02d}_{name}"
@@ -267,7 +271,7 @@ class ModuleGenConfig(Entity):
         return self.examples_path / name
 
     def terraform_docs_config_path(self) -> Path:
-        return self.module_out_path / ".terraform-docs.yml"
+        return self.module_out_path / TERRAFORM_DOCS_CONFIG_FILENAME
 
 
 @dataclass

atlas_init/tf_ext/tf_dep.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+from functools import total_ordering
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import Iterable, NamedTuple
+from threading import RLock
+from typing import Callable, Iterable, NamedTuple
 
 import pydot
 from ask_shell import ShellError, new_task, run_and_wait
 from ask_shell._run import stop_runs_and_pool
 from ask_shell.run_pool import run_pool
 from model_lib import Entity, dump
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 from typer import Typer
 from zero_3rdparty.file_utils import ensure_parents_write_text
@@ -57,13 +59,25 @@ def tf_dep_graph(
     example_dirs = get_example_directories(repo_path, skip_names)
     logger.info(f"example_dirs: \n{'\n'.join(str(d) for d in sorted(example_dirs))}")
     with new_task("Find terraform graphs", total=len(example_dirs)) as task:
-        atlas_graph = parse_graphs(example_dirs, task)
+        atlas_graph = create_atlas_graph(example_dirs, task)
     with new_task("Dump graph"):
         graph_yaml = atlas_graph.dump_yaml()
         ensure_parents_write_text(settings.atlas_graph_path, graph_yaml)
         logger.info(f"Atlas graph dumped to {settings.atlas_graph_path}")
 
 
+def create_atlas_graph(example_dirs: list[Path], task: new_task) -> AtlasGraph:
+    atlas_graph = AtlasGraph()
+
+    def on_graph(example_dir: Path, graph: pydot.Dot):
+        atlas_graph.add_edges(graph.get_edges())
+        atlas_graph.add_variable_edges(example_dir)
+
+    parse_graphs(on_graph, example_dirs, task)
+
+    return atlas_graph
+
+
 def print_edges(graph: pydot.Dot):
     edges = graph.get_edges()
     for edge in edges:
@@ -79,9 +93,15 @@ class ResourceParts(NamedTuple):
         return self.resource_type.split("_")[0]
 
 
+@total_ordering
 class ResourceRef(BaseModel):
     full_ref: str
 
+    @model_validator(mode="after")
+    def ensure_plain(self):
+        self.full_ref = plain_name(self.full_ref)
+        return self
+
     def _resource_parts(self) -> ResourceParts:
         match self.full_ref.split("."):
             case [resource_type, resource_name] if "_" in resource_type:
@@ -106,6 +126,11 @@ class ResourceRef(BaseModel):
     def is_module(self) -> bool:
         return self.full_ref.startswith(MODULE_PREFIX)
 
+    @property
+    def module_name(self) -> str:
+        assert self.is_module, f"ResourceRef {self.full_ref} is not a module"
+        return self.full_ref.removeprefix(MODULE_PREFIX).split(".")[0]
+
     @property
     def is_data(self) -> bool:
         return self.full_ref.startswith(DATA_PREFIX)
@@ -114,6 +139,22 @@ class ResourceRef(BaseModel):
     def resource_type(self) -> str:
         return self._resource_parts().resource_type
 
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            raise TypeError(f"cannot compare {type(self)} with {type(other)}")
+        return self.full_ref < other.full_ref
+
+    def __hash__(self) -> int:
+        return hash(self.full_ref)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            return NotImplemented
+        return self.full_ref == other.full_ref
+
+    def __str__(self) -> str:
+        return self.full_ref
+
 
 class EdgeParsed(BaseModel):
     parent: ResourceRef
@@ -149,7 +190,15 @@ class EdgeParsed(BaseModel):
 
 
 def edge_plain(edge_endpoint: pydot.EdgeEndpoint) -> str:
-    return str(edge_endpoint).strip('"').strip()
+    return plain_name(str(edge_endpoint))
+
+
+def node_plain(node: pydot.Node) -> str:
+    return plain_name(node.get_name())
+
+
+def plain_name(name: str) -> str:
+    return name.strip('"').strip()
 
 
 def edge_src_dest(edge: pydot.Edge) -> tuple[str, str]:
@@ -235,34 +284,27 @@ class AtlasGraph(Entity):
                         self.parent_child_edges[parent_type].add(child_type)
 
 
-def parse_graphs(example_dirs: list[Path], task: new_task, max_workers: int = 16, max_dirs: int = 9999) -> AtlasGraph:
-    atlas_graph = AtlasGraph()
+def parse_graphs(
+    on_graph: Callable[[Path, pydot.Dot], None], example_dirs: list[Path], task: new_task, max_dirs: int = 1_000
+) -> None:
     with run_pool("parse example graphs", total=len(example_dirs)) as executor:
         futures = {
             executor.submit(parse_graph, example_dir): example_dir
             for i, example_dir in enumerate(example_dirs)
             if i < max_dirs
         }
-        graphs = {}
-        for future in futures:
+        for future, example_dir in futures.items():
             try:
-                example_dir, graph_output = future.result()
+                _, graph = future.result()
             except ShellError as e:
-                logger.error(f"Error parsing graph for {futures[future]}: {e}")
+                logger.error(f"Error parsing graph for {example_dir}: {e}")
                 continue
             except KeyboardInterrupt:
                 logger.error("KeyboardInterrupt received, stopping graph parsing.")
                 stop_runs_and_pool("KeyboardInterrupt", immediate=True)
                 break
-            try:
-                graph = graphs[example_dir] = parse_graph_output(example_dir, graph_output)
-            except GraphParseError as e:
-                logger.error(e)
-                continue
-            atlas_graph.add_edges(graph.get_edges())
-            atlas_graph.add_variable_edges(example_dir)
+            on_graph(example_dir, graph)
             task.update(advance=1)
-    return atlas_graph
 
 
 class GraphParseError(Exception):
@@ -271,9 +313,13 @@ class GraphParseError(Exception):
         super().__init__(f"Failed to parse graph for {example_dir}: {message}")
 
 
+_lock = RLock()
+
+
 def parse_graph_output(example_dir: Path, graph_output: str, verbose: bool = False) -> pydot.Dot:
     assert graph_output, f"Graph output is empty for {example_dir}"
-    dots = pydot.graph_from_dot_data(graph_output)  # not thread safe, so we use the main thread here instead
+    with _lock:
+        dots = pydot.graph_from_dot_data(graph_output)
     if not dots:
         raise GraphParseError(example_dir, f"No graphs found in the output:\n{graph_output}")
     assert len(dots) == 1, f"Expected one graph for {example_dir}, got {len(dots)}"
@@ -297,10 +343,10 @@ class EmptyGraphOutputError(Exception):
 @retry(
     stop=stop_after_attempt(3),
     wait=wait_fixed(1),
-    retry=retry_if_exception_type(EmptyGraphOutputError),
+    retry=retry_if_exception_type((EmptyGraphOutputError, GraphParseError)),
     reraise=True,
 )
-def parse_graph(example_dir: Path) -> tuple[Path, str]:
+def parse_graph(example_dir: Path) -> tuple[Path, pydot.Dot]:
     env_vars = {
         "MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER": "true" if is_v2_example_dir(example_dir) else "false",
     }
@@ -309,7 +355,8 @@ def parse_graph(example_dir: Path) -> tuple[Path, str]:
         run_and_wait("terraform init", cwd=example_dir, env=env_vars)
     run = run_and_wait("terraform graph", cwd=example_dir, env=env_vars)
     if graph_output := run.stdout_one_line:
-        return example_dir, graph_output
+        graph = parse_graph_output(example_dir, graph_output)  # just to make sure we get no errors
+        return example_dir, graph
     raise EmptyGraphOutputError(example_dir)
 
 

atlas_init/tf_ext/tf_example_readme.py

@@ -0,0 +1,392 @@
+from __future__ import annotations
+from collections import defaultdict
+from contextlib import suppress
+from functools import total_ordering
+import logging
+from pathlib import Path
+from typing import Callable, ClassVar, Iterable, Protocol, TypeAlias
+from ask_shell import new_task
+from ask_shell.rich_live import get_live_console
+from model_lib import Entity, parse_dict
+from pydantic import Field, model_validator
+import pydot
+from rich.tree import Tree
+import typer
+
+from atlas_init.settings.rich_utils import tree_text
+from atlas_init.tf_ext.gen_readme import ReadmeMarkers, generate_and_write_readme
+from atlas_init.tf_ext.models import EmojiCounter
+from atlas_init.tf_ext.models_module import README_FILENAME
+from atlas_init.tf_ext.tf_dep import EdgeParsed, ResourceRef, node_plain, parse_graph, parse_graphs
+
+logger = logging.getLogger(__name__)
+MODULES_JSON_RELATIVE_PATH = ".terraform/modules/modules.json"
+
+
+def tf_example_readme(
+    example_path: Path = typer.Option(
+        ..., "-e", "--example-path", help="Path to the example directory", default_factory=Path.cwd
+    ),
+    skip_module_details: list[str] = typer.Option(
+        ..., "-s", "--skip-module-details", help="List of module details to skip", default_factory=list
+    ),
+):
+    with new_task("parse example graph"):
+        _, example_graph_dot = parse_graph(example_path)  # ensures init is called
+        example_graph = ResourceGraph.from_graph(example_graph_dot)
+    with new_task("parse module graphs") as task:
+        modules_config = parse_modules_json(example_path, skip_module_details)
+        module_paths = modules_config.module_paths
+        module_graphs: dict[Path, ResourceGraph] = {}
+
+        def on_graph(example_dir: Path, graph: pydot.Dot):
+            module_graphs[example_dir] = ResourceGraph.from_graph(graph)
+
+        parse_graphs(on_graph, module_paths, task)
+    with new_task("create example module graph"):
+        # a graph when all resources in a module are treated as a single node.
+        modules_graph, emoji_counter = create_module_graph(example_graph)
+    with new_task(f"update {README_FILENAME}"):
+        modules_section = []
+        modules_trees_texts = []
+        module_dirs_used: set[Path] = set()
+
+        def add_module_tree(module_dir: Path):
+            # trees are only once per module, not per module instance
+            if module_dir in module_dirs_used:
+                return
+            module_dirs_used.add(module_dir)
+            module_graph = module_graphs[module_dir]
+            module_config = modules_config.get_by_path(module_dir)
+            emojis = ", ".join(emoji_counter.get_emoji(key) for key in module_config.keys)
+            if modules_config.skip_details(module_config):
+                tree = Tree(f"{module_dir.name} ({emojis})")
+                tree.add("details skipped")
+            else:
+                tree = module_graph.to_tree(f"{module_dir.name} ({emojis})", include_orphans=True)
+            get_live_console().print(tree)
+            modules_trees_texts.append(tree_text(tree))
+
+        for _, module_key in emoji_counter.emoji_name():
+            module_config = modules_config.get_by_key(module_key)
+            module_dir = module_config.absolute_path(example_path)
+            add_module_tree(module_dir)
+
+        def add_module_src(node: Tree, name: str) -> None:
+            config = modules_config.get_by_key(name)
+            node.add(f"{config.source}")
+
+        module_index_tree = emoji_tree(emoji_counter, tree_processor=add_module_src, name="Module Instances")
+        modules_section.extend(
+            [
+                "## Modules",
+                "",
+                "### Modules Instances",
+                "```sh",
+                tree_text(module_index_tree),
+                "```",
+                "### Module Definitions",
+                "",
+                "```sh",
+                "\n".join(modules_trees_texts),
+                "```",
+                "",
+                "### Graph with Dependencies",
+                "Any resource without a number prefix is defined at the root level.",
+                "",
+                as_mermaid(modules_graph),
+            ]
+        )
+        generators = ReadmeMarkers.readme_generators()
+        generators.insert(1, (ReadmeMarkers.MODULES, lambda _: "\n".join(modules_section)))
+        generate_and_write_readme(
+            example_path,
+            generators=generators,
+        )
+
+
+def create_module_graph(example_graph: ResourceGraph) -> tuple[ResourceGraph, EmojiCounter]:
+    emoji_counter = EmojiCounter()
+
+    def as_module_edge(parent: ResourceRef, child: ResourceRef) -> bool | ParentChild:
+        if not child.is_module:
+            return False
+        new_parent = add_emoji_prefix(parent, emoji_counter)
+        new_child = add_emoji_prefix(child, emoji_counter)
+        return new_parent, new_child
+
+    return create_subgraph(example_graph, as_module_edge), emoji_counter
+
+
+def emoji_tree(
+    counter: EmojiCounter, *, tree_processor: Callable[[Tree, str], None] | None = None, name: str = "Emoji Tree"
+) -> Tree:
+    tree = Tree(name)
+    for emoji, name in counter.emoji_name():
+        child = tree.add(f"{emoji}  {name}")
+        if tree_processor:
+            tree_processor(child, name)
+    return tree
+
+
+def add_emoji_prefix(ref: ResourceRef, emoji_counter: EmojiCounter) -> ResourceRef:
+    if ref.is_module:
+        return ResourceRef(full_ref=f"{emoji_counter.get_emoji(ref.module_name)} {ref.module_name}")
+    return ref
+
+
+def strip_emoji_prefix(ref: ResourceRef) -> ResourceRef:
+    old_ref = ref.full_ref
+    return ResourceRef(full_ref=old_ref.split(" ")[1]) if " " in old_ref else ref
+
+
+def as_module_name(ref: ResourceRef) -> str:
+    if ref.is_module:
+        return ref.module_name
+    return ""
+
+
+def as_module_ref(ref: ResourceRef) -> ResourceRef:
+    if name := as_module_name(ref):
+        return ResourceRef(full_ref=f"module.{name}")
+    return ref
+
+
+class _RootModuleIgnored(Exception):
+    pass
+
+
+ParentChild: TypeAlias = tuple[ResourceRef, ResourceRef]
+
+
+class ResourceGraph(Entity):
+    IGNORED_ORPHANS: ClassVar[set[str]] = {"node"}  # some extra output from `terraform graph` command
+    parent_children: dict[ResourceRef, set[ResourceRef]] = Field(default_factory=lambda: defaultdict(set))
+    children_parents: dict[ResourceRef, set[ResourceRef]] = Field(default_factory=lambda: defaultdict(set))
+    orphans: set[ResourceRef] = Field(default_factory=set)
+
+    @classmethod
+    def from_graph(cls, graph: pydot.Dot) -> "ResourceGraph":
+        resource_graph = cls()
+        resource_graph.add_edges(graph.get_edges())
+        for orphan in graph.get_node_list():
+            name = node_plain(orphan)
+            if name in cls.IGNORED_ORPHANS:
+                continue
+            ref = ResourceRef(full_ref=name)
+            resource_graph.add_orphan_if_not_found(ref)
+        return resource_graph
+
+    def add_orphan_if_not_found(self, orphan: ResourceRef):
+        if orphan not in self.parent_children and orphan not in self.children_parents:
+            self.orphans.add(orphan)
+
+    def add_edges(self, edges: list[pydot.Edge]):
+        for edge in edges:
+            parsed = EdgeParsed.from_edge(edge)
+            parent = parsed.parent
+            child = parsed.child
+            if str(parent) == ".this":
+                logger.info(f"parent: {parent} child: {child}")
+            self.add_edge(parent, child)
+
+    def add_edge(self, parent: ResourceRef, child: ResourceRef):
+        self.parent_children[parent].add(child)
+        self.children_parents[child].add(parent)
+
+    def all_edges(self) -> list[ParentChild]:
+        return [(parent, child) for parent in self.parent_children for child in self.parent_children[parent]]
+
+    @property
+    def all_parents(self) -> set[ResourceRef]:
+        return set(self.parent_children.keys())
+
+    def sorted_parents(self) -> Iterable[ResourceRef]:
+        used_parents = set()
+        remaining_parents = self.all_parents
+
+        def next_parent() -> ResourceRef | None:
+            candidates = [parent for parent in remaining_parents if not self.children_parents[parent] - used_parents]
+            return min(candidates) if candidates else None
+
+        while remaining_parents:
+            parent = next_parent()
+            if parent is None:
+                break
+            used_parents.add(parent)
+            yield parent
+            remaining_parents.remove(parent)
+
+    def to_tree(self, example_dir_name: str, include_orphans: bool = False) -> Tree:
+        root = Tree(example_dir_name)
+        trees: dict[ResourceRef, Tree] = {}
+
+        for parent in self.sorted_parents():
+            parent_tree = trees.setdefault(parent, Tree(parent.full_ref))
+            for child_ref in sorted(self.parent_children[parent]):
+                child_tree = trees.setdefault(child_ref, Tree(child_ref.full_ref))
+                parent_tree.add(child_tree)
+            if not self.children_parents[parent]:
+                root.add(parent_tree)
+        if include_orphans:
+            for orphan in sorted(self.orphans):
+                if orphan not in trees:
+                    root.add(Tree(orphan.full_ref))
+        return root
+
+
+def as_mermaid(graph: ResourceGraph) -> str:
+    nodes: dict[str, str] = {}
+
+    def mermaid_ref(ref: ResourceRef) -> str:
+        no_emoji = strip_emoji_prefix(ref)
+        mermaid = str(no_emoji)
+        assert '"' not in mermaid, f"mermaid ref should not contain quotes: {mermaid}"
+        nodes[mermaid] = str(ref)  # want original ref in label
+        return mermaid
+
+    def add_mermaid_edge(parent: ResourceRef, child: ResourceRef) -> str:
+        parent_ref = mermaid_ref(parent)
+        child_ref = mermaid_ref(child)
+        return f"{parent_ref} --> {child_ref}"
+
+    edges = [add_mermaid_edge(parent, child) for parent, child in graph.all_edges()]
+    return "\n".join(
+        [
+            "```mermaid",
+            "graph TD",
+            "    " + "\n    ".join(f'{mermaid}["{label}"]' for mermaid, label in sorted(nodes.items())),
+            "    " + "\n    ".join(sorted(edges)),
+            "```",
+        ]
+    )
+
+
+class EdgeFilter(Protocol):
+    def __call__(self, parent: ResourceRef, child: ResourceRef) -> bool | ParentChild: ...
+
+
+def create_subgraph(graph: ResourceGraph, edge_filter: EdgeFilter) -> ResourceGraph:
+    subgraph = ResourceGraph()
+    for parent in graph.sorted_parents():
+        for child in sorted(graph.parent_children[parent]):
+            filter_response = edge_filter(parent, child)
+            match filter_response:
+                case True:
+                    subgraph.add_edge(parent, child)
+                case False:
+                    continue
+                case (parent, child) if parent != child:
+                    subgraph.add_edge(parent, child)
+    return subgraph
+
+
+@total_ordering
+class ModuleExampleConfig(Entity):
+    keys: list[str]
+    rel_path: str = Field(alias="Dir", description="Relative path to the module example")
+    source: str = Field(
+        alias="Source",
+        description="Source of the module, for example: registry.terraform.io/terraform-aws-modules/vpc/aws",
+    )
+    version: str = Field(
+        alias="Version", description="Version of the module example, unset for local modules", default=""
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def move_key(cls, v: dict):
+        key = v.pop("Key", None)
+        if key:
+            v["keys"] = [key]
+        if v.get("Dir", "") == ".":
+            raise _RootModuleIgnored()
+        return v
+
+    @model_validator(mode="after")
+    def validate_keys(self):
+        if not self.keys:
+            raise ValueError("keys is required")
+        return self
+
+    @property
+    def key(self) -> str:
+        return ",".join(sorted(self.keys))
+
+    def absolute_path(self, example_path: Path) -> Path:
+        path = example_path / self.rel_path
+        if not path.exists():
+            raise ValueError(f"module path not found for {self.key}: {path}")
+        return path
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, ModuleExampleConfig):
+            raise TypeError(f"cannot compare {type(self)} with {type(other)}")
+        return self.key < other.key
+
+
+class ModuleExampleConfigs(Entity):
+    example_path: Path
+    modules: dict[str, ModuleExampleConfig] = Field(default_factory=dict)
+    skip_module_details: set[str] = Field(default_factory=set)
+
+    @model_validator(mode="after")
+    def ensure_paths_exists(self):
+        not_exists: dict[str, Path] = {}
+        for config in self.modules.values():
+            path = config.absolute_path(self.example_path)
+            if not path.exists():
+                not_exists[config.key] = path
+        if not_exists:
+            raise ValueError(f"module paths not found: {not_exists}")
+        return self
+
+    @property
+    def module_paths(self) -> list[Path]:
+        return [config.absolute_path(self.example_path) for config in self.modules.values()]
+
+    def skip_details(self, config: ModuleExampleConfig) -> bool:
+        return any(key in self.skip_module_details for key in config.keys)
+
+    def get_by_path(self, module_dir: Path) -> ModuleExampleConfig:
+        for config in self.modules.values():
+            if config.absolute_path(self.example_path) == module_dir:
+                return config
+        raise ValueError(f"module not found for {module_dir}")
+
+    def get_by_key_or_none(self, key: str) -> ModuleExampleConfig | None:
+        return self.modules.get(key)
+
+    def get_by_key(self, key: str) -> ModuleExampleConfig:
+        return self.modules[key]
+
+    def modules_included(self, *, skip_keys: list[str]) -> list[ModuleExampleConfig]:
+        return [config for config in self.modules.values() if all(key not in skip_keys for key in config.keys)]
+
+    def add_module(self, config: ModuleExampleConfig):
+        key = config.keys[0]
+        assert len(config.keys) == 1, "only one key can be added at a time"
+        source = config.source
+        existing_config = next(
+            (existing_config for existing_config in self.modules.values() if source == existing_config.source),
+            None,
+        )
+        if existing_config:
+            existing_config.keys.append(key)
+            existing_config.keys.sort()
+            self.modules[key] = existing_config
+        else:
+            self.modules[key] = config
+
+
+def parse_modules_json(example_path: Path, skip_module_details: list[str] | None = None) -> ModuleExampleConfigs:
+    configs = ModuleExampleConfigs(example_path=example_path, skip_module_details=set(skip_module_details or []))
+    module_json_path = example_path / MODULES_JSON_RELATIVE_PATH
+    if not module_json_path.exists():
+        return configs
+    module_json = parse_dict(module_json_path)
+    for raw in module_json.get("Modules", []):
+        with suppress(_RootModuleIgnored):
+            config = ModuleExampleConfig(**raw)
+            configs.add_module(config)
+    return configs

atlas_init/tf_ext/tf_mod_gen.py

@@ -12,7 +12,7 @@ from zero_3rdparty.file_utils import clean_dir, copy, ensure_parents_write_text
 from atlas_init.cli_tf.example_update import UpdateExamples, update_examples
 from atlas_init.tf_ext.args import TF_CLI_CONFIG_FILE_ARG
 from atlas_init.tf_ext.gen_examples import generate_module_examples, read_example_dirs
-from atlas_init.tf_ext.gen_module_readme import generate_readme
+from atlas_init.tf_ext.gen_readme import generate_and_write_readme
 from atlas_init.tf_ext.gen_resource_main import generate_resource_main
 from atlas_init.tf_ext.gen_resource_output import generate_resource_output
 from atlas_init.tf_ext.gen_resource_variables import generate_module_variables
@@ -192,12 +192,10 @@ def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Pat
     if out_event.changes:
         logger.info(f"Updated attribute descriptions: {len(out_event.changes)}")
         run_and_wait("terraform fmt -recursive .", cwd=path, ansi_content=False, allow_non_zero_exit=True)
-    if readme_path := config.readme_path():
-        with new_task("Generating README.md"):
-            readme_content = generate_readme(config)
-            ensure_parents_write_text(readme_path, readme_content)
+    with new_task("Generating README.md"):
+        generate_and_write_readme(config.module_out_path)
     if example_var_file:
-        examples = read_example_dirs(config.module_out_path)
+        examples = read_example_dirs(config.examples_path)
         if examples:
             failed_examples: list[Path] = []
             with run_pool("Running terraform plan on examples", total=len(examples), exit_wait_timeout=60) as pool:

atlas_init/tf_ext/tf_modules.py

@@ -175,7 +175,9 @@ def tf_modules(
         logger.info(f"Missing modules: \n{'\n'.join(missing_resources)}")
 
 
-def generate_module_graphs(skipped_module_resource_types, settings, atlas_graph):
+def generate_module_graphs(
+    skipped_module_resource_types: Iterable[str], settings: TfExtSettings, atlas_graph: AtlasGraph
+):
     tree = Tree(
         "Module graphs",
     )

atlas_init/tf_ext/typer_app.py

@@ -1,7 +1,7 @@
 from ask_shell import configure_logging
 from typer import Typer
 
-from atlas_init.tf_ext import api_call, settings, tf_desc_gen, tf_mod_gen_provider
+from atlas_init.tf_ext import api_call, settings, tf_desc_gen, tf_example_readme, tf_mod_gen_provider
 
 
 def typer_main():
@@ -20,6 +20,7 @@ def typer_main():
     app.command(name="api-config")(api_call.api_config)
     app.command(name="mod-gen-provider")(tf_mod_gen_provider.tf_mod_gen_provider_resource_modules)
     app.command(name="check-env-vars")(settings.init_tf_ext_settings)
+    app.command(name="example-readme")(tf_example_readme.tf_example_readme)
     configure_logging(app)
     app()
 

{atlas_init-0.8.1.dist-info → atlas_init-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-init
-Version: 0.8.1
+Version: 0.9.0
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init

{atlas_init-0.8.1.dist-info → atlas_init-0.9.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atlas_init/__init__.py,sha256=bE8OXKBhQOEYQBlOStpyAixc8Il2l2NRdaFMFXjCLq4,213
+atlas_init/__init__.py,sha256=SxwCA6YCJ-sDX3qOsnEH3L4WjI9tXH14OB1liYWsnHw,213
 atlas_init/__main__.py,sha256=dY1dWWvwxRZMmnOFla6RSfti-hMeLeKdoXP7SVYqMUc,52
 atlas_init/atlas_init.yaml,sha256=aCCg6PJgLcqh75v3tdEEG4UPrcJlbmbRaLj0c7PCumE,2373
 atlas_init/cli.py,sha256=0b4_osi67Oj-hZnrtK70IbSKQDREjwUFv1_30_A03Zg,9447
@@ -23,7 +23,7 @@ atlas_init/cli_helper/tf_runner.py,sha256=V8pfxPSDBSeQWkR27kI8JLu-MEadUR9TGnoXBj
 atlas_init/cli_root/__init__.py,sha256=Mf0wqy4kqq8pmbjLa98zOGuUWv0bLk2OYGc1n1_ZmZ4,223
 atlas_init/cli_root/go_test.py,sha256=roQIOS-qVfNhJMztR-V3hjtxFMf7-Ioy3e1ffqtTRyo,4601
 atlas_init/cli_root/mms_released.py,sha256=gaUWzY4gqb1Tuo7u-8HqkOC6pW8QSjcwTz2UJMCV3Cw,1765
-atlas_init/cli_root/trigger.py,sha256=5TMc_depk9MOkfEfq3WWWgyCpPmAvoZywmeNnU0QZGg,8520
+atlas_init/cli_root/trigger.py,sha256=vT32qeWq946r8UaU8ZUKtWafpU0kodbUJpQXctFGjjk,8526
 atlas_init/cli_tf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_tf/app.py,sha256=SDIWy1B8cEIA5ieqL8fEhVHwQdK919MjdCSiWgB1M8M,8522
 atlas_init/cli_tf/changelog.py,sha256=biWYKf1pZvXZ-jEgcZ5q9sY7nTGrL2PuI0h9mCILf_g,3181
@@ -82,7 +82,7 @@ atlas_init/settings/env_vars_modules.py,sha256=fqVndOn8ZesuudijakzPZKsrXnm-iezGH
 atlas_init/settings/interactive.py,sha256=Xy1Z5WMAOSaJ-vQI_4xjAbSR92rWQgnffwVoDT27L68,340
 atlas_init/settings/interactive2.py,sha256=Z0-AptV19Eq1GizouOUhD-vA8IWnTejo_urnS7D4gdY,3856
 atlas_init/settings/path.py,sha256=vFRmoUHQplWuKsICu8uHPNAYhsuJu_xoVhmTnii_p7M,2301
-atlas_init/settings/rich_utils.py,sha256=mjYCZ7Sx-OTsjv7AZ4qyjXqvq3TOaczLfDjOQK_dKWk,2012
+atlas_init/settings/rich_utils.py,sha256=MSk-sqj-5B0dMBL4Dvq-B1nursXbxOxKZzqEBRogcQU,2765
 atlas_init/tf/.terraform.lock.hcl,sha256=8petzLKtZy0Yi-Iy5QmkdkkuwRfHnPJSskQhlHROpyo,6880
 atlas_init/tf/always.tf,sha256=WbatrKzdmeV_fWNjO8u3dw96o5nvfB1ciZ4H4E7aSvM,1398
 atlas_init/tf/main.tf,sha256=O-Q72u8iJ9bS8tQPx7VR5dwG925QcyuUCxY92VR7Vng,4623
@@ -124,14 +124,14 @@ atlas_init/tf_ext/__main__.py,sha256=pCZ1j_TIo1lFCBdfrqw-EGta2dEs_oONd_IktW3jlOA
 atlas_init/tf_ext/api_call.py,sha256=vJmCkP-fMLM4qLcFxzbTZUKSxk5Yx1U3tHQZAKFyJ0I,12868
 atlas_init/tf_ext/args.py,sha256=m77eU6DPPD4vFpMWPR7UqH65vBp3AGV7fyXDIe-uORE,1031
 atlas_init/tf_ext/constants.py,sha256=FU8JA-JvbkbtpRykilMCKk7YiREPFpZI0hW_My4kvJE,202
-atlas_init/tf_ext/gen_examples.py,sha256=nN-KhPFs3PnW4PWgEop4fXFTQf1jP6A3pGdKFRWBmCA,4803
-atlas_init/tf_ext/gen_module_readme.py,sha256=04Lu9dYRMnYeKoQrIclpxUCT5YZctNzK71mzhFr_07E,4421
+atlas_init/tf_ext/gen_examples.py,sha256=b_1d7rdLofqmH8d-Q6hjpbhcUXRrkIQYqvj51CeVZLI,4842
+atlas_init/tf_ext/gen_readme.py,sha256=9CjQRJnMHeLJMGfF9k9DvcApk9I64vaB5cKwVPA6gNs,5153
 atlas_init/tf_ext/gen_resource_main.py,sha256=C-aqkOFXH5ffT51igPtNW7szIH-MkY3HnMcSZ0Q4OdE,7263
 atlas_init/tf_ext/gen_resource_output.py,sha256=_LM3mKMVmcqm2uWokvIfMxOnyCD4aCDUNGUp5DujF7E,3518
 atlas_init/tf_ext/gen_resource_variables.py,sha256=hgaHJlgnk1bNCV46TlvUIVZ5PVUQvxxnfo-hKN05SEU,6954
 atlas_init/tf_ext/gen_versions.py,sha256=yYX4jIgQfAZuG2wfu6OsIbDAqt9g8btiLcgKm7DvNqM,469
-atlas_init/tf_ext/models.py,sha256=8tJmLtwzzuNp_waG2eZ9bFmqRx6dJeYivu-ygvJUiLM,3631
-atlas_init/tf_ext/models_module.py,sha256=JpMGSdjgS6D8NlTJNcSIOdq0dDagKyIfSeoFI_-eVMM,18113
+atlas_init/tf_ext/models.py,sha256=vS725jiEc0t61IO1Db0mpxWVQPGaJKIGeZLOb2SOf4I,4430
+atlas_init/tf_ext/models_module.py,sha256=JngaAxN-54hYPffx1rZ-_j-hYT_GELq7lf24e-EQHb0,18276
 atlas_init/tf_ext/newres.py,sha256=quMSLlkJRuvA3attTvJ-DQNSwRPFyT_XJ32ucmEhA-s,3104
 atlas_init/tf_ext/paths.py,sha256=3VQri_VKS5sVKdkHu2XKgHziB1uQpB8hBGBY7GwiAwU,4917
 atlas_init/tf_ext/plan_diffs.py,sha256=Sc-VFrq2k7p01ZZzgtAfqRXCog2h6bbFlonGfaxSzog,5237
@@ -139,16 +139,17 @@ atlas_init/tf_ext/provider_schema.py,sha256=6XrJ3UHjpQ_8yBqWeZ1b4xIPLOqPL9L1ljn4
 atlas_init/tf_ext/py_gen.py,sha256=orZLX2c-tw9eJb8MyTunhkdfqs-f6uOngu5RlLUb9uI,11243
 atlas_init/tf_ext/schema_to_dataclass.py,sha256=kqg0OxqYZwmNGvlayZkXPwU2CueiHsnAAgLyoXkDUtY,19797
 atlas_init/tf_ext/settings.py,sha256=qRSm8e_TquEKoBpYE6VFmXSpiRaHExR00cVykpK1_OI,6814
-atlas_init/tf_ext/tf_dep.py,sha256=mHCfI0LpL1bCru9m2Rh7otGbCexh0lUiZoDAguB1A4g,12229
+atlas_init/tf_ext/tf_dep.py,sha256=UFrNmolw3qW02E7mq23vVGY0C8HOSSY3-6AHmSEqYno,13453
 atlas_init/tf_ext/tf_desc_gen.py,sha256=G6Hv9kKJDNbNL8nRRkPA7w8c7MCDgijjdTv-2BjKrZ0,2575
 atlas_init/tf_ext/tf_desc_update.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atlas_init/tf_ext/tf_mod_gen.py,sha256=wsn9gxu2IKpOEE_KS09e5e-3VT4xjs73cjL_HSWmIo4,12250
+atlas_init/tf_ext/tf_example_readme.py,sha256=A8yBJq9Ts0NzBnurWbbmADrfursjhxfdWX3ZxkYZ5cs,14992
+atlas_init/tf_ext/tf_mod_gen.py,sha256=mOXFfKoghkeV-CCYbqknwII5LmY1iguoUsTjYxndwi4,12141
 atlas_init/tf_ext/tf_mod_gen_provider.py,sha256=M7jJeH0VR1uN440d2H5-9lnj0s0d9QtaMeYsLwWW3Ak,5866
-atlas_init/tf_ext/tf_modules.py,sha256=_JokgP-i1iS-iEXVI0MMNorrBHjFG6XcJRKiUe6JbvY,16200
+atlas_init/tf_ext/tf_modules.py,sha256=h_E8Po1fmOcIjU_d92dCjKZytBij4N9Mux70yLW7xVQ,16248
 atlas_init/tf_ext/tf_vars.py,sha256=A__zcIBNvJ5y0l5F5G7KMQsYCuyr3iBZtRzXm4e6DQU,7875
-atlas_init/tf_ext/typer_app.py,sha256=gGVdttrSzUX2uzTAeXzvCF4BQnMxrkltAAc3_BCsiGc,968
-atlas_init-0.8.1.dist-info/METADATA,sha256=s0HRa9fecuwCA269e3NhIF-DkyTJtPGGK5Ddgfy0Lk0,5863
-atlas_init-0.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atlas_init-0.8.1.dist-info/entry_points.txt,sha256=l38KdfCjY2v5q8Ves1qkWNvPTPND6Tp2EKX-RL-MN3c,200
-atlas_init-0.8.1.dist-info/licenses/LICENSE,sha256=aKnucPyXnK1A-aXn4vac71zRpcB5BXjDyl4PDyi_hZg,1069
-atlas_init-0.8.1.dist-info/RECORD,,
+atlas_init/tf_ext/typer_app.py,sha256=5Zqgs5RMYitlGmPyyXq3WctRcWMRC6RQa7EWJhVeQlA,1063
+atlas_init-0.9.0.dist-info/METADATA,sha256=uOYn0ME4hd1OvjjpfksJWWCC_aV0XNqklFTFbjzygh0,5863
+atlas_init-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atlas_init-0.9.0.dist-info/entry_points.txt,sha256=l38KdfCjY2v5q8Ves1qkWNvPTPND6Tp2EKX-RL-MN3c,200
+atlas_init-0.9.0.dist-info/licenses/LICENSE,sha256=aKnucPyXnK1A-aXn4vac71zRpcB5BXjDyl4PDyi_hZg,1069
+atlas_init-0.9.0.dist-info/RECORD,,